From 3744b19dc13faa3db2c1133e353c3ce7d5322d35 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 18:20:44 +0200 Subject: [PATCH 01/59] feat: run popen Message IO on Trio host threads Move coordinator and worker framed protocol IO into dedicated Trio host threads for local popen + import bootstrap, while keeping the sync Channel/Gateway API and WorkerPool remote_exec. Adds a trio dependency; disable with EXECNET_TRIO_HOST=0. Co-authored-by: Cursor AI Co-authored-by: Cursor Grok 4.5 --- CHANGELOG.rst | 4 + doc/implnotes.rst | 43 ++- pyproject.toml | 3 + src/execnet/_trio_host.py | 527 ++++++++++++++++++++++++++++++++++++ src/execnet/_trio_worker.py | 117 ++++++++ src/execnet/gateway.py | 17 +- src/execnet/gateway_base.py | 43 ++- src/execnet/multi.py | 18 +- uv.lock | 61 +++++ 9 files changed, 818 insertions(+), 15 deletions(-) create mode 100644 src/execnet/_trio_host.py create mode 100644 src/execnet/_trio_worker.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a7af6272..eac3c8d6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,10 @@ ------------------ * `#380 `__: Add support for Python 3.13 and 3.14, and drop EOL 3.8 and 3.9. +* Trio host-thread Message IO for local ``popen`` + import bootstrap (coordinator and + worker). Adds a hard ``trio`` dependency. Disable with ``EXECNET_TRIO_HOST=0``. + Other gateway types and greenlet execmodels keep the legacy thread path. + 2.1.2 (2025-11-11) ------------------ diff --git a/doc/implnotes.rst b/doc/implnotes.rst index d13d9e87..3dae82d6 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -7,7 +7,7 @@ capable of receiving and executing code, and routing data through channels. Gateways operate on InputOutput objects offering -a write and a read(n) method. + a write and a read(n) method. Once bootstrapped a higher level protocol based on Messages is used. Messages are serialized @@ -15,18 +15,43 @@ to and from InputOutput objects. The details of this protocol are locally defined in this module. There is no need for standardizing or versioning the protocol. -After bootstrapping the BaseGateway opens a receiver thread which +Trio host-thread IO (popen / import bootstrap) +---------------------------------------------- + +For local ``popen`` gateways that use import-based bootstrap +(same installed ``execnet`` + ``trio`` on both sides), Message +protocol IO runs inside a dedicated OS thread hosting a Trio +event loop (``execnet._trio_host.TrioHost``): + +* Coordinator: ``trio.lowlevel.open_process`` plus async framed + reader/writer tasks per gateway (one host thread per ``Group``). +* Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio + streams; ``remote_exec`` still runs on the existing WorkerPool + (including ``main_thread_only`` primary-thread integration). + +Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from +non-host threads wait until the frame is written (so abrupt +``os._exit`` cannot drop queued data). Sends from the Trio host +thread (receiver callbacks) only enqueue, to avoid deadlocking +the writer task. + +Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types +(``ssh``, ``socket``, ``via``, ``python=…``, greenlet execmodels) +still use the legacy thread receiver and sync ``Popen`` path. + +Legacy thread model +------------------- + +After bootstrapping, ``BaseGateway`` opens a receiver thread which accepts encoded messages and triggers actions to interpret them. Sending of channel data items happens directly through write operations to InputOutput objects so there is no -separate thread. +separate send thread. -Code execution messages are put into an execqueue from -which they will be taken for execution. gateway.serve() -will take and execute such items, one by one. This means -that by incoming default execution is single-threaded. +Code execution messages are scheduled on a WorkerPool. +On the worker, ``serve()`` integrates the main thread as the +primary executor when using the ``thread`` / ``main_thread_only`` +models. The receiver thread terminates if the remote side sends a gateway termination message or if the IO-connection drops. -It puts an end symbol into the execqueue so -that serve() can cleanly finish as well. diff --git a/pyproject.toml b/pyproject.toml index fcd2b4b5..709f887b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,9 @@ description = "execnet: rapid multi-Python deployment" readme = {"file" = "README.rst", "content-type" = "text/x-rst"} license = "MIT" requires-python = ">=3.10" +dependencies = [ + "trio>=0.32", +] authors = [ { name = "holger krekel and others" }, ] diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py new file mode 100644 index 00000000..6ece94c4 --- /dev/null +++ b/src/execnet/_trio_host.py @@ -0,0 +1,527 @@ +"""Trio host thread for execnet Message-protocol IO. + +Coordinator and worker both run framed read/write loops here. +Sync Channel/Gateway APIs talk to this host via thread-safe queues and +``trio.from_thread``. +""" + +from __future__ import annotations + +import os +import queue +import subprocess +import threading +from collections.abc import Callable +from typing import TYPE_CHECKING +from typing import Any +from typing import Protocol +from typing import TypeVar + +import trio + +from .gateway_base import ExecModel +from .gateway_base import GatewayReceivedTerminate +from .gateway_base import Message +from .gateway_base import trace + +if TYPE_CHECKING: + from .gateway_base import BaseGateway + +T = TypeVar("T") + +_CLOSE_WRITE = object() +_ENABLED_ENV = "EXECNET_TRIO_HOST" + + +def trio_host_enabled() -> bool: + """Return whether the Trio IO path should be used when applicable.""" + value = os.environ.get(_ENABLED_ENV, "1").strip().lower() + return value not in ("0", "false", "no", "off") + + +def should_use_trio_popen(spec: Any) -> bool: + """PoC: Trio path only for local popen with import bootstrap.""" + if not trio_host_enabled(): + return False + if not getattr(spec, "popen", False): + return False + if getattr(spec, "via", None): + return False + if getattr(spec, "python", None): + return False + # Worker exec still uses WorkerPool; greenlet models stay on the legacy path. + execmodel = getattr(spec, "execmodel", None) + if execmodel not in (None, "thread", "main_thread_only"): + return False + return True + + +class AsyncByteIO(Protocol): + async def read_exact(self, n: int) -> bytes: ... + + async def write_all(self, data: bytes) -> None: ... + + async def aclose_read(self) -> None: ... + + async def aclose_write(self) -> None: ... + + +async def read_exact_receive_stream(stream: trio.abc.ReceiveStream, n: int) -> bytes: + buf = bytearray() + while len(buf) < n: + chunk = await stream.receive_some(n - len(buf)) + if not chunk: + raise EOFError("expected %d bytes, got %d" % (n, len(buf))) + buf += chunk + return bytes(buf) + + +async def read_message(io: AsyncByteIO) -> Message: + header = await io.read_exact(9) + msgtype, channel, payload = Message.from_header(header) + data = await io.read_exact(payload) if payload else b"" + return Message.from_parts(msgtype, channel, data) + + +class ProcessStreamsIO: + """Async IO over a Trio Process stdin/stdout pair.""" + + def __init__(self, process: trio.Process) -> None: + assert process.stdin is not None + assert process.stdout is not None + self.process = process + self._stdin = process.stdin + self._stdout = process.stdout + + async def read_exact(self, n: int) -> bytes: + return await read_exact_receive_stream(self._stdout, n) + + async def write_all(self, data: bytes) -> None: + await self._stdin.send_all(data) + + async def aclose_read(self) -> None: + await self._stdout.aclose() + + async def aclose_write(self) -> None: + await self._stdin.aclose() + + +class FdStreamsIO: + """Async IO over OS file descriptors (worker stdio pipes).""" + + def __init__(self, read_fd: int, write_fd: int) -> None: + self._read = trio.lowlevel.FdStream(read_fd) + self._write = trio.lowlevel.FdStream(write_fd) + + async def read_exact(self, n: int) -> bytes: + return await read_exact_receive_stream(self._read, n) + + async def write_all(self, data: bytes) -> None: + await self._write.send_all(data) + + async def aclose_read(self) -> None: + await self._read.aclose() + + async def aclose_write(self) -> None: + await self._write.aclose() + + +class SyncIOHandle: + """Sync IO facade for Group.terminate wait/kill/close_write.""" + + remoteaddress: str + + def __init__( + self, + execmodel: ExecModel, + session: ProtocolSession, + *, + remoteaddress: str | None = None, + ) -> None: + self.execmodel = execmodel + self._session = session + if remoteaddress is not None: + self.remoteaddress = remoteaddress + + def read(self, numbytes: int) -> bytes: + raise RuntimeError("sync read not supported on Trio IO handle") + + def write(self, data: bytes) -> None: + raise RuntimeError("sync write not supported on Trio IO handle") + + def close_read(self) -> None: + self._session.request_close_read() + + def close_write(self) -> None: + self._session.request_close_write() + + def wait(self) -> int | None: + return self._session.wait_process() + + def kill(self) -> None: + self._session.kill_process() + + +class ProtocolSession: + """Reader/writer tasks for one gateway connection.""" + + def __init__( + self, + gateway: BaseGateway, + io: AsyncByteIO, + *, + process: trio.Process | None = None, + host: TrioHost, + ) -> None: + self.gateway = gateway + self.io = io + self.process = process + self.host = host + self._outbound: queue.SimpleQueue[object] = queue.SimpleQueue() + self._done = threading.Event() + self._process_exitcode: int | None = None + self._process_done = threading.Event() + self._send_closed = False + self._lock = threading.Lock() + + def enqueue_message(self, message: Message) -> None: + """Enqueue a frame; wait until written when safe to block. + + Non-host threads wait for the OS write so abrupt ``os._exit`` (xdist + crash tests) cannot drop already-"sent" data still in the queue. + + The Trio host thread (receiver callbacks) must not wait — that would + deadlock the writer task on the same event loop. + """ + wait = not self.host.is_host_thread() + done = threading.Event() if wait else None + errors: list[BaseException] = [] + with self._lock: + if self._send_closed or self._done.is_set(): + raise OSError("cannot send (already closed?)") + self._outbound.put((message.pack(), done, errors)) + if done is None: + return + if not done.wait(timeout=120.0): + raise OSError("cannot send (write timed out)") + if errors: + raise OSError("cannot send (already closed?)") from errors[0] + + def request_close_write(self) -> None: + with self._lock: + if self._send_closed: + return + self._send_closed = True + self._outbound.put(_CLOSE_WRITE) + + def request_close_read(self) -> None: + # Reader observes EOF / cancel; nothing required from callers. + return + + def wait_done(self, timeout: float | None = None) -> bool: + return self._done.wait(timeout) + + def wait_process(self) -> int | None: + if self.process is None: + return None + + async def _wait() -> int | None: + assert self.process is not None + # Always await wait() so the child is reaped (no zombies). + code = await self.process.wait() + self._process_exitcode = code + self._process_done.set() + return code + + try: + return self.host.call(_wait) + except Exception: + self._process_done.wait() + return self._process_exitcode + + def kill_process(self) -> None: + if self.process is None: + return + + async def _kill() -> None: + assert self.process is not None + with trio.move_on_after(5): + self.process.kill() + self._process_exitcode = await self.process.wait() + self._process_done.set() + + try: + self.host.call(_kill) + except Exception as exc: + trace("ERROR killing trio process:", exc) + + def is_alive(self) -> bool: + return not self._done.is_set() + + async def task(self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED) -> None: + try: + async with trio.open_nursery() as nursery: + nursery.start_soon(self._writer) + if self.process is not None: + nursery.start_soon(self._supervisor) + task_status.started() + await self._reader() + nursery.cancel_scope.cancel() + finally: + await self._finish() + + async def _reader(self) -> None: + gateway = self.gateway + + def log(*msg: object) -> None: + gateway._trace("[trio-receiver]", *msg) + + log("RECEIVER: starting") + try: + while True: + msg = await read_message(self.io) + log("received", msg) + with gateway._receivelock: + msg.received(gateway) + del msg + except GatewayReceivedTerminate: + log("GATEWAY_TERMINATE") + except EOFError as exc: + log("EOF without prior gateway termination message") + gateway._error = exc + except Exception as exc: + log(gateway._geterrortext(exc)) + log("finishing receiver") + + async def _writer(self) -> None: + while True: + # abandon_on_cancel: queue.get blocks forever until a sentinel; + # nursery shutdown must not wait on it. + item = await trio.to_thread.run_sync( + self._outbound.get, abandon_on_cancel=True + ) + if item is _CLOSE_WRITE: + try: + await self.io.aclose_write() + except Exception as exc: + self.gateway._trace("aclose_write failed", exc) + return + assert isinstance(item, tuple) + blob, done, errors = item + assert isinstance(blob, bytes) + try: + await self.io.write_all(blob) + except Exception as exc: + self.gateway._trace("write failed", exc) + errors.append(exc) + with self._lock: + self._send_closed = True + finally: + if done is not None: + done.set() + + async def _supervisor(self) -> None: + assert self.process is not None + try: + self._process_exitcode = await self.process.wait() + finally: + self._process_done.set() + + async def _finish(self) -> None: + gateway = self.gateway + # Unblock a writer thread left in queue.get after cancel. + with self._lock: + self._send_closed = True + self._outbound.put(_CLOSE_WRITE) + gateway._trace("[trio-receiver] finishing channels") + gateway._channelfactory._finished_receiving() + # Unblock WorkerGateway.serve()/join before heavy exec-pool shutdown + # so the primary thread is not waiting on _done while terminate waits + # on the primary thread draining work. + self._done.set() + gateway._trace("[trio-receiver] terminating execution") + # May sleep/SIGINT; keep it off the Trio scheduling thread. + await trio.to_thread.run_sync( + gateway._terminate_execution, abandon_on_cancel=True + ) + try: + await self.io.aclose_read() + except Exception: + pass + try: + await self.io.aclose_write() + except Exception: + pass + + +class TrioHost: + """Dedicated OS thread running ``trio.run`` for protocol IO.""" + + def __init__(self, name: str = "execnet-trio-host") -> None: + self._name = name + self._thread: threading.Thread | None = None + self._token: trio.lowlevel.TrioToken | None = None + self._nursery: trio.Nursery | None = None + self._ready = threading.Event() + self._shutdown: trio.Event | None = None + self._started = False + + def start(self) -> None: + if self._started: + return + self._thread = threading.Thread(target=self._run, name=self._name, daemon=True) + self._thread.start() + if not self._ready.wait(timeout=30): + raise RuntimeError("TrioHost failed to start") + self._started = True + + def is_host_thread(self) -> bool: + return self._thread is not None and threading.current_thread() is self._thread + + def _run(self) -> None: + trio.run(self._main) + + async def _main(self) -> None: + self._token = trio.lowlevel.current_trio_token() + self._shutdown = trio.Event() + try: + async with trio.open_nursery() as nursery: + self._nursery = nursery + self._ready.set() + await self._shutdown.wait() + nursery.cancel_scope.cancel() + finally: + self._nursery = None + + def call(self, async_fn: Callable[..., Any], *args: Any) -> Any: + if self._token is None: + raise RuntimeError("TrioHost is not running") + return trio.from_thread.run(async_fn, *args, trio_token=self._token) + + def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: + if self._token is None: + raise RuntimeError("TrioHost is not running") + return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + + async def start_session( + self, + gateway: BaseGateway, + io: AsyncByteIO, + *, + process: trio.Process | None = None, + ) -> ProtocolSession: + if self._nursery is None: + raise RuntimeError("TrioHost nursery is not available") + session = ProtocolSession(gateway, io, process=process, host=self) + await self._nursery.start(session.task) + return session + + def stop(self, timeout: float | None = 5.0) -> None: + if not self._started or self._token is None or self._shutdown is None: + return + + def _set() -> None: + assert self._shutdown is not None + self._shutdown.set() + + try: + trio.from_thread.run_sync(_set, trio_token=self._token) + except Exception: + pass + if self._thread is not None: + self._thread.join(timeout=timeout) + self._started = False + + +async def open_popen_process(args: list[str]) -> trio.Process: + return await trio.lowlevel.open_process( + args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + +async def bootstrap_import_async( + process: trio.Process, + *, + importdir: str, + execmodel: str, + gateway_id: str, +) -> ProcessStreamsIO: + """Send import-bootstrap source and wait for the handshake byte.""" + io = ProcessStreamsIO(process) + sources = [ + "import sys", + "if %r not in sys.path:" % importdir, + " sys.path.insert(0, %r)" % importdir, + "from execnet._trio_worker import serve_popen_trio", + "sys.stdout.write('1')", + "sys.stdout.flush()", + "serve_popen_trio(id=%r, execmodel=%r)" % (f"{gateway_id}-worker", execmodel), + ] + source = "\n".join(sources) + await io.write_all((repr(source) + "\n").encode("utf-8")) + ack = await io.read_exact(1) + if ack != b"1": + raise EOFError(f"bad bootstrap handshake: {ack!r}") + return io + + +class _TempIO: + """Placeholder IO used only while constructing a Trio-backed Gateway.""" + + def __init__(self, execmodel: ExecModel) -> None: + self.execmodel = execmodel + + def read(self, numbytes: int) -> bytes: + raise RuntimeError("sync read not supported on Trio temp IO") + + def write(self, data: bytes) -> None: + raise RuntimeError("sync write not supported on Trio temp IO") + + def close_read(self) -> None: + return + + def close_write(self) -> None: + return + + def wait(self) -> int | None: + return None + + def kill(self) -> None: + return + + +def makegateway_popen_trio(group: Any, spec: Any) -> Any: + """Create a popen Gateway using Trio process + protocol IO on both sides.""" + import execnet + + from . import gateway_io + from .gateway_bootstrap import importdir + + host: TrioHost = group._ensure_trio_host() + args = gateway_io.popen_args(spec) + remote_execmodel = spec.execmodel + + async def _create_and_attach() -> Any: + process = await open_popen_process(args) + try: + async_io = await bootstrap_import_async( + process, + importdir=importdir, + execmodel=remote_execmodel, + gateway_id=spec.id, + ) + except BaseException: + with trio.move_on_after(5): + process.kill() + await process.wait() + raise + + gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + session = await host.start_session(gw, async_io, process=process) + gw._attach_trio_session(session) + gw._io = SyncIOHandle(group.execmodel, session) + return gw + + return host.call(_create_and_attach) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py new file mode 100644 index 00000000..70338f1c --- /dev/null +++ b/src/execnet/_trio_worker.py @@ -0,0 +1,117 @@ +"""Worker-side Trio networking entry for popen/import bootstrap.""" + +from __future__ import annotations + +import os +import sys + +from . import gateway_base +from .gateway_base import WorkerGateway +from .gateway_base import get_execmodel +from .gateway_base import trace + + +def _prepare_protocol_fds() -> tuple[int, int]: + """Dup protocol pipes off stdin/stdout and redirect stdio to /dev/null. + + Returns ``(read_fd, write_fd)`` for the Message protocol (child reads + coordinator stdin writes; child writes go to coordinator stdout reads). + """ + if not hasattr(os, "dup"): # pragma: no cover - jython legacy + raise RuntimeError("Trio worker requires os.dup") + + try: + devnull = os.devnull + except AttributeError: + devnull = "NUL" if os.name == "nt" else "/dev/null" + + # Protocol read end: former stdin (fed by coordinator stdout write / our stdin) + read_fd = os.dup(0) + fd = os.open(devnull, os.O_RDONLY) + os.dup2(fd, 0) + os.close(fd) + + # Protocol write end: former stdout + write_fd = os.dup(1) + fd = os.open(devnull, os.O_WRONLY) + os.dup2(fd, 1) + + if os.name == "nt": + # Match init_popen_io: keep a stderr handle then point fd 2 at null. + sys.stderr = os.fdopen(os.dup(2), "w", 1) + os.dup2(fd, 2) + os.close(fd) + + # Replace sys.stdin/out with the null fds (closefd=False). + sys.stdin = os.fdopen(0, "r", 1, closefd=False) + sys.stdout = os.fdopen(1, "w", 1, closefd=False) + return read_fd, write_fd + + +class _WorkerIOStub: + """Minimal IO stub so WorkerGateway can be constructed without sync pipes.""" + + def __init__(self, execmodel: gateway_base.ExecModel) -> None: + self.execmodel = execmodel + + def read(self, numbytes: int) -> bytes: + raise RuntimeError("sync read not used on Trio worker") + + def write(self, data: bytes) -> None: + raise RuntimeError("sync write not used on Trio worker") + + def close_read(self) -> None: + return + + def close_write(self) -> None: + return + + def wait(self) -> int | None: + return None + + def kill(self) -> None: + return + + +def serve_popen_trio(id: str, execmodel: str = "thread") -> None: + """Serve a WorkerGateway with Message IO on a Trio host thread.""" + from . import _trio_host + + model = get_execmodel(execmodel) + read_fd, write_fd = _prepare_protocol_fds() + # Keep the historic trace token so tests looking for workergateway still pass. + trace(f"creating workergateway on trio id={id!r}") + + host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") + host.start() + try: + io_stub = _WorkerIOStub(model) + gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) + + hasprimary = model.backend in ("thread", "main_thread_only") + gateway._execpool = gateway_base.WorkerPool(model, hasprimary=hasprimary) + gateway._executetask_complete = None + if model.backend == "main_thread_only": + gateway._executetask_complete = model.Event() + gateway._executetask_complete.set() + + async def _start() -> _trio_host.ProtocolSession: + async_io = _trio_host.FdStreamsIO(read_fd, write_fd) + return await host.start_session(gateway, async_io) + + session = host.call(_start) + gateway._attach_trio_session(session) + + try: + if hasprimary: + trace("integrating as primary thread (trio worker)") + gateway._execpool.integrate_as_primary_thread() + gateway.join() + except KeyboardInterrupt: + # Match WorkerGateway.serve(): swallow in the worker. + trace("swallowing keyboardinterrupt, serve finished") + finally: + host.stop(timeout=5.0) + # Trio's to_thread cache uses non-daemon threads that would otherwise + # keep this disposable worker process alive after serve returns. + os._exit(0) diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 1d3eb59d..360b9865 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -26,11 +26,21 @@ class Gateway(gateway_base.BaseGateway): _group: Group - def __init__(self, io: IO, spec: XSpec) -> None: + def __init__( + self, + io: IO, + spec: XSpec, + *, + trio_session: object | None = None, + defer_receive: bool = False, + ) -> None: """:private:""" super().__init__(io=io, id=spec.id, _startcount=1) self.spec = spec - self._initreceive() + if trio_session is not None: + self._attach_trio_session(trio_session) + elif not defer_receive: + self._initreceive() @property def remoteaddress(self) -> str: @@ -91,6 +101,9 @@ def _rinfo(self, update: bool = False) -> RInfo: def hasreceiver(self) -> bool: """Whether gateway is able to receive data.""" + session = self._trio_session + if session is not None: + return session.is_alive() return self._receivepool.active_count() > 0 def remote_status(self) -> RemoteStatus: diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 73dc7175..0829a33d 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -563,6 +563,23 @@ def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: self.channelid = channelid self.data = data + def pack(self) -> bytes: + """Return the full wire frame (9-byte header + payload).""" + header = struct.pack("!bii", self.msgcode, self.channelid, len(self.data)) + return header + self.data + + @staticmethod + def from_header(header: bytes) -> tuple[int, int, int]: + """Unpack a 9-byte header into (msgtype, channelid, payload_len).""" + if len(header) != 9: + raise EOFError("couldn't load message header, short read") + msgtype, channel, payload = struct.unpack("!bii", header) + return msgtype, channel, payload + + @staticmethod + def from_parts(msgtype: int, channel: int, data: bytes) -> Message: + return Message(msgtype, channel, data) + @staticmethod def from_io(io: ReadIO) -> Message: try: @@ -571,12 +588,11 @@ def from_io(io: ReadIO) -> Message: raise EOFError("empty read") except EOFError as e: raise EOFError("couldn't load message header, " + e.args[0]) from None - msgtype, channel, payload = struct.unpack("!bii", header) + msgtype, channel, payload = Message.from_header(header) return Message(msgtype, channel, io.read(payload)) def to_io(self, io: WriteIO) -> None: - header = struct.pack("!bii", self.msgcode, self.channelid, len(self.data)) - io.write(header + self.data) + io.write(self.pack()) def received(self, gateway: BaseGateway) -> None: handler = self._types[self.msgcode][1] @@ -1125,6 +1141,7 @@ def readline(self) -> str: class BaseGateway: _sysex = sysex id = "" + _trio_session: Any = None def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel @@ -1137,11 +1154,18 @@ def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.__trace = trace self._geterrortext = geterrortext self._receivepool = WorkerPool(self.execmodel) + self._trio_session = None def _trace(self, *msg: object) -> None: self.__trace(self.id, *msg) + def _attach_trio_session(self, session: Any) -> None: + """Attach a Trio ProtocolSession for Message IO (no receiver thread).""" + self._trio_session = session + def _initreceive(self) -> None: + if self._trio_session is not None: + return self._receivepool.spawn(self._thread_receiver) def _thread_receiver(self) -> None: @@ -1181,6 +1205,15 @@ def _terminate_execution(self) -> None: def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: message = Message(msgcode, channelid, data) + session = self._trio_session + if session is not None: + try: + session.enqueue_message(message) + self._trace("sent", message) + except (OSError, ValueError) as e: + self._trace("failed to send", message, e) + raise OSError("cannot send (already closed?)") from e + return try: message.to_io(self._io) self._trace("sent", message) @@ -1204,6 +1237,10 @@ def newchannel(self) -> Channel: def join(self, timeout: float | None = None) -> None: """Wait for receiverthread to terminate.""" self._trace("waiting for receiver thread to finish") + session = self._trio_session + if session is not None: + session.wait_done(timeout) + return self._receivepool.waitall(timeout) diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 4dbf8b89..8079f4c4 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -52,6 +52,7 @@ def __init__( self._autoidcounter = 0 self._autoidlock = Lock() self._gateways_to_join: list[Gateway] = [] + self._trio_host: Any = None # we use the same execmodel for all of the Gateway objects # we spawn on our side. Probably we should not allow different # execmodels between different groups but not clear. @@ -62,6 +63,14 @@ def __init__( self.makegateway(xspec) atexit.register(self._cleanup_atexit) + def _ensure_trio_host(self) -> Any: + if self._trio_host is None: + from . import _trio_host + + self._trio_host = _trio_host.TrioHost(name="execnet-trio-group") + self._trio_host.start() + return self._trio_host + @property def execmodel(self) -> ExecModel: return self._execmodel @@ -143,7 +152,11 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: self.allocate_id(spec) if spec.execmodel is None: spec.execmodel = self.remote_execmodel.backend - if spec.via: + from . import _trio_host + + if _trio_host.should_use_trio_popen(spec): + gw = _trio_host.makegateway_popen_trio(self, spec) + elif spec.via: assert not spec.socket master = self[spec.via] proxy_channel = master.remote_exec(gateway_io) @@ -207,6 +220,9 @@ def _unregister(self, gateway: Gateway) -> None: def _cleanup_atexit(self) -> None: trace(f"=== atexit cleanup {self!r} ===") self.terminate(timeout=1.0) + if self._trio_host is not None: + self._trio_host.stop(timeout=1.0) + self._trio_host = None def terminate(self, timeout: float | None = None) -> None: """Trigger exit of member gateways and wait for termination diff --git a/uv.lock b/uv.lock index 893f1543..fc2da46b 100644 --- a/uv.lock +++ b/uv.lock @@ -16,6 +16,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "backports-tarfile" version = "1.2.0" @@ -333,6 +342,9 @@ wheels = [ [[package]] name = "execnet" source = { editable = "." } +dependencies = [ + { name = "trio" }, +] [package.optional-dependencies] testing = [ @@ -358,6 +370,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, { name = "pytest-timeout", marker = "extra == 'testing'" }, { name = "tox", marker = "extra == 'testing'" }, + { name = "trio", specifier = ">=0.32" }, { name = "uv", marker = "extra == 'testing'" }, ] provides-extras = ["testing"] @@ -723,6 +736,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" }, ] +[[package]] +name = "outcome" +version = "1.3.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/df/77698abfac98571e65ffeb0c1fba8ffd692ab8458d617a0eed7d9a8d38f2/outcome-1.3.0.post0.tar.gz", hash = "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8", size = 21060, upload-time = "2023-10-26T04:26:04.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/8b/5ab7257531a5d830fc8000c476e63c935488d74609b50f9384a643ec0a62/outcome-1.3.0.post0-py2.py3-none-any.whl", hash = "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b", size = 10692, upload-time = "2023-10-26T04:26:02.532Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -977,6 +1002,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "tomli" version = "2.4.1" @@ -1072,6 +1115,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6b/3d/7ba55871e9d794d40b6c8424f2e5d1b267ea5d9a4bd2175e08b57960ba13/tox-4.58.0-py3-none-any.whl", hash = "sha256:dcae21f5f015f3a67658e35644cce0d1aa0dedcd06f3927f95d84e1717f6cea5", size = 223298, upload-time = "2026-07-21T13:10:34.731Z" }, ] +[[package]] +name = "trio" +version = "0.33.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "cffi", marker = "implementation_name != 'pypy' and os_name == 'nt'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "outcome" }, + { name = "sniffio" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/b6/c744031c6f89b18b3f5f4f7338603ab381d740a7f45938c4607b2302481f/trio-0.33.0.tar.gz", hash = "sha256:a29b92b73f09d4b48ed249acd91073281a7f1063f09caba5dc70465b5c7aa970", size = 605109, upload-time = "2026-02-14T18:40:55.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/93/dab25dc87ac48da0fe0f6419e07d0bfd98799bed4e05e7b9e0f85a1a4b4b/trio-0.33.0-py3-none-any.whl", hash = "sha256:3bd5d87f781d9b0192d592aef28691f8951d6c2e41b7e1da4c25cde6c180ae9b", size = 510294, upload-time = "2026-02-14T18:40:53.313Z" }, +] + [[package]] name = "trove-classifiers" version = "2025.5.9.12" From beb36b92d05bab211017e5324519e05eb37faba7 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 18:26:19 +0200 Subject: [PATCH 02/59] feat: schedule worker remote_exec from the Trio nursery Replace WorkerPool on the Trio popen worker with TrioWorkerExec: thread-model tasks run via trio.to_thread, main_thread_only hands off to the process main thread. Also wake the protocol writer with a Trio Event instead of blocking to_thread queue.get. Co-authored-by: Cursor AI Co-authored-by: Cursor Grok 4.5 --- doc/implnotes.rst | 5 +- src/execnet/_trio_host.py | 81 +++++++++++++----- src/execnet/_trio_worker.py | 159 +++++++++++++++++++++++++++++++++--- src/execnet/gateway_base.py | 7 ++ 4 files changed, 217 insertions(+), 35 deletions(-) diff --git a/doc/implnotes.rst b/doc/implnotes.rst index 3dae82d6..c8229865 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -26,8 +26,9 @@ event loop (``execnet._trio_host.TrioHost``): * Coordinator: ``trio.lowlevel.open_process`` plus async framed reader/writer tasks per gateway (one host thread per ``Group``). * Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio - streams; ``remote_exec`` still runs on the existing WorkerPool - (including ``main_thread_only`` primary-thread integration). + streams; ``remote_exec`` is scheduled from the Trio nursery + (``trio.to_thread`` for ``thread``, main-thread handoff for + ``main_thread_only``). Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from non-host threads wait until the frame is written (so abrupt diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 6ece94c4..b0db9ad7 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -178,6 +178,7 @@ def __init__( self.process = process self.host = host self._outbound: queue.SimpleQueue[object] = queue.SimpleQueue() + self._wake: trio.Event | None = None self._done = threading.Event() self._process_exitcode: int | None = None self._process_done = threading.Event() @@ -200,6 +201,7 @@ def enqueue_message(self, message: Message) -> None: if self._send_closed or self._done.is_set(): raise OSError("cannot send (already closed?)") self._outbound.put((message.pack(), done, errors)) + self._wake_writer() if done is None: return if not done.wait(timeout=120.0): @@ -207,12 +209,25 @@ def enqueue_message(self, message: Message) -> None: if errors: raise OSError("cannot send (already closed?)") from errors[0] + def _wake_writer(self) -> None: + wake = self._wake + if wake is None: + return + if self.host.is_host_thread(): + wake.set() + else: + try: + self.host.call_sync(wake.set) + except Exception: + pass + def request_close_write(self) -> None: with self._lock: if self._send_closed: return self._send_closed = True self._outbound.put(_CLOSE_WRITE) + self._wake_writer() def request_close_read(self) -> None: # Reader observes EOF / cancel; nothing required from callers. @@ -294,31 +309,47 @@ def log(*msg: object) -> None: log("finishing receiver") async def _writer(self) -> None: + self._wake = trio.Event() while True: - # abandon_on_cancel: queue.get blocks forever until a sentinel; - # nursery shutdown must not wait on it. - item = await trio.to_thread.run_sync( - self._outbound.get, abandon_on_cancel=True - ) - if item is _CLOSE_WRITE: + while True: try: - await self.io.aclose_write() - except Exception as exc: - self.gateway._trace("aclose_write failed", exc) + item = self._outbound.get_nowait() + except queue.Empty: + break + if not await self._writer_handle_item(item): + return + # Reset wake before re-check to avoid losing a notification. + self._wake = trio.Event() + try: + item = self._outbound.get_nowait() + except queue.Empty: + await self._wake.wait() + continue + if not await self._writer_handle_item(item): return - assert isinstance(item, tuple) - blob, done, errors = item - assert isinstance(blob, bytes) + + async def _writer_handle_item(self, item: object) -> bool: + """Handle one outbound queue item. Return False when writer should stop.""" + if item is _CLOSE_WRITE: try: - await self.io.write_all(blob) + await self.io.aclose_write() except Exception as exc: - self.gateway._trace("write failed", exc) - errors.append(exc) - with self._lock: - self._send_closed = True - finally: - if done is not None: - done.set() + self.gateway._trace("aclose_write failed", exc) + return False + assert isinstance(item, tuple) + blob, done, errors = item + assert isinstance(blob, bytes) + try: + await self.io.write_all(blob) + except Exception as exc: + self.gateway._trace("write failed", exc) + errors.append(exc) + with self._lock: + self._send_closed = True + finally: + if done is not None: + done.set() + return True async def _supervisor(self) -> None: assert self.process is not None @@ -329,10 +360,10 @@ async def _supervisor(self) -> None: async def _finish(self) -> None: gateway = self.gateway - # Unblock a writer thread left in queue.get after cancel. with self._lock: self._send_closed = True self._outbound.put(_CLOSE_WRITE) + self._wake_writer() gateway._trace("[trio-receiver] finishing channels") gateway._channelfactory._finished_receiving() # Unblock WorkerGateway.serve()/join before heavy exec-pool shutdown @@ -403,6 +434,14 @@ def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: raise RuntimeError("TrioHost is not running") return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: + """Schedule a task on the root nursery (must be called on the host thread).""" + if not self.is_host_thread(): + raise RuntimeError("start_soon requires the Trio host thread") + if self._nursery is None: + raise RuntimeError("TrioHost nursery is not available") + self._nursery.start_soon(async_fn, *args) + async def start_session( self, gateway: BaseGateway, diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 70338f1c..6d247194 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -1,15 +1,151 @@ -"""Worker-side Trio networking entry for popen/import bootstrap.""" +"""Worker-side Trio networking and exec scheduling for popen/import bootstrap.""" from __future__ import annotations import os +import queue import sys +import threading +from typing import TYPE_CHECKING +from typing import Any + +import trio from . import gateway_base +from .gateway_base import MAIN_THREAD_ONLY_DEADLOCK_TEXT from .gateway_base import WorkerGateway from .gateway_base import get_execmodel +from .gateway_base import loads_internal from .gateway_base import trace +if TYPE_CHECKING: + from . import _trio_host + from .gateway_base import Channel + from .gateway_base import ExecModel + +ExecItem = tuple[Any, ...] + + +class TrioWorkerExec: + """Schedule ``remote_exec`` work from the Trio host nursery. + + * ``thread``: run ``executetask`` via ``trio.to_thread`` (concurrent). + * ``main_thread_only``: hand off to the process main thread (GUI-safe). + """ + + def __init__( + self, + host: _trio_host.TrioHost, + gateway: WorkerGateway, + *, + main_thread_only: bool, + ) -> None: + self.host = host + self.gateway = gateway + self.main_thread_only = main_thread_only + self._lock = threading.Lock() + self._running = 0 + self._shutting_down = False + self._idle = threading.Event() + self._idle.set() + self._primary_q: queue.SimpleQueue[ + tuple[Channel, ExecItem, threading.Event] | None + ] = queue.SimpleQueue() + self._primary_wake = threading.Event() + # Serialize main_thread_only admission (wait+clear) so two tasks cannot + # both observe the idle Event before either clears it. + self._admit_lock = trio.Lock() + + def active_count(self) -> int: + with self._lock: + return self._running + + def _track_start(self) -> None: + with self._lock: + self._running += 1 + self._idle.clear() + + def _track_finish(self) -> None: + with self._lock: + self._running -= 1 + if self._running == 0: + self._idle.set() + + def schedule(self, channel: Channel, sourcetask: bytes) -> None: + """Called from the Trio receiver while holding ``_receivelock``. + + Must not block: deadlock checks and exec run in a nursery task. + """ + item = loads_internal(sourcetask) + assert isinstance(item, tuple) + with self._lock: + if self._shutting_down: + channel.close("execution disallowed") + return + # Already on the Trio host thread (Message handler). + self.host.start_soon(self._run_exec, channel, item) + + async def _run_exec(self, channel: Channel, item: ExecItem) -> None: + if self.main_thread_only: + complete = self.gateway._executetask_complete + assert complete is not None + + def _wait_slot() -> bool: + return complete.wait(timeout=1) + + async with self._admit_lock: + if not await trio.to_thread.run_sync( + _wait_slot, abandon_on_cancel=True + ): + channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) + return + complete.clear() + + self._track_start() + try: + if self.main_thread_only: + done = threading.Event() + self._primary_q.put((channel, item, done)) + self._primary_wake.set() + await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) + else: + await trio.to_thread.run_sync( + self.gateway.executetask, + (channel, item), + abandon_on_cancel=True, + ) + finally: + self._track_finish() + + def integrate_as_primary_thread(self) -> None: + """Block the main thread running main_thread_only exec tasks.""" + while True: + self._primary_wake.wait() + try: + task = self._primary_q.get_nowait() + except queue.Empty: + self._primary_wake.clear() + try: + task = self._primary_q.get_nowait() + except queue.Empty: + continue + if task is None: + break + channel, item, done = task + try: + self.gateway.executetask((channel, item)) + finally: + done.set() + + def trigger_shutdown(self) -> None: + with self._lock: + self._shutting_down = True + self._primary_q.put(None) + self._primary_wake.set() + + def waitall(self, timeout: float | None = None) -> bool: + return self._idle.wait(timeout) + def _prepare_protocol_fds() -> tuple[int, int]: """Dup protocol pipes off stdin/stdout and redirect stdio to /dev/null. @@ -25,24 +161,20 @@ def _prepare_protocol_fds() -> tuple[int, int]: except AttributeError: devnull = "NUL" if os.name == "nt" else "/dev/null" - # Protocol read end: former stdin (fed by coordinator stdout write / our stdin) read_fd = os.dup(0) fd = os.open(devnull, os.O_RDONLY) os.dup2(fd, 0) os.close(fd) - # Protocol write end: former stdout write_fd = os.dup(1) fd = os.open(devnull, os.O_WRONLY) os.dup2(fd, 1) if os.name == "nt": - # Match init_popen_io: keep a stderr handle then point fd 2 at null. sys.stderr = os.fdopen(os.dup(2), "w", 1) os.dup2(fd, 2) os.close(fd) - # Replace sys.stdin/out with the null fds (closefd=False). sys.stdin = os.fdopen(0, "r", 1, closefd=False) sys.stdout = os.fdopen(1, "w", 1, closefd=False) return read_fd, write_fd @@ -51,7 +183,7 @@ def _prepare_protocol_fds() -> tuple[int, int]: class _WorkerIOStub: """Minimal IO stub so WorkerGateway can be constructed without sync pipes.""" - def __init__(self, execmodel: gateway_base.ExecModel) -> None: + def __init__(self, execmodel: ExecModel) -> None: self.execmodel = execmodel def read(self, numbytes: int) -> bytes: @@ -74,7 +206,7 @@ def kill(self) -> None: def serve_popen_trio(id: str, execmodel: str = "thread") -> None: - """Serve a WorkerGateway with Message IO on a Trio host thread.""" + """Serve a WorkerGateway with Message IO + exec scheduling on Trio.""" from . import _trio_host model = get_execmodel(execmodel) @@ -88,10 +220,13 @@ def serve_popen_trio(id: str, execmodel: str = "thread") -> None: io_stub = _WorkerIOStub(model) gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) - hasprimary = model.backend in ("thread", "main_thread_only") - gateway._execpool = gateway_base.WorkerPool(model, hasprimary=hasprimary) + main_thread_only = model.backend == "main_thread_only" + trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) + # Duck-type as WorkerPool for STATUS / _terminate_execution. + gateway._execpool = trio_exec # type: ignore[assignment] + gateway._trio_exec = trio_exec gateway._executetask_complete = None - if model.backend == "main_thread_only": + if main_thread_only: gateway._executetask_complete = model.Event() gateway._executetask_complete.set() @@ -103,9 +238,9 @@ async def _start() -> _trio_host.ProtocolSession: gateway._attach_trio_session(session) try: - if hasprimary: + if main_thread_only: trace("integrating as primary thread (trio worker)") - gateway._execpool.integrate_as_primary_thread() + trio_exec.integrate_as_primary_thread() gateway.join() except KeyboardInterrupt: # Match WorkerGateway.serve(): swallow in the worker. diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 0829a33d..8cf3cb09 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1245,7 +1245,14 @@ def join(self, timeout: float | None = None) -> None: class WorkerGateway(BaseGateway): + _trio_exec: Any = None + def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: + trio_exec = getattr(self, "_trio_exec", None) + if trio_exec is not None: + trio_exec.schedule(channel, sourcetask) + return + if self._execpool.execmodel.backend == "main_thread_only": assert self._executetask_complete is not None # It's necessary to wait for a short time in order to ensure From 37a184ff425f1f560e5da21d8c4ac4ceace069f0 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 21:53:03 +0200 Subject: [PATCH 03/59] refactor: drop eventlet and gevent execmodels Remove the EventletExecModel and GeventExecModel backends and their get_execmodel branches, leaving only the stdlib thread and main_thread_only models. Narrow the test execmodel fixture, drop the gevent test dependency and the eventlet/gevent mypy overrides, and scrub the docs of the removed backends. This is phase 1 of moving execnet onto Trio: the ExecModel surface is kept intact for now and collapsed further once Trio drives IO on both sides. Co-Authored-By: Claude Opus 4.8 --- doc/basics.rst | 23 ++--- pyproject.toml | 8 -- src/execnet/gateway_base.py | 121 ------------------------- src/execnet/multi.py | 2 +- testing/conftest.py | 12 +-- testing/test_channel.py | 2 - testing/test_gateway.py | 30 ------- testing/test_multi.py | 2 +- testing/test_xspec.py | 4 +- uv.lock | 172 +----------------------------------- 10 files changed, 15 insertions(+), 361 deletions(-) diff --git a/doc/basics.rst b/doc/basics.rst index 78672647..723de83f 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -57,10 +57,6 @@ Examples for valid gateway specifications same interpreter as the one it is initiated from and additionally remotely sets an environment variable ``NAME`` to ``value``. -* ``popen//execmodel=eventlet`` specifies a subprocess that uses the - same interpreter as the one it is initiated from but will run the - other side using eventlet for handling IO and dispatching threads. - * ``socket=192.168.1.4:8888`` specifies a Python Socket server process that listens on ``192.168.1.4:8888`` @@ -138,30 +134,29 @@ processes then you often want to call ``group.terminate()`` yourself and specify a larger or not timeout. -threading models: gevent, eventlet, thread, main_thread_only +threading models: thread, main_thread_only ==================================================================== .. versionadded:: 1.2 (status: experimental!) -execnet supports "main_thread_only", "thread", "eventlet" and "gevent" -as thread models on each of the two sides. You need to decide which -model to use before you create any gateways:: +execnet supports "thread" and "main_thread_only" as thread models on +each of the two sides. You need to decide which model to use before +you create any gateways:: # content of threadmodel.py import execnet - # locally use "eventlet", remotely use "thread" model - execnet.set_execmodel("eventlet", "thread") + # locally use "thread", remotely use "main_thread_only" model + execnet.set_execmodel("thread", "main_thread_only") gw = execnet.makegateway() print (gw) print (gw.remote_status()) print (gw.remote_exec("channel.send(1)").receive()) -You need to have eventlet installed in your environment and then -you can execute this little test file:: +You can execute this little test file:: $ python threadmodel.py - - + + 1 How to execute in the main thread diff --git a/pyproject.toml b/pyproject.toml index 709f887b..2c73a32b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,6 @@ testing = [ "tox", "hatch", "uv", - "gevent", ] [dependency-groups] @@ -122,10 +121,3 @@ warn_unused_ignores = false disallow_untyped_calls = false disallow_untyped_defs = false disallow_incomplete_defs = false - -[[tool.mypy.overrides]] -module = [ - "eventlet.*", - "gevent.thread.*", -] -ignore_missing_imports = true diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 8cf3cb09..c9933fe0 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -179,123 +179,6 @@ class MainThreadOnlyExecModel(ThreadExecModel): backend = "main_thread_only" -class EventletExecModel(ExecModel): - backend = "eventlet" - - @property - def queue(self): - import eventlet - - return eventlet.queue - - @property - def subprocess(self): - import eventlet.green.subprocess - - return eventlet.green.subprocess - - @property - def socket(self): - import eventlet.green.socket - - return eventlet.green.socket - - def get_ident(self) -> int: - import eventlet.green.thread - - return eventlet.green.thread.get_ident() # type: ignore[no-any-return] - - def sleep(self, delay: float) -> None: - import eventlet - - eventlet.sleep(delay) - - def start(self, func, args=()) -> None: - import eventlet - - eventlet.spawn_n(func, *args) - - def fdopen(self, fd, mode, bufsize=1, closefd=True): - import eventlet.green.os - - return eventlet.green.os.fdopen(fd, mode, bufsize, closefd=closefd) - - def Lock(self): - import eventlet.green.threading - - return eventlet.green.threading.RLock() - - def RLock(self): - import eventlet.green.threading - - return eventlet.green.threading.RLock() - - def Event(self): - import eventlet.green.threading - - return eventlet.green.threading.Event() - - -class GeventExecModel(ExecModel): - backend = "gevent" - - @property - def queue(self): - import gevent.queue - - return gevent.queue - - @property - def subprocess(self): - import gevent.subprocess - - return gevent.subprocess - - @property - def socket(self): - import gevent - - return gevent.socket - - def get_ident(self) -> int: - import gevent.thread - - return gevent.thread.get_ident() # type: ignore[no-any-return] - - def sleep(self, delay: float) -> None: - import gevent - - gevent.sleep(delay) - - def start(self, func, args=()) -> None: - import gevent - - gevent.spawn(func, *args) - - def fdopen(self, fd, mode, bufsize=1, closefd=True): - import gevent.fileobject - - # Prefer FileObject (FileObjectPosix on Unix). FileObjectThread keeps a - # native threadpool alive and can prevent interpreter shutdown, which - # hangs tests/scripts that open stdio via init_popen_io and then exit. - return gevent.fileobject.FileObject(fd, mode, bufsize, closefd=closefd) - - def Lock(self): - import gevent.lock - - return gevent.lock.RLock() - - def RLock(self): - import gevent.lock - - return gevent.lock.RLock() - - def Event(self): - import gevent.event - - return gevent.event.Event() - - def get_execmodel(backend: str | ExecModel) -> ExecModel: if isinstance(backend, ExecModel): return backend @@ -303,10 +186,6 @@ def get_execmodel(backend: str | ExecModel) -> ExecModel: return ThreadExecModel() elif backend == "main_thread_only": return MainThreadOnlyExecModel() - elif backend == "eventlet": - return EventletExecModel() - elif backend == "gevent": - return GeventExecModel() else: raise ValueError(f"unknown execmodel {backend!r}") diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 8079f4c4..687f1840 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -138,7 +138,7 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: id= specifies the gateway id python= specifies which python interpreter to execute - execmodel=model 'thread', 'main_thread_only', 'eventlet', 'gevent' execution model + execmodel=model 'thread' or 'main_thread_only' execution model chdir= specifies to which directory to change nice= specifies process priority of new process env:NAME=value specifies a remote environment variable setting. diff --git a/testing/conftest.py b/testing/conftest.py index c75f96c7..4d97bf35 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -128,10 +128,6 @@ def anypython(request: pytest.FixtureRequest) -> str: executable = getexecutable(name) if executable is None: pytest.skip(f"no {name} found") - if "execmodel" in request.fixturenames and name != "sys.executable": - backend = request.getfixturevalue("execmodel").backend - if backend not in ("thread", "main_thread_only"): - pytest.xfail(f"cannot run {backend!r} execmodel with bare {name}") return executable @@ -182,14 +178,8 @@ def gw( return gw -@pytest.fixture( - params=["thread", "main_thread_only", "eventlet", "gevent"], scope="session" -) +@pytest.fixture(params=["thread", "main_thread_only"], scope="session") def execmodel(request: pytest.FixtureRequest) -> ExecModel: - if request.param not in ("thread", "main_thread_only"): - pytest.importorskip(request.param) - if request.param in ("eventlet", "gevent") and sys.platform == "win32": - pytest.xfail(request.param + " does not work on win32") return get_execmodel(request.param) diff --git a/testing/test_channel.py b/testing/test_channel.py index d4712277..ba02b57b 100644 --- a/testing/test_channel.py +++ b/testing/test_channel.py @@ -214,8 +214,6 @@ def test_channel_callback_stays_active(self, gw: Gateway) -> None: def check_channel_callback_stays_active( self, gw: Gateway, earlyfree: bool = True ) -> Channel | None: - if gw.spec.execmodel == "gevent": - pytest.xfail("investigate gevent failure") # with 'earlyfree==True', this tests the "sendonly" channel state. l: list[int] = [] channel = gw.remote_exec( diff --git a/testing/test_gateway.py b/testing/test_gateway.py index 634f237a..83a50f44 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -535,36 +535,6 @@ def test_popen_args(spec: str, expected_args: list[str]) -> None: assert args == expected_args -@pytest.mark.parametrize( - "interleave_getstatus", - [ - pytest.param(True, id="interleave-remote-status"), - pytest.param( - False, - id="no-interleave-remote-status", - marks=pytest.mark.xfail( - reason="https://github.com/pytest-dev/execnet/issues/123", - ), - ), - ], -) -def test_regression_gevent_hangs( - group: execnet.Group, interleave_getstatus: bool -) -> None: - pytest.importorskip("gevent") - gw = group.makegateway("popen//execmodel=gevent") - - print(gw.remote_status()) - - def sendback(channel) -> None: - channel.send(1234) - - ch = gw.remote_exec(sendback) - if interleave_getstatus: - print(gw.remote_status()) - assert ch.receive(timeout=0.5) == 1234 - - def test_assert_main_thread_only( execmodel: gateway_base.ExecModel, makegateway: Callable[[str], Gateway] ) -> None: diff --git a/testing/test_multi.py b/testing/test_multi.py index 12e0ed3d..f3166503 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -69,7 +69,7 @@ def test_Group_execmodel_setting(self) -> None: gm._gateways.append(1) # type: ignore[arg-type] try: with pytest.raises(ValueError): - gm.set_execmodel("eventlet") + gm.set_execmodel("main_thread_only") assert gm.execmodel.backend == "thread" finally: gm._gateways.pop() diff --git a/testing/test_xspec.py b/testing/test_xspec.py index f837b07a..5240d72d 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -64,8 +64,8 @@ def test_ssh_options(self) -> None: def test_execmodel(self) -> None: spec = XSpec("execmodel=thread") assert spec.execmodel == "thread" - spec = XSpec("execmodel=eventlet") - assert spec.execmodel == "eventlet" + spec = XSpec("execmodel=main_thread_only") + assert spec.execmodel == "main_thread_only" def test_ssh_options_and_config(self) -> None: spec = XSpec("ssh=-p 22100 user@host//python=python3") diff --git a/uv.lock b/uv.lock index fc2da46b..5dbf4f10 100644 --- a/uv.lock +++ b/uv.lock @@ -332,7 +332,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -348,7 +348,6 @@ dependencies = [ [package.optional-dependencies] testing = [ - { name = "gevent" }, { name = "hatch" }, { name = "pre-commit" }, { name = "pytest" }, @@ -364,7 +363,6 @@ testing = [ [package.metadata] requires-dist = [ - { name = "gevent", marker = "extra == 'testing'" }, { name = "hatch", marker = "extra == 'testing'" }, { name = "pre-commit", marker = "extra == 'testing'" }, { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, @@ -387,118 +385,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z" }, ] -[[package]] -name = "gevent" -version = "26.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, - { name = "greenlet", marker = "platform_python_implementation == 'CPython'" }, - { name = "zope-event" }, - { name = "zope-interface" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c4/cb/98aa3a299e2fc4a2372b5d124863e02965b64579ffc29fe54d0641e65b2f/gevent-26.5.0.tar.gz", hash = "sha256:1655eb04c1e20d71b2aa4a3c7528162dd58ff6cc46a037af1f01f534c80fefba", size = 6712354, upload-time = "2026-05-20T21:22:45.132Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/72/b7/01a5880e01702f39fb09e3616c624054a0dc9a82561a865f3b1eff4bfc80/gevent-26.5.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:2ba673dcbf7747513b58fa64ca7e9d6a828bc5c604d1552d23db89006d7911df", size = 2181491, upload-time = "2026-05-20T20:35:19.326Z" }, - { url = "https://files.pythonhosted.org/packages/8d/fe/035ec5fa58a886740a744380118f03a90ac2da3f6c9cba248f28074ce40a/gevent-26.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:271b1474d81bb33036631adb16a35e5a1ee9dc414b05c999d6b01dc839a89975", size = 2212161, upload-time = "2026-05-20T20:43:25.678Z" }, - { url = "https://files.pythonhosted.org/packages/2a/ea/ea87c08931c9e4c6c40bb05a2cb19c2d6f93fe6e0052f9152ea5ade6d037/gevent-26.5.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:cd3dc60581687e2618286108f8e2f820d8446be4b34131065011c066e911d39c", size = 1768295, upload-time = "2026-05-20T21:17:29.438Z" }, - { url = "https://files.pythonhosted.org/packages/da/92/1d0e7287ae55700a8d25153ac736896bd9bcc3f85a12d374ef398db4b33c/gevent-26.5.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:dc7fa28b2d627f8e87595f39043b6dec71e8e7fb97e685e5506c47cf3ff8cb2e", size = 1862627, upload-time = "2026-05-20T21:15:59.365Z" }, - { url = "https://files.pythonhosted.org/packages/3a/4c/7f5ed67e52dfdef4ff91ae1a6fb28186d52e2496962edc8f17bdea9ab2c0/gevent-26.5.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:68c5fc21cef80268cdff88a4ae6c025fabb019b071f6f8ee4d20a7bccbddb873", size = 1804690, upload-time = "2026-05-20T21:30:51.713Z" }, - { url = "https://files.pythonhosted.org/packages/4c/75/0f5da6ca045f8a052203e1810058029f4b682507a789b413cac7d28bae28/gevent-26.5.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d325502eb0695708ef8c899f605573ed6847f3961f8159627dba267fbf3ce457", size = 2119054, upload-time = "2026-05-20T20:35:22.678Z" }, - { url = "https://files.pythonhosted.org/packages/6e/f1/fcff7f7fad2bb33f3742db6b2145825a2191c0cd31d75789b0741fd28faf/gevent-26.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a11daf3a588b932c8bf965fb18444c69aff48badec88435e988cf8d67137075a", size = 1778784, upload-time = "2026-05-20T21:16:38.182Z" }, - { url = "https://files.pythonhosted.org/packages/98/57/151314f00bdc6ba77333febb3e9dc97fdf94d79426559b4fa8332f0c2b6e/gevent-26.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1101b5ef82a3fb178550cfd80f32293dc8dd2f3d0828292223ebba29d6f76e33", size = 2145373, upload-time = "2026-05-20T20:43:27.255Z" }, - { url = "https://files.pythonhosted.org/packages/1a/b5/7a02f711db62cbed1c1a00e1f9ff50eef95ccc78d4c04a0f93636655d1b7/gevent-26.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:5233109ad4f3af16393ba9888f238919a05ce15ce68d6831ac8a0da8dfb750ae", size = 1696576, upload-time = "2026-05-20T20:15:49.62Z" }, - { url = "https://files.pythonhosted.org/packages/f8/9b/5022adc310697ef25c6fb22eb9bf0ebcad3427b51776e882709de9a8b6d7/gevent-26.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:3be804565168ffacebeb21af9f1cd689831a89f0f12fc0c3f423c730c3c9eb31", size = 1552095, upload-time = "2026-05-20T20:16:54.81Z" }, - { url = "https://files.pythonhosted.org/packages/37/0b/1a530b2db55c97cc0cf44116201f538f3033c04c1d2aca143979b412f4be/gevent-26.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:e80ad2a8a1e8bdaa5605e3bf4929e0cebf9ea7b8237c83362f7257698bb14280", size = 2929714, upload-time = "2026-05-20T20:13:24.656Z" }, - { url = "https://files.pythonhosted.org/packages/b9/df/32fe851ed5f68493f354e09b19bdebae0de1185be4db0b2988e71e737fd3/gevent-26.5.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fe42c037253580a3386fce275f8a2a845e540f5a729916934a732f13d42e72cc", size = 1784838, upload-time = "2026-05-20T21:17:31.063Z" }, - { url = "https://files.pythonhosted.org/packages/e5/9a/21332674f9a10e8cdf13b41b52e9d663647a1c6e1dc3c62b07c0aeefd360/gevent-26.5.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:9f463c7d6f69d13b6fe8e3b832a6175a6e95328a940f38495d25496d1ae8ad88", size = 1880440, upload-time = "2026-05-20T21:16:00.881Z" }, - { url = "https://files.pythonhosted.org/packages/9f/b1/5f8a4196113cf7f3fdd987b483f7e6b10c28ea3930c4727e31ba8cce51b6/gevent-26.5.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:96d5e96b1b14a4c1023dcfcc114533217f13febc3b6169254f23fc18d19fee29", size = 1831592, upload-time = "2026-05-20T21:30:53.832Z" }, - { url = "https://files.pythonhosted.org/packages/4e/69/1559b1f6b5107a9118fccd300240879bd581b6d87b03d568d0d155ea702c/gevent-26.5.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:bccff69c462e3650a0fd1d4e9cfc8b6effe15f3e9b1cad20a7bb5ce14b057efd", size = 2114915, upload-time = "2026-05-20T20:35:25.041Z" }, - { url = "https://files.pythonhosted.org/packages/e4/32/602c499d54472f64e5cdf6013aeab5ce6aa6fed005387e8b4f2d22f5dc8d/gevent-26.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f519139354d5ca7625df9ddb1b2ffada885c14abc5b4dbae3682e967ddf79669", size = 1796906, upload-time = "2026-05-20T21:16:39.65Z" }, - { url = "https://files.pythonhosted.org/packages/f9/3c/2fe77ee6e3d381b3c50c0b7d6c4c08c08b8ff5e8c0d9dd51a3b426d61eec/gevent-26.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0bf57df54f1c66273bf3601c2a1e41b12138fe848933718369663bc54f177ca2", size = 2140806, upload-time = "2026-05-20T20:43:28.895Z" }, - { url = "https://files.pythonhosted.org/packages/22/d5/4620797bbd9c88f4541188efc138b0d615f9834db540da36a2249ee929c5/gevent-26.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:e49ce0de007dfd7412edbc2b5d41cce33b049bb1b7086f50be5a09e601bde603", size = 1699995, upload-time = "2026-05-20T20:15:39.311Z" }, - { url = "https://files.pythonhosted.org/packages/cb/83/ac3477dfc0f9fd80c88110102c73cefc35dcded2b248544f45a8fa5412df/gevent-26.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:5c5ff29495a2eed2a244de8150f21893d6c1b15d8b4b5719ab4bbfa06db1e28f", size = 1547433, upload-time = "2026-05-20T20:15:51.656Z" }, - { url = "https://files.pythonhosted.org/packages/7d/47/5b992ab9c8037633cfd0fe698a97a878f59d8eb53c381e91e9a1a76fd215/gevent-26.5.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:9b4d3f34c913d1a6bec6d030365a517f3b527a9773b12e58cf56c3339bbe96e6", size = 2952523, upload-time = "2026-05-20T20:13:04.698Z" }, - { url = "https://files.pythonhosted.org/packages/74/11/c7dfc773eb43331a682efed610b49df6e976331f1b0e1c592a0c35d29872/gevent-26.5.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1d8da4e799431feeb4c9e441ac7431f0baabb9106976790d884289d08ac08359", size = 1787044, upload-time = "2026-05-20T21:17:32.845Z" }, - { url = "https://files.pythonhosted.org/packages/ae/28/9812933dac93560f46910a9e834805fe76f822c408bd1c20cdf299d7c311/gevent-26.5.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:51becdb4c30a8f45c1c028ad7a97bf5a1ed141f74b159a31aa9cc6aa1e6263a6", size = 1882342, upload-time = "2026-05-20T21:16:02.645Z" }, - { url = "https://files.pythonhosted.org/packages/96/4b/514f248f69b2230b69b0bb17f4158b0b05dd4b2cb469a60ab206e9fe7496/gevent-26.5.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:c42bbcd3d453b08ad8915fd3feaf3d44a3562cdf1c7b208f9837149711e16d9d", size = 1834136, upload-time = "2026-05-20T21:30:55.739Z" }, - { url = "https://files.pythonhosted.org/packages/53/67/f5f30716efca99b6200ae89a9303a7e94dae085b7de6f6d0033c52a37f4b/gevent-26.5.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:bd3445e4fbeeb46690ed8efe94b8d1d46b14aa04af8866ae7a8da5997828d1c6", size = 2115349, upload-time = "2026-05-20T20:35:28.132Z" }, - { url = "https://files.pythonhosted.org/packages/09/d8/60e8809bde7986e6c4e6d106080b3603fa09b3bb0255fed1a4d8282e3ca2/gevent-26.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b573d5b2826edc705f31f07da6889ad483a6a0d64944ebd8d32205f7c5bf46fb", size = 1799443, upload-time = "2026-05-20T21:16:41.928Z" }, - { url = "https://files.pythonhosted.org/packages/f8/41/b388b2b1f0a026ea30687e51ddf81dbb783dfb55fac0a16708d2821d99e5/gevent-26.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4d53b1b28f2082a151bded2850b53f6baed02f742d2a1584029e8bd42d457fb4", size = 2141117, upload-time = "2026-05-20T20:43:30.694Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f3/ac9a4b0de487e390c5d53a908a9347c0df0102de2bbf3e8603087769191d/gevent-26.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:23569ce0c254eb821fc3dcfe250843dde8b3180b09bae9e222e41aa3fa4885b7", size = 1699862, upload-time = "2026-05-20T20:15:33.642Z" }, - { url = "https://files.pythonhosted.org/packages/2a/cf/1ef1fc9b390563c0f97702f94a557d1649b7bbb5724f9b86c2122747e92f/gevent-26.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:40cdcdb2e404b6c82b82a4576bdb33958f23fc2deb0d933e9e022b362001e647", size = 1545341, upload-time = "2026-05-20T20:16:26.229Z" }, - { url = "https://files.pythonhosted.org/packages/17/55/7d98d3888e7bb9ad4656420dec69232ecbbea48792aff9295d0ad7cf8435/gevent-26.5.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:75a0050e4b87f08ddee7e56f59e6014cd7fcdc3153046c09a847940515d12c85", size = 2968223, upload-time = "2026-05-20T20:13:17.223Z" }, - { url = "https://files.pythonhosted.org/packages/f8/b4/e8e116fcbcb9dc0bf3acc50037f86e1204c217c8ed5defde68be11b3aab6/gevent-26.5.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:fd1a0b83a04e19378d9466ae0ee2b5937cf1d7fbfdcb916b2aea82179a208574", size = 1793926, upload-time = "2026-05-20T21:17:34.321Z" }, - { url = "https://files.pythonhosted.org/packages/28/07/7b267e9754b661defb93542e97731a4df21f8a40dc0f6c853faa717cf124/gevent-26.5.0-cp314-cp314-manylinux_2_28_ppc64le.whl", hash = "sha256:4c964c15076e76391d523ec24202f579a2535f7e301a40efb1656ae046d3eb69", size = 1887632, upload-time = "2026-05-20T21:16:04.158Z" }, - { url = "https://files.pythonhosted.org/packages/5c/50/b47d29e99449bd13b557ffa451401dc13d397a9923f562ef90a4e8514502/gevent-26.5.0-cp314-cp314-manylinux_2_28_s390x.whl", hash = "sha256:45d5438d1c84da5df7e832434627624709543630977332bb4e2d05ecca362cc9", size = 1838688, upload-time = "2026-05-20T21:30:57.979Z" }, - { url = "https://files.pythonhosted.org/packages/8b/eb/5b54ccff11bc7d7bebd40a24571ccc115d5cdae4f6c32ab457b43b436e42/gevent-26.5.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:354f35924113abc954819216c2a6ee16751958c615681e0490946e31b437bd2f", size = 2120351, upload-time = "2026-05-20T20:35:32.699Z" }, - { url = "https://files.pythonhosted.org/packages/9c/70/30fd325c30e04b1e5174c61945e17421d53ddb2450366cc52cef234f8c4b/gevent-26.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a47cd2d32f6404212d374ad8014a3491d7477dcf0cc09c5a2308ad6d325fd663", size = 1806684, upload-time = "2026-05-20T21:16:43.87Z" }, - { url = "https://files.pythonhosted.org/packages/cd/e8/fbf911ac3f9524ecfaed174d100fde671904ab8db92ceaf07faaebd13386/gevent-26.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:032157cebdedb84f2f52cdd980f2f5f2623eed6a8f083aadf44b44c47f628642", size = 2146606, upload-time = "2026-05-20T20:43:32.216Z" }, - { url = "https://files.pythonhosted.org/packages/9e/4d/284fcbbfde66fd978c2980c1fbe0eabd586af6e4b728649e9cf459e8b38f/gevent-26.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:9c414935ba5fc88359110968851d3616f119082c937390d00a1c0f4f59be814f", size = 1722497, upload-time = "2026-05-20T20:16:44.274Z" }, - { url = "https://files.pythonhosted.org/packages/15/d2/9f66eb53434704402be0ba733bf3320bf589671a4b76fac52a7d6077e972/gevent-26.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:2a0f5993a04b95a35b3a118b1a58ba272833f9b547b774001dea29f90620882f", size = 1574249, upload-time = "2026-05-20T20:15:50.873Z" }, - { url = "https://files.pythonhosted.org/packages/dc/d5/b4c50adb761878e3c96642b9f79bf44cee3120f3df55cd40876f51d89866/gevent-26.5.0-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:2e117df896a2660c9ebd4e2b5afc02dfd6e2ddf9b495e787e67c72d105432b09", size = 2971993, upload-time = "2026-05-20T20:12:50.845Z" }, - { url = "https://files.pythonhosted.org/packages/03/83/71c2a945e80198422d1d93dbe67355f249fb456b451bf9201199d3ef6a1a/gevent-26.5.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:af5ffe9c11ffb8a39b6bef2e8b722aa2043ae4980977915c6aa8c68b4bc26e46", size = 1796658, upload-time = "2026-05-20T21:17:35.968Z" }, - { url = "https://files.pythonhosted.org/packages/42/96/548ca77aed5cb9a44e855a6c23ebceeb3554a0ea9ca0c01c311878899a3e/gevent-26.5.0-cp315-cp315-manylinux_2_28_ppc64le.whl", hash = "sha256:7da34aef7e87c43dd3662e5785e79ed505c01399a7cb42876d2d8925969fd75f", size = 1891473, upload-time = "2026-05-20T21:16:05.657Z" }, - { url = "https://files.pythonhosted.org/packages/f6/4f/f48bd47d5287afb0fbcc56165f3ed47583f1803bad401653fe27e71ade2d/gevent-26.5.0-cp315-cp315-manylinux_2_28_s390x.whl", hash = "sha256:1c6293a7046bcc6f3d8972a74b19cd7a4cfd02d3881edf0fcf827aa514bd247b", size = 1841429, upload-time = "2026-05-20T21:30:59.907Z" }, - { url = "https://files.pythonhosted.org/packages/a0/72/1925215fc720d2561fa3ec8d4af5f098f8d0cbfa76a45fafed6e5ade7718/gevent-26.5.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:d3bde0f140a275b2fa88e4b6516bda85551930e10bc2fd95e18c1b7d11cb780c", size = 2123895, upload-time = "2026-05-20T20:35:34.964Z" }, - { url = "https://files.pythonhosted.org/packages/83/59/0f584f6b1170c9a6abd9b70ccf5e9cc5ead34eabafabc0e21876ef0fe6f7/gevent-26.5.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:e29fb4b17d9958ec8cb7f6339a111b29bc23f2c2efbef86189d1248bb4862d17", size = 1809047, upload-time = "2026-05-20T21:16:45.977Z" }, - { url = "https://files.pythonhosted.org/packages/82/88/61e854bfd98ac22eac78a97fc6db10de0f9ace46514072b435c217168729/gevent-26.5.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b2239df2f7570efa03736678f3f053bb1bdd22a8a16cd28a2feb7d32ea5f533f", size = 2150764, upload-time = "2026-05-20T20:43:33.781Z" }, - { url = "https://files.pythonhosted.org/packages/b9/f5/af048b97433d7f9a7df7f5510b2c46918b7d073dcfb3bf6d0ef0e5a83dcc/gevent-26.5.0-cp315-cp315-win_amd64.whl", hash = "sha256:aae214952fd38d27a42dc416bb70193962ec932384b63445d29bbb5817a1c042", size = 1722600, upload-time = "2026-05-20T20:19:56.81Z" }, - { url = "https://files.pythonhosted.org/packages/11/95/fb74a2299c6a2d78d9de12deaaac640ab5d2ef96a8e0f97a3ff84b9ca84b/gevent-26.5.0-cp315-cp315-win_arm64.whl", hash = "sha256:f7067564f139e33bf26a31ee3b13d168d76eb99a44b85ced626652b158baa80c", size = 1574406, upload-time = "2026-05-20T20:17:12.125Z" }, -] - -[[package]] -name = "greenlet" -version = "3.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/34/c1/a82edae11d46c0d83481aacaa1e578fea21d94a1ef400afd734d47ad95ad/greenlet-3.2.2.tar.gz", hash = "sha256:ad053d34421a2debba45aa3cc39acf454acbcd025b3fc1a9f8a0dee237abd485", size = 185797, upload-time = "2025-05-09T19:47:35.066Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/66/910217271189cc3f32f670040235f4bf026ded8ca07270667d69c06e7324/greenlet-3.2.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:c49e9f7c6f625507ed83a7485366b46cbe325717c60837f7244fc99ba16ba9d6", size = 267395, upload-time = "2025-05-09T14:50:45.357Z" }, - { url = "https://files.pythonhosted.org/packages/a8/36/8d812402ca21017c82880f399309afadb78a0aa300a9b45d741e4df5d954/greenlet-3.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3cc1a3ed00ecfea8932477f729a9f616ad7347a5e55d50929efa50a86cb7be7", size = 625742, upload-time = "2025-05-09T15:23:58.293Z" }, - { url = "https://files.pythonhosted.org/packages/7b/77/66d7b59dfb7cc1102b2f880bc61cb165ee8998c9ec13c96606ba37e54c77/greenlet-3.2.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c9896249fbef2c615853b890ee854f22c671560226c9221cfd27c995db97e5c", size = 637014, upload-time = "2025-05-09T15:24:47.025Z" }, - { url = "https://files.pythonhosted.org/packages/36/a7/ff0d408f8086a0d9a5aac47fa1b33a040a9fca89bd5a3f7b54d1cd6e2793/greenlet-3.2.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7409796591d879425997a518138889d8d17e63ada7c99edc0d7a1c22007d4907", size = 632874, upload-time = "2025-05-09T15:29:20.014Z" }, - { url = "https://files.pythonhosted.org/packages/a1/75/1dc2603bf8184da9ebe69200849c53c3c1dca5b3a3d44d9f5ca06a930550/greenlet-3.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7791dcb496ec53d60c7f1c78eaa156c21f402dda38542a00afc3e20cae0f480f", size = 631652, upload-time = "2025-05-09T14:53:30.961Z" }, - { url = "https://files.pythonhosted.org/packages/7b/74/ddc8c3bd4c2c20548e5bf2b1d2e312a717d44e2eca3eadcfc207b5f5ad80/greenlet-3.2.2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d8009ae46259e31bc73dc183e402f548e980c96f33a6ef58cc2e7865db012e13", size = 580619, upload-time = "2025-05-09T14:53:42.049Z" }, - { url = "https://files.pythonhosted.org/packages/7e/f2/40f26d7b3077b1c7ae7318a4de1f8ffc1d8ccbad8f1d8979bf5080250fd6/greenlet-3.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fd9fb7c941280e2c837b603850efc93c999ae58aae2b40765ed682a6907ebbc5", size = 1109809, upload-time = "2025-05-09T15:26:59.063Z" }, - { url = "https://files.pythonhosted.org/packages/c5/21/9329e8c276746b0d2318b696606753f5e7b72d478adcf4ad9a975521ea5f/greenlet-3.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:00cd814b8959b95a546e47e8d589610534cfb71f19802ea8a2ad99d95d702057", size = 1133455, upload-time = "2025-05-09T14:53:55.823Z" }, - { url = "https://files.pythonhosted.org/packages/bb/1e/0dca9619dbd736d6981f12f946a497ec21a0ea27262f563bca5729662d4d/greenlet-3.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:d0cb7d47199001de7658c213419358aa8937df767936506db0db7ce1a71f4a2f", size = 294991, upload-time = "2025-05-09T15:05:56.847Z" }, - { url = "https://files.pythonhosted.org/packages/a3/9f/a47e19261747b562ce88219e5ed8c859d42c6e01e73da6fbfa3f08a7be13/greenlet-3.2.2-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:dcb9cebbf3f62cb1e5afacae90761ccce0effb3adaa32339a0670fe7805d8068", size = 268635, upload-time = "2025-05-09T14:50:39.007Z" }, - { url = "https://files.pythonhosted.org/packages/11/80/a0042b91b66975f82a914d515e81c1944a3023f2ce1ed7a9b22e10b46919/greenlet-3.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf3fc9145141250907730886b031681dfcc0de1c158f3cc51c092223c0f381ce", size = 628786, upload-time = "2025-05-09T15:24:00.692Z" }, - { url = "https://files.pythonhosted.org/packages/38/a2/8336bf1e691013f72a6ebab55da04db81a11f68e82bb691f434909fa1327/greenlet-3.2.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:efcdfb9df109e8a3b475c016f60438fcd4be68cd13a365d42b35914cdab4bb2b", size = 640866, upload-time = "2025-05-09T15:24:48.153Z" }, - { url = "https://files.pythonhosted.org/packages/f8/7e/f2a3a13e424670a5d08826dab7468fa5e403e0fbe0b5f951ff1bc4425b45/greenlet-3.2.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4bd139e4943547ce3a56ef4b8b1b9479f9e40bb47e72cc906f0f66b9d0d5cab3", size = 636752, upload-time = "2025-05-09T15:29:23.182Z" }, - { url = "https://files.pythonhosted.org/packages/fd/5d/ce4a03a36d956dcc29b761283f084eb4a3863401c7cb505f113f73af8774/greenlet-3.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71566302219b17ca354eb274dfd29b8da3c268e41b646f330e324e3967546a74", size = 636028, upload-time = "2025-05-09T14:53:32.854Z" }, - { url = "https://files.pythonhosted.org/packages/4b/29/b130946b57e3ceb039238413790dd3793c5e7b8e14a54968de1fe449a7cf/greenlet-3.2.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3091bc45e6b0c73f225374fefa1536cd91b1e987377b12ef5b19129b07d93ebe", size = 583869, upload-time = "2025-05-09T14:53:43.614Z" }, - { url = "https://files.pythonhosted.org/packages/ac/30/9f538dfe7f87b90ecc75e589d20cbd71635531a617a336c386d775725a8b/greenlet-3.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:44671c29da26539a5f142257eaba5110f71887c24d40df3ac87f1117df589e0e", size = 1112886, upload-time = "2025-05-09T15:27:01.304Z" }, - { url = "https://files.pythonhosted.org/packages/be/92/4b7deeb1a1e9c32c1b59fdca1cac3175731c23311ddca2ea28a8b6ada91c/greenlet-3.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c23ea227847c9dbe0b3910f5c0dd95658b607137614eb821e6cbaecd60d81cc6", size = 1138355, upload-time = "2025-05-09T14:53:58.011Z" }, - { url = "https://files.pythonhosted.org/packages/c5/eb/7551c751a2ea6498907b2fcbe31d7a54b602ba5e8eb9550a9695ca25d25c/greenlet-3.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:0a16fb934fcabfdfacf21d79e6fed81809d8cd97bc1be9d9c89f0e4567143d7b", size = 295437, upload-time = "2025-05-09T15:00:57.733Z" }, - { url = "https://files.pythonhosted.org/packages/2c/a1/88fdc6ce0df6ad361a30ed78d24c86ea32acb2b563f33e39e927b1da9ea0/greenlet-3.2.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:df4d1509efd4977e6a844ac96d8be0b9e5aa5d5c77aa27ca9f4d3f92d3fcf330", size = 270413, upload-time = "2025-05-09T14:51:32.455Z" }, - { url = "https://files.pythonhosted.org/packages/a6/2e/6c1caffd65490c68cd9bcec8cb7feb8ac7b27d38ba1fea121fdc1f2331dc/greenlet-3.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da956d534a6d1b9841f95ad0f18ace637668f680b1339ca4dcfb2c1837880a0b", size = 637242, upload-time = "2025-05-09T15:24:02.63Z" }, - { url = "https://files.pythonhosted.org/packages/98/28/088af2cedf8823b6b7ab029a5626302af4ca1037cf8b998bed3a8d3cb9e2/greenlet-3.2.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9c7b15fb9b88d9ee07e076f5a683027bc3befd5bb5d25954bb633c385d8b737e", size = 651444, upload-time = "2025-05-09T15:24:49.856Z" }, - { url = "https://files.pythonhosted.org/packages/4a/9f/0116ab876bb0bc7a81eadc21c3f02cd6100dcd25a1cf2a085a130a63a26a/greenlet-3.2.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:752f0e79785e11180ebd2e726c8a88109ded3e2301d40abced2543aa5d164275", size = 646067, upload-time = "2025-05-09T15:29:24.989Z" }, - { url = "https://files.pythonhosted.org/packages/35/17/bb8f9c9580e28a94a9575da847c257953d5eb6e39ca888239183320c1c28/greenlet-3.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ae572c996ae4b5e122331e12bbb971ea49c08cc7c232d1bd43150800a2d6c65", size = 648153, upload-time = "2025-05-09T14:53:34.716Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ee/7f31b6f7021b8df6f7203b53b9cc741b939a2591dcc6d899d8042fcf66f2/greenlet-3.2.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02f5972ff02c9cf615357c17ab713737cccfd0eaf69b951084a9fd43f39833d3", size = 603865, upload-time = "2025-05-09T14:53:45.738Z" }, - { url = "https://files.pythonhosted.org/packages/b5/2d/759fa59323b521c6f223276a4fc3d3719475dc9ae4c44c2fe7fc750f8de0/greenlet-3.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4fefc7aa68b34b9224490dfda2e70ccf2131368493add64b4ef2d372955c207e", size = 1119575, upload-time = "2025-05-09T15:27:04.248Z" }, - { url = "https://files.pythonhosted.org/packages/30/05/356813470060bce0e81c3df63ab8cd1967c1ff6f5189760c1a4734d405ba/greenlet-3.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a31ead8411a027c2c4759113cf2bd473690517494f3d6e4bf67064589afcd3c5", size = 1147460, upload-time = "2025-05-09T14:54:00.315Z" }, - { url = "https://files.pythonhosted.org/packages/07/f4/b2a26a309a04fb844c7406a4501331b9400e1dd7dd64d3450472fd47d2e1/greenlet-3.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:b24c7844c0a0afc3ccbeb0b807adeefb7eff2b5599229ecedddcfeb0ef333bec", size = 296239, upload-time = "2025-05-09T14:57:17.633Z" }, - { url = "https://files.pythonhosted.org/packages/89/30/97b49779fff8601af20972a62cc4af0c497c1504dfbb3e93be218e093f21/greenlet-3.2.2-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:3ab7194ee290302ca15449f601036007873028712e92ca15fc76597a0aeb4c59", size = 269150, upload-time = "2025-05-09T14:50:30.784Z" }, - { url = "https://files.pythonhosted.org/packages/21/30/877245def4220f684bc2e01df1c2e782c164e84b32e07373992f14a2d107/greenlet-3.2.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dc5c43bb65ec3669452af0ab10729e8fdc17f87a1f2ad7ec65d4aaaefabf6bf", size = 637381, upload-time = "2025-05-09T15:24:12.893Z" }, - { url = "https://files.pythonhosted.org/packages/8e/16/adf937908e1f913856b5371c1d8bdaef5f58f251d714085abeea73ecc471/greenlet-3.2.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:decb0658ec19e5c1f519faa9a160c0fc85a41a7e6654b3ce1b44b939f8bf1325", size = 651427, upload-time = "2025-05-09T15:24:51.074Z" }, - { url = "https://files.pythonhosted.org/packages/ad/49/6d79f58fa695b618654adac64e56aff2eeb13344dc28259af8f505662bb1/greenlet-3.2.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6fadd183186db360b61cb34e81117a096bff91c072929cd1b529eb20dd46e6c5", size = 645795, upload-time = "2025-05-09T15:29:26.673Z" }, - { url = "https://files.pythonhosted.org/packages/5a/e6/28ed5cb929c6b2f001e96b1d0698c622976cd8f1e41fe7ebc047fa7c6dd4/greenlet-3.2.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1919cbdc1c53ef739c94cf2985056bcc0838c1f217b57647cbf4578576c63825", size = 648398, upload-time = "2025-05-09T14:53:36.61Z" }, - { url = "https://files.pythonhosted.org/packages/9d/70/b200194e25ae86bc57077f695b6cc47ee3118becf54130c5514456cf8dac/greenlet-3.2.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3885f85b61798f4192d544aac7b25a04ece5fe2704670b4ab73c2d2c14ab740d", size = 606795, upload-time = "2025-05-09T14:53:47.039Z" }, - { url = "https://files.pythonhosted.org/packages/f8/c8/ba1def67513a941154ed8f9477ae6e5a03f645be6b507d3930f72ed508d3/greenlet-3.2.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:85f3e248507125bf4af607a26fd6cb8578776197bd4b66e35229cdf5acf1dfbf", size = 1117976, upload-time = "2025-05-09T15:27:06.542Z" }, - { url = "https://files.pythonhosted.org/packages/c3/30/d0e88c1cfcc1b3331d63c2b54a0a3a4a950ef202fb8b92e772ca714a9221/greenlet-3.2.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:1e76106b6fc55fa3d6fe1c527f95ee65e324a13b62e243f77b48317346559708", size = 1145509, upload-time = "2025-05-09T14:54:02.223Z" }, - { url = "https://files.pythonhosted.org/packages/90/2e/59d6491834b6e289051b252cf4776d16da51c7c6ca6a87ff97e3a50aa0cd/greenlet-3.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:fe46d4f8e94e637634d54477b0cfabcf93c53f29eedcbdeecaf2af32029b4421", size = 296023, upload-time = "2025-05-09T14:53:24.157Z" }, - { url = "https://files.pythonhosted.org/packages/65/66/8a73aace5a5335a1cba56d0da71b7bd93e450f17d372c5b7c5fa547557e9/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba30e88607fb6990544d84caf3c706c4b48f629e18853fc6a646f82db9629418", size = 629911, upload-time = "2025-05-09T15:24:22.376Z" }, - { url = "https://files.pythonhosted.org/packages/48/08/c8b8ebac4e0c95dcc68ec99198842e7db53eda4ab3fb0a4e785690883991/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:055916fafad3e3388d27dd68517478933a97edc2fc54ae79d3bec827de2c64c4", size = 635251, upload-time = "2025-05-09T15:24:52.205Z" }, - { url = "https://files.pythonhosted.org/packages/37/26/7db30868f73e86b9125264d2959acabea132b444b88185ba5c462cb8e571/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2593283bf81ca37d27d110956b79e8723f9aa50c4bcdc29d3c0543d4743d2763", size = 632620, upload-time = "2025-05-09T15:29:28.051Z" }, - { url = "https://files.pythonhosted.org/packages/10/ec/718a3bd56249e729016b0b69bee4adea0dfccf6ca43d147ef3b21edbca16/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89c69e9a10670eb7a66b8cef6354c24671ba241f46152dd3eed447f79c29fb5b", size = 628851, upload-time = "2025-05-09T14:53:38.472Z" }, - { url = "https://files.pythonhosted.org/packages/9b/9d/d1c79286a76bc62ccdc1387291464af16a4204ea717f24e77b0acd623b99/greenlet-3.2.2-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02a98600899ca1ca5d3a2590974c9e3ec259503b2d6ba6527605fcd74e08e207", size = 593718, upload-time = "2025-05-09T14:53:48.313Z" }, - { url = "https://files.pythonhosted.org/packages/cd/41/96ba2bf948f67b245784cd294b84e3d17933597dffd3acdb367a210d1949/greenlet-3.2.2-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:b50a8c5c162469c3209e5ec92ee4f95c8231b11db6a04db09bbe338176723bb8", size = 1105752, upload-time = "2025-05-09T15:27:08.217Z" }, - { url = "https://files.pythonhosted.org/packages/68/3b/3b97f9d33c1f2eb081759da62bd6162159db260f602f048bc2f36b4c453e/greenlet-3.2.2-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:45f9f4853fb4cc46783085261c9ec4706628f3b57de3e68bae03e8f8b3c0de51", size = 1125170, upload-time = "2025-05-09T14:54:04.082Z" }, - { url = "https://files.pythonhosted.org/packages/31/df/b7d17d66c8d0f578d2885a3d8f565e9e4725eacc9d3fdc946d0031c055c4/greenlet-3.2.2-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:9ea5231428af34226c05f927e16fc7f6fa5e39e3ad3cd24ffa48ba53a47f4240", size = 269899, upload-time = "2025-05-09T14:54:01.581Z" }, -] - [[package]] name = "h11" version = "0.16.0" @@ -984,15 +870,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/24/b4293291fa1dd830f353d2cb163295742fa87f179fcc8a20a306a81978b7/SecretStorage-3.3.3-py3-none-any.whl", hash = "sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99", size = 15221, upload-time = "2022-08-13T16:22:44.457Z" }, ] -[[package]] -name = "setuptools" -version = "80.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8d/d2/ec1acaaff45caed5c2dedb33b67055ba9d4e96b091094df90762e60135fe/setuptools-80.8.0.tar.gz", hash = "sha256:49f7af965996f26d43c8ae34539c8d99c5042fbff34302ea151eaa9c207cd257", size = 1319720, upload-time = "2025-05-20T14:02:53.503Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/29/93c53c098d301132196c3238c312825324740851d77a8500a2462c0fd888/setuptools-80.8.0-py3-none-any.whl", hash = "sha256:95a60484590d24103af13b686121328cc2736bee85de8936383111e421b9edc0", size = 1201470, upload-time = "2025-05-20T14:02:51.348Z" }, -] - [[package]] name = "shellingham" version = "1.5.4" @@ -1222,50 +1099,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/3f/50/bad581df71744867e wheels = [ { url = "https://files.pythonhosted.org/packages/b7/1a/7e4798e9339adc931158c9d69ecc34f5e6791489d469f5e50ec15e35f458/zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931", size = 9630, upload-time = "2024-11-10T15:05:19.275Z" }, ] - -[[package]] -name = "zope-event" -version = "5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/46/c2/427f1867bb96555d1d34342f1dd97f8c420966ab564d58d18469a1db8736/zope.event-5.0.tar.gz", hash = "sha256:bac440d8d9891b4068e2b5a2c5e2c9765a9df762944bda6955f96bb9b91e67cd", size = 17350, upload-time = "2023-06-23T06:28:35.709Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/42/f8dbc2b9ad59e927940325a22d6d3931d630c3644dae7e2369ef5d9ba230/zope.event-5.0-py3-none-any.whl", hash = "sha256:2832e95014f4db26c47a13fdaef84cef2f4df37e66b59d8f1f4a8f319a632c26", size = 6824, upload-time = "2023-06-23T06:28:32.652Z" }, -] - -[[package]] -name = "zope-interface" -version = "7.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/30/93/9210e7606be57a2dfc6277ac97dcc864fd8d39f142ca194fdc186d596fda/zope.interface-7.2.tar.gz", hash = "sha256:8b49f1a3d1ee4cdaf5b32d2e738362c7f5e40ac8b46dd7d1a65e82a4872728fe", size = 252960, upload-time = "2024-11-28T08:45:39.224Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/76/71/e6177f390e8daa7e75378505c5ab974e0bf59c1d3b19155638c7afbf4b2d/zope.interface-7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ce290e62229964715f1011c3dbeab7a4a1e4971fd6f31324c4519464473ef9f2", size = 208243, upload-time = "2024-11-28T08:47:29.781Z" }, - { url = "https://files.pythonhosted.org/packages/52/db/7e5f4226bef540f6d55acfd95cd105782bc6ee044d9b5587ce2c95558a5e/zope.interface-7.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:05b910a5afe03256b58ab2ba6288960a2892dfeef01336dc4be6f1b9ed02ab0a", size = 208759, upload-time = "2024-11-28T08:47:31.908Z" }, - { url = "https://files.pythonhosted.org/packages/28/ea/fdd9813c1eafd333ad92464d57a4e3a82b37ae57c19497bcffa42df673e4/zope.interface-7.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:550f1c6588ecc368c9ce13c44a49b8d6b6f3ca7588873c679bd8fd88a1b557b6", size = 254922, upload-time = "2024-11-28T09:18:11.795Z" }, - { url = "https://files.pythonhosted.org/packages/3b/d3/0000a4d497ef9fbf4f66bb6828b8d0a235e690d57c333be877bec763722f/zope.interface-7.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ef9e2f865721553c6f22a9ff97da0f0216c074bd02b25cf0d3af60ea4d6931d", size = 249367, upload-time = "2024-11-28T08:48:24.238Z" }, - { url = "https://files.pythonhosted.org/packages/3e/e5/0b359e99084f033d413419eff23ee9c2bd33bca2ca9f4e83d11856f22d10/zope.interface-7.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27f926f0dcb058211a3bb3e0e501c69759613b17a553788b2caeb991bed3b61d", size = 254488, upload-time = "2024-11-28T08:48:28.816Z" }, - { url = "https://files.pythonhosted.org/packages/7b/90/12d50b95f40e3b2fc0ba7f7782104093b9fd62806b13b98ef4e580f2ca61/zope.interface-7.2-cp310-cp310-win_amd64.whl", hash = "sha256:144964649eba4c5e4410bb0ee290d338e78f179cdbfd15813de1a664e7649b3b", size = 211947, upload-time = "2024-11-28T08:48:18.831Z" }, - { url = "https://files.pythonhosted.org/packages/98/7d/2e8daf0abea7798d16a58f2f3a2bf7588872eee54ac119f99393fdd47b65/zope.interface-7.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1909f52a00c8c3dcab6c4fad5d13de2285a4b3c7be063b239b8dc15ddfb73bd2", size = 208776, upload-time = "2024-11-28T08:47:53.009Z" }, - { url = "https://files.pythonhosted.org/packages/a0/2a/0c03c7170fe61d0d371e4c7ea5b62b8cb79b095b3d630ca16719bf8b7b18/zope.interface-7.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:80ecf2451596f19fd607bb09953f426588fc1e79e93f5968ecf3367550396b22", size = 209296, upload-time = "2024-11-28T08:47:57.993Z" }, - { url = "https://files.pythonhosted.org/packages/49/b4/451f19448772b4a1159519033a5f72672221e623b0a1bd2b896b653943d8/zope.interface-7.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:033b3923b63474800b04cba480b70f6e6243a62208071fc148354f3f89cc01b7", size = 260997, upload-time = "2024-11-28T09:18:13.935Z" }, - { url = "https://files.pythonhosted.org/packages/65/94/5aa4461c10718062c8f8711161faf3249d6d3679c24a0b81dd6fc8ba1dd3/zope.interface-7.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a102424e28c6b47c67923a1f337ede4a4c2bba3965b01cf707978a801fc7442c", size = 255038, upload-time = "2024-11-28T08:48:26.381Z" }, - { url = "https://files.pythonhosted.org/packages/9f/aa/1a28c02815fe1ca282b54f6705b9ddba20328fabdc37b8cf73fc06b172f0/zope.interface-7.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25e6a61dcb184453bb00eafa733169ab6d903e46f5c2ace4ad275386f9ab327a", size = 259806, upload-time = "2024-11-28T08:48:30.78Z" }, - { url = "https://files.pythonhosted.org/packages/a7/2c/82028f121d27c7e68632347fe04f4a6e0466e77bb36e104c8b074f3d7d7b/zope.interface-7.2-cp311-cp311-win_amd64.whl", hash = "sha256:3f6771d1647b1fc543d37640b45c06b34832a943c80d1db214a37c31161a93f1", size = 212305, upload-time = "2024-11-28T08:49:14.525Z" }, - { url = "https://files.pythonhosted.org/packages/68/0b/c7516bc3bad144c2496f355e35bd699443b82e9437aa02d9867653203b4a/zope.interface-7.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:086ee2f51eaef1e4a52bd7d3111a0404081dadae87f84c0ad4ce2649d4f708b7", size = 208959, upload-time = "2024-11-28T08:47:47.788Z" }, - { url = "https://files.pythonhosted.org/packages/a2/e9/1463036df1f78ff8c45a02642a7bf6931ae4a38a4acd6a8e07c128e387a7/zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:21328fcc9d5b80768bf051faa35ab98fb979080c18e6f84ab3f27ce703bce465", size = 209357, upload-time = "2024-11-28T08:47:50.897Z" }, - { url = "https://files.pythonhosted.org/packages/07/a8/106ca4c2add440728e382f1b16c7d886563602487bdd90004788d45eb310/zope.interface-7.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6dd02ec01f4468da0f234da9d9c8545c5412fef80bc590cc51d8dd084138a89", size = 264235, upload-time = "2024-11-28T09:18:15.56Z" }, - { url = "https://files.pythonhosted.org/packages/fc/ca/57286866285f4b8a4634c12ca1957c24bdac06eae28fd4a3a578e30cf906/zope.interface-7.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e7da17f53e25d1a3bde5da4601e026adc9e8071f9f6f936d0fe3fe84ace6d54", size = 259253, upload-time = "2024-11-28T08:48:29.025Z" }, - { url = "https://files.pythonhosted.org/packages/96/08/2103587ebc989b455cf05e858e7fbdfeedfc3373358320e9c513428290b1/zope.interface-7.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cab15ff4832580aa440dc9790b8a6128abd0b88b7ee4dd56abacbc52f212209d", size = 264702, upload-time = "2024-11-28T08:48:37.363Z" }, - { url = "https://files.pythonhosted.org/packages/5f/c7/3c67562e03b3752ba4ab6b23355f15a58ac2d023a6ef763caaca430f91f2/zope.interface-7.2-cp312-cp312-win_amd64.whl", hash = "sha256:29caad142a2355ce7cfea48725aa8bcf0067e2b5cc63fcf5cd9f97ad12d6afb5", size = 212466, upload-time = "2024-11-28T08:49:14.397Z" }, - { url = "https://files.pythonhosted.org/packages/c6/3b/e309d731712c1a1866d61b5356a069dd44e5b01e394b6cb49848fa2efbff/zope.interface-7.2-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:3e0350b51e88658d5ad126c6a57502b19d5f559f6cb0a628e3dc90442b53dd98", size = 208961, upload-time = "2024-11-28T08:48:29.865Z" }, - { url = "https://files.pythonhosted.org/packages/49/65/78e7cebca6be07c8fc4032bfbb123e500d60efdf7b86727bb8a071992108/zope.interface-7.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15398c000c094b8855d7d74f4fdc9e73aa02d4d0d5c775acdef98cdb1119768d", size = 209356, upload-time = "2024-11-28T08:48:33.297Z" }, - { url = "https://files.pythonhosted.org/packages/11/b1/627384b745310d082d29e3695db5f5a9188186676912c14b61a78bbc6afe/zope.interface-7.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:802176a9f99bd8cc276dcd3b8512808716492f6f557c11196d42e26c01a69a4c", size = 264196, upload-time = "2024-11-28T09:18:17.584Z" }, - { url = "https://files.pythonhosted.org/packages/b8/f6/54548df6dc73e30ac6c8a7ff1da73ac9007ba38f866397091d5a82237bd3/zope.interface-7.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb23f58a446a7f09db85eda09521a498e109f137b85fb278edb2e34841055398", size = 259237, upload-time = "2024-11-28T08:48:31.71Z" }, - { url = "https://files.pythonhosted.org/packages/b6/66/ac05b741c2129fdf668b85631d2268421c5cd1a9ff99be1674371139d665/zope.interface-7.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a71a5b541078d0ebe373a81a3b7e71432c61d12e660f1d67896ca62d9628045b", size = 264696, upload-time = "2024-11-28T08:48:41.161Z" }, - { url = "https://files.pythonhosted.org/packages/0a/2f/1bccc6f4cc882662162a1158cda1a7f616add2ffe322b28c99cb031b4ffc/zope.interface-7.2-cp313-cp313-win_amd64.whl", hash = "sha256:4893395d5dd2ba655c38ceb13014fd65667740f09fa5bb01caa1e6284e48c0cd", size = 212472, upload-time = "2024-11-28T08:49:56.587Z" }, -] From 60e6ce9dbb720e76aab84e6caed6c8c9cd2826fb Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 22:50:05 +0200 Subject: [PATCH 04/59] feat: bootstrap same-python popen worker via `python -m` Launch the Trio popen worker as `python -m execnet._trio_worker ` instead of sending bootstrap source over the wire. The worker imports the installed execnet + trio, writes the `1` handshake after adopting its stdio fds, and serves; the coordinator just waits for the handshake. A rough major/minor version check warns on a real execnet mismatch between coordinator and worker while tolerating patch-level drift. Drops the import-bootstrap source send and the importdir/PYTHONPATH plumbing (no more uninstalled running). Foreign-python and remote transports stay on the legacy path pending the uv-provisioned bootstrap. Co-Authored-By: Claude Opus 4.8 --- doc/implnotes.rst | 18 +++++---- src/execnet/_trio_host.py | 79 ++++++++++++++++++------------------- src/execnet/_trio_worker.py | 57 +++++++++++++++++++++++++- 3 files changed, 105 insertions(+), 49 deletions(-) diff --git a/doc/implnotes.rst b/doc/implnotes.rst index c8229865..84492656 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -18,17 +18,21 @@ for standardizing or versioning the protocol. Trio host-thread IO (popen / import bootstrap) ---------------------------------------------- -For local ``popen`` gateways that use import-based bootstrap -(same installed ``execnet`` + ``trio`` on both sides), Message -protocol IO runs inside a dedicated OS thread hosting a Trio -event loop (``execnet._trio_host.TrioHost``): +For local same-interpreter ``popen`` gateways, Message protocol IO +runs inside a dedicated OS thread hosting a Trio event loop +(``execnet._trio_host.TrioHost``). No source is sent over the wire: +the worker is launched as ``python -m execnet._trio_worker`` and +imports the installed ``execnet`` + ``trio`` (a rough major/minor +version check guards against an incompatible install): * Coordinator: ``trio.lowlevel.open_process`` plus async framed reader/writer tasks per gateway (one host thread per ``Group``). + It waits for the worker's ``b"1"`` handshake before starting the + Message protocol. * Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio - streams; ``remote_exec`` is scheduled from the Trio nursery - (``trio.to_thread`` for ``thread``, main-thread handoff for - ``main_thread_only``). + streams and writes the handshake byte; ``remote_exec`` is + scheduled from the Trio nursery (``trio.to_thread`` for ``thread``, + main-thread handoff for ``main_thread_only``). Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from non-host threads wait until the frame is written (so abrupt diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index b0db9ad7..ce9ef1cb 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -10,6 +10,7 @@ import os import queue import subprocess +import sys import threading from collections.abc import Callable from typing import TYPE_CHECKING @@ -40,20 +41,19 @@ def trio_host_enabled() -> bool: def should_use_trio_popen(spec: Any) -> bool: - """PoC: Trio path only for local popen with import bootstrap.""" + """Trio path only for local same-interpreter popen (module bootstrap).""" if not trio_host_enabled(): return False if not getattr(spec, "popen", False): return False if getattr(spec, "via", None): return False + # A foreign interpreter (python=) is handled by the future uv-provisioned + # remote path, not the same-interpreter module launch. if getattr(spec, "python", None): return False - # Worker exec still uses WorkerPool; greenlet models stay on the legacy path. execmodel = getattr(spec, "execmodel", None) - if execmodel not in (None, "thread", "main_thread_only"): - return False - return True + return execmodel in (None, "thread", "main_thread_only") class AsyncByteIO(Protocol): @@ -273,7 +273,9 @@ async def _kill() -> None: def is_alive(self) -> bool: return not self._done.is_set() - async def task(self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED) -> None: + async def task( + self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED + ) -> None: try: async with trio.open_nursery() as nursery: nursery.start_soon(self._writer) @@ -480,30 +482,27 @@ async def open_popen_process(args: list[str]) -> trio.Process: ) -async def bootstrap_import_async( - process: trio.Process, - *, - importdir: str, - execmodel: str, - gateway_id: str, -) -> ProcessStreamsIO: - """Send import-bootstrap source and wait for the handshake byte.""" - io = ProcessStreamsIO(process) - sources = [ - "import sys", - "if %r not in sys.path:" % importdir, - " sys.path.insert(0, %r)" % importdir, - "from execnet._trio_worker import serve_popen_trio", - "sys.stdout.write('1')", - "sys.stdout.flush()", - "serve_popen_trio(id=%r, execmodel=%r)" % (f"{gateway_id}-worker", execmodel), +def popen_module_args(spec: Any) -> list[str]: + """Launch the Trio worker as a module: ``python -m execnet._trio_worker``. + + No source is sent over the wire; the worker imports the installed execnet + + trio. Only valid for same-interpreter popen (see ``should_use_trio_popen``). + The coordinator version is passed so the worker can do a rough compatibility + check against its own installed version. + """ + import execnet + + args = [sys.executable, "-u"] + if getattr(spec, "dont_write_bytecode", False): + args.append("-B") + args += [ + "-m", + "execnet._trio_worker", + f"{spec.id}-worker", + spec.execmodel, + execnet.__version__, ] - source = "\n".join(sources) - await io.write_all((repr(source) + "\n").encode("utf-8")) - ack = await io.read_exact(1) - if ack != b"1": - raise EOFError(f"bad bootstrap handshake: {ack!r}") - return io + return args class _TempIO: @@ -532,25 +531,23 @@ def kill(self) -> None: def makegateway_popen_trio(group: Any, spec: Any) -> Any: - """Create a popen Gateway using Trio process + protocol IO on both sides.""" - import execnet + """Create a same-interpreter popen Gateway on the Trio IO path. - from . import gateway_io - from .gateway_bootstrap import importdir + The worker is launched as ``python -m execnet._trio_worker`` and imports the + installed execnet + trio; nothing is sent over the wire to bootstrap it. + """ + import execnet host: TrioHost = group._ensure_trio_host() - args = gateway_io.popen_args(spec) - remote_execmodel = spec.execmodel + args = popen_module_args(spec) async def _create_and_attach() -> Any: process = await open_popen_process(args) try: - async_io = await bootstrap_import_async( - process, - importdir=importdir, - execmodel=remote_execmodel, - gateway_id=spec.id, - ) + async_io = ProcessStreamsIO(process) + ack = await async_io.read_exact(1) + if ack != b"1": + raise EOFError(f"bad bootstrap handshake: {ack!r}") except BaseException: with trio.move_on_after(5): process.kill() diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 6d247194..2c297589 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -11,7 +11,6 @@ import trio -from . import gateway_base from .gateway_base import MAIN_THREAD_ONLY_DEADLOCK_TEXT from .gateway_base import WorkerGateway from .gateway_base import get_execmodel @@ -211,6 +210,10 @@ def serve_popen_trio(id: str, execmodel: str = "thread") -> None: model = get_execmodel(execmodel) read_fd, write_fd = _prepare_protocol_fds() + # Bootstrap handshake: the coordinator waits for this byte on our stdout + # before starting the Message protocol. We are launched as a plain module + # (``python -m execnet._trio_worker``), so nothing was sent to bootstrap us. + os.write(write_fd, b"1") # Keep the historic trace token so tests looking for workergateway still pass. trace(f"creating workergateway on trio id={id!r}") @@ -250,3 +253,55 @@ async def _start() -> _trio_host.ProtocolSession: # Trio's to_thread cache uses non-daemon threads that would otherwise # keep this disposable worker process alive after serve returns. os._exit(0) + + +def _rough_version(version: str) -> tuple[int, ...]: + """Leading numeric (major, minor) of a version string; ``()`` if unparsable.""" + parts: list[int] = [] + for chunk in version.split(".")[:2]: + number = "" + for char in chunk: + if char.isdigit(): + number += char + else: + break + if not number: + break + parts.append(int(number)) + return tuple(parts) + + +def _check_version(coordinator_version: str) -> None: + """Warn on a real (major/minor) execnet version mismatch across the wire. + + A minimal (patch-level) mismatch is tolerated. For same-interpreter popen + the versions are always identical; this guards the future remote paths. + """ + import execnet + + ours = _rough_version(execnet.__version__) + theirs = _rough_version(coordinator_version) + if ours and theirs and ours != theirs: + sys.stderr.write( + "WARNING: execnet version mismatch: coordinator %s worker %s\n" + % (coordinator_version, execnet.__version__) + ) + sys.stderr.flush() + + +def _main() -> None: + """Entry point for ``python -m execnet._trio_worker [ver]``. + + The worker imports execnet + trio from the environment; no source is sent + over the wire to bootstrap it. + """ + argv = sys.argv + worker_id = argv[1] if len(argv) > 1 else "worker" + execmodel = argv[2] if len(argv) > 2 else "thread" + if len(argv) > 3: + _check_version(argv[3]) + serve_popen_trio(id=worker_id, execmodel=execmodel) + + +if __name__ == "__main__": + _main() From dfe7391024badcd7ecc15d18e440d193cd9f5aae Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 22:53:28 +0200 Subject: [PATCH 05/59] add claude settings --- .claude/settings.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .claude/settings.json diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000..489c3d39 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,7 @@ +{ + "permissions": { + "allow": [ + "Bash(uv run pytest:*)" + ] + } +} From 603dfc6551afdc66251a77463846ff8b1ff3674e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 00:21:13 +0200 Subject: [PATCH 06/59] feat: provision foreign-python popen workers via uv Route `popen//python=` through the Trio path. If the target interpreter already imports execnet + trio, launch the worker module directly on it (preserving sys.executable); otherwise provision an ephemeral environment with uv and run the worker there. New _provision.py builds the launch: - coordinator_requirement(): `execnet==` for a released coordinator, else a wheel built from the editable source (located via PEP 610 direct_url.json) and cached keyed by version. The wheel path is read from uv's "Successfully built" output rather than reconstructed, since the build-time version can differ from the import-time one. - uv_run_argv(): `uv run --no-project [--python X] --with python -u -m execnet._trio_worker ...`; trio comes in transitively. - target_has_execnet(): cached probe deciding direct vs provisioned. Falls back to the legacy source-copy path when the target lacks execnet and uv is unavailable. Co-Authored-By: Claude Opus 4.8 --- src/execnet/_provision.py | 180 ++++++++++++++++++++++++++++++++++++++ src/execnet/_trio_host.py | 50 ++++++++--- 2 files changed, 217 insertions(+), 13 deletions(-) create mode 100644 src/execnet/_provision.py diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py new file mode 100644 index 00000000..55a7a055 --- /dev/null +++ b/src/execnet/_provision.py @@ -0,0 +1,180 @@ +"""Coordinator-side worker provisioning via ``uv``. + +Non-same-interpreter Trio workers (foreign-python popen, later ssh/socket) are +launched inside an environment that ``uv`` provisions with a matching execnet + +trio. The worker itself is always ``python -m execnet._trio_worker`` and does +the usual ``b"1"`` stdio handshake; only the launch prefix differs. + +Delivery of execnet into that environment is version-aware: + +* released coordinator (``X.Y.Z``) -> ``uv run --with execnet==X.Y.Z`` +* dev coordinator (``X.Y.Z.devN+g...``) -> build a wheel from the editable + install's source tree, cache it keyed by version, and ``uv run --with `` + +trio is pulled transitively as an execnet dependency. +""" + +from __future__ import annotations + +import json +import re +import shutil +import subprocess +import tempfile +from functools import cache +from pathlib import Path +from typing import Any + +_RELEASED_RE = re.compile(r"^\d+\.\d+\.\d+$") + + +def uv_available() -> bool: + """Whether the ``uv`` launcher is on PATH.""" + return shutil.which("uv") is not None + + +@cache +def target_has_execnet(python: str) -> bool: + """Whether interpreter ``python`` can already import execnet + trio. + + When true the worker can be launched directly on that interpreter + (preserving ``sys.executable``); otherwise it must be uv-provisioned. + """ + from .gateway_io import shell_split_path + + argv = [*shell_split_path(python), "-c", "import execnet, trio"] + try: + completed = subprocess.run( + argv, capture_output=True, timeout=30, check=False + ) + return completed.returncode == 0 + except (OSError, subprocess.SubprocessError): + return False + + +def _version_slug(version: str) -> str: + """Filesystem-safe slug for a version string (may contain ``+``/``.``).""" + return re.sub(r"[^0-9A-Za-z]+", "_", version) + + +def _wheel_cache_dir() -> Path: + d = Path(tempfile.gettempdir()) / "execnet-bootstrap-wheels" + d.mkdir(parents=True, exist_ok=True) + return d + + +def _editable_source_root() -> Path | None: + """Source tree of an editable execnet install, via PEP 610 ``direct_url.json``. + + Returns ``None`` when execnet is not installed editable (nothing to build). + """ + from importlib.metadata import PackageNotFoundError + from importlib.metadata import distribution + from urllib.parse import urlparse + from urllib.request import url2pathname + + try: + dist = distribution("execnet") + except PackageNotFoundError: + return None + raw = dist.read_text("direct_url.json") + if not raw: + return None + info = json.loads(raw) + if not info.get("dir_info", {}).get("editable"): + return None + url = info.get("url", "") + if not url.startswith("file:"): + return None + return Path(url2pathname(urlparse(url).path)) + + +_BUILT_RE = re.compile(r"^Successfully built (?P.+\.whl)\s*$", re.MULTILINE) + + +def _parse_built_wheel(uv_build_stderr: str) -> Path | None: + """Extract the exact wheel path from ``uv build``'s ``Successfully built`` line. + + Building the filename ourselves is unsafe: the build-time version (dirty tree + -> ``.dYYYYMMDD``/differing dev count) can differ from the import-time + ``__version__``, so we trust the path uv reports. + """ + matches = _BUILT_RE.findall(uv_build_stderr) + if len(matches) != 1: + return None + return Path(matches[0].strip()) + + +def _build_wheel(version: str) -> Path: + """Build (and cache) a wheel of the editable execnet source for ``version``.""" + version_dir = _wheel_cache_dir() / _version_slug(version) + if version_dir.exists(): + cached = sorted(version_dir.glob("*.whl")) + if len(cached) == 1: + return cached[0] + + root = _editable_source_root() + if root is None: + raise RuntimeError( + f"cannot provision dev execnet {version!r}: no editable source tree " + "found (install a released execnet or an editable checkout)" + ) + version_dir.mkdir(parents=True, exist_ok=True) + proc = subprocess.run( + ["uv", "build", "--wheel", "-o", str(version_dir), str(root)], + check=True, + capture_output=True, + text=True, + ) + wheel = _parse_built_wheel(proc.stderr) + if wheel is None or not wheel.exists(): + raise RuntimeError( + f"could not determine wheel path from uv build for {version!r}:\n" + f"{proc.stderr}" + ) + return wheel + + +def coordinator_requirement() -> str: + """A ``uv --with`` requirement that installs this coordinator's execnet.""" + import execnet + + version = execnet.__version__ + if _RELEASED_RE.match(version): + return f"execnet=={version}" + return str(_build_wheel(version)) + + +def uv_run_argv( + *, python: str | None, requirement: str, module_args: list[str] +) -> list[str]: + """Build ``uv run [--python X] --with python -u -m execnet._trio_worker …``. + + ``--no-project`` keeps the surrounding execnet checkout from being synced, so + the ephemeral env holds only the requirement (+ trio). + """ + argv = ["uv", "run", "--no-project"] + if python: + argv += ["--python", python] + argv += [ + "--with", + requirement, + "python", + "-u", + "-m", + "execnet._trio_worker", + *module_args, + ] + return argv + + +def uv_worker_argv(spec: Any) -> list[str]: + """Full ``uv run`` argv to launch the Trio worker for ``spec``.""" + import execnet + + module_args = [f"{spec.id}-worker", spec.execmodel, execnet.__version__] + return uv_run_argv( + python=spec.python, + requirement=coordinator_requirement(), + module_args=module_args, + ) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index ce9ef1cb..d72b681d 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -41,19 +41,27 @@ def trio_host_enabled() -> bool: def should_use_trio_popen(spec: Any) -> bool: - """Trio path only for local same-interpreter popen (module bootstrap).""" + """Trio path for local popen. + + Same-interpreter popen launches the worker module directly; a foreign + interpreter (``python=``) is provisioned via ``uv`` and only taken when + ``uv`` is available (otherwise the legacy source-copy path handles it). + """ if not trio_host_enabled(): return False if not getattr(spec, "popen", False): return False if getattr(spec, "via", None): return False - # A foreign interpreter (python=) is handled by the future uv-provisioned - # remote path, not the same-interpreter module launch. - if getattr(spec, "python", None): - return False execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") + if execmodel not in (None, "thread", "main_thread_only"): + return False + if getattr(spec, "python", None): + from . import _provision + + # Direct launch if the interpreter already has execnet; else uv-provision. + return _provision.target_has_execnet(spec.python) or _provision.uv_available() + return True class AsyncByteIO(Protocol): @@ -486,13 +494,21 @@ def popen_module_args(spec: Any) -> list[str]: """Launch the Trio worker as a module: ``python -m execnet._trio_worker``. No source is sent over the wire; the worker imports the installed execnet + - trio. Only valid for same-interpreter popen (see ``should_use_trio_popen``). - The coordinator version is passed so the worker can do a rough compatibility + trio. Used for same-interpreter popen and for a ``python=`` interpreter that + already has execnet (so ``sys.executable`` stays that interpreter). The + coordinator version is passed so the worker can do a rough compatibility check against its own installed version. """ import execnet - args = [sys.executable, "-u"] + if getattr(spec, "python", None): + from .gateway_io import shell_split_path + + interpreter = shell_split_path(spec.python) + else: + interpreter = [sys.executable] + + args = [*interpreter, "-u"] if getattr(spec, "dont_write_bytecode", False): args.append("-B") args += [ @@ -531,15 +547,23 @@ def kill(self) -> None: def makegateway_popen_trio(group: Any, spec: Any) -> Any: - """Create a same-interpreter popen Gateway on the Trio IO path. + """Create a popen Gateway on the Trio IO path. - The worker is launched as ``python -m execnet._trio_worker`` and imports the - installed execnet + trio; nothing is sent over the wire to bootstrap it. + Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; + a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the + worker imports execnet + trio; nothing is sent over the wire to bootstrap it. """ import execnet + from . import _provision + host: TrioHost = group._ensure_trio_host() - args = popen_module_args(spec) + if spec.python and not _provision.target_has_execnet(spec.python): + # bare interpreter: provision execnet + trio via uv + args = _provision.uv_worker_argv(spec) + else: + # same interpreter, or a python= that already has execnet + args = popen_module_args(spec) async def _create_and_attach() -> Any: process = await open_popen_process(args) From 76563685af7204c19b07213b620260c833dcc138 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 07:32:11 +0200 Subject: [PATCH 07/59] test: add local ssh-connect harness + typing cleanups Add an in-process asyncssh server (asyncio-loop thread) with binary-safe command passthrough so the ssh transport can be tested against the system ssh client without an external host. Committed, intentionally-insecure ed25519 test keys back it (see testing/sshkeys/README.md); the fixture copies the client key to a 0600 temp file since git does not preserve it. asyncssh joins the testing extra. The initial test exercises the existing legacy ssh path, validating the harness before the Trio ssh transport. Also make mypy green at the source instead of casting at call sites: TrioHost.call is now generic (Callable[..., Awaitable[T]] -> T) and makegateway_popen_trio returns Gateway. Drop the stale types-gevent from the mypy hook. Co-Authored-By: Claude Opus 4.8 --- .pre-commit-config.yaml | 1 - pyproject.toml | 5 + src/execnet/_provision.py | 4 +- src/execnet/_trio_host.py | 17 ++- src/execnet/gateway.py | 2 +- testing/sshkeys/README.md | 17 +++ testing/sshkeys/insecure_client_ed25519 | 7 + testing/sshkeys/insecure_client_ed25519.pub | 1 + testing/sshkeys/insecure_host_ed25519 | 7 + testing/sshkeys/insecure_host_ed25519.pub | 1 + testing/test_ssh_local.py | 147 ++++++++++++++++++++ uv.lock | 44 +++++- 12 files changed, 241 insertions(+), 12 deletions(-) create mode 100644 testing/sshkeys/README.md create mode 100644 testing/sshkeys/insecure_client_ed25519 create mode 100644 testing/sshkeys/insecure_client_ed25519.pub create mode 100644 testing/sshkeys/insecure_host_ed25519 create mode 100644 testing/sshkeys/insecure_host_ed25519.pub create mode 100644 testing/test_ssh_local.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index be4b6e07..37ccd44d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,4 +32,3 @@ repos: additional_dependencies: - pytest - types-pywin32 - - types-gevent diff --git a/pyproject.toml b/pyproject.toml index 2c73a32b..853ba3d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ testing = [ "tox", "hatch", "uv", + "asyncssh", ] [dependency-groups] @@ -121,3 +122,7 @@ warn_unused_ignores = false disallow_untyped_calls = false disallow_untyped_defs = false disallow_incomplete_defs = false + +[[tool.mypy.overrides]] +module = ["asyncssh.*"] +ignore_missing_imports = true diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 55a7a055..b7f6ac56 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -44,9 +44,7 @@ def target_has_execnet(python: str) -> bool: argv = [*shell_split_path(python), "-c", "import execnet, trio"] try: - completed = subprocess.run( - argv, capture_output=True, timeout=30, check=False - ) + completed = subprocess.run(argv, capture_output=True, timeout=30, check=False) return completed.returncode == 0 except (OSError, subprocess.SubprocessError): return False diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index d72b681d..42b45047 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -12,11 +12,13 @@ import subprocess import sys import threading +from collections.abc import Awaitable from collections.abc import Callable from typing import TYPE_CHECKING from typing import Any from typing import Protocol from typing import TypeVar +from typing import cast import trio @@ -26,6 +28,7 @@ from .gateway_base import trace if TYPE_CHECKING: + from .gateway import Gateway from .gateway_base import BaseGateway T = TypeVar("T") @@ -251,7 +254,7 @@ def wait_process(self) -> int | None: async def _wait() -> int | None: assert self.process is not None # Always await wait() so the child is reaped (no zombies). - code = await self.process.wait() + code: int | None = await self.process.wait() self._process_exitcode = code self._process_done.set() return code @@ -434,15 +437,17 @@ async def _main(self) -> None: finally: self._nursery = None - def call(self, async_fn: Callable[..., Any], *args: Any) -> Any: + def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: if self._token is None: raise RuntimeError("TrioHost is not running") - return trio.from_thread.run(async_fn, *args, trio_token=self._token) + return cast("T", trio.from_thread.run(async_fn, *args, trio_token=self._token)) def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: if self._token is None: raise RuntimeError("TrioHost is not running") - return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + return cast( + "T", trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + ) def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: """Schedule a task on the root nursery (must be called on the host thread).""" @@ -546,7 +551,7 @@ def kill(self) -> None: return -def makegateway_popen_trio(group: Any, spec: Any) -> Any: +def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: """Create a popen Gateway on the Trio IO path. Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; @@ -565,7 +570,7 @@ def makegateway_popen_trio(group: Any, spec: Any) -> Any: # same interpreter, or a python= that already has execnet args = popen_module_args(spec) - async def _create_and_attach() -> Any: + async def _create_and_attach() -> Gateway: process = await open_popen_process(args) try: async_io = ProcessStreamsIO(process) diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 360b9865..6e336d6f 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -103,7 +103,7 @@ def hasreceiver(self) -> bool: """Whether gateway is able to receive data.""" session = self._trio_session if session is not None: - return session.is_alive() + return bool(session.is_alive()) return self._receivepool.active_count() > 0 def remote_status(self) -> RemoteStatus: diff --git a/testing/sshkeys/README.md b/testing/sshkeys/README.md new file mode 100644 index 00000000..d0133e2c --- /dev/null +++ b/testing/sshkeys/README.md @@ -0,0 +1,17 @@ +# Intentionally insecure test SSH keys + +These ed25519 keypairs are **committed on purpose** and are **not secret**. They +exist only so the local ssh-connect tests (`testing/test_ssh_local.py`) can run +an in-process asyncssh server that the system `ssh` client authenticates against, +without generating keys at runtime. + +- `insecure_host_ed25519[.pub]` — the test SSH **server** host key. +- `insecure_client_ed25519[.pub]` — the test **client** identity; its public key + is the server's sole authorized key. + +**Never** use these anywhere real. They grant nothing beyond a throwaway server +bound to `127.0.0.1` on an ephemeral port during a test run. + +Note: git does not preserve `0600` permissions, and OpenSSH refuses a +world-readable private key, so the tests copy the client key to a temp dir and +`chmod 0600` it before use. diff --git a/testing/sshkeys/insecure_client_ed25519 b/testing/sshkeys/insecure_client_ed25519 new file mode 100644 index 00000000..6f5650c8 --- /dev/null +++ b/testing/sshkeys/insecure_client_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACB6MZDwubY5LfTxiuBxhoOmRZaebHcTuRC/ahHgDDDXzwAAAKBzGIF1cxiB +dQAAAAtzc2gtZWQyNTUxOQAAACB6MZDwubY5LfTxiuBxhoOmRZaebHcTuRC/ahHgDDDXzw +AAAEBCIAXAZCBC6mZFORLaloyPr6HZsRkWpVxVd/vKSZpIhXoxkPC5tjkt9PGK4HGGg6ZF +lp5sdxO5EL9qEeAMMNfPAAAAHGV4ZWNuZXQtaW5zZWN1cmUtdGVzdC1jbGllbnQB +-----END OPENSSH PRIVATE KEY----- diff --git a/testing/sshkeys/insecure_client_ed25519.pub b/testing/sshkeys/insecure_client_ed25519.pub new file mode 100644 index 00000000..b4d824b9 --- /dev/null +++ b/testing/sshkeys/insecure_client_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHoxkPC5tjkt9PGK4HGGg6ZFlp5sdxO5EL9qEeAMMNfP execnet-insecure-test-client diff --git a/testing/sshkeys/insecure_host_ed25519 b/testing/sshkeys/insecure_host_ed25519 new file mode 100644 index 00000000..0a31038e --- /dev/null +++ b/testing/sshkeys/insecure_host_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACBqqoHPiDB2eUJlGVCLM2eRtfelu138wnGLDRHd1AG9RwAAAKDBS525wUud +uQAAAAtzc2gtZWQyNTUxOQAAACBqqoHPiDB2eUJlGVCLM2eRtfelu138wnGLDRHd1AG9Rw +AAAECXfzL6Ua9c9U+UkQn+Q7A1aKlBboBhABFALH+/ojmbMGqqgc+IMHZ5QmUZUIszZ5G1 +96W7XfzCcYsNEd3UAb1HAAAAGmV4ZWNuZXQtaW5zZWN1cmUtdGVzdC1ob3N0AQID +-----END OPENSSH PRIVATE KEY----- diff --git a/testing/sshkeys/insecure_host_ed25519.pub b/testing/sshkeys/insecure_host_ed25519.pub new file mode 100644 index 00000000..93d3cf93 --- /dev/null +++ b/testing/sshkeys/insecure_host_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGqqgc+IMHZ5QmUZUIszZ5G196W7XfzCcYsNEd3UAb1H execnet-insecure-test-host diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py new file mode 100644 index 00000000..196c5d76 --- /dev/null +++ b/testing/test_ssh_local.py @@ -0,0 +1,147 @@ +"""Local ssh-connect tests backed by an in-process asyncssh server. + +The coordinator shells out to the system ``ssh`` client, which connects to an +asyncssh server running in its own asyncio-loop thread; the server runs each +requested command as a subprocess with binary-safe stdio passthrough (the +execnet Message protocol needs raw bytes). +""" + +from __future__ import annotations + +import asyncio +import os +import shutil +import sys +import threading +from collections.abc import Iterator +from pathlib import Path + +import asyncssh +import pytest + +import execnet + +pytestmark = [ + pytest.mark.skipif( + shutil.which("ssh") is None, reason="system ssh client required" + ), + # asyncssh leaves an un-awaited internal Queue.join coroutine on shutdown, + # surfaced by pytest's unraisable-exception hook during GC; harmless here. + pytest.mark.filterwarnings("ignore:coroutine 'Queue.join' was never awaited"), +] + +# Committed, intentionally-insecure test keys (see sshkeys/README.md). +SSHKEYS = Path(__file__).parent / "sshkeys" +HOST_KEY = SSHKEYS / "insecure_host_ed25519" +CLIENT_KEY = SSHKEYS / "insecure_client_ed25519" +CLIENT_PUBKEY = SSHKEYS / "insecure_client_ed25519.pub" + + +class SSHServerThread: + """asyncssh server on an ephemeral port, driven from its own asyncio thread.""" + + def __init__(self, client_key_path: str) -> None: + # OpenSSH refuses a world-readable private key; git does not preserve + # 0600, so the caller hands us a temp copy already chmod'd 0600. + self.client_key_path = client_key_path + self._loop: asyncio.AbstractEventLoop | None = None + self._server: asyncssh.SSHAcceptor | None = None + self.port: int | None = None + self._ready = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + async def _handle(self, process: asyncssh.SSHServerProcess) -> None: + proc = await asyncio.create_subprocess_shell( + process.command or "", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await process.redirect(stdin=proc.stdin, stdout=proc.stdout, stderr=proc.stderr) + process.exit(await proc.wait()) + + async def _serve(self) -> None: + self._server = await asyncssh.listen( + "127.0.0.1", + 0, + server_host_keys=[str(HOST_KEY)], + authorized_client_keys=str(CLIENT_PUBKEY), + process_factory=self._handle, + encoding=None, # binary stdio + ) + self.port = self._server.get_port() + self._ready.set() + await self._server.wait_closed() + + def _run(self) -> None: + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + try: + self._loop.run_until_complete(self._serve()) + finally: + # Drain asyncssh's shutdown coroutines so GC does not surface an + # "un-awaited coroutine" RuntimeWarning. + pending = asyncio.all_tasks(self._loop) + for task in pending: + task.cancel() + self._loop.run_until_complete( + asyncio.gather(*pending, return_exceptions=True) + ) + self._loop.run_until_complete(self._loop.shutdown_asyncgens()) + self._loop.close() + + def start(self) -> None: + self._thread.start() + assert self._ready.wait(timeout=10), "ssh server did not start" + + def stop(self) -> None: + if self._loop is not None and self._server is not None: + self._loop.call_soon_threadsafe(self._server.close) + self._thread.join(timeout=5) + + def write_ssh_config(self, path: str) -> None: + """Write an ssh config with a ``testhost`` alias pointing at this server.""" + with open(path, "w") as f: + f.write( + "Host testhost\n" + " HostName 127.0.0.1\n" + f" Port {self.port}\n" + " User testuser\n" + f" IdentityFile {self.client_key_path}\n" + " IdentitiesOnly yes\n" + " StrictHostKeyChecking no\n" + " UserKnownHostsFile /dev/null\n" + " LogLevel ERROR\n" + ) + + +@pytest.fixture +def ssh_server(tmp_path) -> Iterator[SSHServerThread]: + # OpenSSH rejects the committed key's checkout permissions; use a 0600 copy. + client_key = tmp_path / "client_ed25519" + client_key.write_bytes(CLIENT_KEY.read_bytes()) + client_key.chmod(0o600) + server = SSHServerThread(str(client_key)) + server.start() + yield server + server.stop() + + +@pytest.fixture +def ssh_config(ssh_server: SSHServerThread, tmp_path) -> str: + path = str(tmp_path / "ssh_config") + ssh_server.write_ssh_config(path) + return path + + +def test_ssh_roundtrip(ssh_config: str) -> None: + group = execnet.Group() + try: + gw = group.makegateway( + f"ssh=testhost//ssh_config={ssh_config}//python={sys.executable}//id=ssh" + ) + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(timeout=5.0) diff --git a/uv.lock b/uv.lock index 5dbf4f10..e4657f7b 100644 --- a/uv.lock +++ b/uv.lock @@ -16,6 +16,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] +[[package]] +name = "asyncssh" +version = "2.24.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/7f/2d79247bacc562104f312d27efe541673aa177feac89de291bd61bca52be/asyncssh-2.24.0.tar.gz", hash = "sha256:4064c590e59ce2e8d82a2f66d35f3120d765828b4df5e3dbfb07b4a8c24686c9", size = 550148, upload-time = "2026-06-27T20:34:44.755Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/29/908ce0ca5e8cae76662e354a0f08df552d6d221844748b9e5ca06051cc44/asyncssh-2.24.0-py3-none-any.whl", hash = "sha256:9abd46300adcb6d4b73269b34c53cd0d17a138b9a22b5b38008ce7d5808734b7", size = 381237, upload-time = "2026-06-27T20:34:43.198Z" }, +] + [[package]] name = "attrs" version = "26.1.0" @@ -145,6 +158,8 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/e9/6d7724983b3d5a0908dbf74f64038ade77c18646ff6636ec7894fd392ce1/cffi-2.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:b65f590ef2a44640f9a05dbb548a429b4ade77913ce683ac8b1480777658a6c0", size = 183837, upload-time = "2026-07-06T21:32:09.655Z" }, + { url = "https://files.pythonhosted.org/packages/69/aa/24580a278de21fd7322635556334d9b535f1cbc00b0a3919447cdf464c65/cffi-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:164bff1657b2a74f0b6d54e11c9b375bc97b931f2ca9c43fcf875838da1570dd", size = 184226, upload-time = "2026-07-06T21:32:11.196Z" }, { url = "https://files.pythonhosted.org/packages/88/a9/02cae418ec4beb282ace11958d9d4737793439d561fadc7e6d56f2e2b354/cffi-2.1.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:c941bb58d5a6e1c3892d86e42927ed6c180302f07e6d395d08c416e594b98b46", size = 211107, upload-time = "2026-07-06T21:32:12.328Z" }, { url = "https://files.pythonhosted.org/packages/3b/30/c806937ed5e4c2c7ac30d9d6b76b5dc57ff8b75d83800d9bb11a8253cf2a/cffi-2.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a016194dbe13d14ee9556e734b772d8d67b947092b268d757fd4290e3ba2dfc2", size = 218733, upload-time = "2026-07-06T21:32:13.67Z" }, { url = "https://files.pythonhosted.org/packages/f9/cf/398272b8bbfd58aa314fda5a7f1cdbb26d1d78ae324a11211521315dd1f0/cffi-2.1.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:03e9810d18c646077e501f661b682fbf5dee4676048527ca3cffe66faa9960dd", size = 205543, upload-time = "2026-07-06T21:32:15.148Z" }, @@ -155,6 +170,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/75/74dfb7c3fc6ebbd408038476bd4c1d7e925c62614e7b9c534ecc34218288/cffi-2.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:11b3fb55f4f8ad92274ed26705f65d8f91457de71f5380061eb6d125a768fecd", size = 220341, upload-time = "2026-07-06T21:32:21.9Z" }, { url = "https://files.pythonhosted.org/packages/70/b6/9003c33a3e7d2c1306f5962e646457dcfe5a8cd8fce6bbe02d7af25db783/cffi-2.1.0-cp310-cp310-win32.whl", hash = "sha256:9d72af0cf10a76a600a9690078fe31c63b9588c8e86bf9fd353f713c84b5db0f", size = 174578, upload-time = "2026-07-06T21:32:23.073Z" }, { url = "https://files.pythonhosted.org/packages/8a/26/710688310447531c7a22f857c7f79d9855ec18b03e04494ced723fb37e2f/cffi-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:fb62edb5bb52cca65fab91a63afa7561607120d26090a7e8fda6fb9f064726da", size = 185071, upload-time = "2026-07-06T21:32:24.671Z" }, + { url = "https://files.pythonhosted.org/packages/d3/67/85c89a59ba36a671e79638f44d466749f08179266a57e4f2ffdf92174072/cffi-2.1.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:02cb7ff33ded4f1532476731f89ede53e2e488a8e6205515a82144246ffa7dcc", size = 183845, upload-time = "2026-07-06T21:32:26.32Z" }, + { url = "https://files.pythonhosted.org/packages/ea/dd/e3b0baa2d3d6a857ac72b7efbf18e32e487c9cdafcc13049ad765495b15e/cffi-2.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f5bce581e6b8c235e566a14768a943b172ada3ed73537bb0c0be1edee312d4e7", size = 184186, upload-time = "2026-07-06T21:32:28.025Z" }, { url = "https://files.pythonhosted.org/packages/65/68/9f3ef890cf3c6ab97bd531c5677f67613d302165d16f8142b2811782a614/cffi-2.1.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:30b65779d598c370374fefabf138d456fd6f3216bfa7bedfab1ba82025b0cd93", size = 211892, upload-time = "2026-07-06T21:32:29.565Z" }, { url = "https://files.pythonhosted.org/packages/22/d7/1a74539db16d8bfd839ff1515948948efbb162e574650fd3d846896eea95/cffi-2.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2", size = 218793, upload-time = "2026-07-06T21:32:30.951Z" }, { url = "https://files.pythonhosted.org/packages/ec/d1/9a5b7169499e8e8d8e636de70b97ac7c9447104d2ff1a2cd94790cea5162/cffi-2.1.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0a96b74cda968eebbad56d973efe5098974f0a9fb323865bf99ea1fd24e3e64c", size = 205737, upload-time = "2026-07-06T21:32:32.216Z" }, @@ -166,6 +183,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/da/5c4918a2d61d86fa927d716cb3d8e4626ef8dc8f605a599d32f33897f59a/cffi-2.1.0-cp311-cp311-win32.whl", hash = "sha256:64c753a0f87a256020004f37a1c8c02c480e725f910f0b2a0f3f07debd1b2479", size = 174496, upload-time = "2026-07-06T21:32:40.467Z" }, { url = "https://files.pythonhosted.org/packages/f9/c8/6c2de1d55cf35ef8b92885d5ef280790f0fb9634d87ea1cc315176aecd61/cffi-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:4f26194e3d95e06501b942642855aed4f953d55e95d7d01b7c4483db3ecff458", size = 185113, upload-time = "2026-07-06T21:32:41.761Z" }, { url = "https://files.pythonhosted.org/packages/9e/4e/e8d7cb5783f1841a3c8fb3a7735838d7484d08ec08c9f984b14cac1ac0e9/cffi-2.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:35aaea0c7ee0e58a5cd8c2fd1a48fdf7ece0d2699b7ecdda08194e9ce5dd9b3d", size = 179927, upload-time = "2026-07-06T21:32:42.961Z" }, + { url = "https://files.pythonhosted.org/packages/1e/85/990925db5df586ec90beb97529c853497e7f85ba0234830447faf41c3057/cffi-2.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f", size = 184829, upload-time = "2026-07-06T21:32:44.324Z" }, + { url = "https://files.pythonhosted.org/packages/4b/92/e7bb136ad6b5352603732cf907ef862ca103f20f2031c1735a46300c20c9/cffi-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde", size = 184728, upload-time = "2026-07-06T21:32:45.683Z" }, { url = "https://files.pythonhosted.org/packages/c3/c0/d1ec30ffb370f748f2fb54425972bfef9871e0132e82fb589c46b6676049/cffi-2.1.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d", size = 214815, upload-time = "2026-07-06T21:32:48.557Z" }, { url = "https://files.pythonhosted.org/packages/1b/dc/5620cf930688be01f2d673804291de757a934c90b946dbdc3d84130c2ea4/cffi-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7", size = 222429, upload-time = "2026-07-06T21:32:49.848Z" }, { url = "https://files.pythonhosted.org/packages/4b/a4/77b53abbf7a1e0beb9637edbef2a94d15f9c822f591e85d439ffd91519a6/cffi-2.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b", size = 210315, upload-time = "2026-07-06T21:32:51.221Z" }, @@ -178,6 +197,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/2c/400ea43e721727dca8a65c4521390e9196757caba4a45643acb2b63271b8/cffi-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6", size = 180088, upload-time = "2026-07-06T21:33:02.278Z" }, { url = "https://files.pythonhosted.org/packages/96/88/a996879e2eeccb815f6e3a5967b12a308257412acec882039d386bd2aa7b/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda", size = 194331, upload-time = "2026-07-06T21:33:03.697Z" }, { url = "https://files.pythonhosted.org/packages/58/85/7ae00d5c8dd6266f4e944c3db630f3c5c9a98b61d469c714d848b1d8138a/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b", size = 196966, upload-time = "2026-07-06T21:33:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e9/45c3a76ad8d43ad9261f4c95436da61128d3ca545d72b9612c0ab5be0b1c/cffi-2.1.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a", size = 184795, upload-time = "2026-07-06T21:33:06.699Z" }, + { url = "https://files.pythonhosted.org/packages/84/4c/82f132cb4418ee6d953d982b19191e87e2a6372c8a4ce36e50b69d6ade4a/cffi-2.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea", size = 184746, upload-time = "2026-07-06T21:33:08.071Z" }, { url = "https://files.pythonhosted.org/packages/a0/1c/4ed5a0e5bdca6cbc275556de3328dd1b76fd0c11cc13c88fe66d1d8715f2/cffi-2.1.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db", size = 214747, upload-time = "2026-07-06T21:33:09.671Z" }, { url = "https://files.pythonhosted.org/packages/3a/a6/e879bb68cc23a2bc9ba8f4b7d8019f0c2694bad2ab6c4a3701d429439f58/cffi-2.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f", size = 222392, upload-time = "2026-07-06T21:33:10.896Z" }, { url = "https://files.pythonhosted.org/packages/88/f6/01890cfd63c08f8eb96a8319b0443690197d240a8bd6346048cf7bde9190/cffi-2.1.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d", size = 210285, upload-time = "2026-07-06T21:33:12.251Z" }, @@ -190,6 +211,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/af/34fee85c48f8d94efc8597bc09470c9dd274c145f1c12e0fbc6ab6d38d74/cffi-2.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f", size = 180114, upload-time = "2026-07-06T21:33:22.515Z" }, { url = "https://files.pythonhosted.org/packages/d8/f0/81478e482afa03f6d18dc8f2afb5edc45b3080853b634b5ed91961be0998/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a", size = 194142, upload-time = "2026-07-06T21:33:23.657Z" }, { url = "https://files.pythonhosted.org/packages/7d/95/8de304305cd9204974b0ca051b86d307cafca13aa575a0ef1b44d92c0d8c/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3", size = 196819, upload-time = "2026-07-06T21:33:25.007Z" }, + { url = "https://files.pythonhosted.org/packages/20/71/7c8372d30e42415602ed9f268f7cfd66f1b855fed881ecd168bcb45dbc0b/cffi-2.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d", size = 184965, upload-time = "2026-07-06T21:33:26.605Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5c/584e626835f0375c928176c04137c96927165cb8733cdb3150ec04e5ee5e/cffi-2.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c4165821e131d6d4ca444347c2b694e2311bcfa3fe5a861cc72968f28867beac", size = 184952, upload-time = "2026-07-06T21:33:27.823Z" }, { url = "https://files.pythonhosted.org/packages/2e/d2/065fcae1c73979fac8e054462478d0ff8a29c40cdc2ed7ea5676a061df53/cffi-2.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6", size = 222353, upload-time = "2026-07-06T21:33:29.178Z" }, { url = "https://files.pythonhosted.org/packages/ed/a5/e8bbb1ce5b3ac2f53ad6a10bde44318a5a8d99d4f4a000d44a6e39aeb3e4/cffi-2.1.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913", size = 210051, upload-time = "2026-07-06T21:33:30.534Z" }, { url = "https://files.pythonhosted.org/packages/28/ed/c127d3ac36e899c965e3361357c3befacd6578c03f40125183e41c3b219e/cffi-2.1.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d", size = 208630, upload-time = "2026-07-06T21:33:31.753Z" }, @@ -199,6 +222,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0f/6f/ade5ce9863a57992a6ea3d0d10d7e29b8749fc127204b3d493d667b2815f/cffi-2.1.0-cp314-cp314-win32.whl", hash = "sha256:1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd", size = 177723, upload-time = "2026-07-06T21:33:51.626Z" }, { url = "https://files.pythonhosted.org/packages/41/de/92b9eeed4ae4a21d6fd9b2a2c8505cbed573299902ea73981cc13f7ff62c/cffi-2.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb", size = 187937, upload-time = "2026-07-06T21:33:53.403Z" }, { url = "https://files.pythonhosted.org/packages/2e/1a/cc6ae6c2913a03aab8898eee57963cf1035b8df5872ed8b9115fcc7e2be8/cffi-2.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804", size = 183001, upload-time = "2026-07-06T21:33:54.74Z" }, + { url = "https://files.pythonhosted.org/packages/14/f0/134c00ce0779ec86dea2aa1aac69339c2741a8045072676763512363a2ea/cffi-2.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7ea6b3e2c4250ff1de21c630fe72d0f63eb95c2c32ffbf64a358cf4a8836d714", size = 188538, upload-time = "2026-07-06T21:33:36.792Z" }, + { url = "https://files.pythonhosted.org/packages/50/d8/3b86aba791cb610d24e8a3e1b2cd529e71fa15096b04e4d4e360049d4a4c/cffi-2.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6af371f3767faeffc6ac1ef57cdfd25844403e9d3f476c5537caee499de96376", size = 188230, upload-time = "2026-07-06T21:33:38.011Z" }, { url = "https://files.pythonhosted.org/packages/14/d0/117dcd9209255ad8571fbc8c92ef32593a1d294dcec91ddc4e4db50606f2/cffi-2.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98", size = 223899, upload-time = "2026-07-06T21:33:39.514Z" }, { url = "https://files.pythonhosted.org/packages/b6/3d/f20f8b886b254e3ad10e15cd4186d3aed49f3e6a35ab37aab9f8f25f7c03/cffi-2.1.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13", size = 211652, upload-time = "2026-07-06T21:33:40.851Z" }, { url = "https://files.pythonhosted.org/packages/28/3b/fad54de07260b93ddeef4b96d0131d57ea900675df1d410ae1deee52d7a6/cffi-2.1.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d", size = 210755, upload-time = "2026-07-06T21:33:42.183Z" }, @@ -210,6 +235,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/dd/0c7dbf815a579ff005008a2d815a55d6bb047c349eef536d9dc53d3f0a8d/cffi-2.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8", size = 186404, upload-time = "2026-07-06T21:33:50.309Z" }, { url = "https://files.pythonhosted.org/packages/55/c7/8c8c50cb11c6750051daf12164098a9a6f027ac4356967fd4d800a07f242/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c", size = 194121, upload-time = "2026-07-06T21:33:56.109Z" }, { url = "https://files.pythonhosted.org/packages/99/e2/67680bf19a6b60d2bb7ff83baefa2a4c3d2d7dc0f3277034b802e1fc504c/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001", size = 196820, upload-time = "2026-07-06T21:33:57.288Z" }, + { url = "https://files.pythonhosted.org/packages/ed/da/4bbe583a3b3a5c8c60892124fe17f3fa3656523faf0d3484eae90f091853/cffi-2.1.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:95f2954c2c9473d892eca6e0409f3568b37ab62a8eedb122461f73cc273476e3", size = 184936, upload-time = "2026-07-06T21:33:58.765Z" }, + { url = "https://files.pythonhosted.org/packages/e5/4b/1f4c36ab273980d7aa75bb126ea4f8971f24a96108acad3a0a084028c57b/cffi-2.1.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:cdf2448aab5f661c9315308ec8b93f4e8a1a67a3c733f8631067a2b67d5913dc", size = 185045, upload-time = "2026-07-06T21:34:00.085Z" }, { url = "https://files.pythonhosted.org/packages/ef/c3/ad299dc38f3583f8d916b299f028af418a9ec98bc695fcbebeae7420691c/cffi-2.1.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699", size = 222342, upload-time = "2026-07-06T21:34:01.814Z" }, { url = "https://files.pythonhosted.org/packages/eb/d8/df4543cc087245044ed02ef3ad8e0a26619d0075ac7a77a12dc81177851b/cffi-2.1.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022", size = 210073, upload-time = "2026-07-06T21:34:03.255Z" }, { url = "https://files.pythonhosted.org/packages/2c/0e/fac738d73728c6cea2a88a2883dca54892496cbba88a1dc1f2909cb8a6f5/cffi-2.1.0-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0", size = 208551, upload-time = "2026-07-06T21:34:04.433Z" }, @@ -219,6 +246,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/d8/772b8259bf75749adffb1c546828978381fb516f60cf701f6c83daf60c85/cffi-2.1.0-cp315-cp315-win32.whl", hash = "sha256:0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6", size = 177696, upload-time = "2026-07-06T21:34:26.355Z" }, { url = "https://files.pythonhosted.org/packages/2f/dd/afa2191fc6d57fedd26e5844a2fe2fcc0bbfa00961bbaa5a41e4921e7cca/cffi-2.1.0-cp315-cp315-win_amd64.whl", hash = "sha256:bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853", size = 187914, upload-time = "2026-07-06T21:34:27.58Z" }, { url = "https://files.pythonhosted.org/packages/05/ef/6cd4f8c671517162379dc79cfae5aea9106bc38abb89628d5c16adf6a838/cffi-2.1.0-cp315-cp315-win_arm64.whl", hash = "sha256:8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda", size = 183004, upload-time = "2026-07-06T21:34:28.905Z" }, + { url = "https://files.pythonhosted.org/packages/11/b6/12fc55092817a5faa26fb8c40c7f9d662e11a46ee248c137aafc42517d92/cffi-2.1.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:f9912624a0c0b834b7520d7769b3644453aabc0a7e1c839da7359f050750e9bc", size = 188378, upload-time = "2026-07-06T21:34:09.926Z" }, + { url = "https://files.pythonhosted.org/packages/8d/2e/cdac88979f295fde5daa69622c7d2111e56e7ceb94f211357fbe452339e4/cffi-2.1.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:df92f2aba50eb4d96718b68ef76f2e57a57b54f2fa62333496d16c6d585a85ca", size = 188319, upload-time = "2026-07-06T21:34:11.101Z" }, { url = "https://files.pythonhosted.org/packages/e0/27/1d0b408497e41a74795af122d7b603c418c5fed0171450f899afd04e594f/cffi-2.1.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d", size = 223904, upload-time = "2026-07-06T21:34:12.606Z" }, { url = "https://files.pythonhosted.org/packages/8b/31/e115c985105dd7ffb32444505f18ceb874bb42d992af05d5dced7ecf1980/cffi-2.1.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8", size = 211554, upload-time = "2026-07-06T21:34:13.987Z" }, { url = "https://files.pythonhosted.org/packages/5a/67/9e6e09409336d9e515c58367e7cfcf4f89df06ad25252675595a58eb59d5/cffi-2.1.0-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd", size = 210795, upload-time = "2026-07-06T21:34:15.972Z" }, @@ -270,6 +299,7 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/12/45/870e7f4bef50e5f53b9f51d4428aee5290eedf58ba443f16b1ebb7ab8e66/cryptography-48.0.1.tar.gz", hash = "sha256:266f4ee051abb2f725b74ef8072b521ce1feacf685a3364fa6a6b45548db791a", size = 832989, upload-time = "2026-06-09T22:32:31.8Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/bc/ee4137cbbe105652c0ee4252792b78fc8e7afa4b8e61d9d5dc05a7f45731/cryptography-48.0.1-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3e4a1a3232eef2e6c732827d5722db29a0cc8b27af2a4d865b094cf954be9ca1", size = 8008324, upload-time = "2026-06-09T22:31:00.702Z" }, { url = "https://files.pythonhosted.org/packages/d5/85/6379d42181bfc713094f081360fc5784d6c816b599d45e7f082502d173ce/cryptography-48.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32143b24adb918f078134e1e230f1eb8cc04886b92c28b5f0041aaf3e5699225", size = 4696243, upload-time = "2026-06-09T22:32:33.446Z" }, { url = "https://files.pythonhosted.org/packages/9c/87/c85d147b53323c7eb4d850920c8901377323c2a0ff8d79c262d4fee89aa2/cryptography-48.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0d27a5696721ef7a672b8c810f6aded391058e0b9486e63e6d93baf765da691", size = 4713235, upload-time = "2026-06-09T22:31:40.141Z" }, { url = "https://files.pythonhosted.org/packages/79/58/67cbf8cf1ee7c54b439ca07bbecf8362c07afc11a3724fea70f745784add/cryptography-48.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb86ce1af36fe65041b6db9a8bb064ee621a7e5fded0f80d475ec243477cd242", size = 4702323, upload-time = "2026-06-09T22:31:42.191Z" }, @@ -281,6 +311,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/13/8b765e2e12b07c74941caadb9d1c8fdc006c4dfbf2b8f2d610519758954d/cryptography-48.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:0ee6ea481db1ab889cba043ec1eda17bb9c1ea79db6722f779c3667f9f70322f", size = 4745488, upload-time = "2026-06-09T22:32:30.07Z" }, { url = "https://files.pythonhosted.org/packages/2e/aa/48972bce55049b32a94f4907eda4d75fa385aad8a39506cc2fc72196ecf0/cryptography-48.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f2ceef93cb096aa3c4cc4b5c94ca6131f9196d28c64d6111533402a9b2054d41", size = 4830256, upload-time = "2026-06-09T22:31:43.868Z" }, { url = "https://files.pythonhosted.org/packages/47/a2/e5079a032fb85cf6005046ca92bbd78b0c82dad2b5751ab8c311659da06f/cryptography-48.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9bd3f92d76217892b15df84ca256c2c113d386fdda7a7d8691aeeced976507c6", size = 4979117, upload-time = "2026-06-09T22:31:05.845Z" }, + { url = "https://files.pythonhosted.org/packages/b7/a0/8f50cae9c74e718ed769d63ed5c74bd0ea830c9550a74629cebd1b9c7bc7/cryptography-48.0.1-cp311-abi3-win32.whl", hash = "sha256:b9a32b876490d66c8bcc9963ef220199569748434ab01a9d6aaeabf88e7f5158", size = 3304154, upload-time = "2026-06-09T22:32:16.845Z" }, + { url = "https://files.pythonhosted.org/packages/c5/69/0572c77dbace6fef72f33755bd52ea399c71367250d366237f8691826b9e/cryptography-48.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:39489bfca54c7a1f6b297efcd8bc608ab92d16c4ca631b0cad4da46724588b24", size = 3817138, upload-time = "2026-06-09T22:32:00.388Z" }, + { url = "https://files.pythonhosted.org/packages/42/06/3e768b4c3bc78201583fa35a0e18f640dd782ff41afba88f8545481a8874/cryptography-48.0.1-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:f817adc181390bd54f2f700107a7419040fb7c1bdf2fc26f36551a06a68c3345", size = 7989830, upload-time = "2026-06-09T22:31:07.8Z" }, { url = "https://files.pythonhosted.org/packages/8a/13/6476736484b94041110c8340a3eb63962fea4975baea8cb4a512adb44d4d/cryptography-48.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d5d30989c6917b478b5817902e85fddaea2261efa8648383d965381ccb9e1ac4", size = 4689201, upload-time = "2026-06-09T22:31:09.745Z" }, { url = "https://files.pythonhosted.org/packages/79/62/65a87f34d2a431546e2509b85d55e8c90df86d668f6731da64d538512ac2/cryptography-48.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df637c05205ea7c1d7fbcbe54bbfea648a52951155f997af13d895d0ecc96991", size = 4702822, upload-time = "2026-06-09T22:32:24.409Z" }, { url = "https://files.pythonhosted.org/packages/7f/59/810b5204b0a9b10f4b6bc06bd551a8b609803cd931806bc3b71884b225e5/cryptography-48.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:869c3b8a53bfe27147832df48b32adadf558249d50e76cb3769d40e986b13265", size = 4694875, upload-time = "2026-06-09T22:32:08.737Z" }, @@ -292,6 +325,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/04/618f4115cfc0add0838c82507aa18a346089428da8653ad38b3ff36f5cb3/cryptography-48.0.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:b4e391975f038e66432328639620a4aff2d307513b004f1ca06d6225bced815c", size = 4736660, upload-time = "2026-06-09T22:32:12.676Z" }, { url = "https://files.pythonhosted.org/packages/24/9c/06e062462a0de28a3b3911322eded4c16deb9f441b1b7575d3dc59488ab5/cryptography-48.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:42fcd8e26fe555d9b3577a135f5091fefa0aa4e99129c23fb56787a1bd4ada72", size = 4822229, upload-time = "2026-06-09T22:31:17.062Z" }, { url = "https://files.pythonhosted.org/packages/f4/be/0561971eaaee4b8a0e7d5113c536921063ab91aaf23278ac374eaf881e11/cryptography-48.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c1400da5e32a43253392277eac7490a60e497d810a63dd5608d71bbd7af507c9", size = 4966364, upload-time = "2026-06-09T22:31:32.842Z" }, + { url = "https://files.pythonhosted.org/packages/a4/27/728c77876f12b000820b69ae490f3c4083775e79e07827e9e60be07ad209/cryptography-48.0.1-cp314-cp314t-win32.whl", hash = "sha256:0df56b056bc17c1b7d6821dfa65216e62bd232d8ab05eb3db44e71d235651471", size = 3278498, upload-time = "2026-06-09T22:31:29.154Z" }, + { url = "https://files.pythonhosted.org/packages/06/e3/79a612c6d7b1e6ee0edd43633d53035bec2cfb78c82b76f7864f39e36f34/cryptography-48.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:9de21387aa95e2a895823d0745b430bed4f33503ba9ab5e0b5311f33e37d66d2", size = 3798790, upload-time = "2026-06-09T22:31:56.697Z" }, + { url = "https://files.pythonhosted.org/packages/ca/6c/00fa2a95997164c8b2072ce327c23d4ab20809ccc323ea5fab91e53a4bba/cryptography-48.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:4fdc69f8e4316bcf0c8c8ec1f26f285d12e8142d88d96c876a59a03be3f6ae67", size = 7987408, upload-time = "2026-06-09T22:32:20.777Z" }, { url = "https://files.pythonhosted.org/packages/b0/d9/45f309a7e4e5f3f8f121d6d3be9e94024a7726ec598d6e08ae04edb2f04d/cryptography-48.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48fe40804d4caa2288f24e70ca8c64c42dd826da0ad7e4f1b41b2128d679e6c8", size = 4690196, upload-time = "2026-06-09T22:31:54.74Z" }, { url = "https://files.pythonhosted.org/packages/5f/9f/a1bc8bcc798811b8527eb374bbccf30a3f3e806829d967118222bf1125eb/cryptography-48.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:86be3b1b0b6bf09482fb50a979c508d2950ed95f5621ec77f4e385962006b83a", size = 4696782, upload-time = "2026-06-09T22:31:45.615Z" }, { url = "https://files.pythonhosted.org/packages/66/c2/81a4fb4e4373c500bb526bc337ac5719dd31dd15b970b84a238168c6aa08/cryptography-48.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4ab0a343c807bbcd90c971cd1ecf072937cd01847a9e002bef88fb47ac6be577", size = 4696618, upload-time = "2026-06-09T22:31:11.564Z" }, @@ -303,10 +339,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/ab/8aaa12e4516ec4464033ab79b6f3b592bd5a92102467c4ace8a0d970203f/cryptography-48.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b74ca3b8e5ecdd833bf6a002ca41b4793bb27fb8f1c06ffaf2643c9e9140e31b", size = 4731388, upload-time = "2026-06-09T22:32:04.019Z" }, { url = "https://files.pythonhosted.org/packages/1b/24/50027ea4dca85ec1f40688f3c24fb32ccacd520583c9592c3cc95628e6fb/cryptography-48.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2c37f2461406063b417837f5f3daab668652acd82423efcd7f0a9f04be972de1", size = 4824186, upload-time = "2026-06-09T22:32:18.707Z" }, { url = "https://files.pythonhosted.org/packages/52/41/04cb5eb17085ade6f50cc611fb657df6a0f5885350de8764ece89c050197/cryptography-48.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:86fe77abb1bd87afb251d4d02ada7ecf53a32cee9b67d976abb2e45a13297475", size = 4964539, upload-time = "2026-06-09T22:31:18.793Z" }, + { url = "https://files.pythonhosted.org/packages/36/bf/ed70785c496e89d7e73b7cda2d21f2447fd6d4e821714b8d04ff217fed92/cryptography-48.0.1-cp39-abi3-win32.whl", hash = "sha256:6b2c0c3e6ccf3ade7750f836ef3ee36eea250cc467d45c256895573ac08cc6f1", size = 3282307, upload-time = "2026-06-09T22:30:53.162Z" }, + { url = "https://files.pythonhosted.org/packages/b3/ff/371ea7d252656ee1eb6d83eeeef3d1d0c6baf1d6497687d081ea03814670/cryptography-48.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:9a49ca6c81417f6a5edb50375a60cccdd70fa0a91a5211829dbea74eba94d2ac", size = 3793408, upload-time = "2026-06-09T22:32:15.191Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d3/eb4e394e587341fdad09a09101fa76478ead3a78b0ad63e55c22f0d75c02/cryptography-48.0.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:08a597acce1ff37f347400087776599e2348a3a8bc53b44120e463cd274efe4a", size = 3951747, upload-time = "2026-06-09T22:31:23.871Z" }, { url = "https://files.pythonhosted.org/packages/e0/4a/3f43451b4f858bfceaaaffc649e6e787e8d4fb332a1d443af39ab02cc8f1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:735824ec41b7f74a7c45fb1591349333e4c696cb6c044e5f46356e560143e4cd", size = 4641226, upload-time = "2026-06-09T22:31:02.532Z" }, { url = "https://files.pythonhosted.org/packages/73/4e/855584c2c23b09e4ce2d3b9c30e983e679cd60b068c513c6bbdb91e11782/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:92a46e1d638daa264ba2971c0b0489c9409787943efae4d60ffda3d091ef832c", size = 4668958, upload-time = "2026-06-09T22:32:06.213Z" }, { url = "https://files.pythonhosted.org/packages/42/3b/d35750e41d803d1e516fd6d6011f065424924da7af1748cef4cc9cb3ede1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7e234ac052af99f2700826a5c29ea99d9c1b1f80341cde62d11c8154dc8e0bd9", size = 4640793, upload-time = "2026-06-09T22:32:26.331Z" }, { url = "https://files.pythonhosted.org/packages/ca/aa/cdb7181fe865285e87e96825aaab239400f1de0c3bfba9bd9769b79f1a92/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:33842cf0888951cef5bc7ac724ab844a42044c1727b967b7f8997289a0464f92", size = 4668505, upload-time = "2026-06-09T22:31:27.534Z" }, + { url = "https://files.pythonhosted.org/packages/5d/8c/ce3823c06c2804f194f9e64f0d67fa3f4094a39f2bb1a990cd03603af8fc/cryptography-48.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6184ca7b174f28d7c703f1290d4b297217c45355f77a98f67e9b7f14549ac54a", size = 3742204, upload-time = "2026-06-09T22:31:34.773Z" }, ] [[package]] @@ -332,7 +372,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -348,6 +388,7 @@ dependencies = [ [package.optional-dependencies] testing = [ + { name = "asyncssh" }, { name = "hatch" }, { name = "pre-commit" }, { name = "pytest" }, @@ -363,6 +404,7 @@ testing = [ [package.metadata] requires-dist = [ + { name = "asyncssh", marker = "extra == 'testing'" }, { name = "hatch", marker = "extra == 'testing'" }, { name = "pre-commit", marker = "extra == 'testing'" }, { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, From f153271a5f2feb4369b73ddf487c88d8c5c99439 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 08:04:26 +0200 Subject: [PATCH 08/59] feat: run ssh gateways on the Trio path Route ssh gateways through the Trio host: `ssh -C [-F cfg] ''` spawns the uv-provisioned worker module on the remote, then the coordinator does the usual b"1" handshake and attaches a Trio session. HostNotFound is raised when ssh exits 255. The popen and ssh factories now share _open_trio_gateway (spawn + handshake + attach); ssh_trio_args builds the shell-quoted remote worker command. test_ssh_roundtrip is parametrized over the trio and legacy paths against the in-process asyncssh server. test_sshconfig_config_parsing (white-box over legacy Popen2IOMaster) is pinned to EXECNET_TRIO_HOST=0, with a new ssh_trio_args test covering -F on the Trio path. Co-Authored-By: Claude Opus 4.8 --- doc/implnotes.rst | 66 +++++++++++++++++------------- src/execnet/_trio_host.py | 85 +++++++++++++++++++++++++++++++++------ src/execnet/multi.py | 2 + testing/test_gateway.py | 13 ++++++ testing/test_ssh_local.py | 8 +++- 5 files changed, 131 insertions(+), 43 deletions(-) diff --git a/doc/implnotes.rst b/doc/implnotes.rst index 84492656..84bcc065 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -15,34 +15,44 @@ to and from InputOutput objects. The details of this protocol are locally defined in this module. There is no need for standardizing or versioning the protocol. -Trio host-thread IO (popen / import bootstrap) ----------------------------------------------- - -For local same-interpreter ``popen`` gateways, Message protocol IO -runs inside a dedicated OS thread hosting a Trio event loop -(``execnet._trio_host.TrioHost``). No source is sent over the wire: -the worker is launched as ``python -m execnet._trio_worker`` and -imports the installed ``execnet`` + ``trio`` (a rough major/minor -version check guards against an incompatible install): - -* Coordinator: ``trio.lowlevel.open_process`` plus async framed - reader/writer tasks per gateway (one host thread per ``Group``). - It waits for the worker's ``b"1"`` handshake before starting the - Message protocol. -* Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio - streams and writes the handshake byte; ``remote_exec`` is - scheduled from the Trio nursery (``trio.to_thread`` for ``thread``, - main-thread handoff for ``main_thread_only``). - -Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from -non-host threads wait until the frame is written (so abrupt -``os._exit`` cannot drop queued data). Sends from the Trio host -thread (receiver callbacks) only enqueue, to avoid deadlocking -the writer task. - -Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types -(``ssh``, ``socket``, ``via``, ``python=…``, greenlet execmodels) -still use the legacy thread receiver and sync ``Popen`` path. +Trio host-thread IO +------------------- + +``popen`` and ``ssh`` gateways run their Message protocol IO inside a +dedicated OS thread hosting a Trio event loop +(``execnet._trio_host.TrioHost``). No source is sent over the wire; the +worker is launched as ``python -m execnet._trio_worker`` and imports the +installed ``execnet`` + ``trio`` (a rough major/minor version check guards +against an incompatible install). How the worker environment is obtained +depends on the target: + +* Same-interpreter ``popen`` -> ``sys.executable -m execnet._trio_worker``. +* A ``python=`` interpreter that already has execnet -> that interpreter + directly (so ``sys.executable`` is preserved). +* A bare ``python=`` interpreter or an ``ssh`` remote -> provisioned via + ``uv`` (``execnet._provision``): ``uv run --with `` where ```` + is ``execnet==`` for a released coordinator or a locally-built, + version-cached wheel for a dev coordinator. + +Coordinator and worker roles: + +* Coordinator: ``trio.lowlevel.open_process`` (directly, or wrapped in + ``ssh``) plus async framed reader/writer tasks per gateway (one host + thread per ``Group``). It waits for the worker's ``b"1"`` handshake + before starting the Message protocol. +* Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio streams and + writes the handshake byte; ``remote_exec`` is scheduled from the Trio + nursery (``trio.to_thread`` for ``thread``, main-thread handoff for + ``main_thread_only``). + +Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from non-host +threads wait until the frame is written (so abrupt ``os._exit`` cannot drop +queued data). Sends from the Trio host thread (receiver callbacks) only +enqueue, to avoid deadlocking the writer task. + +Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types (``socket``, +``via``, greenlet execmodels) still use the legacy thread receiver and sync +``Popen`` path. Legacy thread model ------------------- diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 42b45047..68755d50 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -551,24 +551,19 @@ def kill(self) -> None: return -def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: - """Create a popen Gateway on the Trio IO path. +def _open_trio_gateway( + group: Any, spec: Any, args: list[str], *, remoteaddress: str | None = None +) -> Gateway: + """Spawn ``args``, do the worker handshake, and attach a Trio session. - Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; - a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the - worker imports execnet + trio; nothing is sent over the wire to bootstrap it. + Shared by the popen and ssh factories; ``args`` already encodes how the + worker is launched (direct module, uv-provisioned, or wrapped in ssh). """ import execnet - from . import _provision + from .gateway_bootstrap import HostNotFound host: TrioHost = group._ensure_trio_host() - if spec.python and not _provision.target_has_execnet(spec.python): - # bare interpreter: provision execnet + trio via uv - args = _provision.uv_worker_argv(spec) - else: - # same interpreter, or a python= that already has execnet - args = popen_module_args(spec) async def _create_and_attach() -> Gateway: process = await open_popen_process(args) @@ -577,6 +572,16 @@ async def _create_and_attach() -> Gateway: ack = await async_io.read_exact(1) if ack != b"1": raise EOFError(f"bad bootstrap handshake: {ack!r}") + except EOFError: + with trio.move_on_after(5): + code = await process.wait() + # ssh exits 255 when it cannot reach/authenticate the host. + if remoteaddress is not None and code == 255: + raise HostNotFound(remoteaddress) from None + with trio.move_on_after(5): + process.kill() + await process.wait() + raise except BaseException: with trio.move_on_after(5): process.kill() @@ -586,7 +591,61 @@ async def _create_and_attach() -> Gateway: gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) session = await host.start_session(gw, async_io, process=process) gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session) + gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) return gw return host.call(_create_and_attach) + + +def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: + """Create a popen Gateway on the Trio IO path. + + Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; + a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the + worker imports execnet + trio; nothing is sent over the wire to bootstrap it. + """ + from . import _provision + + if spec.python and not _provision.target_has_execnet(spec.python): + # bare interpreter: provision execnet + trio via uv + args = _provision.uv_worker_argv(spec) + else: + # same interpreter, or a python= that already has execnet + args = popen_module_args(spec) + return _open_trio_gateway(group, spec, args) + + +def ssh_trio_args(spec: Any) -> list[str]: + """``ssh [-F cfg] ''`` for the Trio ssh path. + + The remote runs the uv-provisioned worker module; the worker command is + shell-quoted for the remote shell. + """ + import shlex + + from . import _provision + + remote_command = shlex.join(_provision.uv_worker_argv(spec)) + args = ["ssh", "-C"] + if getattr(spec, "ssh_config", None): + args += ["-F", spec.ssh_config] + assert spec.ssh is not None + args += spec.ssh.split() + args.append(remote_command) + return args + + +def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: + """Create an ssh Gateway on the Trio IO path (uv-provisioned worker).""" + args = ssh_trio_args(spec) + return _open_trio_gateway(group, spec, args, remoteaddress=spec.ssh) + + +def should_use_trio_ssh(spec: Any) -> bool: + """Trio path for ssh gateways (worker provisioned on the remote via uv).""" + if not trio_host_enabled(): + return False + if not getattr(spec, "ssh", None): + return False + execmodel = getattr(spec, "execmodel", None) + return execmodel in (None, "thread", "main_thread_only") diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 687f1840..29cea870 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -156,6 +156,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: if _trio_host.should_use_trio_popen(spec): gw = _trio_host.makegateway_popen_trio(self, spec) + elif _trio_host.should_use_trio_ssh(spec): + gw = _trio_host.makegateway_ssh_trio(self, spec) elif spec.via: assert not spec.socket master = self[spec.via] diff --git a/testing/test_gateway.py b/testing/test_gateway.py index 83a50f44..f7f882ae 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -384,6 +384,8 @@ class TestSshPopenGateway: def test_sshconfig_config_parsing( self, monkeypatch: pytest.MonkeyPatch, makegateway: Callable[[str], Gateway] ) -> None: + # white-box test of the legacy Popen2IOMaster arg construction + monkeypatch.setenv("EXECNET_TRIO_HOST", "0") l = [] monkeypatch.setattr( gateway_io, "Popen2IOMaster", lambda *args, **kwargs: l.append(args[0]) @@ -396,6 +398,17 @@ def test_sshconfig_config_parsing( i = popen_args.index("-F") assert popen_args[i + 1] == "qwe" + def test_ssh_trio_args_include_config( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from execnet import _provision + from execnet import _trio_host + + monkeypatch.setattr(_provision, "coordinator_requirement", lambda: "execnet") + args = _trio_host.ssh_trio_args(execnet.XSpec("ssh=xyz//ssh_config=qwe")) + assert args[args.index("-F") + 1] == "qwe" + assert "xyz" in args + def test_sshaddress(self, gw: Gateway, specssh: execnet.XSpec) -> None: assert gw.remoteaddress == specssh.ssh diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index 196c5d76..72dea348 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -9,7 +9,6 @@ from __future__ import annotations import asyncio -import os import shutil import sys import threading @@ -134,7 +133,12 @@ def ssh_config(ssh_server: SSHServerThread, tmp_path) -> str: return path -def test_ssh_roundtrip(ssh_config: str) -> None: +@pytest.mark.parametrize("trio_host", ["1", "0"], ids=["trio", "legacy"]) +def test_ssh_roundtrip( + ssh_config: str, monkeypatch: pytest.MonkeyPatch, trio_host: str +) -> None: + # trio=1 provisions the worker over ssh with uv; legacy=0 source-copies it. + monkeypatch.setenv("EXECNET_TRIO_HOST", trio_host) group = execnet.Group() try: gw = group.makegateway( From 5197b1eee591461b959fea9be07363e50f91d630 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 08:43:07 +0200 Subject: [PATCH 09/59] feat: ship the execnet wheel to ssh remotes for dev coordinators A dev coordinator's wheel is not on the remote filesystem, so the ssh remote command becomes a POSIX-sh prelude that reads the wheel bytes from stdin (`head -c N` into a temp dir) and execs uv against it. The coordinator streams those bytes as a preamble before the Message protocol; _open_trio_gateway grew a `preamble` argument. Released coordinators still use `uv run --with execnet==` with no shipping. Also collapse the worker CLI contract: id, execmodel and coordinator version now travel as a single JSON argument (_provision.worker_cli_arg) consumed by _trio_worker._main, instead of scattered positional args built in three launchers. Co-Authored-By: Claude Opus 4.8 --- doc/implnotes.rst | 10 ++++- src/execnet/_provision.py | 87 +++++++++++++++++++++++++++---------- src/execnet/_trio_host.py | 45 +++++++++---------- src/execnet/_trio_worker.py | 18 ++++---- testing/test_gateway.py | 10 ++++- testing/test_provision.py | 40 +++++++++++++++++ 6 files changed, 152 insertions(+), 58 deletions(-) create mode 100644 testing/test_provision.py diff --git a/doc/implnotes.rst b/doc/implnotes.rst index 84bcc065..29df051c 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -32,7 +32,15 @@ depends on the target: * A bare ``python=`` interpreter or an ``ssh`` remote -> provisioned via ``uv`` (``execnet._provision``): ``uv run --with `` where ```` is ``execnet==`` for a released coordinator or a locally-built, - version-cached wheel for a dev coordinator. + version-cached wheel for a dev coordinator. For an ``ssh`` remote on a + dev coordinator the wheel is not on the remote filesystem, so the remote + command is a POSIX-sh prelude that reads the wheel bytes from stdin + (``head -c N`` into a temp dir) and ``exec``s ``uv`` against it; the + coordinator streams those bytes before the Message protocol. + +The worker configuration (id, execmodel, coordinator version) is passed as a +single JSON CLI argument (``_provision.worker_cli_arg``), so every launcher +shares one contract instead of scattered positional args. Coordinator and worker roles: diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index b7f6ac56..b24bfe8b 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -18,6 +18,7 @@ import json import re +import shlex import shutil import subprocess import tempfile @@ -143,36 +144,74 @@ def coordinator_requirement() -> str: return str(_build_wheel(version)) -def uv_run_argv( - *, python: str | None, requirement: str, module_args: list[str] -) -> list[str]: - """Build ``uv run [--python X] --with python -u -m execnet._trio_worker …``. +def worker_cli_arg(spec: Any) -> str: + """Single JSON CLI argument carrying the worker config (the whole 'spec thing'). - ``--no-project`` keeps the surrounding execnet checkout from being synced, so - the ephemeral env holds only the requirement (+ trio). + Passed to ``python -m execnet._trio_worker`` by every launcher (popen, uv, + ssh) so the worker config lives in one place rather than scattered positional + args. """ - argv = ["uv", "run", "--no-project"] - if python: - argv += ["--python", python] - argv += [ + import execnet + + return json.dumps( + { + "id": f"{spec.id}-worker", + "execmodel": spec.execmodel, + "coordinator_version": execnet.__version__, + } + ) + + +def worker_module_tokens(spec: Any) -> list[str]: + """``python -u -m execnet._trio_worker `` tokens.""" + return ["python", "-u", "-m", "execnet._trio_worker", worker_cli_arg(spec)] + + +def _uv_prefix(spec: Any) -> list[str]: + # --no-project keeps the surrounding execnet checkout from being synced. + prefix = ["uv", "run", "--no-project"] + if spec.python: + prefix += ["--python", spec.python] + return prefix + + +def uv_worker_argv(spec: Any) -> list[str]: + """``uv run`` argv to launch the Trio worker locally (wheel path is local).""" + return [ + *_uv_prefix(spec), "--with", - requirement, - "python", - "-u", - "-m", - "execnet._trio_worker", - *module_args, + coordinator_requirement(), + *worker_module_tokens(spec), ] - return argv -def uv_worker_argv(spec: Any) -> list[str]: - """Full ``uv run`` argv to launch the Trio worker for ``spec``.""" +def ssh_remote_command(spec: Any) -> tuple[str, bytes]: + """Remote shell command + stdin preamble to launch the worker over ssh. + + Released coordinator -> ``uv run --with execnet== …`` with no preamble. + Dev coordinator -> a POSIX-sh prelude that receives the wheel bytes from + stdin (``head -c N``) into a temp dir and ``exec``s uv against it; the wheel + bytes are returned as the preamble to stream before the Message protocol. + """ import execnet - module_args = [f"{spec.id}-worker", spec.execmodel, execnet.__version__] - return uv_run_argv( - python=spec.python, - requirement=coordinator_requirement(), - module_args=module_args, + version = execnet.__version__ + worker = worker_module_tokens(spec) + if _RELEASED_RE.match(version): + command = shlex.join( + [*_uv_prefix(spec), "--with", f"execnet=={version}", *worker] + ) + return command, b"" + + wheel = _build_wheel(version) + data = wheel.read_bytes() + # "$d/": expand the temp dir, concatenate the (quoted) wheel filename. + remote_wheel = '"$d/"' + shlex.quote(wheel.name) + uv_run = " ".join(shlex.quote(token) for token in [*_uv_prefix(spec), "--with"]) + worker_cmd = " ".join(shlex.quote(token) for token in worker) + prelude = ( + f"d=$(mktemp -d) && " + f"head -c {len(data)} > {remote_wheel} && " + f"exec {uv_run} {remote_wheel} {worker_cmd}" ) + return prelude, data diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 68755d50..f9cf7a71 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -500,11 +500,9 @@ def popen_module_args(spec: Any) -> list[str]: No source is sent over the wire; the worker imports the installed execnet + trio. Used for same-interpreter popen and for a ``python=`` interpreter that - already has execnet (so ``sys.executable`` stays that interpreter). The - coordinator version is passed so the worker can do a rough compatibility - check against its own installed version. + already has execnet (so ``sys.executable`` stays that interpreter). """ - import execnet + from . import _provision if getattr(spec, "python", None): from .gateway_io import shell_split_path @@ -516,13 +514,7 @@ def popen_module_args(spec: Any) -> list[str]: args = [*interpreter, "-u"] if getattr(spec, "dont_write_bytecode", False): args.append("-B") - args += [ - "-m", - "execnet._trio_worker", - f"{spec.id}-worker", - spec.execmodel, - execnet.__version__, - ] + args += ["-m", "execnet._trio_worker", _provision.worker_cli_arg(spec)] return args @@ -552,12 +544,19 @@ def kill(self) -> None: def _open_trio_gateway( - group: Any, spec: Any, args: list[str], *, remoteaddress: str | None = None + group: Any, + spec: Any, + args: list[str], + *, + remoteaddress: str | None = None, + preamble: bytes = b"", ) -> Gateway: """Spawn ``args``, do the worker handshake, and attach a Trio session. Shared by the popen and ssh factories; ``args`` already encodes how the worker is launched (direct module, uv-provisioned, or wrapped in ssh). + ``preamble`` is streamed to the worker's stdin before the handshake (used to + ship a wheel to a remote that receives it with ``head -c``). """ import execnet @@ -569,6 +568,8 @@ async def _create_and_attach() -> Gateway: process = await open_popen_process(args) try: async_io = ProcessStreamsIO(process) + if preamble: + await async_io.write_all(preamble) ack = await async_io.read_exact(1) if ack != b"1": raise EOFError(f"bad bootstrap handshake: {ack!r}") @@ -615,30 +616,30 @@ def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: return _open_trio_gateway(group, spec, args) -def ssh_trio_args(spec: Any) -> list[str]: - """``ssh [-F cfg] ''`` for the Trio ssh path. +def ssh_trio_args(spec: Any) -> tuple[list[str], bytes]: + """``(ssh argv, stdin preamble)`` for the Trio ssh path. - The remote runs the uv-provisioned worker module; the worker command is - shell-quoted for the remote shell. + The remote runs the uv-provisioned worker; for a dev coordinator the + preamble carries the wheel bytes that the remote command receives. """ - import shlex - from . import _provision - remote_command = shlex.join(_provision.uv_worker_argv(spec)) + remote_command, preamble = _provision.ssh_remote_command(spec) args = ["ssh", "-C"] if getattr(spec, "ssh_config", None): args += ["-F", spec.ssh_config] assert spec.ssh is not None args += spec.ssh.split() args.append(remote_command) - return args + return args, preamble def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: """Create an ssh Gateway on the Trio IO path (uv-provisioned worker).""" - args = ssh_trio_args(spec) - return _open_trio_gateway(group, spec, args, remoteaddress=spec.ssh) + args, preamble = ssh_trio_args(spec) + return _open_trio_gateway( + group, spec, args, remoteaddress=spec.ssh, preamble=preamble + ) def should_use_trio_ssh(spec: Any) -> bool: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 2c297589..f90db607 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -290,17 +290,17 @@ def _check_version(coordinator_version: str) -> None: def _main() -> None: - """Entry point for ``python -m execnet._trio_worker [ver]``. + """Entry point for ``python -m execnet._trio_worker ``. - The worker imports execnet + trio from the environment; no source is sent - over the wire to bootstrap it. + ```` is the coordinator's ``_provision.worker_cli_arg`` payload: + ``{"id", "execmodel", "coordinator_version"}``. The worker imports execnet + + trio from the environment; no source is sent over the wire to bootstrap it. """ - argv = sys.argv - worker_id = argv[1] if len(argv) > 1 else "worker" - execmodel = argv[2] if len(argv) > 2 else "thread" - if len(argv) > 3: - _check_version(argv[3]) - serve_popen_trio(id=worker_id, execmodel=execmodel) + import json + + config = json.loads(sys.argv[1]) + _check_version(config["coordinator_version"]) + serve_popen_trio(id=config["id"], execmodel=config["execmodel"]) if __name__ == "__main__": diff --git a/testing/test_gateway.py b/testing/test_gateway.py index f7f882ae..e123bbe2 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -404,10 +404,16 @@ def test_ssh_trio_args_include_config( from execnet import _provision from execnet import _trio_host - monkeypatch.setattr(_provision, "coordinator_requirement", lambda: "execnet") - args = _trio_host.ssh_trio_args(execnet.XSpec("ssh=xyz//ssh_config=qwe")) + monkeypatch.setattr( + _provision, "ssh_remote_command", lambda spec: ("worker-cmd", b"") + ) + args, preamble = _trio_host.ssh_trio_args( + execnet.XSpec("ssh=xyz//ssh_config=qwe") + ) assert args[args.index("-F") + 1] == "qwe" assert "xyz" in args + assert args[-1] == "worker-cmd" + assert preamble == b"" def test_sshaddress(self, gw: Gateway, specssh: execnet.XSpec) -> None: assert gw.remoteaddress == specssh.ssh diff --git a/testing/test_provision.py b/testing/test_provision.py new file mode 100644 index 00000000..8df8775a --- /dev/null +++ b/testing/test_provision.py @@ -0,0 +1,40 @@ +"""Unit tests for coordinator-side uv worker provisioning.""" + +from __future__ import annotations + +import json +import re + +import pytest + +import execnet +from execnet import _provision + +released = re.fullmatch(r"\d+\.\d+\.\d+", execnet.__version__) is not None + + +def test_worker_cli_arg_carries_config() -> None: + spec = execnet.XSpec("popen//id=gw5//execmodel=thread") + config = json.loads(_provision.worker_cli_arg(spec)) + assert config["id"] == "gw5-worker" + assert config["execmodel"] == "thread" + assert config["coordinator_version"] == execnet.__version__ + + +def test_ssh_remote_command_released(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(execnet, "__version__", "9.9.9") + spec = execnet.XSpec("ssh=host//id=gw0//execmodel=thread") + command, preamble = _provision.ssh_remote_command(spec) + assert "execnet==9.9.9" in command + assert "head -c" not in command # no shipping + assert preamble == b"" + + +@pytest.mark.skipif(released, reason="released execnet resolves from an index") +@pytest.mark.skipif(not _provision.uv_available(), reason="uv required to build wheel") +def test_ssh_remote_command_dev_ships_wheel() -> None: + spec = execnet.XSpec("ssh=host//id=gw0//execmodel=thread") + command, preamble = _provision.ssh_remote_command(spec) + assert "mktemp -d" in command + assert f"head -c {len(preamble)}" in command + assert preamble[:2] == b"PK" # a wheel is a zip archive From 21096da19f4a2be00290b81a32286502b8f8a501 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 09:48:12 +0200 Subject: [PATCH 10/59] feat: add execnet-socketserver console command Expose the socket server as a `[project.scripts]` entry point so it can be started directly, e.g. provisioned anywhere with `uvx --from execnet execnet-socketserver :8888`. Refactor the __main__ block into a main() with an argparse CLI (hostport + --once), and thread execmodel explicitly through startserver/exec_from_one_connection instead of relying on a module global (which main()'s local scope broke). Co-Authored-By: Claude Opus 4.8 --- doc/basics.rst | 4 +- pyproject.toml | 3 ++ src/execnet/script/socketserver.py | 43 +++++++++++++++----- testing/test_socketserver_cli.py | 63 ++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 testing/test_socketserver_cli.py diff --git a/doc/basics.rst b/doc/basics.rst index 723de83f..a6b256b7 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -58,7 +58,9 @@ Examples for valid gateway specifications remotely sets an environment variable ``NAME`` to ``value``. * ``socket=192.168.1.4:8888`` specifies a Python Socket server - process that listens on ``192.168.1.4:8888`` + process that listens on ``192.168.1.4:8888``. Such a server can be + started with the ``execnet-socketserver`` console command, e.g. run + anywhere with ``uvx --from execnet execnet-socketserver :8888``. .. versionadded:: 1.5 diff --git a/pyproject.toml b/pyproject.toml index 853ba3d8..f419c31d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,9 @@ classifiers = [ "Topic :: System :: Networking", ] +[project.scripts] +execnet-socketserver = "execnet.script.socketserver:main" + [project.optional-dependencies] testing = [ "pre-commit", diff --git a/src/execnet/script/socketserver.py b/src/execnet/script/socketserver.py index fa98743c..43d82147 100644 --- a/src/execnet/script/socketserver.py +++ b/src/execnet/script/socketserver.py @@ -48,7 +48,7 @@ def print_(*args) -> None: ) -def exec_from_one_connection(serversock) -> None: +def exec_from_one_connection(serversock, execmodel: ExecModel) -> None: print_(progname, "Entering Accept loop", serversock.getsockname()) clientsock, address = serversock.accept() print_(progname, "got new connection from {} {}".format(*address)) @@ -89,12 +89,12 @@ def bind_and_listen(hostport: str | tuple[str, int], execmodel: ExecModel): return serversock -def startserver(serversock, loop: bool = False) -> None: +def startserver(serversock, execmodel: ExecModel, loop: bool = False) -> None: execute_path = os.getcwd() try: while 1: try: - exec_from_one_connection(serversock) + exec_from_one_connection(serversock, execmodel) except (KeyboardInterrupt, SystemExit): raise except BaseException as exc: @@ -112,15 +112,40 @@ def startserver(serversock, loop: bool = False) -> None: serversock.shutdown(2) -if __name__ == "__main__": - import sys +def main(argv: list[str] | None = None) -> None: + """Console entry point (``execnet-socketserver``). + + Bind a socket and serve gateway connections. Intended to be run directly, + e.g. provisioned on a host with ``uvx --from execnet execnet-socketserver``. + """ + import argparse - hostport = sys.argv[1] if len(sys.argv) > 1 else ":8888" from execnet.gateway_base import get_execmodel + parser = argparse.ArgumentParser( + prog="execnet-socketserver", + description="Serve execnet gateway connections over a socket.", + ) + parser.add_argument( + "hostport", + nargs="?", + default=":8888", + help="address to bind as HOST:PORT or :PORT (default: :8888)", + ) + parser.add_argument( + "--once", + action="store_true", + help="serve a single connection and exit instead of looping", + ) + args = parser.parse_args(argv) + execmodel = get_execmodel("thread") - serversock = bind_and_listen(hostport, execmodel) - startserver(serversock, loop=True) + serversock = bind_and_listen(args.hostport, execmodel) + startserver(serversock, execmodel, loop=not args.once) + + +if __name__ == "__main__": + main() elif __name__ == "__channelexec__": chan: Channel = globals()["channel"] @@ -130,4 +155,4 @@ def startserver(serversock, loop: bool = False) -> None: sock = bind_and_listen(bindname, execmodel) port = sock.getsockname() chan.send(port) - startserver(sock) + startserver(sock, execmodel) diff --git a/testing/test_socketserver_cli.py b/testing/test_socketserver_cli.py new file mode 100644 index 00000000..039803fa --- /dev/null +++ b/testing/test_socketserver_cli.py @@ -0,0 +1,63 @@ +"""Test the ``execnet-socketserver`` console entry point end to end.""" + +from __future__ import annotations + +import shutil +import socket +import subprocess +import time +from collections.abc import Iterator + +import pytest + +import execnet + +SERVER = shutil.which("execnet-socketserver") + +pytestmark = pytest.mark.skipif( + SERVER is None, reason="execnet-socketserver console script not installed" +) + + +def _free_port() -> int: + s = socket.socket() + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + return port + + +@pytest.fixture +def socketserver_port() -> Iterator[int]: + assert SERVER is not None + port = _free_port() + proc = subprocess.Popen( + [SERVER, f"127.0.0.1:{port}"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + # loop mode: a throwaway probe just consumes one accept iteration + for _ in range(100): + try: + socket.create_connection(("127.0.0.1", port), timeout=0.2).close() + break + except OSError: + time.sleep(0.1) + else: + pytest.fail("execnet-socketserver did not start") + yield port + finally: + proc.kill() + proc.wait(timeout=5) + + +def test_socketserver_cli_roundtrip(socketserver_port: int) -> None: + group = execnet.Group() + try: + gw = group.makegateway(f"socket=127.0.0.1:{socketserver_port}//id=sock") + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(timeout=5.0) From 420693ac3d939af96929a2e8b433b5d0e2066ff1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 10:53:35 +0200 Subject: [PATCH 11/59] feat: run direct socket gateways on Trio with subprocess workers Move the direct `socket=host:port` transport onto the Trio host: - The socketserver becomes a Trio TCP listener that spawns a `python -m execnet._trio_worker --socket-fd N` subprocess per connection (passing the accepted socket by fd) instead of exec'ing sent source inline. - The worker gains a socket serve mode: it adopts an inherited socket fd into a Trio SocketStream and serves the Message protocol over it. The worker CLI now always carries its config as args, so a future popen socketpair can reuse the same path instead of hijacking stdio. - The coordinator connects a Trio TCP stream, waits for the b"1" handshake, and attaches a Trio session (no local process). A failed connect raises HostNotFound. The worker serve setup is refactored into shared _build_worker_gateway / _run_worker helpers. `installvia` still uses the legacy path for now. Co-Authored-By: Claude Opus 4.8 --- src/execnet/_trio_host.py | 95 ++++++++++++++++++++++ src/execnet/_trio_worker.py | 124 ++++++++++++++++++++--------- src/execnet/multi.py | 2 + src/execnet/script/socketserver.py | 70 ++++++++++++++-- testing/test_socketserver_cli.py | 41 +++++----- 5 files changed, 263 insertions(+), 69 deletions(-) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index f9cf7a71..c58d0c34 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -137,6 +137,45 @@ async def aclose_write(self) -> None: await self._write.aclose() +class SocketStreamIO: + """Async IO over a single bidirectional Trio stream (a socket). + + ``read_exact`` never over-reads (``receive_some(k)`` returns at most ``k`` + bytes), so no cross-call buffering is needed. ``aclose`` on a Trio stream is + idempotent, so close-read and close-write both just close the socket. + """ + + def __init__(self, stream: trio.abc.Stream) -> None: + self._stream = stream + + async def read_exact(self, n: int) -> bytes: + return await read_exact_receive_stream(self._stream, n) + + async def write_all(self, data: bytes) -> None: + await self._stream.send_all(data) + + async def aclose_read(self) -> None: + await self._stream.aclose() + + async def aclose_write(self) -> None: + await self._stream.aclose() + + +async def adopt_socket(socket_fd: int) -> SocketStreamIO: + """Worker side: wrap an inherited socket fd and send the handshake. + + Runs on the Trio host loop. The coordinator waits for ``b"1"`` before + starting the Message protocol; the worker config comes from the CLI. + """ + import socket as _socket + + sock = _socket.socket(fileno=socket_fd) + stream = trio.SocketStream(trio.socket.from_stdlib_socket(sock)) + io = SocketStreamIO(stream) + await io.write_all(b"1") + return io + + class SyncIOHandle: """Sync IO facade for Group.terminate wait/kill/close_write.""" @@ -650,3 +689,59 @@ def should_use_trio_ssh(spec: Any) -> bool: return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") + + +def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: + """Connect a Trio TCP stream to a running ``execnet-socketserver``. + + The server spawns the worker and synthesises its config, so the coordinator + just connects, waits for the worker's ``b"1"`` handshake, and attaches a Trio + session (no local process). + """ + import execnet + + from .gateway_bootstrap import HostNotFound + + host: TrioHost = group._ensure_trio_host() + assert spec.socket is not None + host_str, _, port_str = spec.socket.rpartition(":") + address = (host_str, int(port_str)) + + async def _create_and_attach() -> Gateway: + try: + stream = await trio.open_tcp_stream(*address) + except OSError as exc: + raise HostNotFound(spec.socket) from exc + io = SocketStreamIO(stream) + try: + ack = await io.read_exact(1) + if ack != b"1": + raise EOFError(f"bad socket handshake: {ack!r}") + except BaseException: + with trio.move_on_after(5): + await stream.aclose() + raise + + gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + session = await host.start_session(gw, io) + gw._attach_trio_session(session) + gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=spec.socket) + return gw + + return host.call(_create_and_attach) + + +def should_use_trio_socket(spec: Any) -> bool: + """Trio path for direct ``socket=host:port`` gateways. + + ``installvia`` (start a server through another gateway) still uses the legacy + path for now. + """ + if not trio_host_enabled(): + return False + if not getattr(spec, "socket", None): + return False + if getattr(spec, "installvia", None): + return False + execmodel = getattr(spec, "execmodel", None) + return execmodel in (None, "thread", "main_thread_only") diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index f90db607..5fc137fb 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -204,8 +204,59 @@ def kill(self) -> None: return +def _build_worker_gateway( + host: _trio_host.TrioHost, id: str, model: ExecModel +) -> tuple[WorkerGateway, TrioWorkerExec, bool]: + """Construct the WorkerGateway + Trio exec pool (no IO yet).""" + trace(f"creating workergateway on trio id={id!r}") + io_stub = _WorkerIOStub(model) + gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) + + main_thread_only = model.backend == "main_thread_only" + trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) + # Duck-type as WorkerPool for STATUS / _terminate_execution. + gateway._execpool = trio_exec # type: ignore[assignment] + gateway._trio_exec = trio_exec + gateway._executetask_complete = None + if main_thread_only: + gateway._executetask_complete = model.Event() + gateway._executetask_complete.set() + return gateway, trio_exec, main_thread_only + + +def _run_worker(host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel) -> None: + """Attach ``io`` as the gateway session and serve until shutdown.""" + gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model) + + async def _start() -> _trio_host.ProtocolSession: + return await host.start_session(gateway, io) + + session = host.call(_start) + gateway._attach_trio_session(session) + + try: + if main_thread_only: + trace("integrating as primary thread (trio worker)") + trio_exec.integrate_as_primary_thread() + gateway.join() + except KeyboardInterrupt: + # Match WorkerGateway.serve(): swallow in the worker. + trace("swallowing keyboardinterrupt, serve finished") + finally: + host.stop(timeout=5.0) + # Trio's to_thread cache uses non-daemon threads that would otherwise + # keep this disposable worker process alive after serve returns. + os._exit(0) + + +async def _make_fd_io(read_fd: int, write_fd: int) -> Any: + from . import _trio_host + + return _trio_host.FdStreamsIO(read_fd, write_fd) + + def serve_popen_trio(id: str, execmodel: str = "thread") -> None: - """Serve a WorkerGateway with Message IO + exec scheduling on Trio.""" + """Serve a WorkerGateway over the stdio pipes (popen / ssh worker).""" from . import _trio_host model = get_execmodel(execmodel) @@ -214,45 +265,27 @@ def serve_popen_trio(id: str, execmodel: str = "thread") -> None: # before starting the Message protocol. We are launched as a plain module # (``python -m execnet._trio_worker``), so nothing was sent to bootstrap us. os.write(write_fd, b"1") - # Keep the historic trace token so tests looking for workergateway still pass. - trace(f"creating workergateway on trio id={id!r}") host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") host.start() - try: - io_stub = _WorkerIOStub(model) - gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) - - main_thread_only = model.backend == "main_thread_only" - trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) - # Duck-type as WorkerPool for STATUS / _terminate_execution. - gateway._execpool = trio_exec # type: ignore[assignment] - gateway._trio_exec = trio_exec - gateway._executetask_complete = None - if main_thread_only: - gateway._executetask_complete = model.Event() - gateway._executetask_complete.set() + io = host.call(_make_fd_io, read_fd, write_fd) + _run_worker(host, io, id, model) - async def _start() -> _trio_host.ProtocolSession: - async_io = _trio_host.FdStreamsIO(read_fd, write_fd) - return await host.start_session(gateway, async_io) - session = host.call(_start) - gateway._attach_trio_session(session) +def serve_socket_trio(id: str, execmodel: str, socket_fd: int) -> None: + """Serve a WorkerGateway over an inherited socket fd. - try: - if main_thread_only: - trace("integrating as primary thread (trio worker)") - trio_exec.integrate_as_primary_thread() - gateway.join() - except KeyboardInterrupt: - # Match WorkerGateway.serve(): swallow in the worker. - trace("swallowing keyboardinterrupt, serve finished") - finally: - host.stop(timeout=5.0) - # Trio's to_thread cache uses non-daemon threads that would otherwise - # keep this disposable worker process alive after serve returns. - os._exit(0) + Used for the socketserver (an accepted TCP connection) and, in future, a + popen socketpair. The socket is adopted inside Trio and the handshake is + written on the host loop; config comes from the CLI. + """ + from . import _trio_host + + model = get_execmodel(execmodel) + host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") + host.start() + io = host.call(_trio_host.adopt_socket, socket_fd) + _run_worker(host, io, id, model) def _rough_version(version: str) -> tuple[int, ...]: @@ -290,17 +323,30 @@ def _check_version(coordinator_version: str) -> None: def _main() -> None: - """Entry point for ``python -m execnet._trio_worker ``. + """Entry point for ``python -m execnet._trio_worker [--socket-fd N]``. ```` is the coordinator's ``_provision.worker_cli_arg`` payload: - ``{"id", "execmodel", "coordinator_version"}``. The worker imports execnet + - trio from the environment; no source is sent over the wire to bootstrap it. + ``{"id", "execmodel", "coordinator_version"}``. With ``--socket-fd`` the + worker serves over that inherited socket (socketserver); otherwise over the + stdio pipes (popen / ssh). The worker imports execnet + trio from the + environment; no source is sent over the wire to bootstrap it. """ + import argparse import json - config = json.loads(sys.argv[1]) + parser = argparse.ArgumentParser(prog="execnet._trio_worker") + parser.add_argument("config", help="JSON worker config") + # Protocol transport: an inherited socket fd (socketserver, or a future popen + # socketpair); without it the worker serves over the stdio pipes (ssh). + parser.add_argument("--socket-fd", type=int, default=None) + ns = parser.parse_args() + + config = json.loads(ns.config) _check_version(config["coordinator_version"]) - serve_popen_trio(id=config["id"], execmodel=config["execmodel"]) + if ns.socket_fd is not None: + serve_socket_trio(config["id"], config["execmodel"], ns.socket_fd) + else: + serve_popen_trio(id=config["id"], execmodel=config["execmodel"]) if __name__ == "__main__": diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 29cea870..d64722ee 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -158,6 +158,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: gw = _trio_host.makegateway_popen_trio(self, spec) elif _trio_host.should_use_trio_ssh(spec): gw = _trio_host.makegateway_ssh_trio(self, spec) + elif _trio_host.should_use_trio_socket(spec): + gw = _trio_host.makegateway_socket_trio(self, spec) elif spec.via: assert not spec.socket master = self[spec.via] diff --git a/src/execnet/script/socketserver.py b/src/execnet/script/socketserver.py index 43d82147..ea580bcd 100644 --- a/src/execnet/script/socketserver.py +++ b/src/execnet/script/socketserver.py @@ -112,15 +112,74 @@ def startserver(serversock, execmodel: ExecModel, loop: bool = False) -> None: serversock.shutdown(2) +async def _trio_serve(hostport: str, once: bool) -> None: + """Trio TCP server that spawns a worker subprocess per connection. + + No code is executed inline: each accepted socket is handed (by fd) to a + fresh ``python -m execnet._trio_worker --socket-fd`` process which serves the + gateway over it. + """ + import itertools + import json + import subprocess + + import trio + + import execnet + + host, _, port_str = hostport.rpartition(":") + listeners = await trio.open_tcp_listeners(int(port_str), host=host or None) + addr = listeners[0].socket.getsockname() + # Report the bound address (port may be ephemeral) for callers to read. + print("execnet-socketserver listening on %s %s" % (addr[0], addr[1]), flush=True) + + counter = itertools.count() + + with trio.CancelScope() as scope: + + async def handler(stream: trio.SocketStream) -> None: + fd = stream.socket.fileno() + # Synthesise the worker config (socketserver workers default to the + # thread model, matching the legacy socket server). + config = json.dumps( + { + "id": "socketworker%d" % next(counter), + "execmodel": "thread", + "coordinator_version": execnet.__version__, + } + ) + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "execnet._trio_worker", + config, + "--socket-fd", + str(fd), + ], + pass_fds=[fd], + ) + # The child forked with a copy of the fd; release ours. + await stream.aclose() + if once: + scope.cancel() + return + # Reap the worker without blocking other connections. + await trio.to_thread.run_sync(proc.wait) + + await trio.serve_listeners(handler, listeners) + + def main(argv: list[str] | None = None) -> None: """Console entry point (``execnet-socketserver``). - Bind a socket and serve gateway connections. Intended to be run directly, - e.g. provisioned on a host with ``uvx --from execnet execnet-socketserver``. + Serve execnet gateway connections over a socket, spawning a worker + subprocess per connection. Intended to be run directly, e.g. provisioned on + a host with ``uvx --from execnet execnet-socketserver``. """ import argparse - from execnet.gateway_base import get_execmodel + import trio parser = argparse.ArgumentParser( prog="execnet-socketserver", @@ -138,10 +197,7 @@ def main(argv: list[str] | None = None) -> None: help="serve a single connection and exit instead of looping", ) args = parser.parse_args(argv) - - execmodel = get_execmodel("thread") - serversock = bind_and_listen(args.hostport, execmodel) - startserver(serversock, execmodel, loop=not args.once) + trio.run(_trio_serve, args.hostport, args.once) if __name__ == "__main__": diff --git a/testing/test_socketserver_cli.py b/testing/test_socketserver_cli.py index 039803fa..2b6d60dd 100644 --- a/testing/test_socketserver_cli.py +++ b/testing/test_socketserver_cli.py @@ -1,11 +1,14 @@ -"""Test the ``execnet-socketserver`` console entry point end to end.""" +"""Test the ``execnet-socketserver`` console entry point end to end. + +The Trio socketserver binds a port and spawns a ``python -m execnet._trio_worker`` +subprocess per connection (no inline code execution); the coordinator connects +over a Trio TCP stream. +""" from __future__ import annotations import shutil -import socket import subprocess -import time from collections.abc import Iterator import pytest @@ -19,33 +22,25 @@ ) -def _free_port() -> int: - s = socket.socket() - s.bind(("127.0.0.1", 0)) - port = s.getsockname()[1] - s.close() - return port - - @pytest.fixture def socketserver_port() -> Iterator[int]: assert SERVER is not None - port = _free_port() proc = subprocess.Popen( - [SERVER, f"127.0.0.1:{port}"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + [SERVER, ":0"], # ephemeral port; it prints the one it bound + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, ) try: - # loop mode: a throwaway probe just consumes one accept iteration - for _ in range(100): - try: - socket.create_connection(("127.0.0.1", port), timeout=0.2).close() + assert proc.stdout is not None + port = None + while True: + line = proc.stdout.readline() + if not line: + pytest.fail("execnet-socketserver exited before binding") + if "listening on" in line: + port = int(line.split()[-1]) break - except OSError: - time.sleep(0.1) - else: - pytest.fail("execnet-socketserver did not start") yield port finally: proc.kill() From b83ff3be6e62bdf829a56e5a79758dd5885f9c60 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 11:34:48 +0200 Subject: [PATCH 12/59] feat: migrate installvia to a GATEWAY_START_SOCKET protocol message Since execnet is now always installed on both sides, replace the remote_exec source-shipping used by `socket//installvia=` with a first-class protocol message. The coordinator sends GATEWAY_START_SOCKET (with the bind host) on a request channel; the via gateway's Trio host binds an ephemeral port, replies with the (host, port) on that channel, and serves the one connection by spawning a worker subprocess. The coordinator then connects to it over the Trio socket path. This makes the inline-exec socketserver dead, so drop it: the `execnet-socketserver` script is now only the Trio server + CLI, and the Windows service wrapper launches that. The legacy `__channelexec__` mode, `bind_and_listen`, `startserver`, and the inline `exec` are gone. Co-Authored-By: Claude Opus 4.8 --- src/execnet/_trio_host.py | 107 +++++++++++-- src/execnet/gateway_base.py | 11 ++ src/execnet/script/socketserver.py | 184 ++-------------------- src/execnet/script/socketserverservice.py | 7 +- 4 files changed, 124 insertions(+), 185 deletions(-) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index c58d0c34..ed78dd7d 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -7,6 +7,8 @@ from __future__ import annotations +import itertools +import json import os import queue import subprocess @@ -25,6 +27,8 @@ from .gateway_base import ExecModel from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message +from .gateway_base import dumps_internal +from .gateway_base import loads_internal from .gateway_base import trace if TYPE_CHECKING: @@ -703,15 +707,21 @@ def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: from .gateway_bootstrap import HostNotFound host: TrioHost = group._ensure_trio_host() - assert spec.socket is not None - host_str, _, port_str = spec.socket.rpartition(":") - address = (host_str, int(port_str)) + if getattr(spec, "installvia", None): + realhost, realport = start_socketserver_via(group[spec.installvia]) + address = (realhost, realport) + remoteaddress = "%s:%d" % (realhost, realport) + else: + assert spec.socket is not None + host_str, _, port_str = spec.socket.rpartition(":") + address = (host_str, int(port_str)) + remoteaddress = spec.socket async def _create_and_attach() -> Gateway: try: stream = await trio.open_tcp_stream(*address) except OSError as exc: - raise HostNotFound(spec.socket) from exc + raise HostNotFound(remoteaddress) from exc io = SocketStreamIO(stream) try: ack = await io.read_exact(1) @@ -725,23 +735,96 @@ async def _create_and_attach() -> Gateway: gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) session = await host.start_session(gw, io) gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=spec.socket) + gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) return gw return host.call(_create_and_attach) def should_use_trio_socket(spec: Any) -> bool: - """Trio path for direct ``socket=host:port`` gateways. - - ``installvia`` (start a server through another gateway) still uses the legacy - path for now. - """ + """Trio path for ``socket=host:port`` gateways, including ``installvia``.""" if not trio_host_enabled(): return False if not getattr(spec, "socket", None): return False - if getattr(spec, "installvia", None): - return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") + + +_socket_worker_counter = itertools.count() + + +def _spawn_socket_worker(fd: int) -> subprocess.Popen[bytes]: + """Spawn a worker subprocess serving over the inherited socket ``fd``.""" + import execnet + + config = json.dumps( + { + "id": "socketworker%d" % next(_socket_worker_counter), + "execmodel": "thread", + "coordinator_version": execnet.__version__, + } + ) + return subprocess.Popen( + [sys.executable, "-m", "execnet._trio_worker", config, "--socket-fd", str(fd)], + pass_fds=[fd], + ) + + +async def serve_socket_connection(stream: trio.SocketStream, *, reap: bool) -> None: + """Hand an accepted socket to a fresh worker subprocess (server side). + + ``reap`` waits for the worker (loop server); when false the worker outlives + this task (one-shot / installvia). + """ + proc = _spawn_socket_worker(stream.socket.fileno()) + # The child forked with a copy of the fd; release ours. + await stream.aclose() + if reap: + await trio.to_thread.run_sync(proc.wait) + + +async def _start_socket_and_reply( + gateway: BaseGateway, channelid: int, bind_host: str +) -> None: + """Bind an ephemeral port, reply with its address, then serve one connection. + + Runs as a task on the worker's Trio host (scheduled from the message + handler). The reply travels back on ``channelid`` like a STATUS reply. + """ + listeners = await trio.open_tcp_listeners(0, host=bind_host) + addr = listeners[0].socket.getsockname() + gateway._send(Message.CHANNEL_DATA, channelid, dumps_internal((addr[0], addr[1]))) + gateway._send(Message.CHANNEL_CLOSE, channelid) + + stream = await listeners[0].accept() + for listener in listeners: + await listener.aclose() + await serve_socket_connection(stream, reap=True) + + +def handle_start_socket(gateway: BaseGateway, channelid: int, data: bytes) -> None: + """Worker handler for ``Message.GATEWAY_START_SOCKET`` (on the host thread).""" + bind_host = loads_internal(data) + assert isinstance(bind_host, str) + host: TrioHost = gateway._trio_exec.host # type: ignore[attr-defined] + # The receiver runs on the host thread, so schedule the async work directly. + host.start_soon(_start_socket_and_reply, gateway, channelid, bind_host) + + +def start_socketserver_via( + via_gateway: Any, bind_host: str = "localhost" +) -> tuple[str, int]: + """Ask ``via_gateway`` (protocol message) to start a one-shot socket listener. + + Returns the ``(host, port)`` the coordinator should connect to. + """ + channel = via_gateway.newchannel() + via_gateway._send( + Message.GATEWAY_START_SOCKET, channel.id, dumps_internal(bind_host) + ) + realhost, realport = channel.receive() + channel.waitclose() + if not realhost or realhost in ("0.0.0.0", "::"): + realhost = "localhost" + return realhost, int(realport) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index c9933fe0..764f6a0a 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -548,6 +548,17 @@ def _channel_last_message(message: Message, gateway: BaseGateway) -> None: CHANNEL_LAST_MESSAGE = 7 _types[CHANNEL_LAST_MESSAGE] = ("CHANNEL_LAST_MESSAGE", _channel_last_message) + def _gateway_start_socket(message: Message, gateway: BaseGateway) -> None: + # Start a one-shot socketserver on this (Trio) gateway's host and reply + # with the bound (host, port) on the request channel. Handled natively + # instead of shipping source via remote_exec. + from . import _trio_host + + _trio_host.handle_start_socket(gateway, message.channelid, message.data) + + GATEWAY_START_SOCKET = 8 + _types[GATEWAY_START_SOCKET] = ("GATEWAY_START_SOCKET", _gateway_start_socket) + class GatewayReceivedTerminate(Exception): """Receiverthread got termination message.""" diff --git a/src/execnet/script/socketserver.py b/src/execnet/script/socketserver.py index ea580bcd..eb2a86ae 100644 --- a/src/execnet/script/socketserver.py +++ b/src/execnet/script/socketserver.py @@ -1,131 +1,19 @@ #! /usr/bin/env python -""" -start socket based minimal readline exec server - -it can exeuted in 2 modes of operation - -1. as normal script, that listens for new connections - -2. via existing_gateway.remote_exec (as imported module) +"""Trio socket server for execnet gateways. +Listens on a TCP port and hands each accepted connection (by fd) to a fresh +``python -m execnet._trio_worker`` subprocess that serves the gateway over it. +No code is executed inline. Run directly, e.g. provisioned anywhere with +``uvx --from execnet execnet-socketserver``. """ -# this part of the program only executes on the server side -# from __future__ import annotations -import os -import sys -from typing import TYPE_CHECKING - -try: - import fcntl -except ImportError: - fcntl = None # type: ignore[assignment] - -if TYPE_CHECKING: - from execnet.gateway_base import Channel - from execnet.gateway_base import ExecModel - -progname = "socket_readline_exec_server-1.2" - - -debug = 0 - -if debug: # and not os.isatty(sys.stdin.fileno()) - f = open("/tmp/execnet-socket-pyout.log", "w") - old = sys.stdout, sys.stderr - sys.stdout = sys.stderr = f - - -def print_(*args) -> None: - print(" ".join(str(arg) for arg in args)) - - -exec( - """def exec_(source, locs): - exec(source, locs)""" -) - - -def exec_from_one_connection(serversock, execmodel: ExecModel) -> None: - print_(progname, "Entering Accept loop", serversock.getsockname()) - clientsock, address = serversock.accept() - print_(progname, "got new connection from {} {}".format(*address)) - clientfile = clientsock.makefile("rb") - print_("reading line") - # rstrip so that we can use \r\n for telnet testing - source = clientfile.readline().rstrip() - clientfile.close() - g = {"clientsock": clientsock, "address": address, "execmodel": execmodel} - source = eval(source) - if source: - co = compile(source + "\n", "", "exec") - print_(progname, "compiled source, executing") - try: - exec_(co, g) # type: ignore[name-defined] # noqa: F821 - finally: - print_(progname, "finished executing code") - # background thread might hold a reference to this (!?) - # clientsock.close() - - -def bind_and_listen(hostport: str | tuple[str, int], execmodel: ExecModel): - socket = execmodel.socket - if isinstance(hostport, str): - host, port = hostport.split(":") - hostport = (host, int(port)) - serversock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - # set close-on-exec - if hasattr(fcntl, "FD_CLOEXEC"): - old = fcntl.fcntl(serversock.fileno(), fcntl.F_GETFD) - fcntl.fcntl(serversock.fileno(), fcntl.F_SETFD, old | fcntl.FD_CLOEXEC) - # allow the address to be reused in a reasonable amount of time - if os.name == "posix" and sys.platform != "cygwin": - serversock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - - serversock.bind(hostport) - serversock.listen(5) - return serversock - - -def startserver(serversock, execmodel: ExecModel, loop: bool = False) -> None: - execute_path = os.getcwd() - try: - while 1: - try: - exec_from_one_connection(serversock, execmodel) - except (KeyboardInterrupt, SystemExit): - raise - except BaseException as exc: - if debug: - import traceback - - traceback.print_exc() - else: - print_("got exception", exc) - os.chdir(execute_path) - if not loop: - break - finally: - print_("leaving socketserver execloop") - serversock.shutdown(2) - async def _trio_serve(hostport: str, once: bool) -> None: - """Trio TCP server that spawns a worker subprocess per connection. - - No code is executed inline: each accepted socket is handed (by fd) to a - fresh ``python -m execnet._trio_worker --socket-fd`` process which serves the - gateway over it. - """ - import itertools - import json - import subprocess - import trio - import execnet + from execnet import _trio_host host, _, port_str = hostport.rpartition(":") listeners = await trio.open_tcp_listeners(int(port_str), host=host or None) @@ -133,50 +21,22 @@ async def _trio_serve(hostport: str, once: bool) -> None: # Report the bound address (port may be ephemeral) for callers to read. print("execnet-socketserver listening on %s %s" % (addr[0], addr[1]), flush=True) - counter = itertools.count() + if once: + stream = await listeners[0].accept() + for listener in listeners: + await listener.aclose() + # The worker outlives this one-shot server. + await _trio_host.serve_socket_connection(stream, reap=False) + return - with trio.CancelScope() as scope: + async def handler(stream: trio.SocketStream) -> None: + await _trio_host.serve_socket_connection(stream, reap=True) - async def handler(stream: trio.SocketStream) -> None: - fd = stream.socket.fileno() - # Synthesise the worker config (socketserver workers default to the - # thread model, matching the legacy socket server). - config = json.dumps( - { - "id": "socketworker%d" % next(counter), - "execmodel": "thread", - "coordinator_version": execnet.__version__, - } - ) - proc = subprocess.Popen( - [ - sys.executable, - "-m", - "execnet._trio_worker", - config, - "--socket-fd", - str(fd), - ], - pass_fds=[fd], - ) - # The child forked with a copy of the fd; release ours. - await stream.aclose() - if once: - scope.cancel() - return - # Reap the worker without blocking other connections. - await trio.to_thread.run_sync(proc.wait) - - await trio.serve_listeners(handler, listeners) + await trio.serve_listeners(handler, listeners) def main(argv: list[str] | None = None) -> None: - """Console entry point (``execnet-socketserver``). - - Serve execnet gateway connections over a socket, spawning a worker - subprocess per connection. Intended to be run directly, e.g. provisioned on - a host with ``uvx --from execnet execnet-socketserver``. - """ + """Console entry point (``execnet-socketserver``).""" import argparse import trio @@ -202,13 +62,3 @@ def main(argv: list[str] | None = None) -> None: if __name__ == "__main__": main() - -elif __name__ == "__channelexec__": - chan: Channel = globals()["channel"] - execmodel = chan.gateway.execmodel - bindname = chan.receive() - assert isinstance(bindname, (str, tuple)) - sock = bind_and_listen(bindname, execmodel) - port = sock.getsockname() - chan.send(port) - startserver(sock, execmodel) diff --git a/src/execnet/script/socketserverservice.py b/src/execnet/script/socketserverservice.py index 18e375c4..1c991307 100644 --- a/src/execnet/script/socketserverservice.py +++ b/src/execnet/script/socketserverservice.py @@ -15,8 +15,6 @@ import win32service import win32serviceutil -from execnet.gateway_base import get_execmodel - from . import socketserver appname = "ExecNetSocketServer" @@ -65,12 +63,9 @@ def SvcDoRun(self) -> None: hostport = ":8888" print("Starting py.execnet SocketServer on %s" % hostport) - exec_model = get_execmodel("thread") - serversock = socketserver.bind_and_listen(hostport, exec_model) thread = threading.Thread( - target=socketserver.startserver, args=(serversock,), kwargs={"loop": True} + target=socketserver.main, args=([hostport],), daemon=True ) - thread.setDaemon(True) thread.start() # wait to be stopped or self.WAIT_TIME to pass From 62b746b705dece7e79484fcf9db57064aa049734 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 11:49:19 +0200 Subject: [PATCH 13/59] refactor: remove the legacy socket coordinator code With direct socket and installvia both on the Trio path, the legacy socket coordinator is dead. Delete gateway_socket.py (SocketIO, create_io, start_via), drop bootstrap_socket and the socket branch from gateway_bootstrap.bootstrap, and remove the now-unreachable `elif spec.socket:` arm from Group.makegateway. All socket gateways now go through _trio_host.makegateway_socket_trio. Co-Authored-By: Claude Opus 4.8 --- src/execnet/gateway_bootstrap.py | 22 ------- src/execnet/gateway_socket.py | 102 ------------------------------- src/execnet/multi.py | 5 -- 3 files changed, 129 deletions(-) delete mode 100644 src/execnet/gateway_socket.py diff --git a/src/execnet/gateway_bootstrap.py b/src/execnet/gateway_bootstrap.py index e9d7efe1..5707dff4 100644 --- a/src/execnet/gateway_bootstrap.py +++ b/src/execnet/gateway_bootstrap.py @@ -55,26 +55,6 @@ def bootstrap_exec(io: IO, spec: XSpec) -> None: raise HostNotFound(io.remoteaddress) from None -def bootstrap_socket(io: IO, id) -> None: - # XXX: switch to spec - from execnet.gateway_socket import SocketIO - - sendexec( - io, - inspect.getsource(gateway_base), - "import socket", - inspect.getsource(SocketIO), - "try: execmodel", - "except NameError:", - " execmodel = get_execmodel('thread')", - "io = SocketIO(clientsock, execmodel)", - "io.write('1'.encode('ascii'))", - "serve(io, id='%s-worker')" % id, - ) - s = io.read(1) - assert s == b"1" - - def sendexec(io: IO, *sources: str) -> None: source = "\n".join(sources) io.write((repr(source) + "\n").encode("utf-8")) @@ -88,8 +68,6 @@ def bootstrap(io: IO, spec: XSpec) -> execnet.Gateway: bootstrap_import(io, spec) elif spec.ssh or spec.vagrant_ssh: bootstrap_exec(io, spec) - elif spec.socket: - bootstrap_socket(io, spec) else: raise ValueError("unknown gateway type, can't bootstrap") gw = execnet.Gateway(io, spec) diff --git a/src/execnet/gateway_socket.py b/src/execnet/gateway_socket.py deleted file mode 100644 index be42f1ab..00000000 --- a/src/execnet/gateway_socket.py +++ /dev/null @@ -1,102 +0,0 @@ -from __future__ import annotations - -import sys -from typing import cast - -from execnet.gateway import Gateway -from execnet.gateway_base import ExecModel -from execnet.gateway_bootstrap import HostNotFound -from execnet.multi import Group -from execnet.xspec import XSpec - - -class SocketIO: - remoteaddress: str - - def __init__(self, sock, execmodel: ExecModel) -> None: - self.sock = sock - self.execmodel = execmodel - socket = execmodel.socket - try: - # IPTOS_LOWDELAY - sock.setsockopt(socket.SOL_IP, socket.IP_TOS, 0x10) - sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - except (AttributeError, OSError): - sys.stderr.write("WARNING: cannot set socketoption") - - def read(self, numbytes: int) -> bytes: - "Read exactly 'bytes' bytes from the socket." - buf = b"" - while len(buf) < numbytes: - t = self.sock.recv(numbytes - len(buf)) - if not t: - raise EOFError - buf += t - return buf - - def write(self, data: bytes) -> None: - self.sock.sendall(data) - - def close_read(self) -> None: - try: - self.sock.shutdown(0) - except self.execmodel.socket.error: - pass - - def close_write(self) -> None: - try: - self.sock.shutdown(1) - except self.execmodel.socket.error: - pass - - def wait(self) -> None: - pass - - def kill(self) -> None: - pass - - -def start_via( - gateway: Gateway, hostport: tuple[str, int] | None = None -) -> tuple[str, int]: - """Instantiate a socketserver on the given gateway. - - Returns a host, port tuple. - """ - if hostport is None: - host, port = ("localhost", 0) - else: - host, port = hostport - - from execnet.script import socketserver - - # execute the above socketserverbootstrap on the other side - channel = gateway.remote_exec(socketserver) - channel.send((host, port)) - realhost, realport = cast("tuple[str, int]", channel.receive()) - # self._trace("new_remote received" - # "port=%r, hostname = %r" %(realport, hostname)) - if not realhost or realhost == "0.0.0.0": - realhost = "localhost" - return realhost, realport - - -def create_io(spec: XSpec, group: Group, execmodel: ExecModel) -> SocketIO: - assert spec.socket is not None - assert not spec.python, "socket: specifying python executables not yet supported" - gateway_id = spec.installvia - if gateway_id: - host, port = start_via(group[gateway_id]) - else: - host, port_str = spec.socket.split(":") - port = int(port_str) - - socket = execmodel.socket - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - io = SocketIO(sock, execmodel) - io.remoteaddress = "%s:%d" % (host, port) - try: - sock.connect((host, port)) - except execmodel.socket.gaierror as e: - raise HostNotFound() from e - return io diff --git a/src/execnet/multi.py b/src/execnet/multi.py index d64722ee..cc85d757 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -170,11 +170,6 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: elif spec.popen or spec.ssh or spec.vagrant_ssh: io = gateway_io.create_io(spec, execmodel=self.execmodel) gw = gateway_bootstrap.bootstrap(io, spec) - elif spec.socket: - from . import gateway_socket - - sio = gateway_socket.create_io(spec, self, execmodel=self.execmodel) - gw = gateway_bootstrap.bootstrap(sio, spec) else: raise ValueError(f"no gateway type found for {spec._spec!r}") gw.spec = spec From 19d620d976b4a677a7e2596378a0010bd0be6491 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 23:34:36 +0200 Subject: [PATCH 14/59] feat: run popen via-gateways on Trio via GATEWAY_START_POPEN Move the common `popen//via=` proxy transport onto the Trio host with a new protocol message instead of remote_exec'ing gateway_io source. The coordinator sends GATEWAY_START_POPEN (with the sub worker config) on a request channel; the master's Trio host spawns a `python -m execnet._trio_worker` sub-worker and relays its Message protocol raw over that channel (stdin<-channel, stdout->channel). The coordinator wraps the channel as ChannelByteIO and runs a normal Trio session over it. ChannelByteIO is marked an interim hack: it tunnels sub frames as CHANNEL_DATA (double-framing) and should become a proper relayed transport. ssh/foreign-python via sub-gateways still use the legacy path. Co-Authored-By: Claude Opus 4.8 --- doc/implnotes.rst | 13 +++- src/execnet/_trio_host.py | 136 ++++++++++++++++++++++++++++++++++++ src/execnet/gateway_base.py | 10 +++ src/execnet/multi.py | 2 + 4 files changed, 159 insertions(+), 2 deletions(-) diff --git a/doc/implnotes.rst b/doc/implnotes.rst index 29df051c..ba1c8408 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -58,8 +58,17 @@ threads wait until the frame is written (so abrupt ``os._exit`` cannot drop queued data). Sends from the Trio host thread (receiver callbacks) only enqueue, to avoid deadlocking the writer task. -Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types (``socket``, -``via``, greenlet execmodels) still use the legacy thread receiver and sync +``socket`` and ``via`` gateways run on the Trio host too. ``socket`` +connects a Trio TCP stream to an ``execnet-socketserver`` (itself a Trio +listener spawning ``python -m execnet._trio_worker --socket-fd`` subprocess +workers). Infrastructure that used to be driven by ``remote_exec``-ing +source is now expressed as native protocol messages handled on the target's +Trio host: ``GATEWAY_START_SOCKET`` (installvia -> bind a one-shot listener, +reply with its address) and ``GATEWAY_START_POPEN`` (``via`` -> spawn a popen +sub-worker and relay its protocol over the request channel). + +Disable with ``EXECNET_TRIO_HOST=0``. ssh/foreign-python ``via`` sub-gateways +and greenlet execmodels still use the legacy thread receiver and sync ``Popen`` path. Legacy thread model diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index ed78dd7d..e00f3de0 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -9,6 +9,7 @@ import itertools import json +import math import os import queue import subprocess @@ -165,6 +166,49 @@ async def aclose_write(self) -> None: await self._stream.aclose() +_CHANNEL_EOF = object() + + +class ChannelByteIO: + """Async byte IO tunnelled over a sync execnet ``Channel``. + + INTERIM HACK: the ``via`` transport currently runs the sub-gateway protocol + as raw bytes over a channel to the master, which double-frames it (sub frame + -> CHANNEL_DATA -> master frame). This bridges the sync ``Channel`` to the + async protocol; it should be replaced by a proper relayed transport rather + than tunnelling bytes through the channel layer. + + The channel callback (on the host loop) feeds an unbounded memory channel + that ``read_exact`` drains; writes ``channel.send`` raw frames. + """ + + def __init__(self, channel: Any) -> None: + self._channel = channel + self._send, self._recv = trio.open_memory_channel[Any](math.inf) + self._buf = bytearray() + channel.setcallback(self._send.send_nowait, endmarker=_CHANNEL_EOF) + + async def read_exact(self, n: int) -> bytes: + while len(self._buf) < n: + data = await self._recv.receive() + if data is _CHANNEL_EOF: + raise EOFError("channel closed") + assert isinstance(data, bytes) + self._buf += data + out = bytes(self._buf[:n]) + del self._buf[:n] + return out + + async def write_all(self, data: bytes) -> None: + self._channel.send(data) + + async def aclose_read(self) -> None: + return + + async def aclose_write(self) -> None: + self._channel.close() + + async def adopt_socket(socket_fd: int) -> SocketStreamIO: """Worker side: wrap an inherited socket fd and send the handshake. @@ -828,3 +872,95 @@ def start_socketserver_via( if not realhost or realhost in ("0.0.0.0", "::"): realhost = "localhost" return realhost, int(realport) + + +async def _start_popen_and_relay( + gateway: BaseGateway, channelid: int, worker_config: str +) -> None: + """Spawn a popen sub-worker and relay its Message protocol over the channel. + + Runs on the master's Trio host: bytes from the channel go to the sub's + stdin, and the sub's stdout goes back on the channel (the ``via`` transport). + """ + args = [sys.executable, "-u", "-m", "execnet._trio_worker", worker_config] + process = await open_popen_process(args) + channel = gateway._channelfactory.new(channelid) + send_ch, recv_ch = trio.open_memory_channel[Any](math.inf) + channel.setcallback(send_ch.send_nowait, endmarker=_CHANNEL_EOF) + + async def coordinator_to_sub() -> None: + assert process.stdin is not None + async for data in recv_ch: + if data is _CHANNEL_EOF: + break + await process.stdin.send_all(data) + with trio.move_on_after(5): + await process.stdin.aclose() + + async def sub_to_coordinator() -> None: + assert process.stdout is not None + while True: + data = await process.stdout.receive_some(65536) + if not data: + break + channel.send(data) + channel.close() + + try: + async with trio.open_nursery() as nursery: + nursery.start_soon(coordinator_to_sub) + nursery.start_soon(sub_to_coordinator) + finally: + with trio.move_on_after(5): + await process.wait() + + +def handle_start_popen(gateway: BaseGateway, channelid: int, data: bytes) -> None: + """Worker handler for ``Message.GATEWAY_START_POPEN`` (on the host thread).""" + worker_config = loads_internal(data) + assert isinstance(worker_config, str) + host: TrioHost = gateway._trio_exec.host # type: ignore[attr-defined] + host.start_soon(_start_popen_and_relay, gateway, channelid, worker_config) + + +def should_use_trio_via(spec: Any) -> bool: + """Trio path for ``popen//via=`` (a popen sub-gateway on the master). + + Only a same-interpreter popen sub is supported for now (no ssh/foreign sub). + """ + if not trio_host_enabled(): + return False + if not getattr(spec, "via", None): + return False + if getattr(spec, "socket", None) or getattr(spec, "ssh", None): + return False + if getattr(spec, "python", None): + return False + execmodel = getattr(spec, "execmodel", None) + return execmodel in (None, "thread", "main_thread_only") + + +def makegateway_via_trio(group: Any, spec: Any) -> Gateway: + """Create a ``via`` gateway: a popen sub relayed through the master gateway.""" + import execnet + + from . import _provision + + master = group[spec.via] + host: TrioHost = group._ensure_trio_host() + channel = master.newchannel() + worker_config = _provision.worker_cli_arg(spec) + master._send(Message.GATEWAY_START_POPEN, channel.id, dumps_internal(worker_config)) + + async def _create_and_attach() -> Gateway: + io = ChannelByteIO(channel) + ack = await io.read_exact(1) + if ack != b"1": + raise EOFError(f"bad via handshake: {ack!r}") + gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + session = await host.start_session(gw, io) + gw._attach_trio_session(session) + gw._io = SyncIOHandle(group.execmodel, session) + return gw + + return host.call(_create_and_attach) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 764f6a0a..679e8fd5 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -559,6 +559,16 @@ def _gateway_start_socket(message: Message, gateway: BaseGateway) -> None: GATEWAY_START_SOCKET = 8 _types[GATEWAY_START_SOCKET] = ("GATEWAY_START_SOCKET", _gateway_start_socket) + def _gateway_start_popen(message: Message, gateway: BaseGateway) -> None: + # Spawn a popen worker subprocess on this (Trio) gateway's host and relay + # its Message protocol over the request channel (the ``via`` transport). + from . import _trio_host + + _trio_host.handle_start_popen(gateway, message.channelid, message.data) + + GATEWAY_START_POPEN = 9 + _types[GATEWAY_START_POPEN] = ("GATEWAY_START_POPEN", _gateway_start_popen) + class GatewayReceivedTerminate(Exception): """Receiverthread got termination message.""" diff --git a/src/execnet/multi.py b/src/execnet/multi.py index cc85d757..3a9dc602 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -160,6 +160,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: gw = _trio_host.makegateway_ssh_trio(self, spec) elif _trio_host.should_use_trio_socket(spec): gw = _trio_host.makegateway_socket_trio(self, spec) + elif _trio_host.should_use_trio_via(spec): + gw = _trio_host.makegateway_via_trio(self, spec) elif spec.via: assert not spec.socket master = self[spec.via] From 92969c5a269a223a2d2edc8ef55e1c2abdb58b3e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 02:32:10 +0200 Subject: [PATCH 15/59] feat: spawn arbitrary via sub-gateways with GATEWAY_START_SUB Generalize the via relay message from popen-only to a spawn-request dict carrying the sub-spec essentials plus provisioning material: a released coordinator sends a pip requirement, a dev coordinator ships its wheel for the master to materialize into the local wheel cache. The master resolves the launch locally (direct module, uv-provisioned python=, or ssh with the wheel streamed as stdin preamble), so ssh= and python= sub specs now run on the Trio path. Also route ssh=...//via=... through the via path instead of opening a direct ssh connection, and close the request channel with an error on spawn/relay failure instead of crashing the host nursery. Co-Authored-By: Claude Fable 5 --- src/execnet/_provision.py | 182 +++++++++++++++++++++++++++++++----- src/execnet/_trio_host.py | 71 ++++++++------ src/execnet/gateway_base.py | 13 +-- testing/test_multi.py | 18 ++++ testing/test_ssh_local.py | 20 ++++ 5 files changed, 249 insertions(+), 55 deletions(-) diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index b24bfe8b..83fd1515 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -17,10 +17,12 @@ from __future__ import annotations import json +import os import re import shlex import shutil import subprocess +import sys import tempfile from functools import cache from pathlib import Path @@ -162,52 +164,62 @@ def worker_cli_arg(spec: Any) -> str: ) +def _worker_tokens(config: str) -> list[str]: + """``python -u -m execnet._trio_worker `` tokens. + + The literal ``python`` token is resolved by uv (inside the provisioned + environment) or the remote shell. + """ + return ["python", "-u", "-m", "execnet._trio_worker", config] + + def worker_module_tokens(spec: Any) -> list[str]: - """``python -u -m execnet._trio_worker `` tokens.""" - return ["python", "-u", "-m", "execnet._trio_worker", worker_cli_arg(spec)] + """``python -u -m execnet._trio_worker `` tokens for ``spec``.""" + return _worker_tokens(worker_cli_arg(spec)) -def _uv_prefix(spec: Any) -> list[str]: +def _uv_tokens(python: str | None) -> list[str]: # --no-project keeps the surrounding execnet checkout from being synced. prefix = ["uv", "run", "--no-project"] - if spec.python: - prefix += ["--python", spec.python] + if python: + prefix += ["--python", python] return prefix def uv_worker_argv(spec: Any) -> list[str]: """``uv run`` argv to launch the Trio worker locally (wheel path is local).""" return [ - *_uv_prefix(spec), + *_uv_tokens(spec.python), "--with", coordinator_requirement(), *worker_module_tokens(spec), ] -def ssh_remote_command(spec: Any) -> tuple[str, bytes]: - """Remote shell command + stdin preamble to launch the worker over ssh. +def _remote_shell_command( + python: str | None, + config: str, + *, + requirement: str | None = None, + wheel: Path | None = None, +) -> tuple[str, bytes]: + """Remote sh command + stdin preamble launching the worker via uv. - Released coordinator -> ``uv run --with execnet== …`` with no preamble. - Dev coordinator -> a POSIX-sh prelude that receives the wheel bytes from - stdin (``head -c N``) into a temp dir and ``exec``s uv against it; the wheel - bytes are returned as the preamble to stream before the Message protocol. + With ``requirement`` the remote installs from an index and no preamble is + needed. With ``wheel`` a POSIX-sh prelude receives the wheel bytes from + stdin (``head -c N``) into a temp dir and ``exec``s uv against it; the + wheel bytes are returned as the preamble to stream before the protocol. """ - import execnet + worker = _worker_tokens(config) + uv = _uv_tokens(python) + if wheel is None: + assert requirement is not None + return shlex.join([*uv, "--with", requirement, *worker]), b"" - version = execnet.__version__ - worker = worker_module_tokens(spec) - if _RELEASED_RE.match(version): - command = shlex.join( - [*_uv_prefix(spec), "--with", f"execnet=={version}", *worker] - ) - return command, b"" - - wheel = _build_wheel(version) data = wheel.read_bytes() # "$d/": expand the temp dir, concatenate the (quoted) wheel filename. remote_wheel = '"$d/"' + shlex.quote(wheel.name) - uv_run = " ".join(shlex.quote(token) for token in [*_uv_prefix(spec), "--with"]) + uv_run = " ".join(shlex.quote(token) for token in [*uv, "--with"]) worker_cmd = " ".join(shlex.quote(token) for token in worker) prelude = ( f"d=$(mktemp -d) && " @@ -215,3 +227,127 @@ def ssh_remote_command(spec: Any) -> tuple[str, bytes]: f"exec {uv_run} {remote_wheel} {worker_cmd}" ) return prelude, data + + +def ssh_remote_command(spec: Any) -> tuple[str, bytes]: + """Remote shell command + stdin preamble to launch the worker over ssh. + + Released coordinator -> ``uv run --with execnet== …`` with no preamble. + Dev coordinator -> wheel-shipping prelude (see ``_remote_shell_command``). + """ + import execnet + + version = execnet.__version__ + config = worker_cli_arg(spec) + if _RELEASED_RE.match(version): + return _remote_shell_command( + spec.python, config, requirement=f"execnet=={version}" + ) + return _remote_shell_command(spec.python, config, wheel=_build_wheel(version)) + + +def ssh_argv(ssh: str, ssh_config: str | None, remote_command: str) -> list[str]: + """``ssh`` client argv running ``remote_command`` on host ``ssh``.""" + args = ["ssh", "-C"] + if ssh_config: + args += ["-F", ssh_config] + args += ssh.split() + args.append(remote_command) + return args + + +def spawn_request(spec: Any) -> dict[str, Any]: + """Payload for ``GATEWAY_START_SUB``: ask a via master to spawn a sub-worker. + + Carries the sub-spec essentials plus provisioning material when the sub + may need it (ssh or foreign python): a released coordinator sends a pip + requirement; a dev coordinator ships its wheel bytes for the master to + materialize into its local wheel cache. + + TODO: the wheel is shipped eagerly because only the master can tell + whether the target interpreter already has execnet; a wheel-on-demand + round-trip would avoid the transfer in the common provisioned case. + """ + import execnet + + request: dict[str, Any] = { + "config": worker_cli_arg(spec), + "python": spec.python or None, + "ssh": spec.ssh or None, + "ssh_config": spec.ssh_config or None, + } + if spec.ssh or spec.python: + version = execnet.__version__ + if _RELEASED_RE.match(version): + request["requirement"] = f"execnet=={version}" + else: + wheel = _build_wheel(version) + request["wheel"] = (wheel.name, wheel.read_bytes()) + return request + + +def materialize_wheel(name: str, data: bytes) -> Path: + """Write shipped wheel bytes into the local wheel cache (idempotent).""" + target = _wheel_cache_dir() / name + if not target.exists(): + tmp = target.with_name(f"{target.name}.{os.getpid()}.tmp") + tmp.write_bytes(data) + tmp.replace(target) + return target + + +def _requested_requirement(request: dict[str, Any]) -> tuple[str | None, Path | None]: + """(uv requirement, local wheel path) from a spawn request's material.""" + requirement = request.get("requirement") + if isinstance(requirement, str): + return requirement, None + shipped = request.get("wheel") + if shipped is not None: + name, data = shipped + path = materialize_wheel(name, data) + return str(path), path + return None, None + + +def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: + """(argv, stdin preamble) spawning a requested sub-worker on this host. + + Handles a ``GATEWAY_START_SUB`` request on a via master: plain popen runs + this interpreter's worker module, a foreign ``python`` runs directly when + it already has execnet and is uv-provisioned otherwise, and ``ssh`` wraps + the remote uv command (streaming a shipped wheel as the preamble for dev + versions). + """ + from .gateway_io import shell_split_path + + config = request["config"] + assert isinstance(config, str) + python = request.get("python") + ssh = request.get("ssh") + if ssh: + assert isinstance(ssh, str) + requirement, wheel = _requested_requirement(request) + if requirement is None: + raise RuntimeError("ssh spawn request without provisioning material") + command, preamble = _remote_shell_command( + python, config, requirement=requirement, wheel=wheel + ) + return ssh_argv(ssh, request.get("ssh_config"), command), preamble + if python: + assert isinstance(python, str) + if target_has_execnet(python): + argv = [*shell_split_path(python), "-u", "-m", "execnet._trio_worker"] + return [*argv, config], b"" + requirement, _ = _requested_requirement(request) + if requirement is None or not uv_available(): + raise RuntimeError( + f"cannot provision sub-worker for python={python!r}: " + "uv and provisioning material required" + ) + return [ + *_uv_tokens(python), + "--with", + requirement, + *_worker_tokens(config), + ], b"" + return [sys.executable, "-u", "-m", "execnet._trio_worker", config], b"" diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index e00f3de0..709a8b64 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -17,6 +17,7 @@ import threading from collections.abc import Awaitable from collections.abc import Callable +from contextlib import suppress from typing import TYPE_CHECKING from typing import Any from typing import Protocol @@ -712,13 +713,8 @@ def ssh_trio_args(spec: Any) -> tuple[list[str], bytes]: from . import _provision remote_command, preamble = _provision.ssh_remote_command(spec) - args = ["ssh", "-C"] - if getattr(spec, "ssh_config", None): - args += ["-F", spec.ssh_config] assert spec.ssh is not None - args += spec.ssh.split() - args.append(remote_command) - return args, preamble + return _provision.ssh_argv(spec.ssh, spec.ssh_config, remote_command), preamble def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: @@ -730,11 +726,17 @@ def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: def should_use_trio_ssh(spec: Any) -> bool: - """Trio path for ssh gateways (worker provisioned on the remote via uv).""" + """Trio path for ssh gateways (worker provisioned on the remote via uv). + + ``ssh=…//via=…`` is not a direct ssh connection but a sub-gateway spawned + by the master; that goes through the via path instead. + """ if not trio_host_enabled(): return False if not getattr(spec, "ssh", None): return False + if getattr(spec, "via", None): + return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") @@ -874,22 +876,32 @@ def start_socketserver_via( return realhost, int(realport) -async def _start_popen_and_relay( - gateway: BaseGateway, channelid: int, worker_config: str +async def _start_sub_and_relay( + gateway: BaseGateway, channelid: int, request: dict[str, Any] ) -> None: - """Spawn a popen sub-worker and relay its Message protocol over the channel. + """Spawn a requested sub-worker and relay its Message protocol over the channel. Runs on the master's Trio host: bytes from the channel go to the sub's stdin, and the sub's stdout goes back on the channel (the ``via`` transport). + A stdin preamble (shipped wheel for a dev-version ssh sub) is streamed + before the relayed protocol bytes. """ - args = [sys.executable, "-u", "-m", "execnet._trio_worker", worker_config] - process = await open_popen_process(args) + from . import _provision + channel = gateway._channelfactory.new(channelid) + try: + args, preamble = _provision.sub_spawn_argv(request) + process = await open_popen_process(args) + except Exception as exc: + channel.close(f"could not spawn via sub-gateway: {exc}") + return send_ch, recv_ch = trio.open_memory_channel[Any](math.inf) channel.setcallback(send_ch.send_nowait, endmarker=_CHANNEL_EOF) async def coordinator_to_sub() -> None: assert process.stdin is not None + if preamble: + await process.stdin.send_all(preamble) async for data in recv_ch: if data is _CHANNEL_EOF: break @@ -910,38 +922,44 @@ async def sub_to_coordinator() -> None: async with trio.open_nursery() as nursery: nursery.start_soon(coordinator_to_sub) nursery.start_soon(sub_to_coordinator) + except Exception as exc: + # Do not let a relay failure crash the host nursery; surface it on + # the channel so the coordinator does not hang on the handshake. + gateway._trace("via sub relay failed:", exc) + with suppress(Exception): + channel.close(f"via sub-gateway relay failed: {exc}") finally: with trio.move_on_after(5): await process.wait() -def handle_start_popen(gateway: BaseGateway, channelid: int, data: bytes) -> None: - """Worker handler for ``Message.GATEWAY_START_POPEN`` (on the host thread).""" - worker_config = loads_internal(data) - assert isinstance(worker_config, str) +def handle_start_sub(gateway: BaseGateway, channelid: int, data: bytes) -> None: + """Worker handler for ``Message.GATEWAY_START_SUB`` (on the host thread).""" + request = loads_internal(data) + assert isinstance(request, dict) host: TrioHost = gateway._trio_exec.host # type: ignore[attr-defined] - host.start_soon(_start_popen_and_relay, gateway, channelid, worker_config) + host.start_soon(_start_sub_and_relay, gateway, channelid, request) def should_use_trio_via(spec: Any) -> bool: - """Trio path for ``popen//via=`` (a popen sub-gateway on the master). + """Trio path for ``via=`` sub-gateways (popen, foreign python, or ssh). - Only a same-interpreter popen sub is supported for now (no ssh/foreign sub). + The master spawns the sub-worker from a ``GATEWAY_START_SUB`` request and + relays its Message protocol. Socket subs go through ``installvia`` + instead; vagrant stays on the legacy path. """ if not trio_host_enabled(): return False if not getattr(spec, "via", None): return False - if getattr(spec, "socket", None) or getattr(spec, "ssh", None): - return False - if getattr(spec, "python", None): + if getattr(spec, "socket", None) or getattr(spec, "vagrant_ssh", None): return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") def makegateway_via_trio(group: Any, spec: Any) -> Gateway: - """Create a ``via`` gateway: a popen sub relayed through the master gateway.""" + """Create a ``via`` gateway: a sub-worker spawned by and relayed through the master.""" import execnet from . import _provision @@ -949,8 +967,9 @@ def makegateway_via_trio(group: Any, spec: Any) -> Gateway: master = group[spec.via] host: TrioHost = group._ensure_trio_host() channel = master.newchannel() - worker_config = _provision.worker_cli_arg(spec) - master._send(Message.GATEWAY_START_POPEN, channel.id, dumps_internal(worker_config)) + request = _provision.spawn_request(spec) + master._send(Message.GATEWAY_START_SUB, channel.id, dumps_internal(request)) + remoteaddress = f"{spec.ssh}[via {spec.via}]" if spec.ssh else None async def _create_and_attach() -> Gateway: io = ChannelByteIO(channel) @@ -960,7 +979,7 @@ async def _create_and_attach() -> Gateway: gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) session = await host.start_session(gw, io) gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session) + gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) return gw return host.call(_create_and_attach) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 679e8fd5..2736be96 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -559,15 +559,16 @@ def _gateway_start_socket(message: Message, gateway: BaseGateway) -> None: GATEWAY_START_SOCKET = 8 _types[GATEWAY_START_SOCKET] = ("GATEWAY_START_SOCKET", _gateway_start_socket) - def _gateway_start_popen(message: Message, gateway: BaseGateway) -> None: - # Spawn a popen worker subprocess on this (Trio) gateway's host and relay - # its Message protocol over the request channel (the ``via`` transport). + def _gateway_start_sub(message: Message, gateway: BaseGateway) -> None: + # Spawn a sub-gateway worker (popen, foreign python, or ssh) on this + # (Trio) gateway's host and relay its Message protocol over the request + # channel (the ``via`` transport). from . import _trio_host - _trio_host.handle_start_popen(gateway, message.channelid, message.data) + _trio_host.handle_start_sub(gateway, message.channelid, message.data) - GATEWAY_START_POPEN = 9 - _types[GATEWAY_START_POPEN] = ("GATEWAY_START_POPEN", _gateway_start_popen) + GATEWAY_START_SUB = 9 + _types[GATEWAY_START_SUB] = ("GATEWAY_START_SUB", _gateway_start_sub) class GatewayReceivedTerminate(Exception): diff --git a/testing/test_multi.py b/testing/test_multi.py index f3166503..e0390298 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -231,6 +231,24 @@ def test_terminate_with_proxying(self) -> None: group.makegateway("popen//via=master//id=worker") group.terminate(1.0) + def test_via_foreign_python(self) -> None: + # A python= sub-spec through a via master: the master resolves the + # interpreter locally (this interpreter has execnet, so the sub runs + # the worker module directly, no uv provisioning). + import sys + + group = Group() + try: + group.makegateway("popen//id=master") + gw = group.makegateway( + f"popen//python={sys.executable}//via=master//id=sub" + ) + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(1.0) + @pytest.mark.xfail(reason="active_count() has been broken for some time") def test_safe_terminate(execmodel: ExecModel) -> None: diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index 72dea348..c21135d1 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -149,3 +149,23 @@ def test_ssh_roundtrip( assert channel.receive() == 42 finally: group.terminate(timeout=5.0) + + +def test_ssh_via_roundtrip(ssh_config: str) -> None: + """An ssh sub-gateway spawned by a popen master (GATEWAY_START_SUB relay). + + The master runs the ssh client; for a dev coordinator the wheel travels + coordinator -> master (in the spawn request) -> remote (ssh stdin preamble). + """ + group = execnet.Group() + try: + group.makegateway("popen//id=master") + gw = group.makegateway( + f"ssh=testhost//ssh_config={ssh_config}//python={sys.executable}" + "//via=master//id=sshvia" + ) + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(timeout=5.0) From 48d328c6d61f58a342fb6c5bbfe6f860c099d4ef Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 02:53:29 +0200 Subject: [PATCH 16/59] feat: run vagrant_ssh gateways on the Trio path vagrant_ssh= now launches the uv-provisioned worker through `vagrant ssh -- -C ` (mirroring the ssh argv), both as a direct gateway and as a via sub-gateway through GATEWAY_START_SUB. Co-Authored-By: Claude Fable 5 --- src/execnet/_provision.py | 31 ++++++++++++++++++++++++++----- src/execnet/_trio_host.py | 36 +++++++++++++++++++++++++++++++----- src/execnet/multi.py | 2 ++ testing/test_provision.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 10 deletions(-) diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 83fd1515..c55fb431 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -256,6 +256,21 @@ def ssh_argv(ssh: str, ssh_config: str | None, remote_command: str) -> list[str] return args +def vagrant_ssh_argv( + machine: str, ssh_config: str | None, remote_command: str +) -> list[str]: + """``vagrant ssh`` argv running ``remote_command`` on the named VM. + + Everything after ``--`` is passed through to the underlying ssh client, + mirroring ``ssh_argv``. + """ + args = ["vagrant", "ssh", machine, "--", "-C"] + if ssh_config: + args += ["-F", ssh_config] + args.append(remote_command) + return args + + def spawn_request(spec: Any) -> dict[str, Any]: """Payload for ``GATEWAY_START_SUB``: ask a via master to spawn a sub-worker. @@ -274,9 +289,10 @@ def spawn_request(spec: Any) -> dict[str, Any]: "config": worker_cli_arg(spec), "python": spec.python or None, "ssh": spec.ssh or None, + "vagrant_ssh": spec.vagrant_ssh or None, "ssh_config": spec.ssh_config or None, } - if spec.ssh or spec.python: + if spec.ssh or spec.vagrant_ssh or spec.python: version = execnet.__version__ if _RELEASED_RE.match(version): request["requirement"] = f"execnet=={version}" @@ -324,15 +340,20 @@ def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: assert isinstance(config, str) python = request.get("python") ssh = request.get("ssh") - if ssh: - assert isinstance(ssh, str) + vagrant = request.get("vagrant_ssh") + if ssh or vagrant: requirement, wheel = _requested_requirement(request) if requirement is None: - raise RuntimeError("ssh spawn request without provisioning material") + raise RuntimeError("remote spawn request without provisioning material") command, preamble = _remote_shell_command( python, config, requirement=requirement, wheel=wheel ) - return ssh_argv(ssh, request.get("ssh_config"), command), preamble + ssh_config = request.get("ssh_config") + if ssh: + assert isinstance(ssh, str) + return ssh_argv(ssh, ssh_config, command), preamble + assert isinstance(vagrant, str) + return vagrant_ssh_argv(vagrant, ssh_config, command), preamble if python: assert isinstance(python, str) if target_has_execnet(python): diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 709a8b64..2a612996 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -725,6 +725,32 @@ def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: ) +def should_use_trio_vagrant(spec: Any) -> bool: + """Trio path for ``vagrant_ssh=`` gateways (uv-provisioned worker).""" + if not trio_host_enabled(): + return False + if not getattr(spec, "vagrant_ssh", None): + return False + if getattr(spec, "via", None): + return False + execmodel = getattr(spec, "execmodel", None) + return execmodel in (None, "thread", "main_thread_only") + + +def makegateway_vagrant_trio(group: Any, spec: Any) -> Gateway: + """Create a ``vagrant ssh``-wrapped Gateway on the Trio IO path.""" + from . import _provision + + remote_command, preamble = _provision.ssh_remote_command(spec) + assert spec.vagrant_ssh is not None + args = _provision.vagrant_ssh_argv( + spec.vagrant_ssh, spec.ssh_config, remote_command + ) + return _open_trio_gateway( + group, spec, args, remoteaddress=spec.vagrant_ssh, preamble=preamble + ) + + def should_use_trio_ssh(spec: Any) -> bool: """Trio path for ssh gateways (worker provisioned on the remote via uv). @@ -942,17 +968,16 @@ def handle_start_sub(gateway: BaseGateway, channelid: int, data: bytes) -> None: def should_use_trio_via(spec: Any) -> bool: - """Trio path for ``via=`` sub-gateways (popen, foreign python, or ssh). + """Trio path for ``via=`` sub-gateways (popen, python, ssh, vagrant). The master spawns the sub-worker from a ``GATEWAY_START_SUB`` request and - relays its Message protocol. Socket subs go through ``installvia`` - instead; vagrant stays on the legacy path. + relays its Message protocol. Socket subs go through ``installvia`` instead. """ if not trio_host_enabled(): return False if not getattr(spec, "via", None): return False - if getattr(spec, "socket", None) or getattr(spec, "vagrant_ssh", None): + if getattr(spec, "socket", None): return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") @@ -969,7 +994,8 @@ def makegateway_via_trio(group: Any, spec: Any) -> Gateway: channel = master.newchannel() request = _provision.spawn_request(spec) master._send(Message.GATEWAY_START_SUB, channel.id, dumps_internal(request)) - remoteaddress = f"{spec.ssh}[via {spec.via}]" if spec.ssh else None + remote = spec.ssh or spec.vagrant_ssh + remoteaddress = f"{remote}[via {spec.via}]" if remote else None async def _create_and_attach() -> Gateway: io = ChannelByteIO(channel) diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 3a9dc602..29b8bd6a 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -158,6 +158,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: gw = _trio_host.makegateway_popen_trio(self, spec) elif _trio_host.should_use_trio_ssh(spec): gw = _trio_host.makegateway_ssh_trio(self, spec) + elif _trio_host.should_use_trio_vagrant(spec): + gw = _trio_host.makegateway_vagrant_trio(self, spec) elif _trio_host.should_use_trio_socket(spec): gw = _trio_host.makegateway_socket_trio(self, spec) elif _trio_host.should_use_trio_via(spec): diff --git a/testing/test_provision.py b/testing/test_provision.py index 8df8775a..84e660e5 100644 --- a/testing/test_provision.py +++ b/testing/test_provision.py @@ -38,3 +38,39 @@ def test_ssh_remote_command_dev_ships_wheel() -> None: assert "mktemp -d" in command assert f"head -c {len(preamble)}" in command assert preamble[:2] == b"PK" # a wheel is a zip archive + + +def test_vagrant_ssh_argv() -> None: + argv = _provision.vagrant_ssh_argv("default", None, "run-worker") + assert argv == ["vagrant", "ssh", "default", "--", "-C", "run-worker"] + argv = _provision.vagrant_ssh_argv("default", "/tmp/cfg", "run-worker") + assert argv == [ + "vagrant", + "ssh", + "default", + "--", + "-C", + "-F", + "/tmp/cfg", + "run-worker", + ] + + +def test_sub_spawn_argv_plain_popen() -> None: + import sys + + argv, preamble = _provision.sub_spawn_argv({"config": "{}"}) + assert argv == [sys.executable, "-u", "-m", "execnet._trio_worker", "{}"] + assert preamble == b"" + + +def test_sub_spawn_argv_vagrant_released() -> None: + request = { + "config": "{}", + "vagrant_ssh": "default", + "requirement": "execnet==9.9.9", + } + argv, preamble = _provision.sub_spawn_argv(request) + assert argv[:5] == ["vagrant", "ssh", "default", "--", "-C"] + assert "execnet==9.9.9" in argv[-1] + assert preamble == b"" From 3d1d31efb6be1fc56df527d2e4cae8bd4e67e50e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 09:48:52 +0200 Subject: [PATCH 17/59] refactor!: remove the legacy bootstrap stack and EXECNET_TRIO_HOST hatch The Trio host is now the only IO path. Delete gateway_io.py (ProxyIO, Popen2IOMaster, source bootstrap lines) and gateway_bootstrap.py, the Popen2IO sync pipe IO, the thread receiver, and WorkerGateway.serve(); makegateway dispatches directly on the spec. shell_split_path moves to _provision, and HostNotFound moves to gateway_base and now subclasses ConnectionError so generic OSError handling catches unreachable hosts. Co-Authored-By: Claude Fable 5 --- doc/implnotes.rst | 31 +--- src/execnet/__init__.py | 2 +- src/execnet/_provision.py | 15 +- src/execnet/_trio_host.py | 102 +------------ src/execnet/_trio_worker.py | 2 +- src/execnet/gateway.py | 23 +-- src/execnet/gateway_base.py | 190 +++-------------------- src/execnet/gateway_bootstrap.py | 74 --------- src/execnet/gateway_io.py | 255 ------------------------------- src/execnet/multi.py | 28 +--- testing/test_basics.py | 148 ++++-------------- testing/test_gateway.py | 26 +--- testing/test_ssh_local.py | 8 +- testing/test_xspec.py | 26 ++-- 14 files changed, 111 insertions(+), 819 deletions(-) delete mode 100644 src/execnet/gateway_bootstrap.py delete mode 100644 src/execnet/gateway_io.py diff --git a/doc/implnotes.rst b/doc/implnotes.rst index ba1c8408..eed92232 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -64,26 +64,11 @@ listener spawning ``python -m execnet._trio_worker --socket-fd`` subprocess workers). Infrastructure that used to be driven by ``remote_exec``-ing source is now expressed as native protocol messages handled on the target's Trio host: ``GATEWAY_START_SOCKET`` (installvia -> bind a one-shot listener, -reply with its address) and ``GATEWAY_START_POPEN`` (``via`` -> spawn a popen -sub-worker and relay its protocol over the request channel). - -Disable with ``EXECNET_TRIO_HOST=0``. ssh/foreign-python ``via`` sub-gateways -and greenlet execmodels still use the legacy thread receiver and sync -``Popen`` path. - -Legacy thread model -------------------- - -After bootstrapping, ``BaseGateway`` opens a receiver thread which -accepts encoded messages and triggers actions to interpret them. -Sending of channel data items happens directly through -write operations to InputOutput objects so there is no -separate send thread. - -Code execution messages are scheduled on a WorkerPool. -On the worker, ``serve()`` integrates the main thread as the -primary executor when using the ``thread`` / ``main_thread_only`` -models. - -The receiver thread terminates if the remote side sends -a gateway termination message or if the IO-connection drops. +reply with its address) and ``GATEWAY_START_SUB`` (``via`` -> spawn a +sub-worker — popen, foreign python, ssh, or vagrant — and relay its protocol +over the request channel). + +The Trio host is the only IO path; the legacy thread receiver and the +source-shipping bootstrap have been removed. The receiver task terminates +when the remote side sends a gateway termination message or the +IO-connection drops. diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 1edf3254..cb91cb66 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -12,6 +12,7 @@ from .gateway_base import Channel from .gateway_base import DataFormatError from .gateway_base import DumpError +from .gateway_base import HostNotFound from .gateway_base import LoadError from .gateway_base import RemoteError from .gateway_base import TimeoutError @@ -19,7 +20,6 @@ from .gateway_base import dumps from .gateway_base import load from .gateway_base import loads -from .gateway_bootstrap import HostNotFound from .multi import Group from .multi import MultiChannel from .multi import default_group diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index c55fb431..b1c16335 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -36,6 +36,17 @@ def uv_available() -> bool: return shutil.which("uv") is not None +def shell_split_path(path: str) -> list[str]: + """Split a ``python=`` value into argv tokens with shell lexing. + + Takes care to handle Windows' ``\\`` correctly. + """ + if sys.platform.startswith("win"): + # replace \\ by / otherwise shlex will strip them out + path = path.replace("\\", "/") + return shlex.split(path) + + @cache def target_has_execnet(python: str) -> bool: """Whether interpreter ``python`` can already import execnet + trio. @@ -43,8 +54,6 @@ def target_has_execnet(python: str) -> bool: When true the worker can be launched directly on that interpreter (preserving ``sys.executable``); otherwise it must be uv-provisioned. """ - from .gateway_io import shell_split_path - argv = [*shell_split_path(python), "-c", "import execnet, trio"] try: completed = subprocess.run(argv, capture_output=True, timeout=30, check=False) @@ -334,8 +343,6 @@ def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: the remote uv command (streaming a shipped wheel as the preamble for dev versions). """ - from .gateway_io import shell_split_path - config = request["config"] assert isinstance(config, str) python = request.get("python") diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 2a612996..59f6b47f 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -10,7 +10,6 @@ import itertools import json import math -import os import queue import subprocess import sys @@ -40,37 +39,6 @@ T = TypeVar("T") _CLOSE_WRITE = object() -_ENABLED_ENV = "EXECNET_TRIO_HOST" - - -def trio_host_enabled() -> bool: - """Return whether the Trio IO path should be used when applicable.""" - value = os.environ.get(_ENABLED_ENV, "1").strip().lower() - return value not in ("0", "false", "no", "off") - - -def should_use_trio_popen(spec: Any) -> bool: - """Trio path for local popen. - - Same-interpreter popen launches the worker module directly; a foreign - interpreter (``python=``) is provisioned via ``uv`` and only taken when - ``uv`` is available (otherwise the legacy source-copy path handles it). - """ - if not trio_host_enabled(): - return False - if not getattr(spec, "popen", False): - return False - if getattr(spec, "via", None): - return False - execmodel = getattr(spec, "execmodel", None) - if execmodel not in (None, "thread", "main_thread_only"): - return False - if getattr(spec, "python", None): - from . import _provision - - # Direct launch if the interpreter already has execnet; else uv-provision. - return _provision.target_has_execnet(spec.python) or _provision.uv_available() - return True class AsyncByteIO(Protocol): @@ -467,7 +435,7 @@ async def _finish(self) -> None: self._wake_writer() gateway._trace("[trio-receiver] finishing channels") gateway._channelfactory._finished_receiving() - # Unblock WorkerGateway.serve()/join before heavy exec-pool shutdown + # Unblock the worker's join() before heavy exec-pool shutdown # so the primary thread is not waiting on _done while terminate waits # on the primary thread draining work. self._done.set() @@ -593,9 +561,7 @@ def popen_module_args(spec: Any) -> list[str]: from . import _provision if getattr(spec, "python", None): - from .gateway_io import shell_split_path - - interpreter = shell_split_path(spec.python) + interpreter = _provision.shell_split_path(spec.python) else: interpreter = [sys.executable] @@ -648,7 +614,7 @@ def _open_trio_gateway( """ import execnet - from .gateway_bootstrap import HostNotFound + from .gateway_base import HostNotFound host: TrioHost = group._ensure_trio_host() @@ -677,7 +643,7 @@ async def _create_and_attach() -> Gateway: await process.wait() raise - gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, async_io, process=process) gw._attach_trio_session(session) gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) @@ -725,18 +691,6 @@ def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: ) -def should_use_trio_vagrant(spec: Any) -> bool: - """Trio path for ``vagrant_ssh=`` gateways (uv-provisioned worker).""" - if not trio_host_enabled(): - return False - if not getattr(spec, "vagrant_ssh", None): - return False - if getattr(spec, "via", None): - return False - execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") - - def makegateway_vagrant_trio(group: Any, spec: Any) -> Gateway: """Create a ``vagrant ssh``-wrapped Gateway on the Trio IO path.""" from . import _provision @@ -751,22 +705,6 @@ def makegateway_vagrant_trio(group: Any, spec: Any) -> Gateway: ) -def should_use_trio_ssh(spec: Any) -> bool: - """Trio path for ssh gateways (worker provisioned on the remote via uv). - - ``ssh=…//via=…`` is not a direct ssh connection but a sub-gateway spawned - by the master; that goes through the via path instead. - """ - if not trio_host_enabled(): - return False - if not getattr(spec, "ssh", None): - return False - if getattr(spec, "via", None): - return False - execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") - - def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: """Connect a Trio TCP stream to a running ``execnet-socketserver``. @@ -776,7 +714,7 @@ def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: """ import execnet - from .gateway_bootstrap import HostNotFound + from .gateway_base import HostNotFound host: TrioHost = group._ensure_trio_host() if getattr(spec, "installvia", None): @@ -804,7 +742,7 @@ async def _create_and_attach() -> Gateway: await stream.aclose() raise - gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, io) gw._attach_trio_session(session) gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) @@ -813,16 +751,6 @@ async def _create_and_attach() -> Gateway: return host.call(_create_and_attach) -def should_use_trio_socket(spec: Any) -> bool: - """Trio path for ``socket=host:port`` gateways, including ``installvia``.""" - if not trio_host_enabled(): - return False - if not getattr(spec, "socket", None): - return False - execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") - - _socket_worker_counter = itertools.count() @@ -967,22 +895,6 @@ def handle_start_sub(gateway: BaseGateway, channelid: int, data: bytes) -> None: host.start_soon(_start_sub_and_relay, gateway, channelid, request) -def should_use_trio_via(spec: Any) -> bool: - """Trio path for ``via=`` sub-gateways (popen, python, ssh, vagrant). - - The master spawns the sub-worker from a ``GATEWAY_START_SUB`` request and - relays its Message protocol. Socket subs go through ``installvia`` instead. - """ - if not trio_host_enabled(): - return False - if not getattr(spec, "via", None): - return False - if getattr(spec, "socket", None): - return False - execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") - - def makegateway_via_trio(group: Any, spec: Any) -> Gateway: """Create a ``via`` gateway: a sub-worker spawned by and relayed through the master.""" import execnet @@ -1002,7 +914,7 @@ async def _create_and_attach() -> Gateway: ack = await io.read_exact(1) if ack != b"1": raise EOFError(f"bad via handshake: {ack!r}") - gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, io) gw._attach_trio_session(session) gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 5fc137fb..9e1f1be7 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -215,7 +215,7 @@ def _build_worker_gateway( main_thread_only = model.backend == "main_thread_only" trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) # Duck-type as WorkerPool for STATUS / _terminate_execution. - gateway._execpool = trio_exec # type: ignore[assignment] + gateway._execpool = trio_exec gateway._trio_exec = trio_exec gateway._executetask_complete = None if main_thread_only: diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 6e336d6f..593bc6fe 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -26,21 +26,14 @@ class Gateway(gateway_base.BaseGateway): _group: Group - def __init__( - self, - io: IO, - spec: XSpec, - *, - trio_session: object | None = None, - defer_receive: bool = False, - ) -> None: - """:private:""" + def __init__(self, io: IO, spec: XSpec) -> None: + """:private: + + The Trio session doing the Message IO is attached separately via + ``_attach_trio_session`` once the connection is established. + """ super().__init__(io=io, id=spec.id, _startcount=1) self.spec = spec - if trio_session is not None: - self._attach_trio_session(trio_session) - elif not defer_receive: - self._initreceive() @property def remoteaddress(self) -> str: @@ -102,9 +95,7 @@ def _rinfo(self, update: bool = False) -> RInfo: def hasreceiver(self) -> bool: """Whether gateway is able to receive data.""" session = self._trio_session - if session is not None: - return bool(session.is_alive()) - return self._receivepool.active_count() > 0 + return session is not None and bool(session.is_alive()) def remote_status(self) -> RemoteStatus: """Obtain information about the remote execution status.""" diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 2736be96..0260b416 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1,4 +1,4 @@ -"""Base execnet gateway code send to the other side for bootstrapping. +"""Core gateway, channel and serialization code shared by coordinator and worker. :copyright: 2004-2015 :authors: @@ -389,48 +389,6 @@ def trace(*msg: object) -> None: notrace = trace = lambda *msg: None -class Popen2IO: - error = (IOError, OSError, EOFError) - - def __init__(self, outfile, infile, execmodel: ExecModel) -> None: - # we need raw byte streams - self.outfile, self.infile = outfile, infile - if sys.platform == "win32": - import msvcrt - - try: - msvcrt.setmode(infile.fileno(), os.O_BINARY) - msvcrt.setmode(outfile.fileno(), os.O_BINARY) - except (AttributeError, OSError): - pass - self._read = getattr(infile, "buffer", infile).read - self._write = getattr(outfile, "buffer", outfile).write - self.execmodel = execmodel - - def read(self, numbytes: int) -> bytes: - """Read exactly 'numbytes' bytes from the pipe.""" - # a file in non-blocking mode may return less bytes, so we loop - buf = b"" - while numbytes > len(buf): - data = self._read(numbytes - len(buf)) - if not data: - raise EOFError("expected %d bytes, got %d" % (numbytes, len(buf))) - buf += data - return buf - - def write(self, data: bytes) -> None: - """Write out all data bytes.""" - assert isinstance(data, bytes) - self._write(data) - self.outfile.flush() - - def close_read(self) -> None: - self.infile.close() - - def close_write(self) -> None: - self.outfile.close() - - class Message: """Encapsulates Messages and their wire protocol.""" @@ -572,7 +530,11 @@ def _gateway_start_sub(message: Message, gateway: BaseGateway) -> None: class GatewayReceivedTerminate(Exception): - """Receiverthread got termination message.""" + """Receiver got a gateway termination message.""" + + +class HostNotFound(ConnectionError): + """The remote side of a gateway could not be reached.""" def geterrortext( @@ -1043,6 +1005,8 @@ class BaseGateway: _sysex = sysex id = "" _trio_session: Any = None + # Set by the receiver on EOF without a prior termination message. + _error: BaseException | None = None def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel @@ -1054,7 +1018,6 @@ def __init__(self, io: IO, id, _startcount: int = 2) -> None: # globals may be NONE at process-termination self.__trace = trace self._geterrortext = geterrortext - self._receivepool = WorkerPool(self.execmodel) self._trio_session = None def _trace(self, *msg: object) -> None: @@ -1064,43 +1027,6 @@ def _attach_trio_session(self, session: Any) -> None: """Attach a Trio ProtocolSession for Message IO (no receiver thread).""" self._trio_session = session - def _initreceive(self) -> None: - if self._trio_session is not None: - return - self._receivepool.spawn(self._thread_receiver) - - def _thread_receiver(self) -> None: - def log(*msg: object) -> None: - self._trace("[receiver-thread]", *msg) - - log("RECEIVERTHREAD: starting to run") - io = self._io - try: - while 1: - msg = Message.from_io(io) - log("received", msg) - with self._receivelock: - msg.received(self) - del msg - except (KeyboardInterrupt, GatewayReceivedTerminate): - pass - except EOFError as exc: - log("EOF without prior gateway termination message") - self._error = exc - except Exception as exc: - log(self._geterrortext(exc)) - log("finishing receiving thread") - # wake up and terminate any execution waiting to receive - self._channelfactory._finished_receiving() - log("terminating execution") - self._terminate_execution() - log("closing read") - self._io.close_read() - log("closing write") - self._io.close_write() - log("terminating our receive pseudo pool") - self._receivepool.trigger_shutdown() - def _terminate_execution(self) -> None: pass @@ -1136,41 +1062,25 @@ def newchannel(self) -> Channel: return self._channelfactory.new() def join(self, timeout: float | None = None) -> None: - """Wait for receiverthread to terminate.""" - self._trace("waiting for receiver thread to finish") + """Wait for the receiver (Trio session) to terminate.""" + self._trace("waiting for receiver to finish") session = self._trio_session if session is not None: session.wait_done(timeout) - return - self._receivepool.waitall(timeout) class WorkerGateway(BaseGateway): _trio_exec: Any = None + # The exec pool (a TrioWorkerExec duck-typed as WorkerPool for STATUS). + _execpool: Any = None + _executetask_complete: Event | None = None def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: - trio_exec = getattr(self, "_trio_exec", None) - if trio_exec is not None: - trio_exec.schedule(channel, sourcetask) + trio_exec = self._trio_exec + if trio_exec is None: + channel.close("execution disallowed") return - - if self._execpool.execmodel.backend == "main_thread_only": - assert self._executetask_complete is not None - # It's necessary to wait for a short time in order to ensure - # that we do not report a false-positive deadlock error, since - # channel close does not elicit a response that would provide - # a guarantee to remote_exec callers that the previous task - # has released the main thread. If the timeout expires then it - # should be practically impossible to report a false-positive. - if not self._executetask_complete.wait(timeout=1): - channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) - return - # It's only safe to clear here because the above wait proves - # that there is not a previous task about to set it again. - self._executetask_complete.clear() - - sourcetask_ = loads_internal(sourcetask) - self._execpool.spawn(self.executetask, (channel, sourcetask_)) + trio_exec.schedule(channel, sourcetask) def _terminate_execution(self) -> None: # called from receiverthread @@ -1192,31 +1102,6 @@ def _terminate_execution(self) -> None: ) os._exit(1) - def serve(self) -> None: - def trace(msg: str) -> None: - self._trace("[serve] " + msg) - - hasprimary = self.execmodel.backend in ("thread", "main_thread_only") - self._execpool = WorkerPool(self.execmodel, hasprimary=hasprimary) - self._executetask_complete = None - if self.execmodel.backend == "main_thread_only": - self._executetask_complete = self.execmodel.Event() - # Initialize state to indicate that there is no previous task - # executing so that we don't need a separate flag to track this. - self._executetask_complete.set() - trace("spawning receiver thread") - self._initreceive() - try: - if hasprimary: - # this will return when we are in shutdown - trace("integrating as primary thread") - self._execpool.integrate_as_primary_thread() - trace("joining receiver thread") - self.join() - except KeyboardInterrupt: - # in the worker we can't really do anything sensible - trace("swallowing keyboardinterrupt, serve finished") - def executetask( self, item: tuple[Channel, tuple[str, str | None, str | None, dict[str, object]]], @@ -1701,44 +1586,3 @@ def save_frozenset(self, s: frozenset[object]) -> None: def save_Channel(self, channel: Channel) -> None: self._write(opcode.CHANNEL) self._write_int4(channel.id) - - -def init_popen_io(execmodel: ExecModel) -> Popen2IO: - if not hasattr(os, "dup"): # jython - io = Popen2IO(sys.stdout, sys.stdin, execmodel) - import tempfile - - sys.stdin = tempfile.TemporaryFile("r") - sys.stdout = tempfile.TemporaryFile("w") - else: - try: - devnull = os.devnull - except AttributeError: - devnull = "NUL" if os.name == "nt" else "/dev/null" - # stdin - stdin = execmodel.fdopen(os.dup(0), "r", 1) - fd = os.open(devnull, os.O_RDONLY) - os.dup2(fd, 0) - os.close(fd) - - # stdout - stdout = execmodel.fdopen(os.dup(1), "w", 1) - fd = os.open(devnull, os.O_WRONLY) - os.dup2(fd, 1) - - # stderr for win32 - if os.name == "nt": - sys.stderr = execmodel.fdopen(os.dup(2), "w", 1) - os.dup2(fd, 2) - os.close(fd) - io = Popen2IO(stdout, stdin, execmodel) - # Use closefd=False since 0 and 1 are shared with - # sys.__stdin__ and sys.__stdout__. - sys.stdin = execmodel.fdopen(0, "r", 1, closefd=False) - sys.stdout = execmodel.fdopen(1, "w", 1, closefd=False) - return io - - -def serve(io: IO, id) -> None: - trace(f"creating workergateway on {io!r}") - WorkerGateway(io=io, id=id, _startcount=2).serve() diff --git a/src/execnet/gateway_bootstrap.py b/src/execnet/gateway_bootstrap.py deleted file mode 100644 index 5707dff4..00000000 --- a/src/execnet/gateway_bootstrap.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Code to initialize the remote side of a gateway once the IO is created.""" - -from __future__ import annotations - -import inspect -import os - -import execnet - -from . import gateway_base -from .gateway_base import IO -from .xspec import XSpec - -importdir = os.path.dirname(os.path.dirname(execnet.__file__)) - - -class HostNotFound(Exception): - pass - - -def bootstrap_import(io: IO, spec: XSpec) -> None: - # Only insert the importdir into the path if we must. This prevents - # bugs where backports expect to be shadowed by the standard library on - # newer versions of python but would instead shadow the standard library. - sendexec( - io, - "import sys", - "if %r not in sys.path:" % importdir, - " sys.path.insert(0, %r)" % importdir, - "from execnet.gateway_base import serve, init_popen_io, get_execmodel", - "sys.stdout.write('1')", - "sys.stdout.flush()", - "execmodel = get_execmodel(%r)" % spec.execmodel, - "serve(init_popen_io(execmodel), id='%s-worker')" % spec.id, - ) - s = io.read(1) - assert s == b"1", repr(s) - - -def bootstrap_exec(io: IO, spec: XSpec) -> None: - try: - sendexec( - io, - inspect.getsource(gateway_base), - "execmodel = get_execmodel(%r)" % spec.execmodel, - "io = init_popen_io(execmodel)", - "io.write('1'.encode('ascii'))", - "serve(io, id='%s-worker')" % spec.id, - ) - s = io.read(1) - assert s == b"1" - except EOFError: - ret = io.wait() - if ret == 255 and hasattr(io, "remoteaddress"): - raise HostNotFound(io.remoteaddress) from None - - -def sendexec(io: IO, *sources: str) -> None: - source = "\n".join(sources) - io.write((repr(source) + "\n").encode("utf-8")) - - -def bootstrap(io: IO, spec: XSpec) -> execnet.Gateway: - if spec.popen: - if spec.via or spec.python: - bootstrap_exec(io, spec) - else: - bootstrap_import(io, spec) - elif spec.ssh or spec.vagrant_ssh: - bootstrap_exec(io, spec) - else: - raise ValueError("unknown gateway type, can't bootstrap") - gw = execnet.Gateway(io, spec) - return gw diff --git a/src/execnet/gateway_io.py b/src/execnet/gateway_io.py deleted file mode 100644 index 21285ab4..00000000 --- a/src/execnet/gateway_io.py +++ /dev/null @@ -1,255 +0,0 @@ -"""execnet IO initialization code. - -Creates IO instances used for gateway IO. -""" - -from __future__ import annotations - -import shlex -import sys -from typing import TYPE_CHECKING -from typing import cast - -if TYPE_CHECKING: - from execnet.gateway_base import Channel - from execnet.gateway_base import ExecModel - from execnet.xspec import XSpec - -try: - from execnet.gateway_base import Message - from execnet.gateway_base import Popen2IO -except ImportError: - from __main__ import Message # type: ignore[no-redef] - from __main__ import Popen2IO # type: ignore[no-redef] - -from functools import partial - - -class Popen2IOMaster(Popen2IO): - # Set externally, for some specs only. - remoteaddress: str - - def __init__(self, args, execmodel: ExecModel) -> None: - PIPE = execmodel.subprocess.PIPE - self.popen = p = execmodel.subprocess.Popen(args, stdout=PIPE, stdin=PIPE) - super().__init__(p.stdin, p.stdout, execmodel=execmodel) - - def wait(self) -> int | None: - try: - return self.popen.wait() # type: ignore[no-any-return] - except OSError: - return None - - def kill(self) -> None: - try: - self.popen.kill() - except OSError as e: - sys.stderr.write("ERROR killing: %s\n" % e) - sys.stderr.flush() - - -popen_bootstrapline = "import sys;exec(eval(sys.stdin.readline()))" - - -def shell_split_path(path: str) -> list[str]: - """ - Use shell lexer to split the given path into a list of components, - taking care to handle Windows' '\' correctly. - """ - if sys.platform.startswith("win"): - # replace \\ by / otherwise shlex will strip them out - path = path.replace("\\", "/") - return shlex.split(path) - - -def popen_args(spec: XSpec) -> list[str]: - args = shell_split_path(spec.python) if spec.python else [sys.executable] - args.append("-u") - if spec.dont_write_bytecode: - args.append("-B") - args.extend(["-c", popen_bootstrapline]) - return args - - -def ssh_args(spec: XSpec) -> list[str]: - # NOTE: If changing this, you need to sync those changes to vagrant_args - # as well, or, take some time to further refactor the commonalities of - # ssh_args and vagrant_args. - remotepython = spec.python or "python" - args = ["ssh", "-C"] - if spec.ssh_config is not None: - args.extend(["-F", str(spec.ssh_config)]) - - assert spec.ssh is not None - args.extend(spec.ssh.split()) - remotecmd = f'{remotepython} -c "{popen_bootstrapline}"' - args.append(remotecmd) - return args - - -def vagrant_ssh_args(spec: XSpec) -> list[str]: - # This is the vagrant-wrapped version of SSH. Unfortunately the - # command lines are incompatible to just channel through ssh_args - # due to ordering/templating issues. - # NOTE: This should be kept in sync with the ssh_args behaviour. - # spec.vagrant is identical to spec.ssh in that they both carry - # the remote host "address". - assert spec.vagrant_ssh is not None - remotepython = spec.python or "python" - args = ["vagrant", "ssh", spec.vagrant_ssh, "--", "-C"] - if spec.ssh_config is not None: - args.extend(["-F", str(spec.ssh_config)]) - remotecmd = f'{remotepython} -c "{popen_bootstrapline}"' - args.extend([remotecmd]) - return args - - -def create_io(spec: XSpec, execmodel: ExecModel) -> Popen2IOMaster: - if spec.popen: - args = popen_args(spec) - return Popen2IOMaster(args, execmodel) - if spec.ssh: - args = ssh_args(spec) - io = Popen2IOMaster(args, execmodel) - io.remoteaddress = spec.ssh - return io - if spec.vagrant_ssh: - args = vagrant_ssh_args(spec) - io = Popen2IOMaster(args, execmodel) - io.remoteaddress = spec.vagrant_ssh - return io - assert False - - -# -# Proxy Gateway handling code -# -# master: proxy initiator -# forwarder: forwards between master and sub -# sub: sub process that is proxied to the initiator - -RIO_KILL = 1 -RIO_WAIT = 2 -RIO_REMOTEADDRESS = 3 -RIO_CLOSE_WRITE = 4 - - -class ProxyIO: - """A Proxy IO object allows to instantiate a Gateway - through another "via" gateway. - - A master:ProxyIO object provides an IO object effectively connected to the - sub via the forwarder. To achieve this, master:ProxyIO interacts with - forwarder:serve_proxy_io() which itself instantiates and interacts with the - sub. - """ - - def __init__(self, proxy_channel: Channel, execmodel: ExecModel) -> None: - # after exchanging the control channel we use proxy_channel - # for messaging IO - self.controlchan = proxy_channel.gateway.newchannel() - proxy_channel.send(self.controlchan) - self.iochan = proxy_channel - self.iochan_file = self.iochan.makefile("r") - self.execmodel = execmodel - - def read(self, nbytes: int) -> bytes: - # TODO(typing): The IO protocol requires bytes here but ChannelFileRead - # returns str. - return self.iochan_file.read(nbytes) # type: ignore[return-value] - - def write(self, data: bytes) -> None: - self.iochan.send(data) - - def _controll(self, event: int) -> object: - self.controlchan.send(event) - return self.controlchan.receive() - - def close_write(self) -> None: - self._controll(RIO_CLOSE_WRITE) - - def close_read(self) -> None: - raise NotImplementedError() - - def kill(self) -> None: - self._controll(RIO_KILL) - - def wait(self) -> int | None: - response = self._controll(RIO_WAIT) - assert response is None or isinstance(response, int) - return response - - @property - def remoteaddress(self) -> str: - response = self._controll(RIO_REMOTEADDRESS) - assert isinstance(response, str) - return response - - def __repr__(self) -> str: - return f"" - - -class PseudoSpec: - def __init__(self, vars) -> None: - self.__dict__.update(vars) - - def __getattr__(self, name: str) -> None: - return None - - -def serve_proxy_io(proxy_channelX: Channel) -> None: - execmodel = proxy_channelX.gateway.execmodel - log = partial( - proxy_channelX.gateway._trace, "serve_proxy_io:%s" % proxy_channelX.id - ) - spec = cast("XSpec", PseudoSpec(proxy_channelX.receive())) - # create sub IO object which we will proxy back to our proxy initiator - sub_io = create_io(spec, execmodel) - control_chan = cast("Channel", proxy_channelX.receive()) - log("got control chan", control_chan) - - # read data from master, forward it to the sub - # XXX writing might block, thus blocking the receiver thread - def forward_to_sub(data: bytes) -> None: - log("forward data to sub, size %s" % len(data)) - sub_io.write(data) - - proxy_channelX.setcallback(forward_to_sub) - - def control(data: int) -> None: - if data == RIO_WAIT: - control_chan.send(sub_io.wait()) - elif data == RIO_KILL: - sub_io.kill() - control_chan.send(None) - elif data == RIO_REMOTEADDRESS: - control_chan.send(sub_io.remoteaddress) - elif data == RIO_CLOSE_WRITE: - sub_io.close_write() - control_chan.send(None) - - control_chan.setcallback(control) - - # write data to the master coming from the sub - forward_to_master_file = proxy_channelX.makefile("w") - - # read bootstrap byte from sub, send it on to master - log("reading bootstrap byte from sub", spec.id) - initial = sub_io.read(1) - assert initial == b"1", initial - log("forwarding bootstrap byte from sub", spec.id) - forward_to_master_file.write(initial) - - # enter message forwarding loop - while True: - try: - message = Message.from_io(sub_io) - except EOFError: - log("EOF from sub, terminating proxying loop", spec.id) - break - message.to_io(forward_to_master_file) - # proxy_channelX will be closed from remote_exec's finalization code - - -if __name__ == "__channelexec__": - serve_proxy_io(channel) # type: ignore[name-defined] # noqa:F821 diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 29b8bd6a..c4892995 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -20,8 +20,6 @@ from typing import TypeAlias from typing import overload -from . import gateway_bootstrap -from . import gateway_io from .gateway_base import Channel from .gateway_base import ExecModel from .gateway_base import WorkerPool @@ -154,26 +152,16 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: spec.execmodel = self.remote_execmodel.backend from . import _trio_host - if _trio_host.should_use_trio_popen(spec): - gw = _trio_host.makegateway_popen_trio(self, spec) - elif _trio_host.should_use_trio_ssh(spec): - gw = _trio_host.makegateway_ssh_trio(self, spec) - elif _trio_host.should_use_trio_vagrant(spec): - gw = _trio_host.makegateway_vagrant_trio(self, spec) - elif _trio_host.should_use_trio_socket(spec): + if spec.socket: gw = _trio_host.makegateway_socket_trio(self, spec) - elif _trio_host.should_use_trio_via(spec): - gw = _trio_host.makegateway_via_trio(self, spec) elif spec.via: - assert not spec.socket - master = self[spec.via] - proxy_channel = master.remote_exec(gateway_io) - proxy_channel.send(vars(spec)) - proxy_io_master = gateway_io.ProxyIO(proxy_channel, self.execmodel) - gw = gateway_bootstrap.bootstrap(proxy_io_master, spec) - elif spec.popen or spec.ssh or spec.vagrant_ssh: - io = gateway_io.create_io(spec, execmodel=self.execmodel) - gw = gateway_bootstrap.bootstrap(io, spec) + gw = _trio_host.makegateway_via_trio(self, spec) + elif spec.ssh: + gw = _trio_host.makegateway_ssh_trio(self, spec) + elif spec.vagrant_ssh: + gw = _trio_host.makegateway_vagrant_trio(self, spec) + elif spec.popen: + gw = _trio_host.makegateway_popen_trio(self, spec) else: raise ValueError(f"no gateway type found for {spec._spec!r}") gw.spec = spec diff --git a/testing/test_basics.py b/testing/test_basics.py index 1756ec34..d4ffd5d8 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -17,11 +17,9 @@ import execnet from execnet import gateway from execnet import gateway_base -from execnet import gateway_io from execnet.gateway_base import ChannelFactory from execnet.gateway_base import ExecModel from execnet.gateway_base import Message -from execnet.gateway_base import Popen2IO skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), @@ -68,81 +66,29 @@ def test_errors_on_execnet() -> None: assert hasattr(execnet, "DataFormatError") -def test_subprocess_interaction(anypython: str) -> None: - line = gateway_io.popen_bootstrapline - compile(line, "xyz", "exec") - args = [str(anypython), "-c", line] - popen = subprocess.Popen( - args, - bufsize=0, - universal_newlines=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - - assert popen.stdin is not None - assert popen.stdout is not None - - def send(line: str) -> None: - assert popen.stdin is not None - popen.stdin.write(line) - popen.stdin.flush() - - def receive() -> str: - assert popen.stdout is not None - return popen.stdout.readline() - - try: - source = inspect.getsource(read_write_loop) + "read_write_loop()" - send(repr(source) + "\n") - s = receive() - assert s == "ok\n" - send("hello\n") - s = receive() - assert s == "received: hello\n" - send("world\n") - s = receive() - assert s == "received: world\n" - send("\n") # terminate loop - finally: - popen.stdin.close() - popen.stdout.close() - popen.wait() +IO_MESSAGE_EXTRA_SOURCE = """ +from io import BytesIO +class BufIO: + def __init__(self): + self.buf = BytesIO() -def read_write_loop() -> None: - sys.stdout.write("ok\n") - sys.stdout.flush() - while 1: - try: - line = sys.stdin.readline() - if not line.strip(): - break - sys.stdout.write("received: %s" % line) - sys.stdout.flush() - except (OSError, EOFError): - break + def write(self, data): + self.buf.write(data) + def read(self, numbytes): + data = self.buf.read(numbytes) + if len(data) < numbytes: + raise EOFError("expected %d bytes" % numbytes) + return data -IO_MESSAGE_EXTRA_SOURCE = """ -import sys -backend = sys.argv[1] -from io import BytesIO -import tempfile -temp_out = BytesIO() -temp_in = BytesIO() -io = Popen2IO(temp_out, temp_in, get_execmodel(backend)) for i, handler in enumerate(Message._types): print ("checking", i, handler) for data in "hello", "hello".encode('ascii'): + io = BufIO() msg1 = Message(i, i, dumps(data)) msg1.to_io(io) - x = io.outfile.getvalue() - io.outfile.truncate(0) - io.outfile.seek(0) - io.infile.seek(0) - io.infile.write(x) - io.infile.seek(0) + io.buf.seek(0) msg2 = Message.from_io(io) assert msg1.channelid == msg2.channelid, (msg1, msg2) assert msg1.data == msg2.data, (msg1.data, msg2.data) @@ -177,44 +123,12 @@ def checker(anypython: str, tmp_path: Path) -> Checker: return Checker(python=anypython, path=tmp_path) -def test_io_message(checker: Checker, execmodel: ExecModel) -> None: - out = checker.run_check( - inspect.getsource(gateway_base) + IO_MESSAGE_EXTRA_SOURCE, execmodel.backend - ) +def test_io_message(checker: Checker) -> None: + out = checker.run_check(inspect.getsource(gateway_base) + IO_MESSAGE_EXTRA_SOURCE) print(out.stdout) assert "all passed" in out.stdout -def test_popen_io(checker: Checker, execmodel: ExecModel) -> None: - out = checker.run_check( - inspect.getsource(gateway_base) - + f""" -io = init_popen_io(get_execmodel({execmodel.backend!r})) -io.write(b"hello") -s = io.read(1) -assert s == b"x" -""", - input="x", - ) - print(out.stderr) - assert "hello" in out.stdout - - -def test_popen_io_readloop(execmodel: ExecModel) -> None: - sio = BytesIO(b"test") - io = Popen2IO(sio, sio, execmodel) - real_read = io._read - - def newread(numbytes: int) -> bytes: - if numbytes > 1: - numbytes = numbytes - 1 - return real_read(numbytes) # type: ignore[no-any-return] - - io._read = newread - result = io.read(3) - assert result == b"tes" - - def test_rinfo_source(checker: Checker) -> None: out = checker.run_check( f""" @@ -252,19 +166,20 @@ class Arg(Exception): @pytest.mark.skipif("not hasattr(os, 'dup')") -def test_stdouterrin_setnull( - execmodel: ExecModel, capfd: pytest.CaptureFixture[str] -) -> None: - # Backup and restore stdin state, and rely on capfd to handle - # this for stdout and stderr. +def test_stdouterrin_setnull(capfd: pytest.CaptureFixture[str]) -> None: + # _prepare_protocol_fds dups the stdio fds for the Message protocol and + # points fd 0/1 at devnull; writes/reads on the original fds must go + # nowhere. Back up and restore the real fds around the call. + from execnet import _trio_worker + orig_stdin = sys.stdin - orig_stdin_fd = os.dup(0) + orig_stdout = sys.stdout + orig_fd0 = os.dup(0) + orig_fd1 = os.dup(1) try: - # The returned Popen2IO instance can be garbage collected - # prematurely since we don't hold a reference here, but we - # tolerate this because it is intended to leave behind a - # sane state afterwards. - gateway_base.init_popen_io(execmodel) + read_fd, write_fd = _trio_worker._prepare_protocol_fds() + os.close(read_fd) + os.close(write_fd) os.write(1, b"hello") os.read(0, 1) out, err = capfd.readouterr() @@ -272,8 +187,11 @@ def test_stdouterrin_setnull( assert not err finally: sys.stdin = orig_stdin - os.dup2(orig_stdin_fd, 0) - os.close(orig_stdin_fd) + sys.stdout = orig_stdout + os.dup2(orig_fd0, 0) + os.dup2(orig_fd1, 1) + os.close(orig_fd0) + os.close(orig_fd1) class PseudoChannel: diff --git a/testing/test_gateway.py b/testing/test_gateway.py index e123bbe2..c3c87e4a 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -16,7 +16,6 @@ import execnet from execnet import gateway_base -from execnet import gateway_io from execnet.gateway import Gateway TESTTIMEOUT = 10.0 # seconds @@ -381,23 +380,6 @@ def test_socket_gw_host_not_found(makegateway: Callable[[str], Gateway]) -> None class TestSshPopenGateway: gwtype = "ssh" - def test_sshconfig_config_parsing( - self, monkeypatch: pytest.MonkeyPatch, makegateway: Callable[[str], Gateway] - ) -> None: - # white-box test of the legacy Popen2IOMaster arg construction - monkeypatch.setenv("EXECNET_TRIO_HOST", "0") - l = [] - monkeypatch.setattr( - gateway_io, "Popen2IOMaster", lambda *args, **kwargs: l.append(args[0]) - ) - with pytest.raises(AttributeError): - makegateway("ssh=xyz//ssh_config=qwe") - - assert len(l) == 1 - popen_args = l[0] - i = popen_args.index("-F") - assert popen_args[i + 1] == "qwe" - def test_ssh_trio_args_include_config( self, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -549,9 +531,11 @@ def test_no_tracing_by_default(self): ], ) def test_popen_args(spec: str, expected_args: list[str]) -> None: - expected_args = [*expected_args, "-u", "-c", gateway_io.popen_bootstrapline] - args = gateway_io.popen_args(execnet.XSpec(spec)) - assert args == expected_args + from execnet import _trio_host + + args = _trio_host.popen_module_args(execnet.XSpec(spec + "//id=gw0")) + assert args[: len(expected_args)] == expected_args + assert args[len(expected_args) :][:3] == ["-u", "-m", "execnet._trio_worker"] def test_assert_main_thread_only( diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index c21135d1..086ce877 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -133,12 +133,8 @@ def ssh_config(ssh_server: SSHServerThread, tmp_path) -> str: return path -@pytest.mark.parametrize("trio_host", ["1", "0"], ids=["trio", "legacy"]) -def test_ssh_roundtrip( - ssh_config: str, monkeypatch: pytest.MonkeyPatch, trio_host: str -) -> None: - # trio=1 provisions the worker over ssh with uv; legacy=0 source-copies it. - monkeypatch.setenv("EXECNET_TRIO_HOST", trio_host) +def test_ssh_roundtrip(ssh_config: str) -> None: + # The worker is provisioned over ssh with uv. group = execnet.Group() try: gw = group.makegateway( diff --git a/testing/test_xspec.py b/testing/test_xspec.py index 5240d72d..f9681f55 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -11,10 +11,8 @@ import execnet from execnet import XSpec +from execnet import _provision from execnet.gateway import Gateway -from execnet.gateway_io import popen_args -from execnet.gateway_io import ssh_args -from execnet.gateway_io import vagrant_ssh_args skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), @@ -69,22 +67,20 @@ def test_execmodel(self) -> None: def test_ssh_options_and_config(self) -> None: spec = XSpec("ssh=-p 22100 user@host//python=python3") - spec.ssh_config = "/home/user/ssh_config" - assert ssh_args(spec)[:6] == ["ssh", "-C", "-F", spec.ssh_config, "-p", "22100"] + args = _provision.ssh_argv("-p 22100 user@host", "/home/user/ssh_config", "cmd") + assert args[:6] == ["ssh", "-C", "-F", "/home/user/ssh_config", "-p", "22100"] + assert spec.ssh is not None def test_vagrant_options(self) -> None: - spec = XSpec("vagrant_ssh=default//python=python3") - assert vagrant_ssh_args(spec)[:-1] == ["vagrant", "ssh", "default", "--", "-C"] + args = _provision.vagrant_ssh_argv("default", None, "cmd") + assert args[:-1] == ["vagrant", "ssh", "default", "--", "-C"] def test_popen_with_sudo_python(self) -> None: - spec = XSpec("popen//python=sudo python3") - assert popen_args(spec) == [ - "sudo", - "python3", - "-u", - "-c", - "import sys;exec(eval(sys.stdin.readline()))", - ] + from execnet import _trio_host + + spec = XSpec("popen//python=sudo python3//id=gw0") + args = _trio_host.popen_module_args(spec) + assert args[:5] == ["sudo", "python3", "-u", "-m", "execnet._trio_worker"] def test_env(self) -> None: xspec = XSpec("popen//env:NAME=value1") From 0a24039959a02ea96679fe863bc9b9437d0d7abf Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 11:11:22 +0200 Subject: [PATCH 18/59] refactor: unify transports on StapledStream with sans-IO frame decoding Phase B.1+B.2 of the Trio port: - add gateway_base.FrameDecoder, an incremental sans-IO decoder for the 9-byte-header Message framing (feed arbitrary chunks, complete messages come out; close() flags mid-frame EOF) - replace the hand-rolled AsyncByteIO wrappers (ProcessStreamsIO, FdStreamsIO, SocketStreamIO) with trio.StapledStream / plain trio.SocketStream behind a neutral ByteStream protocol (send_all/receive_some/send_eof/aclose) that a future anyio backend can satisfy structurally; the via tunnel keeps its interim bridge as ChannelByteStream - ProtocolSession's reader becomes the uniform receive_some+feed loop; exact reads survive only as the one-byte handshake ack - route worker exec requests through a single FIFO pump task: batched frame decoding removed the per-message awaits that had accidentally serialized main_thread_only exec admission, so admission now happens explicitly in message-arrival order instead of racing tasks - give pre-commit's mypy the trio dependency so trio types are real; drop now-redundant casts Co-Authored-By: Claude Fable 5 --- .pre-commit-config.yaml | 1 + src/execnet/_trio_host.py | 211 +++++++++++++----------------------- src/execnet/_trio_worker.py | 45 ++++---- src/execnet/gateway_base.py | 36 ++++++ testing/test_basics.py | 84 ++++++++++++++ 5 files changed, 223 insertions(+), 154 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 37ccd44d..f7a9f86c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,4 +31,5 @@ repos: - id: mypy additional_dependencies: - pytest + - trio - types-pywin32 diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 59f6b47f..1a8dd197 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -21,11 +21,11 @@ from typing import Any from typing import Protocol from typing import TypeVar -from typing import cast import trio from .gateway_base import ExecModel +from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message from .gateway_base import dumps_internal @@ -41,144 +41,95 @@ _CLOSE_WRITE = object() -class AsyncByteIO(Protocol): - async def read_exact(self, n: int) -> bytes: ... +RECEIVE_CHUNK = 65536 - async def write_all(self, data: bytes) -> None: ... - async def aclose_read(self) -> None: ... +class ByteStream(Protocol): + """Neutral bidirectional byte-stream protocol for gateway transports. - async def aclose_write(self) -> None: ... - - -async def read_exact_receive_stream(stream: trio.abc.ReceiveStream, n: int) -> bytes: - buf = bytearray() - while len(buf) < n: - chunk = await stream.receive_some(n - len(buf)) - if not chunk: - raise EOFError("expected %d bytes, got %d" % (n, len(buf))) - buf += chunk - return bytes(buf) - - -async def read_message(io: AsyncByteIO) -> Message: - header = await io.read_exact(9) - msgtype, channel, payload = Message.from_header(header) - data = await io.read_exact(payload) if payload else b"" - return Message.from_parts(msgtype, channel, data) - - -class ProcessStreamsIO: - """Async IO over a Trio Process stdin/stdout pair.""" - - def __init__(self, process: trio.Process) -> None: - assert process.stdin is not None - assert process.stdout is not None - self.process = process - self._stdin = process.stdin - self._stdout = process.stdout - - async def read_exact(self, n: int) -> bytes: - return await read_exact_receive_stream(self._stdout, n) - - async def write_all(self, data: bytes) -> None: - await self._stdin.send_all(data) - - async def aclose_read(self) -> None: - await self._stdout.aclose() - - async def aclose_write(self) -> None: - await self._stdin.aclose() - - -class FdStreamsIO: - """Async IO over OS file descriptors (worker stdio pipes).""" - - def __init__(self, read_fd: int, write_fd: int) -> None: - self._read = trio.lowlevel.FdStream(read_fd) - self._write = trio.lowlevel.FdStream(write_fd) - - async def read_exact(self, n: int) -> bytes: - return await read_exact_receive_stream(self._read, n) - - async def write_all(self, data: bytes) -> None: - await self._write.send_all(data) + ``trio.StapledStream`` (process/fd pipe pairs) and ``trio.SocketStream`` + satisfy this structurally; a future anyio backend's byte streams use the + same four names. ``send_eof`` signals write-EOF to the peer (half-close + for sockets; for pipe pairs trio falls back to closing the send half). + """ - async def aclose_read(self) -> None: - await self._read.aclose() + async def send_all(self, data: bytes) -> None: ... - async def aclose_write(self) -> None: - await self._write.aclose() + async def receive_some(self, max_bytes: int | None = None) -> bytes: ... + async def send_eof(self) -> None: ... -class SocketStreamIO: - """Async IO over a single bidirectional Trio stream (a socket). + async def aclose(self) -> None: ... - ``read_exact`` never over-reads (``receive_some(k)`` returns at most ``k`` - bytes), so no cross-call buffering is needed. ``aclose`` on a Trio stream is - idempotent, so close-read and close-write both just close the socket. - """ - def __init__(self, stream: trio.abc.Stream) -> None: - self._stream = stream +def staple_process_stream(process: trio.Process) -> ByteStream: + """One bidirectional stream over a Trio Process stdin/stdout pair.""" + assert process.stdin is not None + assert process.stdout is not None + return trio.StapledStream(process.stdin, process.stdout) - async def read_exact(self, n: int) -> bytes: - return await read_exact_receive_stream(self._stream, n) - async def write_all(self, data: bytes) -> None: - await self._stream.send_all(data) +def staple_fd_stream(read_fd: int, write_fd: int) -> ByteStream: + """One bidirectional stream over OS pipe fds (worker stdio pipes).""" + return trio.StapledStream( + trio.lowlevel.FdStream(write_fd), trio.lowlevel.FdStream(read_fd) + ) - async def aclose_read(self) -> None: - await self._stream.aclose() - async def aclose_write(self) -> None: - await self._stream.aclose() +async def read_handshake_ack(stream: ByteStream, what: str) -> None: + """Wait for the worker's single ``b"1"`` ready byte.""" + ack = await stream.receive_some(1) + if ack != b"1": + raise EOFError(f"bad {what} handshake: {ack!r}") _CHANNEL_EOF = object() -class ChannelByteIO: - """Async byte IO tunnelled over a sync execnet ``Channel``. +class ChannelByteStream: + """``ByteStream`` tunnelled over a sync execnet ``Channel``. INTERIM HACK: the ``via`` transport currently runs the sub-gateway protocol as raw bytes over a channel to the master, which double-frames it (sub frame -> CHANNEL_DATA -> master frame). This bridges the sync ``Channel`` to the - async protocol; it should be replaced by a proper relayed transport rather - than tunnelling bytes through the channel layer. + async protocol; it dissolves once low-level raw channels exist (Phase B.4). The channel callback (on the host loop) feeds an unbounded memory channel - that ``read_exact`` drains; writes ``channel.send`` raw frames. + that ``receive_some`` drains; writes ``channel.send`` raw bytes. """ def __init__(self, channel: Any) -> None: self._channel = channel self._send, self._recv = trio.open_memory_channel[Any](math.inf) self._buf = bytearray() + self._eof = False channel.setcallback(self._send.send_nowait, endmarker=_CHANNEL_EOF) - async def read_exact(self, n: int) -> bytes: - while len(self._buf) < n: + async def send_all(self, data: bytes) -> None: + self._channel.send(data) + + async def receive_some(self, max_bytes: int | None = None) -> bytes: + if not self._buf and not self._eof: data = await self._recv.receive() if data is _CHANNEL_EOF: - raise EOFError("channel closed") - assert isinstance(data, bytes) - self._buf += data - out = bytes(self._buf[:n]) - del self._buf[:n] + self._eof = True + else: + assert isinstance(data, bytes) + self._buf += data + if max_bytes is None: + max_bytes = len(self._buf) + out = bytes(self._buf[:max_bytes]) + del self._buf[:max_bytes] return out - async def write_all(self, data: bytes) -> None: - self._channel.send(data) - - async def aclose_read(self) -> None: - return + async def send_eof(self) -> None: + self._channel.close() - async def aclose_write(self) -> None: + async def aclose(self) -> None: self._channel.close() -async def adopt_socket(socket_fd: int) -> SocketStreamIO: +async def adopt_socket(socket_fd: int) -> trio.SocketStream: """Worker side: wrap an inherited socket fd and send the handshake. Runs on the Trio host loop. The coordinator waits for ``b"1"`` before @@ -188,9 +139,8 @@ async def adopt_socket(socket_fd: int) -> SocketStreamIO: sock = _socket.socket(fileno=socket_fd) stream = trio.SocketStream(trio.socket.from_stdlib_socket(sock)) - io = SocketStreamIO(stream) - await io.write_all(b"1") - return io + await stream.send_all(b"1") + return stream class SyncIOHandle: @@ -235,7 +185,7 @@ class ProtocolSession: def __init__( self, gateway: BaseGateway, - io: AsyncByteIO, + io: ByteStream, *, process: trio.Process | None = None, host: TrioHost, @@ -361,13 +311,17 @@ def log(*msg: object) -> None: gateway._trace("[trio-receiver]", *msg) log("RECEIVER: starting") + decoder = FrameDecoder() try: while True: - msg = await read_message(self.io) - log("received", msg) - with gateway._receivelock: - msg.received(gateway) - del msg + data = await self.io.receive_some(RECEIVE_CHUNK) + if not data: + decoder.close() # raises EOFError on a mid-frame EOF + raise EOFError("clean EOF") + for msg in decoder.feed(data): + log("received", msg) + with gateway._receivelock: + msg.received(gateway) except GatewayReceivedTerminate: log("GATEWAY_TERMINATE") except EOFError as exc: @@ -401,15 +355,15 @@ async def _writer_handle_item(self, item: object) -> bool: """Handle one outbound queue item. Return False when writer should stop.""" if item is _CLOSE_WRITE: try: - await self.io.aclose_write() + await self.io.send_eof() except Exception as exc: - self.gateway._trace("aclose_write failed", exc) + self.gateway._trace("send_eof failed", exc) return False assert isinstance(item, tuple) blob, done, errors = item assert isinstance(blob, bytes) try: - await self.io.write_all(blob) + await self.io.send_all(blob) except Exception as exc: self.gateway._trace("write failed", exc) errors.append(exc) @@ -445,11 +399,7 @@ async def _finish(self) -> None: gateway._terminate_execution, abandon_on_cancel=True ) try: - await self.io.aclose_read() - except Exception: - pass - try: - await self.io.aclose_write() + await self.io.aclose() except Exception: pass @@ -496,14 +446,12 @@ async def _main(self) -> None: def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: if self._token is None: raise RuntimeError("TrioHost is not running") - return cast("T", trio.from_thread.run(async_fn, *args, trio_token=self._token)) + return trio.from_thread.run(async_fn, *args, trio_token=self._token) def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: if self._token is None: raise RuntimeError("TrioHost is not running") - return cast( - "T", trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) - ) + return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: """Schedule a task on the root nursery (must be called on the host thread).""" @@ -516,7 +464,7 @@ def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: async def start_session( self, gateway: BaseGateway, - io: AsyncByteIO, + io: ByteStream, *, process: trio.Process | None = None, ) -> ProtocolSession: @@ -621,12 +569,10 @@ def _open_trio_gateway( async def _create_and_attach() -> Gateway: process = await open_popen_process(args) try: - async_io = ProcessStreamsIO(process) + async_io = staple_process_stream(process) if preamble: - await async_io.write_all(preamble) - ack = await async_io.read_exact(1) - if ack != b"1": - raise EOFError(f"bad bootstrap handshake: {ack!r}") + await async_io.send_all(preamble) + await read_handshake_ack(async_io, "bootstrap") except EOFError: with trio.move_on_after(5): code = await process.wait() @@ -732,18 +678,15 @@ async def _create_and_attach() -> Gateway: stream = await trio.open_tcp_stream(*address) except OSError as exc: raise HostNotFound(remoteaddress) from exc - io = SocketStreamIO(stream) try: - ack = await io.read_exact(1) - if ack != b"1": - raise EOFError(f"bad socket handshake: {ack!r}") + await read_handshake_ack(stream, "socket") except BaseException: with trio.move_on_after(5): await stream.aclose() raise gw = execnet.Gateway(_TempIO(group.execmodel), spec) - session = await host.start_session(gw, io) + session = await host.start_session(gw, stream) gw._attach_trio_session(session) gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) return gw @@ -910,10 +853,8 @@ def makegateway_via_trio(group: Any, spec: Any) -> Gateway: remoteaddress = f"{remote}[via {spec.via}]" if remote else None async def _create_and_attach() -> Gateway: - io = ChannelByteIO(channel) - ack = await io.read_exact(1) - if ack != b"1": - raise EOFError(f"bad via handshake: {ack!r}") + io = ChannelByteStream(channel) + await read_handshake_ack(io, "via") gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, io) gw._attach_trio_session(session) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 9e1f1be7..5fc9bceb 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -2,6 +2,7 @@ from __future__ import annotations +import functools import os import queue import sys @@ -51,9 +52,14 @@ def __init__( tuple[Channel, ExecItem, threading.Event] | None ] = queue.SimpleQueue() self._primary_wake = threading.Event() - # Serialize main_thread_only admission (wait+clear) so two tasks cannot - # both observe the idle Event before either clears it. - self._admit_lock = trio.Lock() + # Exec requests flow through a single pump task so admission happens + # strictly in message-arrival order (trio task scheduling order is + # deliberately unordered, so per-request tasks would race for the + # main_thread_only slot). + self._pending_send: trio.MemorySendChannel[tuple[Channel, ExecItem]] + self._pending_recv: trio.MemoryReceiveChannel[tuple[Channel, ExecItem]] + self._pending_send, self._pending_recv = trio.open_memory_channel(float("inf")) + self._pump_started = False def active_count(self) -> int: with self._lock: @@ -82,24 +88,25 @@ def schedule(self, channel: Channel, sourcetask: bytes) -> None: channel.close("execution disallowed") return # Already on the Trio host thread (Message handler). - self.host.start_soon(self._run_exec, channel, item) - - async def _run_exec(self, channel: Channel, item: ExecItem) -> None: - if self.main_thread_only: - complete = self.gateway._executetask_complete - assert complete is not None - - def _wait_slot() -> bool: - return complete.wait(timeout=1) - - async with self._admit_lock: - if not await trio.to_thread.run_sync( - _wait_slot, abandon_on_cancel=True - ): + if not self._pump_started: + self._pump_started = True + self.host.start_soon(self._pump) + self._pending_send.send_nowait((channel, item)) + + async def _pump(self) -> None: + """Admit queued exec requests in FIFO order, then run each as a task.""" + async for channel, item in self._pending_recv: + if self.main_thread_only: + complete = self.gateway._executetask_complete + assert complete is not None + wait_slot = functools.partial(complete.wait, timeout=1) + if not await trio.to_thread.run_sync(wait_slot, abandon_on_cancel=True): channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) - return + continue complete.clear() + self.host.start_soon(self._run_exec, channel, item) + async def _run_exec(self, channel: Channel, item: ExecItem) -> None: self._track_start() try: if self.main_thread_only: @@ -252,7 +259,7 @@ async def _start() -> _trio_host.ProtocolSession: async def _make_fd_io(read_fd: int, write_fd: int) -> Any: from . import _trio_host - return _trio_host.FdStreamsIO(read_fd, write_fd) + return _trio_host.staple_fd_stream(read_fd, write_fd) def serve_popen_trio(id: str, execmodel: str = "thread") -> None: diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 0260b416..d36c7542 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -529,6 +529,42 @@ def _gateway_start_sub(message: Message, gateway: BaseGateway) -> None: _types[GATEWAY_START_SUB] = ("GATEWAY_START_SUB", _gateway_start_sub) +class FrameDecoder: + """Incremental decoder for the 9-byte-header Message framing. + + ``feed(data)`` accepts arbitrary byte chunks and yields every complete + Message; partial frames buffer internally until more bytes arrive. + Pure computation — no IO, no awaits, no knowledge of streams — so + receivers only ever stream bytes in (``receive_some`` loops) and the + decoder owns framing. + """ + + def __init__(self) -> None: + self._buffer = bytearray() + + def feed(self, data: bytes) -> Iterator[Message]: + self._buffer += data + return self._parse() + + def _parse(self) -> Iterator[Message]: + while len(self._buffer) >= 9: + msgtype, channelid, payload_len = Message.from_header( + bytes(self._buffer[:9]) + ) + if len(self._buffer) < 9 + payload_len: + return + payload = bytes(self._buffer[9 : 9 + payload_len]) + del self._buffer[: 9 + payload_len] + yield Message(msgtype, channelid, payload) + + def close(self) -> None: + """Signal EOF; raises EOFError if the stream ended mid-frame.""" + if self._buffer: + raise EOFError( + "connection closed mid-frame (%d buffered bytes)" % len(self._buffer) + ) + + class GatewayReceivedTerminate(Exception): """Receiver got a gateway termination message.""" diff --git a/testing/test_basics.py b/testing/test_basics.py index d4ffd5d8..473cd741 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -236,6 +236,90 @@ def test_wire_protocol(self) -> None: assert isinstance(repr(msg), str) +class TestFrameDecoder: + def _messages(self) -> list[Message]: + return [ + Message(Message.CHANNEL_DATA, 1, b"x" * 20), + Message(Message.STATUS, 42, b""), + Message(Message.CHANNEL_DATA, 7, b"y"), + ] + + def test_single_feed_yields_all(self) -> None: + decoder = gateway_base.FrameDecoder() + blob = b"".join(m.pack() for m in self._messages()) + got = list(decoder.feed(blob)) + assert [(m.msgcode, m.channelid, m.data) for m in got] == [ + (m.msgcode, m.channelid, m.data) for m in self._messages() + ] + decoder.close() + + @pytest.mark.parametrize("chunksize", [1, 2, 3, 8, 9, 10, 13]) + def test_adversarial_chunk_splits(self, chunksize: int) -> None: + decoder = gateway_base.FrameDecoder() + blob = b"".join(m.pack() for m in self._messages()) + got: list[Message] = [] + for start in range(0, len(blob), chunksize): + got.extend(decoder.feed(blob[start : start + chunksize])) + assert [(m.msgcode, m.channelid, m.data) for m in got] == [ + (m.msgcode, m.channelid, m.data) for m in self._messages() + ] + decoder.close() + + def test_close_mid_frame_raises(self) -> None: + decoder = gateway_base.FrameDecoder() + blob = Message(Message.CHANNEL_DATA, 1, b"hello").pack() + assert list(decoder.feed(blob[:-2])) == [] + with pytest.raises(EOFError, match="mid-frame"): + decoder.close() + + def test_feed_buffers_even_when_not_iterated(self) -> None: + decoder = gateway_base.FrameDecoder() + blob = Message(Message.CHANNEL_DATA, 5, b"data").pack() + decoder.feed(blob[:4]) # result deliberately not iterated + (msg,) = decoder.feed(blob[4:]) + assert (msg.msgcode, msg.channelid, msg.data) == ( + Message.CHANNEL_DATA, + 5, + b"data", + ) + + def test_memory_stream_roundtrip(self) -> None: + """Protocol-level: frames sent over a trio memory stream pair arrive + intact through the receive_some + FrameDecoder loop.""" + import trio + import trio.testing + + messages = self._messages() + + async def main() -> list[Message]: + ours, theirs = trio.testing.memory_stream_pair() + received: list[Message] = [] + + async def sender() -> None: + for m in messages: + await theirs.send_all(m.pack()) + await theirs.send_eof() + + async def receiver() -> None: + decoder = gateway_base.FrameDecoder() + while True: + data = await ours.receive_some(4096) + if not data: + decoder.close() + break + received.extend(decoder.feed(data)) + + async with trio.open_nursery() as nursery: + nursery.start_soon(sender) + nursery.start_soon(receiver) + return received + + received = trio.run(main) + assert [(m.msgcode, m.channelid, m.data) for m in received] == [ + (m.msgcode, m.channelid, m.data) for m in messages + ] + + class TestPureChannel: @pytest.fixture def fac(self, execmodel: ExecModel) -> ChannelFactory: From f271da5e379dd23392b39ddcdcc0792bc6fc43c1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 12:19:03 +0200 Subject: [PATCH 19/59] refactor: extract the LoopPortal cross-thread primitive Phase B.3 of the Trio port: one portal module replaces the three ad-hoc cross-thread bridges. - new execnet/portal.py: LoopPortal (trio token holder with run/run_sync/post/is_loop_thread; post = run_sync_soon, strict FIFO) and SyncReceiver (loop-to-thread queue whose get() stays KeyboardInterrupt-interruptible on the main thread) - TrioHost owns a LoopPortal; call/call_sync/is_host_thread delegate - ProtocolSession's outbound queue becomes an unbounded trio memory channel; every send is posted through the portal so loop callbacks and foreign threads share one FIFO, and the writer task is a plain async-for -- the recreated-Event wake dance is gone. Blocking and close semantics are unchanged: non-loop threads still wait for the OS write (120s timeout), closed sends still raise OSError, and a finished loop maps trio.RunFinishedError to the same OSError. - TrioWorkerExec's main-thread exec handoff uses SyncReceiver Because each end only needs the other loop's token, the same primitive will serve two-loop setups (facade host loop + user loop) in B.5/B.6. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_host.py | 85 ++++++++++++++------------------ src/execnet/_trio_worker.py | 23 +++------ src/execnet/portal.py | 96 +++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 66 deletions(-) create mode 100644 src/execnet/portal.py diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 1a8dd197..cd20a0f2 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -10,7 +10,6 @@ import itertools import json import math -import queue import subprocess import sys import threading @@ -31,6 +30,7 @@ from .gateway_base import dumps_internal from .gateway_base import loads_internal from .gateway_base import trace +from .portal import LoopPortal if TYPE_CHECKING: from .gateway import Gateway @@ -194,14 +194,24 @@ def __init__( self.io = io self.process = process self.host = host - self._outbound: queue.SimpleQueue[object] = queue.SimpleQueue() - self._wake: trio.Event | None = None + self._outbound_send: trio.MemorySendChannel[object] + self._outbound_recv: trio.MemoryReceiveChannel[object] + self._outbound_send, self._outbound_recv = trio.open_memory_channel(math.inf) self._done = threading.Event() self._process_exitcode: int | None = None self._process_done = threading.Event() self._send_closed = False self._lock = threading.Lock() + def _post_outbound(self, item: object) -> None: + """Schedule ``item`` onto the outbound channel. + + Goes through the portal even from the host thread so every send — + loop callbacks and foreign threads alike — lands in one global FIFO + order. Raises ``trio.RunFinishedError`` after loop shutdown. + """ + self.host.portal.post(self._outbound_send.send_nowait, item) + def enqueue_message(self, message: Message) -> None: """Enqueue a frame; wait until written when safe to block. @@ -211,14 +221,16 @@ def enqueue_message(self, message: Message) -> None: The Trio host thread (receiver callbacks) must not wait — that would deadlock the writer task on the same event loop. """ - wait = not self.host.is_host_thread() + wait = not self.host.portal.is_loop_thread() done = threading.Event() if wait else None errors: list[BaseException] = [] with self._lock: if self._send_closed or self._done.is_set(): raise OSError("cannot send (already closed?)") - self._outbound.put((message.pack(), done, errors)) - self._wake_writer() + try: + self._post_outbound((message.pack(), done, errors)) + except trio.RunFinishedError: + raise OSError("cannot send (already closed?)") from None if done is None: return if not done.wait(timeout=120.0): @@ -226,25 +238,13 @@ def enqueue_message(self, message: Message) -> None: if errors: raise OSError("cannot send (already closed?)") from errors[0] - def _wake_writer(self) -> None: - wake = self._wake - if wake is None: - return - if self.host.is_host_thread(): - wake.set() - else: - try: - self.host.call_sync(wake.set) - except Exception: - pass - def request_close_write(self) -> None: with self._lock: if self._send_closed: return self._send_closed = True - self._outbound.put(_CLOSE_WRITE) - self._wake_writer() + with suppress(trio.RunFinishedError): + self._post_outbound(_CLOSE_WRITE) def request_close_read(self) -> None: # Reader observes EOF / cancel; nothing required from callers. @@ -332,22 +332,7 @@ def log(*msg: object) -> None: log("finishing receiver") async def _writer(self) -> None: - self._wake = trio.Event() - while True: - while True: - try: - item = self._outbound.get_nowait() - except queue.Empty: - break - if not await self._writer_handle_item(item): - return - # Reset wake before re-check to avoid losing a notification. - self._wake = trio.Event() - try: - item = self._outbound.get_nowait() - except queue.Empty: - await self._wake.wait() - continue + async for item in self._outbound_recv: if not await self._writer_handle_item(item): return @@ -385,8 +370,8 @@ async def _finish(self) -> None: gateway = self.gateway with self._lock: self._send_closed = True - self._outbound.put(_CLOSE_WRITE) - self._wake_writer() + with suppress(trio.RunFinishedError): + self._post_outbound(_CLOSE_WRITE) gateway._trace("[trio-receiver] finishing channels") gateway._channelfactory._finished_receiving() # Unblock the worker's join() before heavy exec-pool shutdown @@ -410,7 +395,7 @@ class TrioHost: def __init__(self, name: str = "execnet-trio-host") -> None: self._name = name self._thread: threading.Thread | None = None - self._token: trio.lowlevel.TrioToken | None = None + self._portal: LoopPortal | None = None self._nursery: trio.Nursery | None = None self._ready = threading.Event() self._shutdown: trio.Event | None = None @@ -425,14 +410,20 @@ def start(self) -> None: raise RuntimeError("TrioHost failed to start") self._started = True + @property + def portal(self) -> LoopPortal: + if self._portal is None: + raise RuntimeError("TrioHost is not running") + return self._portal + def is_host_thread(self) -> bool: - return self._thread is not None and threading.current_thread() is self._thread + return self._portal is not None and self._portal.is_loop_thread() def _run(self) -> None: trio.run(self._main) async def _main(self) -> None: - self._token = trio.lowlevel.current_trio_token() + self._portal = LoopPortal() self._shutdown = trio.Event() try: async with trio.open_nursery() as nursery: @@ -444,14 +435,10 @@ async def _main(self) -> None: self._nursery = None def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: - if self._token is None: - raise RuntimeError("TrioHost is not running") - return trio.from_thread.run(async_fn, *args, trio_token=self._token) + return self.portal.run(async_fn, *args) def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: - if self._token is None: - raise RuntimeError("TrioHost is not running") - return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + return self.portal.run_sync(sync_fn, *args) def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: """Schedule a task on the root nursery (must be called on the host thread).""" @@ -475,7 +462,7 @@ async def start_session( return session def stop(self, timeout: float | None = 5.0) -> None: - if not self._started or self._token is None or self._shutdown is None: + if not self._started or self._portal is None or self._shutdown is None: return def _set() -> None: @@ -483,7 +470,7 @@ def _set() -> None: self._shutdown.set() try: - trio.from_thread.run_sync(_set, trio_token=self._token) + self._portal.run_sync(_set) except Exception: pass if self._thread is not None: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 5fc9bceb..bb71b027 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -4,7 +4,6 @@ import functools import os -import queue import sys import threading from typing import TYPE_CHECKING @@ -17,6 +16,7 @@ from .gateway_base import get_execmodel from .gateway_base import loads_internal from .gateway_base import trace +from .portal import SyncReceiver if TYPE_CHECKING: from . import _trio_host @@ -48,10 +48,9 @@ def __init__( self._shutting_down = False self._idle = threading.Event() self._idle.set() - self._primary_q: queue.SimpleQueue[ + self._primary: SyncReceiver[ tuple[Channel, ExecItem, threading.Event] | None - ] = queue.SimpleQueue() - self._primary_wake = threading.Event() + ] = SyncReceiver() # Exec requests flow through a single pump task so admission happens # strictly in message-arrival order (trio task scheduling order is # deliberately unordered, so per-request tasks would race for the @@ -111,8 +110,7 @@ async def _run_exec(self, channel: Channel, item: ExecItem) -> None: try: if self.main_thread_only: done = threading.Event() - self._primary_q.put((channel, item, done)) - self._primary_wake.set() + self._primary.put((channel, item, done)) await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) else: await trio.to_thread.run_sync( @@ -126,15 +124,7 @@ async def _run_exec(self, channel: Channel, item: ExecItem) -> None: def integrate_as_primary_thread(self) -> None: """Block the main thread running main_thread_only exec tasks.""" while True: - self._primary_wake.wait() - try: - task = self._primary_q.get_nowait() - except queue.Empty: - self._primary_wake.clear() - try: - task = self._primary_q.get_nowait() - except queue.Empty: - continue + task = self._primary.get() if task is None: break channel, item, done = task @@ -146,8 +136,7 @@ def integrate_as_primary_thread(self) -> None: def trigger_shutdown(self) -> None: with self._lock: self._shutting_down = True - self._primary_q.put(None) - self._primary_wake.set() + self._primary.put(None) def waitall(self, timeout: float | None = None) -> bool: return self._idle.wait(timeout) diff --git a/src/execnet/portal.py b/src/execnet/portal.py new file mode 100644 index 00000000..9804ca45 --- /dev/null +++ b/src/execnet/portal.py @@ -0,0 +1,96 @@ +"""Cross-thread / cross-loop communication primitives. + +A :class:`LoopPortal` is a handle to a running trio loop that foreign +threads use to run functions on the loop or push work into it. Because +each direction only needs the *receiving* loop's token, two trio loops in +two threads can communicate by holding each other's portal (the sync +facade's host loop and a user loop; the worker loop and the process main +thread). + +:class:`SyncReceiver` covers the opposite direction: a plain thread +(typically the process main thread) receiving items produced on a loop, +in a way that stays interruptible by KeyboardInterrupt. +""" + +from __future__ import annotations + +import queue +import threading +from collections.abc import Awaitable +from collections.abc import Callable +from typing import Any +from typing import Generic +from typing import TypeVar + +import trio + +T = TypeVar("T") + + +class LoopPortal: + """Handle to a running trio loop, usable from foreign threads. + + Must be constructed on the loop's own thread (it captures the current + trio token). + """ + + def __init__(self) -> None: + self._token = trio.lowlevel.current_trio_token() + + def is_loop_thread(self) -> bool: + """Whether the calling thread is running this portal's loop.""" + try: + return trio.lowlevel.current_trio_token() is self._token + except RuntimeError: + return False + + def run(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: + """Run ``await async_fn(*args)`` on the loop, blocking this thread.""" + return trio.from_thread.run(async_fn, *args, trio_token=self._token) + + def run_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: + """Run ``sync_fn(*args)`` on the loop, blocking this thread.""" + return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + + def post(self, sync_fn: Callable[..., object], *args: Any) -> None: + """Schedule ``sync_fn(*args)`` on the loop without waiting. + + Thread-safe and callable from the loop thread itself; all posts run + in strict FIFO order (``TrioToken.run_sync_soon``). Raises + ``trio.RunFinishedError`` once the loop has shut down. + """ + self._token.run_sync_soon(sync_fn, *args) + + +class SyncReceiver(Generic[T]): + """Receive items on a plain thread, KeyboardInterrupt-friendly. + + ``queue.SimpleQueue.get`` parks in a C-level lock acquire that shields + KeyboardInterrupt on the main thread; ``threading.Event.wait`` does not. + So producers put into an unbounded queue and set a wake event, and + :meth:`get` waits on the event and drains the queue. + """ + + def __init__(self) -> None: + self._items: queue.SimpleQueue[T] = queue.SimpleQueue() + self._wake = threading.Event() + + def put(self, item: T) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + self._items.put(item) + self._wake.set() + + def get(self) -> T: + """Block until an item is available (interruptible on main thread).""" + while True: + self._wake.wait() + try: + return self._items.get_nowait() + except queue.Empty: + # Re-check after clearing so a put between get_nowait and + # clear cannot be lost. + self._wake.clear() + try: + return self._items.get_nowait() + except queue.Empty: + continue From 0eb6cc531774148003de81ef1e84a6174d3eb804 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 14:45:43 +0200 Subject: [PATCH 20/59] feat: add the trio-native AsyncGateway core with low-level RawChannels Phase B.4 starts the async-native inversion: a new _trio_gateway module hosts AsyncGateway, whose single serve task reads the framed Message protocol off a ByteStream (receive_some + sans-IO FrameDecoder) and dispatches inline -- no receiver thread, no receive lock -- plus a writer task draining an unbounded outbound queue. RawChannel is the low-level half of the two-level channel model: id-routed raw byte payloads with the sync Channel close semantics (CHANNEL_CLOSE both ways, CHANNEL_LAST_MESSAGE as write-EOF leaving the peer sendonly, CHANNEL_CLOSE_ERROR surfacing as RemoteError). Errors keep the execnet contract -- OSError on closed sends, EOFError/RemoteError on receive -- so no trio exception types leak into the API. The ByteStream protocol and RECEIVE_CHUNK move here from _trio_host so the async core sits at the bottom of the dependency stack. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 398 +++++++++++++++++++++++++++++++++++ src/execnet/_trio_host.py | 24 +-- testing/test_trio_gateway.py | 202 ++++++++++++++++++ 3 files changed, 602 insertions(+), 22 deletions(-) create mode 100644 src/execnet/_trio_gateway.py create mode 100644 testing/test_trio_gateway.py diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py new file mode 100644 index 00000000..67e103ed --- /dev/null +++ b/src/execnet/_trio_gateway.py @@ -0,0 +1,398 @@ +"""Trio-native gateway core: async dispatch loop and low-level raw channels. + +Async-first counterpart of the sync machinery in ``gateway_base``: an +:class:`AsyncGateway` owns a :class:`ByteStream` and runs a single dispatch +task (stream -> ``FrameDecoder`` -> route). Message handlers execute inline +on that task, so there is no receiver thread and no receive lock. + +Two-level channel model: + +* :class:`RawChannel` (this module) -- id-routed raw byte payload streams + over the gateway: no serialization, no strconfig, no callbacks. + ``CHANNEL_DATA`` payloads route to the channel verbatim; the layer on top + decides what the bytes mean. +* ``AsyncChannel`` -- the serialized object API layered on a RawChannel. + +The code deliberately sticks to idioms an anyio backend can mirror later: +a neutral ``ByteStream`` protocol, the sans-IO ``FrameDecoder``, and +unbounded memory channels. +""" + +from __future__ import annotations + +import math +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from contextlib import suppress +from typing import Protocol + +import trio + +from .gateway_base import FrameDecoder +from .gateway_base import GatewayReceivedTerminate +from .gateway_base import Message +from .gateway_base import RemoteError +from .gateway_base import Unserializer +from .gateway_base import dumps_internal +from .gateway_base import loads_internal +from .gateway_base import trace + +RECEIVE_CHUNK = 65536 + + +class ByteStream(Protocol): + """Neutral bidirectional byte-stream protocol for gateway transports. + + ``trio.StapledStream`` (process/fd pipe pairs) and ``trio.SocketStream`` + satisfy this structurally; a future anyio backend's byte streams use the + same four names. ``send_eof`` signals write-EOF to the peer (half-close + for sockets; for pipe pairs trio falls back to closing the send half). + """ + + async def send_all(self, data: bytes) -> None: ... + + async def receive_some(self, max_bytes: int | None = None) -> bytes: ... + + async def send_eof(self) -> None: ... + + async def aclose(self) -> None: ... + + +class RawChannel: + """Low-level id-routed byte payload stream over an :class:`AsyncGateway`. + + Payload boundaries are preserved: every :meth:`send_bytes` arrives as + one :meth:`receive_bytes` result on the peer. No serialization and no + flow control beyond the gateway's outbound queue. + + Close semantics mirror the sync ``Channel`` state machine: + + * :meth:`aclose` closes both directions (``CHANNEL_CLOSE`` / + ``CHANNEL_CLOSE_ERROR`` to the peer). + * :meth:`send_eof` only ends our payload stream (``CHANNEL_LAST_MESSAGE``); + the peer drains, hits EOF, and may keep sending to us. + """ + + _strconfig: tuple[bool, bool] | None = None + + def __init__(self, gateway: AsyncGateway, id: int) -> None: + self.gateway = gateway + self.id = id + self._closed = False # no more sends (local aclose or remote close) + self._sent_eof = False + self._remote_closed = False + self._remote_error: RemoteError | None = None + self._payload_send, self._payloads = trio.open_memory_channel[bytes](math.inf) + + def __repr__(self) -> str: + state = "closed" if self._closed else "open" + return f"" + + async def send_bytes(self, data: bytes) -> None: + """Send one payload; the peer receives it as a single item. + + OSError is raised when the channel or gateway is closed, matching + the sync ``Channel.send`` contract. + """ + if self._closed or self._sent_eof: + raise OSError(f"cannot send to {self!r}") + await self.gateway._send(Message.CHANNEL_DATA, self.id, data) + + async def receive_bytes(self) -> bytes: + """Receive the next payload. + + Raises EOFError once the peer closed or sent EOF and all payloads + are drained; a peer close-with-error raises that ``RemoteError``. + """ + try: + return await self._payloads.receive() + except (trio.EndOfChannel, trio.ClosedResourceError): + raise self._pending_error() from None + + async def send_eof(self) -> None: + """Signal that no more payloads follow (peer keeps its send side).""" + if self._closed or self._sent_eof: + raise OSError(f"cannot send EOF to {self!r}") + self._sent_eof = True + await self.gateway._send(Message.CHANNEL_LAST_MESSAGE, self.id) + + async def aclose(self, error: str | None = None) -> None: + """Close both directions; ``error`` reaches the peer as a RemoteError.""" + if self._closed: + await trio.lowlevel.checkpoint() + return + self._closed = True + self._payload_send.close() + self.gateway._forget_channel(self.id) + if not self._remote_closed: + # A peer-initiated close needs no reply; a dead gateway is + # already as closed as it gets. + with suppress(OSError): + if error is not None: + await self.gateway._send( + Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(error) + ) + else: + await self.gateway._send(Message.CHANNEL_CLOSE, self.id) + + def __aiter__(self) -> RawChannel: + return self + + async def __anext__(self) -> bytes: + try: + return await self.receive_bytes() + except EOFError: + raise StopAsyncIteration from None + + def _pending_error(self) -> BaseException: + return ( + self._remote_error + or self.gateway._error + or EOFError(f"raw channel {self.id} closed") + ) + + # dispatch-loop internals (inline on the gateway's serve task) + + def _feed(self, data: bytes) -> None: + try: + self._payload_send.send_nowait(data) + except (trio.BrokenResourceError, trio.ClosedResourceError): + pass # locally closed: drop, like the sync channel + + def _close_from_remote( + self, error: RemoteError | None, *, sendonly: bool + ) -> None: + if error is not None: + self._remote_error = error + self._remote_closed = True + if not sendonly: + self._closed = True + self.gateway._forget_channel(self.id) + self._payload_send.close() + + +class AsyncGateway: + """Async-native gateway: the framed Message protocol over a ByteStream. + + Serving (``serve_gateway`` or ``nursery.start(gateway._serve)``) runs a + reader task that dispatches messages inline and a writer task draining + an unbounded outbound queue -- sends never block on the peer. + + ``_startcount`` follows the sync convention: locally allocated channel + ids step by two, coordinators from 1 (odd) and workers from 2 (even), + so the two peers never collide. + """ + + _error: BaseException | None = None + + def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None: + self._stream = stream + self.id = id + self._channels: dict[int, RawChannel] = {} + self._count = _startcount + self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) + self._outbound_send, self._outbound = trio.open_memory_channel[bytes](math.inf) + self._closed = False + self._serve_started = False + self._writer_done = trio.Event() + self._done = trio.Event() + + def __repr__(self) -> str: + state = "closed" if self._closed else "open" + return f"" + + @property + def closed(self) -> bool: + return self._closed + + async def wait_closed(self) -> None: + """Wait until serving has fully shut down.""" + await self._done.wait() + + def _trace(self, *msg: object) -> None: + trace(self.id, *msg) + + def open_raw_channel(self, id: int | None = None) -> RawChannel: + """Return the raw channel for ``id``, allocating a fresh id if None. + + An explicit id attaches to a channel the peer references (e.g. an id + received in a request payload); the same object is returned if the + dispatch loop already routed data to it. + """ + if self._closed: + raise OSError(f"connection already closed: {self!r}") + if id is None: + id = self._count + self._count += 2 + return self._channel_for(id) + + async def terminate(self) -> None: + """Send GATEWAY_TERMINATE to the peer, then close this side.""" + if not self._closed: + with suppress(OSError): + await self._send(Message.GATEWAY_TERMINATE) + await self.aclose() + + async def aclose(self) -> None: + """Flush queued frames, close the stream, and wait for shutdown.""" + if not self._closed: + self._closed = True + self._outbound_send.close() + with trio.move_on_after(5): + await self._writer_done.wait() + with trio.CancelScope(shield=True): + with suppress(Exception): + await self._stream.aclose() + if self._serve_started: + await self._done.wait() + else: + self._finish_channels() + + async def _serve( + self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED + ) -> None: + """Run the reader and writer until EOF, termination, or ``aclose``.""" + self._serve_started = True + try: + async with trio.open_nursery() as nursery: + nursery.start_soon(self._writer) + task_status.started() + await self._reader() + # Reader is done: let the writer flush queued frames + # (close replies), then stop it. + self._closed = True + self._outbound_send.close() + with trio.move_on_after(5): + await self._writer_done.wait() + nursery.cancel_scope.cancel() + finally: + self._closed = True + self._outbound_send.close() + self._finish_channels() + with trio.CancelScope(shield=True): + with suppress(Exception): + await self._stream.aclose() + self._done.set() + + async def _reader(self) -> None: + decoder = FrameDecoder() + try: + while True: + try: + data = await self._stream.receive_some(RECEIVE_CHUNK) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + if not self._closed: + self._error = exc + return + if not data: + decoder.close() # raises EOFError on a mid-frame EOF + raise EOFError("connection closed (no gateway termination)") + for message in decoder.feed(data): + self._trace("received", message) + self._dispatch(message) + except GatewayReceivedTerminate: + self._trace("received GATEWAY_TERMINATE") + except EOFError as exc: + self._trace("EOF without prior gateway termination message") + self._error = exc + + async def _writer(self) -> None: + try: + async for frame in self._outbound: + await self._stream.send_all(frame) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + self._trace("writer failed", exc) + if self._error is None: + self._error = exc + self._outbound_send.close() # fail future sends fast + else: + # Queue closed and drained: signal write-EOF to the peer. + with suppress(Exception): + await self._stream.send_eof() + finally: + self._writer_done.set() + + def _dispatch(self, message: Message) -> None: + """Route one message; runs inline on the serve task.""" + code = message.msgcode + channelid = message.channelid + if code == Message.CHANNEL_DATA: + self._channel_for(channelid)._feed(message.data) + elif code == Message.CHANNEL_CLOSE: + self._channel_for(channelid)._close_from_remote(None, sendonly=False) + elif code == Message.CHANNEL_CLOSE_ERROR: + error_message = loads_internal(message.data) + assert isinstance(error_message, str) + self._channel_for(channelid)._close_from_remote( + RemoteError(error_message), sendonly=False + ) + elif code == Message.CHANNEL_LAST_MESSAGE: + self._channel_for(channelid)._close_from_remote(None, sendonly=True) + elif code == Message.GATEWAY_TERMINATE: + raise GatewayReceivedTerminate(self) + elif code == Message.STATUS: + status = { + "numchannels": len(self._channels), + "numexecuting": 0, + "execmodel": "trio", + } + self._send_nowait(Message.CHANNEL_DATA, channelid, dumps_internal(status)) + self._send_nowait(Message.CHANNEL_CLOSE, channelid) + elif code == Message.RECONFIGURE: + data = loads_internal(message.data) + assert isinstance(data, tuple) + if channelid == 0: + self._strconfig = data + else: + # picked up by the serialized channel layer + self._channel_for(channelid)._strconfig = data + else: + # CHANNEL_EXEC / GATEWAY_START_*: not served by the async core + self._trace("rejecting unsupported message", message) + self._send_nowait( + Message.CHANNEL_CLOSE_ERROR, + channelid, + dumps_internal(f"unsupported message on async gateway: {message!r}"), + ) + + def _channel_for(self, id: int) -> RawChannel: + try: + return self._channels[id] + except KeyError: + channel = self._channels[id] = RawChannel(self, id) + return channel + + def _forget_channel(self, id: int) -> None: + self._channels.pop(id, None) + + async def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: + # The queue is unbounded, so this never waits on the peer -- but it + # is a real checkpoint and raises once the gateway is closed. + try: + await self._outbound_send.send(Message(msgcode, channelid, data).pack()) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + raise OSError("cannot send (already closed?)") from exc + + def _send_nowait(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: + """Enqueue a frame from a dispatch handler (sync, inline on the loop).""" + with suppress(trio.BrokenResourceError, trio.ClosedResourceError): + self._outbound_send.send_nowait(Message(msgcode, channelid, data).pack()) + + def _finish_channels(self) -> None: + for channel in list(self._channels.values()): + channel._close_from_remote(None, sendonly=True) + self._channels.clear() + + +@asynccontextmanager +async def serve_gateway( + stream: ByteStream, *, id: str, _startcount: int = 1 +) -> AsyncIterator[AsyncGateway]: + """Serve an :class:`AsyncGateway` over ``stream`` for the ``with`` body.""" + gateway = AsyncGateway(stream, id=id, _startcount=_startcount) + async with trio.open_nursery() as nursery: + await nursery.start(gateway._serve) + try: + yield gateway + finally: + await gateway.aclose() diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index cd20a0f2..b09dca15 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -18,11 +18,12 @@ from contextlib import suppress from typing import TYPE_CHECKING from typing import Any -from typing import Protocol from typing import TypeVar import trio +from ._trio_gateway import RECEIVE_CHUNK +from ._trio_gateway import ByteStream from .gateway_base import ExecModel from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -41,27 +42,6 @@ _CLOSE_WRITE = object() -RECEIVE_CHUNK = 65536 - - -class ByteStream(Protocol): - """Neutral bidirectional byte-stream protocol for gateway transports. - - ``trio.StapledStream`` (process/fd pipe pairs) and ``trio.SocketStream`` - satisfy this structurally; a future anyio backend's byte streams use the - same four names. ``send_eof`` signals write-EOF to the peer (half-close - for sockets; for pipe pairs trio falls back to closing the send half). - """ - - async def send_all(self, data: bytes) -> None: ... - - async def receive_some(self, max_bytes: int | None = None) -> bytes: ... - - async def send_eof(self) -> None: ... - - async def aclose(self) -> None: ... - - def staple_process_stream(process: trio.Process) -> ByteStream: """One bidirectional stream over a Trio Process stdin/stdout pair.""" assert process.stdin is not None diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py new file mode 100644 index 00000000..872c35a5 --- /dev/null +++ b/testing/test_trio_gateway.py @@ -0,0 +1,202 @@ +"""Protocol tests for the trio-native async gateway core (RawChannel level). + +Two AsyncGateways are wired together over an in-memory stream pair (the +transport harness, as in ``TestFrameDecoder``); the assertions cover execnet +semantics: payload routing by channel id, the close/EOF/sendonly state +machine, RemoteError propagation, and gateway termination. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +import pytest +import trio +import trio.testing + +from execnet._trio_gateway import AsyncGateway +from execnet.gateway_base import Message +from execnet.gateway_base import RemoteError +from execnet.gateway_base import dumps_internal +from execnet.gateway_base import loads_internal + + +@asynccontextmanager +async def gateway_pair() -> AsyncIterator[tuple[AsyncGateway, AsyncGateway]]: + left_stream, right_stream = trio.testing.memory_stream_pair() + async with trio.open_nursery() as nursery: + left = AsyncGateway(left_stream, id="left", _startcount=1) + right = AsyncGateway(right_stream, id="right", _startcount=2) + await nursery.start(left._serve) + await nursery.start(right._serve) + try: + yield left, right + finally: + await left.aclose() + await right.aclose() + + +def test_payload_boundaries_are_preserved() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.send_bytes(b"first payload") + await sender.send_bytes(b"second") + assert await receiver.receive_bytes() == b"first payload" + assert await receiver.receive_bytes() == b"second" + + trio.run(main) + + +def test_send_eof_makes_peer_sendonly() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.send_bytes(b"data") + await sender.send_eof() + assert await receiver.receive_bytes() == b"data" + with pytest.raises(EOFError): + await receiver.receive_bytes() + # the receiver of an EOF may still send back + await receiver.send_bytes(b"reply") + assert await sender.receive_bytes() == b"reply" + with pytest.raises(OSError, match="cannot send"): + await sender.send_bytes(b"after eof") + + trio.run(main) + + +def test_close_drains_then_blocks_both_directions() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.send_bytes(b"x") + await sender.aclose() + # payloads sent before the close still drain + assert await receiver.receive_bytes() == b"x" + with pytest.raises(EOFError): + await receiver.receive_bytes() + with pytest.raises(OSError, match="cannot send"): + await receiver.send_bytes(b"y") + with pytest.raises(OSError, match="cannot send"): + await sender.send_bytes(b"z") + + trio.run(main) + + +def test_close_with_error_raises_remote_error() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.aclose(error="boom happened") + with pytest.raises(RemoteError, match="boom happened"): + await receiver.receive_bytes() + + trio.run(main) + + +def test_async_iteration_yields_payloads_until_eof() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + payloads = [b"a", b"bb", b"ccc"] + for payload in payloads: + await sender.send_bytes(payload) + await sender.send_eof() + assert [data async for data in receiver] == payloads + + trio.run(main) + + +def test_terminate_closes_peer_cleanly() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + channel = right.open_raw_channel() + await left.terminate() + await right.wait_closed() + assert right._error is None + with pytest.raises(EOFError): + await channel.receive_bytes() + + trio.run(main) + + +def test_status_reply_travels_on_raw_channel() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + channel = left.open_raw_channel() + await left._send(Message.STATUS, channel.id) + status = loads_internal(await channel.receive_bytes()) + assert status["execmodel"] == "trio" + assert status["numexecuting"] == 0 + # the peer closes the status channel after the reply + with pytest.raises(EOFError): + await channel.receive_bytes() + + trio.run(main) + + +def test_unsupported_message_is_rejected_with_remote_error() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + channel = left.open_raw_channel() + await left._send( + Message.CHANNEL_EXEC, + channel.id, + dumps_internal(("code", None, None, {})), + ) + with pytest.raises(RemoteError, match="unsupported message"): + await channel.receive_bytes() + + trio.run(main) + + +def test_send_after_gateway_close_raises() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + channel = left.open_raw_channel() + await left.aclose() + with pytest.raises(OSError, match="cannot send"): + await channel.send_bytes(b"x") + with pytest.raises(OSError, match="already closed"): + left.open_raw_channel() + + trio.run(main) + + +def test_peer_disappearing_surfaces_eof_error() -> None: + async def main() -> None: + left_stream, right_stream = trio.testing.memory_stream_pair() + async with trio.open_nursery() as nursery: + right = AsyncGateway(right_stream, id="right", _startcount=2) + await nursery.start(right._serve) + channel = right.open_raw_channel() + # peer vanishes without a termination message + await left_stream.aclose() + await right.wait_closed() + with pytest.raises(EOFError): + await channel.receive_bytes() + + trio.run(main) + + +def test_mid_frame_eof_is_an_error() -> None: + async def main() -> None: + left_stream, right_stream = trio.testing.memory_stream_pair() + async with trio.open_nursery() as nursery: + right = AsyncGateway(right_stream, id="right", _startcount=2) + await nursery.start(right._serve) + frame = Message(Message.CHANNEL_DATA, 1, b"payload").pack() + await left_stream.send_all(frame[:5]) + await left_stream.aclose() + await right.wait_closed() + assert isinstance(right._error, EOFError) + assert "mid-frame" in str(right._error) + + trio.run(main) From ed5f99ffa66d942b16a969e0d2e2c87c2fb85b9d Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 15:06:32 +0200 Subject: [PATCH 21/59] feat: layer the serialized AsyncChannel API on RawChannel The high level of the two-level channel model: AsyncChannel wraps a RawChannel with dumps/loads per item, per-channel strconfig (RECONFIGURE travels the wire, both ends coerce on load), async iteration, receive timeouts via trio.fail_after surfacing execnet's TimeoutError, and wait_closed mirroring the sync waitclose contract (reraise RemoteError). Channel objects serialize over the wire: a duck-typed save_AsyncChannel emits the existing CHANNEL opcode and AsyncGateway grows a factory adapter the Unserializer resolves ids through, so channels received inside items attach to the local gateway like sync channels do. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 144 ++++++++++++++++++++++++++++++++--- src/execnet/gateway_base.py | 6 ++ testing/test_trio_gateway.py | 89 +++++++++++++++++++++- 3 files changed, 227 insertions(+), 12 deletions(-) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 67e103ed..aefb0162 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -24,6 +24,7 @@ from collections.abc import AsyncIterator from contextlib import asynccontextmanager from contextlib import suppress +from typing import Any from typing import Protocol import trio @@ -32,6 +33,7 @@ from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message from .gateway_base import RemoteError +from .gateway_base import TimeoutError from .gateway_base import Unserializer from .gateway_base import dumps_internal from .gateway_base import loads_internal @@ -81,6 +83,7 @@ def __init__(self, gateway: AsyncGateway, id: int) -> None: self._closed = False # no more sends (local aclose or remote close) self._sent_eof = False self._remote_closed = False + self._receive_closed = trio.Event() # no more payloads will arrive self._remote_error: RemoteError | None = None self._payload_send, self._payloads = trio.open_memory_channel[bytes](math.inf) @@ -123,6 +126,7 @@ async def aclose(self, error: str | None = None) -> None: return self._closed = True self._payload_send.close() + self._receive_closed.set() self.gateway._forget_channel(self.id) if not self._remote_closed: # A peer-initiated close needs no reply; a dead gateway is @@ -159,18 +163,129 @@ def _feed(self, data: bytes) -> None: except (trio.BrokenResourceError, trio.ClosedResourceError): pass # locally closed: drop, like the sync channel - def _close_from_remote( - self, error: RemoteError | None, *, sendonly: bool - ) -> None: + def _close_from_remote(self, error: RemoteError | None, *, sendonly: bool) -> None: if error is not None: self._remote_error = error self._remote_closed = True + self._receive_closed.set() if not sendonly: self._closed = True self.gateway._forget_channel(self.id) self._payload_send.close() +class AsyncChannel: + """Serialized object API over a :class:`RawChannel`. + + Every payload is one dumps/loads-serialized item; close/EOF semantics + and error propagation come from the raw layer. Channels are + async-iterable, and :meth:`receive` supports the familiar execnet + timeout (raising ``TimeoutError``). + + Channel objects themselves serialize: sending an AsyncChannel inside an + item transfers a reference the peer receives as its own AsyncChannel + for the same id (the wire CHANNEL opcode, as with sync channels). + """ + + RemoteError = RemoteError + TimeoutError = TimeoutError + + def __init__(self, raw: RawChannel) -> None: + self._raw = raw + self.gateway = raw.gateway + self.id = raw.id + + def __repr__(self) -> str: + flag = "closed" if self.isclosed() else "open" + return f"" + + def isclosed(self) -> bool: + """Return True if the channel is closed for sending.""" + return self._raw._closed + + async def send(self, item: object) -> None: + """Serialize ``item`` and send it to the other side. + + The item must be a simple Python type; OSError is raised when the + channel or gateway is closed. + """ + if self.isclosed(): + raise OSError(f"cannot send to {self!r}") + await self._raw.send_bytes(dumps_internal(item)) + + async def receive(self, timeout: float | None = None) -> Any: + """Receive the next item sent from the other side. + + Raises EOFError once the peer closed or sent EOF, a RemoteError for + a peer close-with-error, and TimeoutError if no item arrived within + ``timeout`` seconds. + """ + if timeout is None: + data = await self._raw.receive_bytes() + else: + try: + with trio.fail_after(timeout): + data = await self._raw.receive_bytes() + except trio.TooSlowError: + raise TimeoutError("no item after %r seconds" % timeout) from None + return loads_internal(data, self) + + async def send_eof(self) -> None: + """Signal that no more items follow (peer keeps its send side).""" + await self._raw.send_eof() + + async def aclose(self, error: str | None = None) -> None: + """Close the channel; ``error`` reaches the peer as a RemoteError.""" + await self._raw.aclose(error) + + async def wait_closed(self) -> None: + """Wait until the peer closed or sent EOF; reraise remote errors.""" + await self._raw._receive_closed.wait() + error = self._raw._remote_error or self.gateway._error + if error is not None: + raise error + + async def reconfigure( + self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False + ) -> None: + """Set the string coercion for both ends of this channel.""" + strconfig = (py2str_as_py3str, py3str_as_py2str) + self._raw._strconfig = strconfig + await self.gateway._send( + Message.RECONFIGURE, self.id, dumps_internal(strconfig) + ) + + def __aiter__(self) -> AsyncChannel: + return self + + async def __anext__(self) -> Any: + try: + return await self.receive() + except EOFError: + raise StopAsyncIteration from None + + # Unserializer duck-type: loads_internal(data, self) reads _strconfig + # and _channelfactory off the object to resolve CHANNEL opcodes. + + @property + def _strconfig(self) -> tuple[bool, bool]: + return self._raw._strconfig or self.gateway._strconfig + + @property + def _channelfactory(self) -> _AsyncChannelFactory: + return self.gateway._channelfactory + + +class _AsyncChannelFactory: + """Duck-typed factory for the Unserializer CHANNEL opcode (``.new(id)``).""" + + def __init__(self, gateway: AsyncGateway) -> None: + self.gateway = gateway + + def new(self, id: int) -> AsyncChannel: + return self.gateway.open_channel(id) + + class AsyncGateway: """Async-native gateway: the framed Message protocol over a ByteStream. @@ -189,6 +304,8 @@ def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None self._stream = stream self.id = id self._channels: dict[int, RawChannel] = {} + self._async_channels: dict[int, AsyncChannel] = {} + self._channelfactory = _AsyncChannelFactory(self) self._count = _startcount self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) self._outbound_send, self._outbound = trio.open_memory_channel[bytes](math.inf) @@ -226,6 +343,15 @@ def open_raw_channel(self, id: int | None = None) -> RawChannel: self._count += 2 return self._channel_for(id) + def open_channel(self, id: int | None = None) -> AsyncChannel: + """Return the serialized channel for ``id``, allocating one if None.""" + raw = self.open_raw_channel(id) + try: + return self._async_channels[raw.id] + except KeyError: + channel = self._async_channels[raw.id] = AsyncChannel(raw) + return channel + async def terminate(self) -> None: """Send GATEWAY_TERMINATE to the peer, then close this side.""" if not self._closed: @@ -240,9 +366,8 @@ async def aclose(self) -> None: self._outbound_send.close() with trio.move_on_after(5): await self._writer_done.wait() - with trio.CancelScope(shield=True): - with suppress(Exception): - await self._stream.aclose() + with trio.CancelScope(shield=True), suppress(Exception): + await self._stream.aclose() if self._serve_started: await self._done.wait() else: @@ -269,9 +394,8 @@ async def _serve( self._closed = True self._outbound_send.close() self._finish_channels() - with trio.CancelScope(shield=True): - with suppress(Exception): - await self._stream.aclose() + with trio.CancelScope(shield=True), suppress(Exception): + await self._stream.aclose() self._done.set() async def _reader(self) -> None: @@ -364,6 +488,7 @@ def _channel_for(self, id: int) -> RawChannel: def _forget_channel(self, id: int) -> None: self._channels.pop(id, None) + self._async_channels.pop(id, None) async def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: # The queue is unbounded, so this never waits on the peer -- but it @@ -382,6 +507,7 @@ def _finish_channels(self) -> None: for channel in list(self._channels.values()): channel._close_from_remote(None, sendonly=True) self._channels.clear() + self._async_channels.clear() @asynccontextmanager diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index d36c7542..9d39cce0 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1622,3 +1622,9 @@ def save_frozenset(self, s: frozenset[object]) -> None: def save_Channel(self, channel: Channel) -> None: self._write(opcode.CHANNEL) self._write_int4(channel.id) + + def save_AsyncChannel(self, channel: Any) -> None: + # trio-native channel (execnet._trio_gateway); same wire opcode, + # duck-typed here to avoid importing the async core. + self._write(opcode.CHANNEL) + self._write_int4(channel.id) diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 872c35a5..d95622af 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -15,6 +15,7 @@ import trio import trio.testing +from execnet import gateway_base from execnet._trio_gateway import AsyncGateway from execnet.gateway_base import Message from execnet.gateway_base import RemoteError @@ -129,7 +130,7 @@ async def main() -> None: def test_status_reply_travels_on_raw_channel() -> None: async def main() -> None: - async with gateway_pair() as (left, right): + async with gateway_pair() as (left, _right): channel = left.open_raw_channel() await left._send(Message.STATUS, channel.id) status = loads_internal(await channel.receive_bytes()) @@ -144,7 +145,7 @@ async def main() -> None: def test_unsupported_message_is_rejected_with_remote_error() -> None: async def main() -> None: - async with gateway_pair() as (left, right): + async with gateway_pair() as (left, _right): channel = left.open_raw_channel() await left._send( Message.CHANNEL_EXEC, @@ -159,7 +160,7 @@ async def main() -> None: def test_send_after_gateway_close_raises() -> None: async def main() -> None: - async with gateway_pair() as (left, right): + async with gateway_pair() as (left, _right): channel = left.open_raw_channel() await left.aclose() with pytest.raises(OSError, match="cannot send"): @@ -200,3 +201,85 @@ async def main() -> None: assert "mid-frame" in str(right._error) trio.run(main) + + +def test_channel_serializes_builtin_items() -> None: + items = [42, "text", b"bytes", [1, 2], ("a", 1), {"key": [True, None]}, {1, 2}] + + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + for item in items: + await sender.send(item) + await sender.send_eof() + assert [item async for item in receiver] == items + + trio.run(main) + + +def test_channel_receive_timeout() -> None: + async def main() -> None: + async with gateway_pair() as (left, _right): + channel = left.open_channel() + with pytest.raises(gateway_base.TimeoutError): + await channel.receive(timeout=0.05) + + trio.run(main) + + +def test_channel_close_with_error_and_wait_closed() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + await sender.aclose(error="exec exploded") + with pytest.raises(RemoteError, match="exec exploded"): + await receiver.receive() + with pytest.raises(RemoteError, match="exec exploded"): + await receiver.wait_closed() + + trio.run(main) + + +def test_channel_wait_closed_on_clean_eof() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + await sender.send(1) + await sender.send_eof() + await receiver.wait_closed() + # items sent before the EOF still drain after waitclose + assert await receiver.receive() == 1 + + trio.run(main) + + +def test_channel_objects_travel_over_the_wire() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + carrier = left.open_channel() + right_carrier = right.open_channel(carrier.id) + extra = left.open_channel() + await carrier.send({"reply-to": extra}) + received = await right_carrier.receive() + remote_extra = received["reply-to"] + assert remote_extra.id == extra.id + await remote_extra.send("over the transferred channel") + assert await extra.receive() == "over the transferred channel" + + trio.run(main) + + +def test_channel_reconfigure_string_coercion() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + await sender.reconfigure(py3str_as_py2str=True) + await receiver.send("text") + # our side now loads py3 strings as bytes + assert await sender.receive() == b"text" + + trio.run(main) From 80ad5dfade4a14bedcece250af1530fcaacafa8e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 15:35:21 +0200 Subject: [PATCH 22/59] feat: run async popen gateways with remote_exec inside the user's trio run open_popen_gateway spawns a _trio_worker subprocess, does the handshake, and serves an AsyncGateway directly in the caller's nursery -- the first end-to-end trio-native path with no host thread. AsyncGateway grows remote_exec accepting the same source kinds (string / pure function / module) as the sync API; exec-finish close, RemoteError propagation, and concurrent execs all flow through the raw/serialized channel layers. Source normalization moves to _exec_source (shared by both coordinators; gateway.py re-exports the old names for its tests), and the transport helpers (staple_*, handshake, popen argv) move down into _trio_gateway so the async core has no dependency on the sync host machinery. Co-Authored-By: Claude Fable 5 --- src/execnet/_exec_source.py | 87 ++++++++++++++++++++++++ src/execnet/_trio_gateway.py | 128 +++++++++++++++++++++++++++++++++++ src/execnet/_trio_host.py | 64 ++---------------- src/execnet/_trio_worker.py | 4 +- src/execnet/gateway.py | 81 ++++------------------ testing/test_gateway.py | 4 +- testing/test_trio_gateway.py | 68 +++++++++++++++++++ testing/test_xspec.py | 4 +- 8 files changed, 307 insertions(+), 133 deletions(-) create mode 100644 src/execnet/_exec_source.py diff --git a/src/execnet/_exec_source.py b/src/execnet/_exec_source.py new file mode 100644 index 00000000..10c55795 --- /dev/null +++ b/src/execnet/_exec_source.py @@ -0,0 +1,87 @@ +"""Normalize ``remote_exec`` sources (string / function / module) to code. + +Shared by the sync coordinator ``Gateway`` and the trio-native +``AsyncGateway`` so both accept the same source kinds with identical +restrictions (pure functions taking ``channel`` first, no closures, no +non-builtin globals). +""" + +from __future__ import annotations + +import inspect +import linecache +import textwrap +import types +from collections.abc import Callable + + +def normalize_exec_source( + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + kwargs: dict[str, object], +) -> tuple[str, str | None, str | None]: + """Return ``(source, file_name, call_name)`` for a CHANNEL_EXEC payload.""" + call_name = None + file_name = None + if isinstance(source, types.ModuleType): + file_name = inspect.getsourcefile(source) + linecache.updatecache(file_name) # type: ignore[arg-type] + source = inspect.getsource(source) + elif isinstance(source, types.FunctionType): + call_name = source.__name__ + file_name = inspect.getsourcefile(source) + source = _source_of_function(source) + else: + source = textwrap.dedent(str(source)) + + if not call_name and kwargs: + raise TypeError("can't pass kwargs to non-function remote_exec") + return source, file_name, call_name + + +def _find_non_builtin_globals(source: str, codeobj: types.CodeType) -> list[str]: + import ast + import builtins + + vars = dict.fromkeys(codeobj.co_varnames) + return [ + node.id + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.Name) + and node.id not in vars + and node.id not in builtins.__dict__ + ] + + +def _source_of_function(function: types.FunctionType | Callable[..., object]) -> str: + if function.__name__ == "": + raise ValueError("can't evaluate lambda functions'") + # XXX: we dont check before remote instantiation + # if arguments are used properly + try: + sig = inspect.getfullargspec(function) + except AttributeError: + args = inspect.getargspec(function)[0] + else: + args = sig.args + if not args or args[0] != "channel": + raise ValueError("expected first function argument to be `channel`") + + closure = function.__closure__ + codeobj = function.__code__ + + if closure is not None: + raise ValueError("functions with closures can't be passed") + + try: + source = inspect.getsource(function) + except OSError as e: + raise ValueError("can't find source file for %s" % function) from e + + source = textwrap.dedent(source) # just for inner functions + + used_globals = _find_non_builtin_globals(source, codeobj) + if used_globals: + raise ValueError("the use of non-builtin globals isn't supported", used_globals) + + leading_ws = "\n" * (codeobj.co_firstlineno - 1) + return leading_ws + source diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index aefb0162..f51eec9d 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -21,7 +21,11 @@ from __future__ import annotations import math +import subprocess +import sys +import types from collections.abc import AsyncIterator +from collections.abc import Callable from contextlib import asynccontextmanager from contextlib import suppress from typing import Any @@ -29,6 +33,7 @@ import trio +from ._exec_source import normalize_exec_source from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message @@ -60,6 +65,69 @@ async def send_eof(self) -> None: ... async def aclose(self) -> None: ... +def staple_process_stream(process: trio.Process) -> ByteStream: + """One bidirectional stream over a Trio Process stdin/stdout pair.""" + assert process.stdin is not None + assert process.stdout is not None + return trio.StapledStream(process.stdin, process.stdout) + + +def staple_fd_stream(read_fd: int, write_fd: int) -> ByteStream: + """One bidirectional stream over OS pipe fds (worker stdio pipes).""" + return trio.StapledStream( + trio.lowlevel.FdStream(write_fd), trio.lowlevel.FdStream(read_fd) + ) + + +async def read_handshake_ack(stream: ByteStream, what: str) -> None: + """Wait for the worker's single ``b"1"`` ready byte.""" + ack = await stream.receive_some(1) + if ack != b"1": + raise EOFError(f"bad {what} handshake: {ack!r}") + + +async def open_popen_process(args: list[str]) -> trio.Process: + return await trio.lowlevel.open_process( + args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + +def popen_module_args(spec: Any) -> list[str]: + """Launch the Trio worker as a module: ``python -m execnet._trio_worker``. + + No source is sent over the wire; the worker imports the installed execnet + + trio. Used for same-interpreter popen and for a ``python=`` interpreter that + already has execnet (so ``sys.executable`` stays that interpreter). + """ + from . import _provision + + if getattr(spec, "python", None): + interpreter = _provision.shell_split_path(spec.python) + else: + interpreter = [sys.executable] + + args = [*interpreter, "-u"] + if getattr(spec, "dont_write_bytecode", False): + args.append("-B") + args += ["-m", "execnet._trio_worker", _provision.worker_cli_arg(spec)] + return args + + +def popen_worker_argv(spec: Any) -> list[str]: + """Argv for a popen worker: direct module launch, or uv-provisioned. + + A bare ``python=`` interpreter without execnet gets execnet + trio + provisioned via ``uv``; otherwise the worker module is launched directly. + """ + from . import _provision + + if spec.python and not _provision.target_has_execnet(spec.python): + return _provision.uv_worker_argv(spec) + return popen_module_args(spec) + + class RawChannel: """Low-level id-routed byte payload stream over an :class:`AsyncGateway`. @@ -352,6 +420,27 @@ def open_channel(self, id: int | None = None) -> AsyncChannel: channel = self._async_channels[raw.id] = AsyncChannel(raw) return channel + async def remote_exec( + self, + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + **kwargs: object, + ) -> AsyncChannel: + """Connect a new channel to remote execution of ``source``. + + Accepts the same source kinds as the sync ``Gateway.remote_exec``: + a source string, a pure function called with ``channel`` and + ``**kwargs``, or a module. The remote end closes the channel when + execution finishes. + """ + source, file_name, call_name = normalize_exec_source(source, kwargs) + channel = self.open_channel() + await self._send( + Message.CHANNEL_EXEC, + channel.id, + dumps_internal((source, file_name, call_name, kwargs)), + ) + return channel + async def terminate(self) -> None: """Send GATEWAY_TERMINATE to the peer, then close this side.""" if not self._closed: @@ -522,3 +611,42 @@ async def serve_gateway( yield gateway finally: await gateway.aclose() + + +@asynccontextmanager +async def open_popen_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: + """Spawn a popen worker and serve an AsyncGateway over its stdio. + + Runs inside the caller's own trio run -- no host thread involved. On + exit the worker is asked to terminate (GATEWAY_TERMINATE), and killed + if it has not exited within a bounded grace period. + """ + from .xspec import XSpec + + if not isinstance(spec, XSpec): + spec = XSpec(spec) + if spec.execmodel is None: + # the sync Group normally stamps this before spawning + spec.execmodel = "thread" + process = await open_popen_process(popen_worker_argv(spec)) + try: + stream = staple_process_stream(process) + await read_handshake_ack(stream, "popen") + except BaseException: + with trio.CancelScope(shield=True), trio.move_on_after(5): + process.kill() + await process.wait() + raise + try: + async with serve_gateway(stream, id=spec.id or "popen-async") as gateway: + try: + yield gateway + finally: + await gateway.terminate() + finally: + with trio.CancelScope(shield=True): + with trio.move_on_after(5): + await process.wait() + if process.returncode is None: + process.kill() + await process.wait() diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index b09dca15..78be2c71 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -24,6 +24,10 @@ from ._trio_gateway import RECEIVE_CHUNK from ._trio_gateway import ByteStream +from ._trio_gateway import open_popen_process +from ._trio_gateway import popen_worker_argv +from ._trio_gateway import read_handshake_ack +from ._trio_gateway import staple_process_stream from .gateway_base import ExecModel from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -42,27 +46,6 @@ _CLOSE_WRITE = object() -def staple_process_stream(process: trio.Process) -> ByteStream: - """One bidirectional stream over a Trio Process stdin/stdout pair.""" - assert process.stdin is not None - assert process.stdout is not None - return trio.StapledStream(process.stdin, process.stdout) - - -def staple_fd_stream(read_fd: int, write_fd: int) -> ByteStream: - """One bidirectional stream over OS pipe fds (worker stdio pipes).""" - return trio.StapledStream( - trio.lowlevel.FdStream(write_fd), trio.lowlevel.FdStream(read_fd) - ) - - -async def read_handshake_ack(stream: ByteStream, what: str) -> None: - """Wait for the worker's single ``b"1"`` ready byte.""" - ack = await stream.receive_some(1) - if ack != b"1": - raise EOFError(f"bad {what} handshake: {ack!r}") - - _CHANNEL_EOF = object() @@ -458,35 +441,6 @@ def _set() -> None: self._started = False -async def open_popen_process(args: list[str]) -> trio.Process: - return await trio.lowlevel.open_process( - args, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - - -def popen_module_args(spec: Any) -> list[str]: - """Launch the Trio worker as a module: ``python -m execnet._trio_worker``. - - No source is sent over the wire; the worker imports the installed execnet + - trio. Used for same-interpreter popen and for a ``python=`` interpreter that - already has execnet (so ``sys.executable`` stays that interpreter). - """ - from . import _provision - - if getattr(spec, "python", None): - interpreter = _provision.shell_split_path(spec.python) - else: - interpreter = [sys.executable] - - args = [*interpreter, "-u"] - if getattr(spec, "dont_write_bytecode", False): - args.append("-B") - args += ["-m", "execnet._trio_worker", _provision.worker_cli_arg(spec)] - return args - - class _TempIO: """Placeholder IO used only while constructing a Trio-backed Gateway.""" @@ -572,15 +526,7 @@ def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the worker imports execnet + trio; nothing is sent over the wire to bootstrap it. """ - from . import _provision - - if spec.python and not _provision.target_has_execnet(spec.python): - # bare interpreter: provision execnet + trio via uv - args = _provision.uv_worker_argv(spec) - else: - # same interpreter, or a python= that already has execnet - args = popen_module_args(spec) - return _open_trio_gateway(group, spec, args) + return _open_trio_gateway(group, spec, popen_worker_argv(spec)) def ssh_trio_args(spec: Any) -> tuple[list[str], bytes]: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index bb71b027..01b0186c 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -246,9 +246,9 @@ async def _start() -> _trio_host.ProtocolSession: async def _make_fd_io(read_fd: int, write_fd: int) -> Any: - from . import _trio_host + from . import _trio_gateway - return _trio_host.staple_fd_stream(read_fd, write_fd) + return _trio_gateway.staple_fd_stream(read_fd, write_fd) def serve_popen_trio(id: str, execmodel: str = "thread") -> None: diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 593bc6fe..b1d3ed46 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -5,21 +5,30 @@ from __future__ import annotations -import inspect -import linecache -import textwrap import types from collections.abc import Callable from typing import TYPE_CHECKING from typing import Any from . import gateway_base +from ._exec_source import _find_non_builtin_globals +from ._exec_source import _source_of_function +from ._exec_source import normalize_exec_source from .gateway_base import IO from .gateway_base import Channel from .gateway_base import Message from .multi import Group from .xspec import XSpec +__all__ = [ + "Gateway", + "RInfo", + "RemoteStatus", + "_find_non_builtin_globals", + "_source_of_function", + "rinfo_source", +] + class Gateway(gateway_base.BaseGateway): """Gateway to a local or remote Python Interpreter.""" @@ -127,22 +136,7 @@ def remote_exec( will be available in the global namespace of the remotely executing code. """ - call_name = None - file_name = None - if isinstance(source, types.ModuleType): - file_name = inspect.getsourcefile(source) - linecache.updatecache(file_name) # type: ignore[arg-type] - source = inspect.getsource(source) - elif isinstance(source, types.FunctionType): - call_name = source.__name__ - file_name = inspect.getsourcefile(source) - source = _source_of_function(source) - else: - source = textwrap.dedent(str(source)) - - if not call_name and kwargs: - raise TypeError("can't pass kwargs to non-function remote_exec") - + source, file_name, call_name = normalize_exec_source(source, kwargs) channel = self.newchannel() self._send( Message.CHANNEL_EXEC, @@ -185,52 +179,3 @@ def rinfo_source(channel) -> None: pid=os.getpid(), ) ) - - -def _find_non_builtin_globals(source: str, codeobj: types.CodeType) -> list[str]: - import ast - import builtins - - vars = dict.fromkeys(codeobj.co_varnames) - return [ - node.id - for node in ast.walk(ast.parse(source)) - if isinstance(node, ast.Name) - and node.id not in vars - and node.id not in builtins.__dict__ - ] - - -def _source_of_function(function: types.FunctionType | Callable[..., object]) -> str: - if function.__name__ == "": - raise ValueError("can't evaluate lambda functions'") - # XXX: we dont check before remote instantiation - # if arguments are used properly - try: - sig = inspect.getfullargspec(function) - except AttributeError: - args = inspect.getargspec(function)[0] - else: - args = sig.args - if not args or args[0] != "channel": - raise ValueError("expected first function argument to be `channel`") - - closure = function.__closure__ - codeobj = function.__code__ - - if closure is not None: - raise ValueError("functions with closures can't be passed") - - try: - source = inspect.getsource(function) - except OSError as e: - raise ValueError("can't find source file for %s" % function) from e - - source = textwrap.dedent(source) # just for inner functions - - used_globals = _find_non_builtin_globals(source, codeobj) - if used_globals: - raise ValueError("the use of non-builtin globals isn't supported", used_globals) - - leading_ws = "\n" * (codeobj.co_firstlineno - 1) - return leading_ws + source diff --git a/testing/test_gateway.py b/testing/test_gateway.py index c3c87e4a..b7098773 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -531,9 +531,9 @@ def test_no_tracing_by_default(self): ], ) def test_popen_args(spec: str, expected_args: list[str]) -> None: - from execnet import _trio_host + from execnet import _trio_gateway - args = _trio_host.popen_module_args(execnet.XSpec(spec + "//id=gw0")) + args = _trio_gateway.popen_module_args(execnet.XSpec(spec + "//id=gw0")) assert args[: len(expected_args)] == expected_args assert args[len(expected_args) :][:3] == ["-u", "-m", "execnet._trio_worker"] diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index d95622af..4825beec 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -17,6 +17,7 @@ from execnet import gateway_base from execnet._trio_gateway import AsyncGateway +from execnet._trio_gateway import open_popen_gateway from execnet.gateway_base import Message from execnet.gateway_base import RemoteError from execnet.gateway_base import dumps_internal @@ -272,6 +273,73 @@ async def main() -> None: trio.run(main) +def _remote_add(channel, a, b) -> None: # type: ignore[no-untyped-def] + channel.send(a + b) + + +class TestPopenAsyncGateway: + """Integration: an AsyncGateway serving a real popen worker inside + the user's own trio run (no host thread).""" + + def test_remote_exec_roundtrip(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + channel = await gateway.remote_exec( + "channel.send(channel.receive() + 1)" + ) + await channel.send(41) + assert await channel.receive() == 42 + await channel.wait_closed() + + trio.run(main) + + def test_remote_exec_function_with_kwargs(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + channel = await gateway.remote_exec(_remote_add, a=40, b=2) + assert await channel.receive() == 42 + + trio.run(main) + + def test_remote_error_propagates(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + channel = await gateway.remote_exec("raise ValueError('kaboom')") + with pytest.raises(RemoteError, match="kaboom"): + await channel.receive() + + trio.run(main) + + def test_exec_finish_closes_channel_ending_iteration(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + channel = await gateway.remote_exec( + "for i in range(3): channel.send(i)" + ) + assert [item async for item in channel] == [0, 1, 2] + + trio.run(main) + + def test_concurrent_remote_execs(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + results = [] + + async def run_one(value: int) -> None: + channel = await gateway.remote_exec( + "channel.send(channel.receive() * 10)" + ) + await channel.send(value) + results.append(await channel.receive()) + + async with trio.open_nursery() as nursery: + for value in range(5): + nursery.start_soon(run_one, value) + assert sorted(results) == [0, 10, 20, 30, 40] + + trio.run(main) + + def test_channel_reconfigure_string_coercion() -> None: async def main() -> None: async with gateway_pair() as (left, right): diff --git a/testing/test_xspec.py b/testing/test_xspec.py index f9681f55..661e50af 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -76,10 +76,10 @@ def test_vagrant_options(self) -> None: assert args[:-1] == ["vagrant", "ssh", "default", "--", "-C"] def test_popen_with_sudo_python(self) -> None: - from execnet import _trio_host + from execnet import _trio_gateway spec = XSpec("popen//python=sudo python3//id=gw0") - args = _trio_host.popen_module_args(spec) + args = _trio_gateway.popen_module_args(spec) assert args[:5] == ["sudo", "python3", "-u", "-m", "execnet._trio_worker"] def test_env(self) -> None: From b3b1ab871c496db9a49730516512fdfe254bd0ca Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 15:51:33 +0200 Subject: [PATCH 23/59] feat: add AsyncGroup owning gateways as nursery child scopes AsyncGroup is the trio-native group: an async context manager whose nursery serves every makegateway() as a child task. Leaving the block terminates all gateways concurrently with the safe_terminate contract -- GATEWAY_TERMINATE plus a timeout grace, then kill, bounded at roughly twice the timeout even when a kill sticks (issues #43/#221) -- with the cleanup shielded so external cancellation cannot leak workers. open_popen_gateway becomes a thin single-gateway AsyncGroup wrapper, so the popen integration tests now exercise the group termination path too. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 142 ++++++++++++++++++++++++++++------- testing/test_trio_gateway.py | 44 +++++++++++ 2 files changed, 159 insertions(+), 27 deletions(-) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index f51eec9d..37be39c8 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -28,11 +28,15 @@ from collections.abc import Callable from contextlib import asynccontextmanager from contextlib import suppress +from typing import TYPE_CHECKING from typing import Any from typing import Protocol import trio +if TYPE_CHECKING: + from typing_extensions import Self + from ._exec_source import normalize_exec_source from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -613,21 +617,8 @@ async def serve_gateway( await gateway.aclose() -@asynccontextmanager -async def open_popen_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: - """Spawn a popen worker and serve an AsyncGateway over its stdio. - - Runs inside the caller's own trio run -- no host thread involved. On - exit the worker is asked to terminate (GATEWAY_TERMINATE), and killed - if it has not exited within a bounded grace period. - """ - from .xspec import XSpec - - if not isinstance(spec, XSpec): - spec = XSpec(spec) - if spec.execmodel is None: - # the sync Group normally stamps this before spawning - spec.execmodel = "thread" +async def _connect_popen_worker(spec: Any) -> tuple[ByteStream, trio.Process]: + """Spawn a popen worker for ``spec`` and complete the ready handshake.""" process = await open_popen_process(popen_worker_argv(spec)) try: stream = staple_process_stream(process) @@ -637,16 +628,113 @@ async def open_popen_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGa process.kill() await process.wait() raise - try: - async with serve_gateway(stream, id=spec.id or "popen-async") as gateway: - try: - yield gateway - finally: - await gateway.terminate() - finally: - with trio.CancelScope(shield=True): - with trio.move_on_after(5): - await process.wait() - if process.returncode is None: - process.kill() + return stream, process + + +class AsyncGroup: + """Trio-native group: an async context manager owning the gateway nursery. + + Gateways created with :meth:`makegateway` are served as child tasks of + the group's nursery. Leaving the ``async with`` block terminates every + gateway with the safe_terminate contract: GATEWAY_TERMINATE plus a + ``timeout`` grace, then kill -- bounded at roughly twice the timeout + even when a kill gets stuck (see issues #43 / #221). + """ + + def __init__(self, termination_timeout: float = 10.0) -> None: + self._termination_timeout = termination_timeout + self._nursery: trio.Nursery | None = None + self._gateways: list[AsyncGateway] = [] + self._processes: dict[AsyncGateway, trio.Process] = {} + + def __repr__(self) -> str: + ids = [gateway.id for gateway in self._gateways] + return f"" + + async def __aenter__(self) -> Self: + self._nursery_manager = trio.open_nursery() + self._nursery = await self._nursery_manager.__aenter__() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: types.TracebackType | None, + ) -> bool | None: + # The serve tasks only end once their gateways shut down, so + # terminate before letting the nursery join its children. + # Shielded: cleanup stays bounded even under cancellation. + terminate_error: BaseException | None = None + try: + with trio.CancelScope(shield=True): + await self.terminate(self._termination_timeout) + except BaseException as error: + terminate_error = error + self._nursery = None + suppress_body_exc = await self._nursery_manager.__aexit__( + exc_type, exc_value, traceback + ) + if terminate_error is not None: + raise terminate_error + return suppress_body_exc + + async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: + """Create a gateway for ``spec`` served on the group's nursery. + + Only popen-style specs (including uv-provisioned ``python=``) are + supported on the async path for now. + """ + from .xspec import XSpec + + if self._nursery is None: + raise RuntimeError(f"{self!r} is not entered") + if not isinstance(spec, XSpec): + spec = XSpec(spec) + if not (spec.popen or spec.python): + raise ValueError(f"unsupported spec for AsyncGroup: {spec!r}") + if spec.execmodel is None: + # the sync Group normally stamps this before spawning + spec.execmodel = "thread" + if spec.id is None: + spec.id = "gw%d" % len(self._gateways) + stream, process = await _connect_popen_worker(spec) + gateway = AsyncGateway(stream, id=spec.id, _startcount=1) + await self._nursery.start(gateway._serve) + self._gateways.append(gateway) + self._processes[gateway] = process + return gateway + + async def terminate(self, timeout: float | None = None) -> None: + """Terminate all gateways; never hangs (kill after ``timeout``).""" + gateways = list(self._gateways) + self._gateways.clear() + async with trio.open_nursery() as nursery: + for gateway in gateways: + nursery.start_soon(self._terminate_one, gateway, timeout) + + async def _terminate_one( + self, gateway: AsyncGateway, timeout: float | None + ) -> None: + grace = math.inf if timeout is None else timeout + await gateway.terminate() + process = self._processes.pop(gateway, None) + if process is None: + return + with trio.move_on_after(grace): + await process.wait() + if process.returncode is None: + process.kill() + with trio.move_on_after(grace): await process.wait() + + +@asynccontextmanager +async def open_popen_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: + """Spawn one popen worker and serve an AsyncGateway over its stdio. + + Runs inside the caller's own trio run -- no host thread involved. + Convenience for a single-gateway :class:`AsyncGroup`. + """ + async with AsyncGroup() as group: + yield await group.makegateway(spec) diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 4825beec..88e52983 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -17,6 +17,7 @@ from execnet import gateway_base from execnet._trio_gateway import AsyncGateway +from execnet._trio_gateway import AsyncGroup from execnet._trio_gateway import open_popen_gateway from execnet.gateway_base import Message from execnet.gateway_base import RemoteError @@ -340,6 +341,49 @@ async def run_one(value: int) -> None: trio.run(main) +class TestAsyncGroup: + def test_multiple_gateways_with_auto_ids(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + first = await group.makegateway() + second = await group.makegateway() + assert {first.id, second.id} == {"gw0", "gw1"} + for gateway in (first, second): + channel = await gateway.remote_exec("channel.send(42)") + assert await channel.receive() == 42 + + trio.run(main) + + def test_group_exit_terminates_and_reaps_workers(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + await group.makegateway() + await group.makegateway() + processes = list(group._processes.values()) + # workers exited on GATEWAY_TERMINATE, nobody had to kill them + assert [process.returncode for process in processes] == [0, 0] + + trio.run(main) + + def test_terminate_kills_hung_worker_within_bound(self) -> None: + async def main() -> None: + async with AsyncGroup(termination_timeout=1.0) as group: + gateway = await group.makegateway() + await gateway.remote_exec("import time\nwhile True: time.sleep(1)") + processes = list(group._processes.values()) + assert all(process.returncode is not None for process in processes) + + trio.run(main) + + def test_unsupported_spec_is_rejected(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + with pytest.raises(ValueError, match="unsupported spec"): + await group.makegateway("ssh=nowhere.example.invalid") + + trio.run(main) + + def test_channel_reconfigure_string_coercion() -> None: async def main() -> None: async with gateway_pair() as (left, right): From 8286b778bef0fe068b61cdbd20bd1b91ac5173bc Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 16:44:50 +0200 Subject: [PATCH 24/59] feat: make the via tunnel frame-native over raw channel routing The via transport no longer double-frames. ChannelFactory grows a raw-receiver registry (CHANNEL_DATA payloads for registered ids route verbatim, no serialization) plus allocate_id, giving the sync gateways the low-level half of the two-level channel model. The master relay forwards the sub-worker's ready byte alone and then runs its stdout through a FrameDecoder, sending exactly one whole sub-protocol frame per CHANNEL_DATA -- and writes coordinator payloads (one frame each, by writer construction) straight to the sub's stdin. Coordinator ends of the tunnel match: RawTunnelStream (sync master, replacing the interim ChannelByteStream hack) and RawChannelStream (async master, wrapping a RawChannel as a ByteStream). AsyncGroup accepts via= specs relayed through a group member, and terminates tunneled gateways before their masters. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 92 ++++++++++++++++++++++++++--- src/execnet/_trio_host.py | 111 +++++++++++++++++++++++------------ src/execnet/gateway_base.py | 45 ++++++++++++++ testing/test_trio_gateway.py | 21 +++++++ 4 files changed, 223 insertions(+), 46 deletions(-) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 37be39c8..94f33838 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -246,6 +246,46 @@ def _close_from_remote(self, error: RemoteError | None, *, sendonly: bool) -> No self._payload_send.close() +class RawChannelStream: + """``ByteStream`` over a :class:`RawChannel` -- the frame-native via tunnel. + + The gateway writer performs one ``send_all`` per frame, so every raw + payload carries exactly one whole sub-protocol frame (the master relay + keeps that invariant in the other direction). ``receive_some`` buffers + payloads and honours ``max_bytes`` for the handshake read. + """ + + def __init__(self, raw: RawChannel) -> None: + self._raw = raw + self._buf = bytearray() + self._eof = False + + async def send_all(self, data: bytes) -> None: + await self._raw.send_bytes(data) + + async def receive_some(self, max_bytes: int | None = None) -> bytes: + if not self._buf and not self._eof: + try: + self._buf += await self._raw.receive_bytes() + except EOFError: + self._eof = True + except RemoteError as exc: + self._eof = True + raise EOFError(f"via tunnel closed: {exc}") from None + if max_bytes is None: + max_bytes = len(self._buf) + out = bytes(self._buf[:max_bytes]) + del self._buf[:max_bytes] + return out + + async def send_eof(self) -> None: + with suppress(OSError): + await self._raw.send_eof() + + async def aclose(self) -> None: + await self._raw.aclose() + + class AsyncChannel: """Serialized object API over a :class:`RawChannel`. @@ -517,7 +557,7 @@ async def _writer(self) -> None: try: async for frame in self._outbound: await self._stream.send_all(frame) - except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + except (trio.BrokenResourceError, trio.ClosedResourceError, OSError) as exc: self._trace("writer failed", exc) if self._error is None: self._error = exc @@ -682,8 +722,9 @@ async def __aexit__( async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: """Create a gateway for ``spec`` served on the group's nursery. - Only popen-style specs (including uv-provisioned ``python=``) are - supported on the async path for now. + Popen-style specs (including uv-provisioned ``python=``) and + ``via=`` sub-gateways relayed through a group member are supported + on the async path for now. """ from .xspec import XSpec @@ -698,20 +739,53 @@ async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: spec.execmodel = "thread" if spec.id is None: spec.id = "gw%d" % len(self._gateways) - stream, process = await _connect_popen_worker(spec) + process: trio.Process | None = None + if spec.via: + stream: ByteStream = await self._open_via_stream(spec) + else: + stream, process = await _connect_popen_worker(spec) gateway = AsyncGateway(stream, id=spec.id, _startcount=1) await self._nursery.start(gateway._serve) self._gateways.append(gateway) - self._processes[gateway] = process + if process is not None: + self._processes[gateway] = process return gateway + async def _open_via_stream(self, spec: Any) -> ByteStream: + """Ask the ``spec.via`` master to spawn a sub-worker; tunnel over a + raw channel (each payload one whole sub-protocol frame).""" + from . import _provision + + master = self._gateway_by_id(spec.via) + raw = master.open_raw_channel() + request = _provision.spawn_request(spec) + await master._send(Message.GATEWAY_START_SUB, raw.id, dumps_internal(request)) + stream = RawChannelStream(raw) + await read_handshake_ack(stream, "via") + return stream + + def _gateway_by_id(self, id: str) -> AsyncGateway: + for gateway in self._gateways: + if gateway.id == id: + return gateway + raise KeyError(f"no gateway {id!r} in {self!r}") + async def terminate(self, timeout: float | None = None) -> None: - """Terminate all gateways; never hangs (kill after ``timeout``).""" + """Terminate all gateways; never hangs (kill after ``timeout``). + + Tunneled (``via``) gateways go first so their termination frames + still travel through a live master. + """ gateways = list(self._gateways) self._gateways.clear() - async with trio.open_nursery() as nursery: - for gateway in gateways: - nursery.start_soon(self._terminate_one, gateway, timeout) + tunneled = [gw for gw in gateways if gw not in self._processes] + spawned = [gw for gw in gateways if gw in self._processes] + for batch in (tunneled, spawned): + if not batch: + continue + async with trio.open_nursery() as nursery: + for gateway in batch: + nursery.start_soon(self._terminate_one, gateway, timeout) async def _terminate_one( self, gateway: AsyncGateway, timeout: float | None diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 78be2c71..1bccc30e 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -49,36 +49,46 @@ _CHANNEL_EOF = object() -class ChannelByteStream: - """``ByteStream`` tunnelled over a sync execnet ``Channel``. +class RawTunnelStream: + """``ByteStream`` over a raw channel id on a sync master ``Gateway``. - INTERIM HACK: the ``via`` transport currently runs the sub-gateway protocol - as raw bytes over a channel to the master, which double-frames it (sub frame - -> CHANNEL_DATA -> master frame). This bridges the sync ``Channel`` to the - async protocol; it dissolves once low-level raw channels exist (Phase B.4). - - The channel callback (on the host loop) feeds an unbounded memory channel - that ``receive_some`` drains; writes ``channel.send`` raw bytes. + The frame-native via tunnel: the master relays whole sub-protocol + frames as verbatim CHANNEL_DATA payloads (no serialization, no + double-framing), so this stream only buffers payloads; the + sub-session's FrameDecoder sees exact frame boundaries. """ - def __init__(self, channel: Any) -> None: - self._channel = channel + def __init__(self, gateway: BaseGateway, channelid: int) -> None: + self._gateway = gateway + self.channelid = channelid self._send, self._recv = trio.open_memory_channel[Any](math.inf) self._buf = bytearray() self._eof = False - channel.setcallback(self._send.send_nowait, endmarker=_CHANNEL_EOF) + self._closed = False + gateway._channelfactory.register_raw_receiver( + channelid, self._on_data, self._on_close + ) + + def _on_data(self, data: bytes) -> None: + self._send.send_nowait(data) + + def _on_close(self, error: Any) -> None: + self._send.send_nowait(_CHANNEL_EOF if error is None else error) async def send_all(self, data: bytes) -> None: - self._channel.send(data) + self._gateway._send(Message.CHANNEL_DATA, self.channelid, data) async def receive_some(self, max_bytes: int | None = None) -> bytes: if not self._buf and not self._eof: - data = await self._recv.receive() - if data is _CHANNEL_EOF: + item = await self._recv.receive() + if item is _CHANNEL_EOF: + self._eof = True + elif isinstance(item, Exception): self._eof = True + raise EOFError(f"via tunnel closed: {item}") from None else: - assert isinstance(data, bytes) - self._buf += data + assert isinstance(item, bytes) + self._buf += item if max_bytes is None: max_bytes = len(self._buf) out = bytes(self._buf[:max_bytes]) @@ -86,10 +96,18 @@ async def receive_some(self, max_bytes: int | None = None) -> bytes: return out async def send_eof(self) -> None: - self._channel.close() + self._close_tunnel() async def aclose(self) -> None: - self._channel.close() + self._close_tunnel() + + def _close_tunnel(self) -> None: + if self._closed: + return + self._closed = True + self._gateway._channelfactory.unregister_raw_receiver(self.channelid) + with suppress(OSError): + self._gateway._send(Message.CHANNEL_CLOSE, self.channelid) async def adopt_socket(socket_fd: int) -> trio.SocketStream: @@ -689,24 +707,35 @@ def start_socketserver_via( async def _start_sub_and_relay( gateway: BaseGateway, channelid: int, request: dict[str, Any] ) -> None: - """Spawn a requested sub-worker and relay its Message protocol over the channel. - - Runs on the master's Trio host: bytes from the channel go to the sub's - stdin, and the sub's stdout goes back on the channel (the ``via`` transport). + """Spawn a requested sub-worker and relay its Message protocol frames. + + Runs on the master's Trio host (the ``via`` transport). The tunnel is + frame-native both ways: coordinator payloads arrive verbatim through the + raw-receiver registry and go to the sub's stdin unchanged (each payload + one whole frame), while the sub's stdout runs through a FrameDecoder so + every CHANNEL_DATA sent back carries exactly one frame -- except the + initial ready byte, which is forwarded on its own for the handshake. A stdin preamble (shipped wheel for a dev-version ssh sub) is streamed before the relayed protocol bytes. """ from . import _provision - channel = gateway._channelfactory.new(channelid) + def send_close_error(text: str) -> None: + with suppress(OSError): + gateway._send(Message.CHANNEL_CLOSE_ERROR, channelid, dumps_internal(text)) + try: args, preamble = _provision.sub_spawn_argv(request) process = await open_popen_process(args) except Exception as exc: - channel.close(f"could not spawn via sub-gateway: {exc}") + send_close_error(f"could not spawn via sub-gateway: {exc}") return send_ch, recv_ch = trio.open_memory_channel[Any](math.inf) - channel.setcallback(send_ch.send_nowait, endmarker=_CHANNEL_EOF) + gateway._channelfactory.register_raw_receiver( + channelid, + send_ch.send_nowait, + lambda error: send_ch.send_nowait(_CHANNEL_EOF), + ) async def coordinator_to_sub() -> None: assert process.stdin is not None @@ -721,12 +750,18 @@ async def coordinator_to_sub() -> None: async def sub_to_coordinator() -> None: assert process.stdout is not None - while True: - data = await process.stdout.receive_some(65536) - if not data: - break - channel.send(data) - channel.close() + ack = bytes(await process.stdout.receive_some(1)) + if ack: + gateway._send(Message.CHANNEL_DATA, channelid, ack) + decoder = FrameDecoder() + while True: + data = bytes(await process.stdout.receive_some(RECEIVE_CHUNK)) + if not data: + break + for message in decoder.feed(data): + gateway._send(Message.CHANNEL_DATA, channelid, message.pack()) + with suppress(OSError): + gateway._send(Message.CHANNEL_CLOSE, channelid) try: async with trio.open_nursery() as nursery: @@ -736,9 +771,9 @@ async def sub_to_coordinator() -> None: # Do not let a relay failure crash the host nursery; surface it on # the channel so the coordinator does not hang on the handshake. gateway._trace("via sub relay failed:", exc) - with suppress(Exception): - channel.close(f"via sub-gateway relay failed: {exc}") + send_close_error(f"via sub-gateway relay failed: {exc}") finally: + gateway._channelfactory.unregister_raw_receiver(channelid) with trio.move_on_after(5): await process.wait() @@ -759,14 +794,16 @@ def makegateway_via_trio(group: Any, spec: Any) -> Gateway: master = group[spec.via] host: TrioHost = group._ensure_trio_host() - channel = master.newchannel() + channelid = master._channelfactory.allocate_id() request = _provision.spawn_request(spec) - master._send(Message.GATEWAY_START_SUB, channel.id, dumps_internal(request)) remote = spec.ssh or spec.vagrant_ssh remoteaddress = f"{remote}[via {spec.via}]" if remote else None async def _create_and_attach() -> Gateway: - io = ChannelByteStream(channel) + # Register the raw receiver before the request goes out so no + # relayed frame can arrive unrouted. + io = RawTunnelStream(master, channelid) + master._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) await read_handshake_ack(io, "via") gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, io) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 9d39cce0..22afa46e 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -890,6 +890,12 @@ def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: self._callbacks: dict[ int, tuple[Callable[[Any], Any], object, tuple[bool, bool]] ] = {} + # Channel ID => (feed, on_close) receiving CHANNEL_DATA verbatim + # (no serialization) -- the low-level raw channel routing used by + # byte-shaped consumers such as the via tunnel. + self._raw_receivers: dict[ + int, tuple[Callable[[bytes], None], Callable[[RemoteError | None], None]] + ] = {} self._writelock = gateway.execmodel.Lock() self.gateway = gateway self.count = startcount @@ -910,6 +916,31 @@ def new(self, id: int | None = None) -> Channel: channel = self._channels[id] = Channel(self.gateway, id) return channel + def allocate_id(self) -> int: + """Reserve a fresh channel id without creating a Channel object.""" + with self._writelock: + if self.finished: + raise OSError(f"connection already closed: {self.gateway}") + id = self.count + self.count += 2 + return id + + def register_raw_receiver( + self, + id: int, + feed: Callable[[bytes], None], + on_close: Callable[[RemoteError | None], None], + ) -> None: + """Route CHANNEL_DATA payloads for ``id`` verbatim to ``feed``. + + ``on_close`` fires once when the channel closes (with the peer's + RemoteError, if any) or when receiving finishes. + """ + self._raw_receivers[id] = (feed, on_close) + + def unregister_raw_receiver(self, id: int) -> None: + self._raw_receivers.pop(id, None) + def channels(self) -> list[Channel]: return self._list(self._channels.values()) @@ -925,6 +956,11 @@ def _no_longer_opened(self, id: int) -> None: callback(endmarker) def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: + raw = self._raw_receivers.pop(id, None) + if raw is not None: + _feed, on_close = raw + on_close(remoteerror) + return channel = self._channels.get(id) if channel is None: # channel already in "deleted" state @@ -945,6 +981,11 @@ def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> Non def _local_receive(self, id: int, data) -> None: # executes in receiver thread + raw = self._raw_receivers.get(id) + if raw is not None: + feed, _on_close = raw + feed(data) + return channel = self._channels.get(id) try: callback, _endmarker, strconfig = self._callbacks[id] @@ -974,6 +1015,10 @@ def _finished_receiving(self) -> None: self._local_close(id, sendonly=True) for id in self._list(self._callbacks): self._no_longer_opened(id) + for id in self._list(self._raw_receivers): + item = self._raw_receivers.pop(id, None) + if item is not None: + item[1](None) class ChannelFile: diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 88e52983..630d3e7d 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -375,6 +375,27 @@ async def main() -> None: trio.run(main) + def test_via_gateway_relays_through_master(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + master = await group.makegateway("popen//id=master") + sub = await group.makegateway("popen//via=master") + master_channel = await master.remote_exec( + "import os; channel.send(os.getpid())" + ) + sub_channel = await sub.remote_exec( + "import os; channel.send(os.getpid())" + ) + master_pid = await master_channel.receive() + sub_pid = await sub_channel.receive() + # a real second process, reached through the master's relay + assert sub_pid != master_pid + echo = await sub.remote_exec("channel.send(channel.receive() * 2)") + await echo.send(21) + assert await echo.receive() == 42 + + trio.run(main) + def test_unsupported_spec_is_rejected(self) -> None: async def main() -> None: async with AsyncGroup() as group: From 57f899f78cc9b20e733c77cbb138deb5ce593f36 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 16:51:16 +0200 Subject: [PATCH 25/59] docs: record B.4 completion in the phase handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-b-async-core.md | 145 ++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 handoff-phase-b-async-core.md diff --git a/handoff-phase-b-async-core.md b/handoff-phase-b-async-core.md new file mode 100644 index 00000000..8ba250ec --- /dev/null +++ b/handoff-phase-b-async-core.md @@ -0,0 +1,145 @@ +# Handoff: Phase B — invert execnet onto an async-native Trio core + +For a fresh session on branch `feat/trio-host-thread-io`. B.1–B.4 are done; +work continues at **B.5**. Plan context lives in session memory +(`trio-port-plan`), but everything needed is restated here. + +Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` +(never grep-filter pre-commit output). ssh paths have a real local harness +in `testing/test_ssh_local.py` (asyncssh server; system ssh client needed). +Known flake: `test_socket_installvia` EOFs rarely under load. + +## Where the repo stands (2026-07-24, after commit `8286b77`) + +Trio is the only IO path; no source is shipped over the wire (workers run +`python -m execnet._trio_worker `, foreign/remote interpreters +are uv-provisioned via `_provision.py`, dev coordinators ship a wheel). All +transports work: popen, `python=`, ssh, vagrant_ssh, socket, and `via` +sub-gateways (`GATEWAY_START_SUB` spawn-request + byte relay on the master). +The architecture is still sync-first: Trio hides behind the sync +`Channel`/`Gateway`/`Group`. Phase B inverts this. + +File map (src/execnet/): + +| file | role | +|---|---| +| `gateway_base.py` | `Message` wire protocol + **`FrameDecoder`** (sans-IO), serializer (incl. duck-typed `save_AsyncChannel` → CHANNEL opcode), sync `Channel`/`ChannelFactory` (now with **raw-receiver registry**: `register_raw_receiver`/`allocate_id`, verbatim CHANNEL_DATA routing), `BaseGateway`/`WorkerGateway`, `ExecModel`, `WorkerPool`, `HostNotFound` | +| `_trio_gateway.py` | **the async-native core (B.4)**: `ByteStream` Protocol, `RawChannel` (id-routed verbatim byte payloads, sync-Channel close semantics: OSError on closed sends, EOFError/RemoteError on receive, `send_eof` = LAST_MESSAGE/sendonly), `AsyncChannel` (dumps/loads per item, strconfig/RECONFIGURE, timeout via `fail_after`, `wait_closed`, channel-passing), `AsyncGateway` (single serve task: reader dispatches inline — no locks; writer drains unbounded queue, one `send_all` per frame), `AsyncGroup` (async CM owning the nursery; `makegateway` popen/`python=`/`via=`; terminate = tunneled first, then GATEWAY_TERMINATE + timeout + kill, ~2×timeout bound), `RawChannelStream` (RawChannel→ByteStream adapter), `open_popen_gateway`, transport helpers (`staple_*`, `read_handshake_ack`, `popen_worker_argv`) | +| `portal.py` | **`LoopPortal`** (trio-token holder) and **`SyncReceiver`** (loop→plain-thread queue, KI-interruptible `get()`) | +| `_trio_host.py` | sync-facade host: `TrioHost` (loop thread), `ProtocolSession`, `RawTunnelStream` (via tunnel over sync master, raw registry based), gateway factories, `GATEWAY_START_*` handlers (`_start_sub_and_relay` is frame-native: ready byte alone, then one whole frame per CHANNEL_DATA) | +| `_trio_worker.py` | worker entry, `TrioWorkerExec` (FIFO `_pump` admission), `_prepare_protocol_fds` | +| `gateway.py` | sync coordinator `Gateway` (remote_exec, exit, rinfo) | +| `_exec_source.py` | remote_exec source normalization shared by sync + async coordinators | +| `multi.py` | sync `Group`, `MultiChannel`, `safe_terminate` (WorkerPool-based) | +| `_provision.py` | uv provisioning, wheel build/ship/materialize, argv builders | + +Async-core tests live in `testing/test_trio_gateway.py` (memory_stream_pair +protocol tests + popen/via integration inside `trio.run`). + +Semantics that MUST survive (xdist depends on them): + +- Sends from non-loop threads block until the frame hit the OS write + (120s → OSError) so abrupt `os._exit` cannot drop "sent" data; sends from + the loop thread only enqueue. After close: `OSError("cannot send (already + closed?)")`; `trio.RunFinishedError` maps to the same. +- exec admission order = message arrival order (see `TrioWorkerExec._pump`; + `test_main_thread_only_concurrent_remote_exec_deadlock` guards this — + trio shuffles its run batch, so never rely on task-spawn order). +- Channel callbacks run on the receiver (loop) thread — keep for now. + +## Phase B goal + +Three public namespaces, all pure Trio (**no anyio/asyncio** — an +`execnet.anyio` backend is deferred Phase E; keep idioms portable: neutral +stream protocol, sans-IO protocol logic, no gratuitous trio-only constructs): + +``` +execnet.sync – today's blocking API, rebuilt as a facade (top-level + execnet.Group etc. stay as compatibility aliases) +execnet.trio – trio-native AsyncGroup/AsyncGateway/AsyncChannel, awaited + directly inside the user's own trio.run +execnet.portal – the communicating API (exists as portal.py; public + exposure happens in B.6) +``` + +## Remaining work + +### B.4 Async-native core — DONE (commits `0eb6cc5`..`8286b77`) + +Everything lives in `_trio_gateway.py` (see file map). Landed: +RawChannel + AsyncChannel two-level model, AsyncGateway single-task +dispatch (no `_receivelock` on the async path), AsyncGroup with bounded +nursery-scoped termination, async popen/`python=`/`via=` gateways with +`remote_exec` inside the user's own `trio.run`, and the frame-native via +tunnel (raw-receiver registry in the sync `ChannelFactory`; +`ChannelByteStream` is gone). + +Leftovers deliberately deferred: + +- **ChannelFile / makefile over RawChannel** — do together with the B.5 + facade (the current `ChannelFileRead` is str-based; decide there whether + a text wrapper stays for backward compat). +- rsync data plane / wheel shipping over raw channels — later. +- async ssh/socket gateways — the async `makegateway` only does + popen-style and `via=`; other transports stay on the sync host path + until B.5/B.6 need them. +- worker-side CHANNEL_EXEC on an AsyncGateway is rejected with a + RemoteError ("unsupported message") — async exec is Phase C + (`exec=task`). + +### B.5 Rebuild the sync API as a facade + +Public surface stays byte-for-byte: `Group`, `makegateway` spec strings, +`Gateway.remote_exec/exit/reconfigure/remote_status/hasreceiver`, `Channel` +send/receive/setcallback/makefile/waitclose/reconfigure, `MultiChannel`, +`RSync`, `execnet.dumps/loads/dump/load`, `HostNotFound`, `TimeoutError`, +`RemoteError`, `DataFormatError`. Existing tests keep running against the +facade unchanged — that is the acceptance bar. + +Known tricky spots: + +- `Channel.__del__` sends CHANNEL_CLOSE/LAST_MESSAGE during GC — must go + through the portal as a non-waiting post, and tolerate interpreter + shutdown (today: `suppress(OSError, ValueError)` + `Message is not None` + guard). +- KeyboardInterrupt lands on the main thread while blocked inside a portal + call — decide propagation (today the sync side blocks in + `threading.Event.wait`, which KI can interrupt). +- `group.execmodel` / `set_execmodel` / `spec.execmodel` stay as deprecated + shims. `main_thread_only` must keep working throughout (pytest-xdist runs + GUI-bound code on the worker main thread via + `TrioWorkerExec.integrate_as_primary_thread`). +- Retire `ExecModel` internals and `WorkerPool` at the END of the phase + (WorkerPool is still the worker exec duck-type target for STATUS and is + used by `multi.safe_terminate` + `testing/test_threadpool.py` + + `testing/conftest.py`'s `pool` fixture). + +### B.6 Namespace split + +Introduce `execnet.sync` / `execnet.trio` / `execnet.portal`; top-level +`execnet.*` aliases into `execnet.sync`. Existing suite pinned to the +facade; add trio-native tests (memory_stream_pair + FrameDecoder for +protocol-level, real popen gateways inside `trio.run` for integration). + +## Invariants (do not regress) + +- No source shipping, ever. Workers import installed execnet+trio; rough + major/minor version check (`_trio_worker._check_version`) warns. +- Wheel-on-demand for `GATEWAY_START_SUB` is a recorded TODO in + `_provision.spawn_request` (currently ships eagerly whenever the sub-spec + has `python=`/`ssh=`). +- Worker teardown: `_trio_worker._run_worker` ends with `os._exit(0)` + because trio's `to_thread` cache uses non-daemon threads. +- `Group.terminate(timeout)` must never hang (bounded even when kill is + stuck). + +## After Phase B (context, don't build now) + +- **C**: worker config axes `loop=thread|main` × `exec=thread|main|task` + (compat: `execmodel=thread` → loop=main+exec=thread; `main_thread_only` → + loop=thread+exec=main); `exec=task` later enables async remote_exec. +- **D**: docs + trio-surface tests, pytest-xdist verification. +- **E** (deferred): `execnet.anyio` — coordinator core on anyio, + `backend="asyncio"` knob for the facade. `FrameDecoder` and the B.4 + channel layers are IO-free by construction, so they port as-is; anyio's + `StapledByteStream` satisfies the `ByteStream` protocol structurally. From f932084db8e0dadd362078a36b74b3c66649096e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 23:34:23 +0200 Subject: [PATCH 26/59] feat: rebuild the sync API as a facade over the async gateway engine The sync coordinator and worker now run their protocol IO on the B.4 async core instead of the parallel ProtocolSession implementation: - AsyncGateway grows per-frame write acknowledgements (outbound queue items carry an optional on_written callback; the writer fails pending frames on shutdown instead of stranding their senders), plus a _finalize hook for subclass shutdown work. - AsyncGroup learns every transport (ssh=, vagrant_ssh=, socket= with installvia=, alongside popen/python=/via=), an overridable gateway factory, per-process reaper tasks, and remoteaddress stamping. - SyncBridgeGateway subclasses AsyncGateway to dispatch messages into the classic sync Message handlers under the receive lock; it keeps the xdist invariant that non-loop-thread sends block until the OS write (120s -> OSError) while loop-thread sends only enqueue, all in one portal-posted FIFO. ProtocolSession is gone; the worker serves on the same bridge. - multi.Group owns a FacadeAsyncGroup task on its TrioHost: makegateway delegates to AsyncGroup.makegateway, and terminate keeps the exit()/join() member contract while the bounded GATEWAY_TERMINATE + grace + kill shutdown runs through AsyncGroup.terminate. - Channel.__del__ posts its close message through the portal without waiting, so GC never blocks on a dying loop. - RawTunnelStream.aclose now feeds EOF to its own reader; previously a terminated via bridge could wait forever on a reader that no longer had a registered raw receiver. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 235 +++++++++++++-- src/execnet/_trio_host.py | 539 +++++++++++++---------------------- src/execnet/_trio_worker.py | 2 +- src/execnet/gateway_base.py | 19 +- src/execnet/multi.py | 61 ++-- testing/test_trio_gateway.py | 2 +- 6 files changed, 460 insertions(+), 398 deletions(-) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 94f33838..fa4f2ba9 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -40,6 +40,7 @@ from ._exec_source import normalize_exec_source from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate +from .gateway_base import HostNotFound from .gateway_base import Message from .gateway_base import RemoteError from .gateway_base import TimeoutError @@ -411,6 +412,8 @@ class AsyncGateway: """ _error: BaseException | None = None + #: where the peer lives, when the transport knows (ssh host, socket addr) + remoteaddress: str | None = None def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None: self._stream = stream @@ -420,7 +423,9 @@ def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None self._channelfactory = _AsyncChannelFactory(self) self._count = _startcount self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) - self._outbound_send, self._outbound = trio.open_memory_channel[bytes](math.inf) + self._outbound_send, self._outbound = trio.open_memory_channel[ + tuple[bytes, Callable[[BaseException | None], None] | None] + ](math.inf) self._closed = False self._serve_started = False self._writer_done = trio.Event() @@ -526,10 +531,20 @@ async def _serve( finally: self._closed = True self._outbound_send.close() - self._finish_channels() - with trio.CancelScope(shield=True), suppress(Exception): - await self._stream.aclose() - self._done.set() + try: + await self._finalize() + finally: + with trio.CancelScope(shield=True), suppress(Exception): + await self._stream.aclose() + self._done.set() + + async def _finalize(self) -> None: + """Serve-shutdown hook: release the channel layer. + + The sync facade overrides this to close its sync channels and shut + down execution instead. + """ + self._finish_channels() async def _reader(self) -> None: decoder = FrameDecoder() @@ -554,21 +569,43 @@ async def _reader(self) -> None: self._error = exc async def _writer(self) -> None: + error: BaseException | None = None try: - async for frame in self._outbound: - await self._stream.send_all(frame) + async for frame, on_written in self._outbound: + try: + await self._stream.send_all(frame) + except BaseException as exc: + error = exc + if on_written is not None: + on_written(exc) + raise + if on_written is not None: + on_written(None) except (trio.BrokenResourceError, trio.ClosedResourceError, OSError) as exc: self._trace("writer failed", exc) if self._error is None: self._error = exc - self._outbound_send.close() # fail future sends fast else: # Queue closed and drained: signal write-EOF to the peer. with suppress(Exception): await self._stream.send_eof() finally: + # No more writes will happen: fail queued frames instead of + # leaving their senders waiting on acknowledgements. + self._outbound_send.close() + self._fail_pending_writes(error) self._writer_done.set() + def _fail_pending_writes(self, error: BaseException | None) -> None: + exc = error if error is not None else OSError("cannot send (already closed?)") + while True: + try: + _frame, on_written = self._outbound.receive_nowait() + except (trio.WouldBlock, trio.EndOfChannel, trio.ClosedResourceError): + return + if on_written is not None: + on_written(exc) + def _dispatch(self, message: Message) -> None: """Route one message; runs inline on the serve task.""" code = message.msgcode @@ -627,14 +664,32 @@ async def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> No # The queue is unbounded, so this never waits on the peer -- but it # is a real checkpoint and raises once the gateway is closed. try: - await self._outbound_send.send(Message(msgcode, channelid, data).pack()) + await self._outbound_send.send( + (Message(msgcode, channelid, data).pack(), None) + ) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + raise OSError("cannot send (already closed?)") from exc + + def enqueue_frame( + self, + frame: bytes, + on_written: Callable[[BaseException | None], None] | None = None, + ) -> None: + """Queue one wire frame (sync, loop thread only). + + ``on_written`` fires exactly once: after the frame reached the OS + write, or with the failure when it never will. Raises OSError when + the outbound side is already closed. + """ + try: + self._outbound_send.send_nowait((frame, on_written)) except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: raise OSError("cannot send (already closed?)") from exc def _send_nowait(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: """Enqueue a frame from a dispatch handler (sync, inline on the loop).""" - with suppress(trio.BrokenResourceError, trio.ClosedResourceError): - self._outbound_send.send_nowait(Message(msgcode, channelid, data).pack()) + with suppress(OSError): + self.enqueue_frame(Message(msgcode, channelid, data).pack()) def _finish_channels(self) -> None: for channel in list(self._channels.values()): @@ -657,18 +712,99 @@ async def serve_gateway( await gateway.aclose() -async def _connect_popen_worker(spec: Any) -> tuple[ByteStream, trio.Process]: - """Spawn a popen worker for ``spec`` and complete the ready handshake.""" - process = await open_popen_process(popen_worker_argv(spec)) +def ssh_transport_args(spec: Any) -> tuple[list[str], bytes]: + """``(ssh argv, stdin preamble)`` for an ssh worker. + + The remote runs the uv-provisioned worker; for a dev coordinator the + preamble carries the wheel bytes that the remote command receives. + """ + from . import _provision + + remote_command, preamble = _provision.ssh_remote_command(spec) + assert spec.ssh is not None + return _provision.ssh_argv(spec.ssh, spec.ssh_config, remote_command), preamble + + +def vagrant_transport_args(spec: Any) -> tuple[list[str], bytes]: + """``(vagrant ssh argv, stdin preamble)`` for a vagrant_ssh worker.""" + from . import _provision + + remote_command, preamble = _provision.ssh_remote_command(spec) + assert spec.vagrant_ssh is not None + argv = _provision.vagrant_ssh_argv( + spec.vagrant_ssh, spec.ssh_config, remote_command + ) + return argv, preamble + + +async def connect_command_worker( + args: list[str], + *, + preamble: bytes = b"", + remoteaddress: str | None = None, +) -> tuple[ByteStream, trio.Process]: + """Spawn ``args`` and complete the worker ready handshake. + + ``preamble`` is streamed to the worker's stdin before the handshake + (a shipped wheel that the remote receives with ``head -c``). With a + ``remoteaddress``, a handshake EOF plus exit code 255 (ssh could not + reach or authenticate the host) becomes :class:`HostNotFound`. + """ + process = await open_popen_process(args) try: stream = staple_process_stream(process) - await read_handshake_ack(stream, "popen") + if preamble: + await stream.send_all(preamble) + await read_handshake_ack(stream, "bootstrap") + except BaseException as exc: + host_not_found = False + with trio.CancelScope(shield=True): + if isinstance(exc, EOFError) and remoteaddress is not None: + with trio.move_on_after(5): + host_not_found = await process.wait() == 255 + with trio.move_on_after(5): + process.kill() + await process.wait() + if host_not_found: + assert remoteaddress is not None + raise HostNotFound(remoteaddress) from None + raise + return stream, process + + +async def connect_socket_worker( + address: tuple[str, int], remoteaddress: str +) -> ByteStream: + """Connect to a running socketserver and complete the ready handshake.""" + try: + stream = await trio.open_tcp_stream(*address) + except OSError as exc: + raise HostNotFound(remoteaddress) from exc + try: + await read_handshake_ack(stream, "socket") except BaseException: with trio.CancelScope(shield=True), trio.move_on_after(5): - process.kill() - await process.wait() + await stream.aclose() raise - return stream, process + return stream + + +async def start_socketserver_via( + gateway: AsyncGateway, bind_host: str = "localhost" +) -> tuple[str, int]: + """Ask ``gateway`` (protocol message) to start a one-shot socket listener. + + Returns the ``(host, port)`` the coordinator should connect to. + """ + channel = gateway.open_channel() + await gateway._send( + Message.GATEWAY_START_SOCKET, channel.id, dumps_internal(bind_host) + ) + realhost, realport = await channel.receive() + await channel.wait_closed() + if not realhost or realhost in ("0.0.0.0", "::"): + realhost = "localhost" + return realhost, int(realport) class AsyncGroup: @@ -722,9 +858,10 @@ async def __aexit__( async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: """Create a gateway for ``spec`` served on the group's nursery. - Popen-style specs (including uv-provisioned ``python=``) and - ``via=`` sub-gateways relayed through a group member are supported - on the async path for now. + All transport types are supported: popen (including uv-provisioned + ``python=``), ``ssh=``, ``vagrant_ssh=``, ``socket=`` (with + ``installvia=``), and ``via=`` sub-gateways relayed through a group + member. """ from .xspec import XSpec @@ -732,25 +869,73 @@ async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: raise RuntimeError(f"{self!r} is not entered") if not isinstance(spec, XSpec): spec = XSpec(spec) - if not (spec.popen or spec.python): - raise ValueError(f"unsupported spec for AsyncGroup: {spec!r}") if spec.execmodel is None: # the sync Group normally stamps this before spawning spec.execmodel = "thread" if spec.id is None: spec.id = "gw%d" % len(self._gateways) process: trio.Process | None = None + remoteaddress: str | None = None if spec.via: stream: ByteStream = await self._open_via_stream(spec) + remote = spec.ssh or spec.vagrant_ssh + if remote: + remoteaddress = f"{remote}[via {spec.via}]" + elif spec.socket: + address, remoteaddress = await self._resolve_socket_address(spec) + stream = await connect_socket_worker(address, remoteaddress) + elif spec.ssh: + args, preamble = ssh_transport_args(spec) + remoteaddress = spec.ssh + stream, process = await connect_command_worker( + args, preamble=preamble, remoteaddress=remoteaddress + ) + elif spec.vagrant_ssh: + args, preamble = vagrant_transport_args(spec) + remoteaddress = spec.vagrant_ssh + stream, process = await connect_command_worker( + args, preamble=preamble, remoteaddress=remoteaddress + ) + elif spec.popen or spec.python: + stream, process = await connect_command_worker(popen_worker_argv(spec)) else: - stream, process = await _connect_popen_worker(spec) - gateway = AsyncGateway(stream, id=spec.id, _startcount=1) + raise ValueError(f"unsupported spec for AsyncGroup: {spec!r}") + gateway = self._make_gateway(stream, spec) + gateway.remoteaddress = remoteaddress await self._nursery.start(gateway._serve) self._gateways.append(gateway) if process is not None: self._processes[gateway] = process + self._nursery.start_soon(self._reap_process, process) return gateway + def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: + """Construct the gateway object for a freshly connected stream. + + Overridden by the sync facade to build bridge gateways instead. + """ + return AsyncGateway(stream, id=spec.id, _startcount=1) + + async def _reap_process(self, process: trio.Process) -> None: + # Prompt reaping for workers that exit on their own (no zombies). + with suppress(Exception): + await process.wait() + + async def _resolve_socket_address(self, spec: Any) -> tuple[tuple[str, int], str]: + """``((host, port), remoteaddress)`` for a ``socket=`` spec. + + ``installvia=`` asks that group member to start a one-shot + socketserver first. The sync facade overrides this to talk to its + sync master gateway. + """ + if getattr(spec, "installvia", None): + master = self._gateway_by_id(spec.installvia) + realhost, realport = await start_socketserver_via(master) + return (realhost, realport), "%s:%d" % (realhost, realport) + assert spec.socket is not None + host_str, _, port_str = spec.socket.rpartition(":") + return (host_str, int(port_str)), spec.socket + async def _open_via_stream(self, spec: Any) -> ByteStream: """Ask the ``spec.via`` master to spawn a sub-worker; tunnel over a raw channel (each payload one whole sub-protocol frame).""" diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 1bccc30e..bf0e3a7e 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -23,11 +23,12 @@ import trio from ._trio_gateway import RECEIVE_CHUNK +from ._trio_gateway import AsyncGateway +from ._trio_gateway import AsyncGroup from ._trio_gateway import ByteStream from ._trio_gateway import open_popen_process -from ._trio_gateway import popen_worker_argv from ._trio_gateway import read_handshake_ack -from ._trio_gateway import staple_process_stream +from ._trio_gateway import ssh_transport_args from .gateway_base import ExecModel from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -40,10 +41,13 @@ if TYPE_CHECKING: from .gateway import Gateway from .gateway_base import BaseGateway + from .multi import Group T = TypeVar("T") -_CLOSE_WRITE = object() + +# Kept name: the ssh argv/preamble builder moved to the async core. +ssh_trio_args = ssh_transport_args _CHANNEL_EOF = object() @@ -106,6 +110,8 @@ def _close_tunnel(self) -> None: return self._closed = True self._gateway._channelfactory.unregister_raw_receiver(self.channelid) + # Unblock our own reader: no more payloads will be routed here. + self._send.send_nowait(_CHANNEL_EOF) with suppress(OSError): self._gateway._send(Message.CHANNEL_CLOSE, self.channelid) @@ -125,19 +131,21 @@ async def adopt_socket(socket_fd: int) -> trio.SocketStream: class SyncIOHandle: - """Sync IO facade for Group.terminate wait/kill/close_write.""" + """Sync IO facade for Gateway.exit close_write and terminate wait/kill.""" remoteaddress: str def __init__( self, execmodel: ExecModel, - session: ProtocolSession, + session: SyncBridgeGateway, *, + process: trio.Process | None = None, remoteaddress: str | None = None, ) -> None: self.execmodel = execmodel self._session = session + self._process = process if remoteaddress is not None: self.remoteaddress = remoteaddress @@ -148,68 +156,134 @@ def write(self, data: bytes) -> None: raise RuntimeError("sync write not supported on Trio IO handle") def close_read(self) -> None: - self._session.request_close_read() + return def close_write(self) -> None: self._session.request_close_write() def wait(self) -> int | None: - return self._session.wait_process() + process = self._process + if process is None: + return None + + async def _wait() -> int | None: + # Always await wait() so the child is reaped (no zombies). + code: int | None = await process.wait() + return code + + try: + return self._session.host.call(_wait) + except Exception: + return process.returncode def kill(self) -> None: - self._session.kill_process() + process = self._process + if process is None: + return + + async def _kill() -> None: + with trio.move_on_after(5): + process.kill() + await process.wait() + + try: + self._session.host.call(_kill) + except Exception as exc: + trace("ERROR killing trio process:", exc) + + +class SyncBridgeGateway(AsyncGateway): + """Async engine serving a sync ``BaseGateway``. + The reader/writer/framing machinery is inherited from + :class:`AsyncGateway`; dispatch is overridden to run the classic sync + ``Message`` handlers (channel queues, callbacks, exec scheduling) under + the gateway's receive lock instead of routing to raw channels. -class ProtocolSession: - """Reader/writer tasks for one gateway connection.""" + Foreign threads send through :meth:`enqueue_message`: every enqueue goes + through the portal so loop callbacks and threads land in one global FIFO, + and non-loop threads wait until the frame hit the OS write (120s -> + OSError) so an abrupt ``os._exit`` cannot drop already-"sent" data. + """ def __init__( self, - gateway: BaseGateway, - io: ByteStream, + stream: ByteStream, *, - process: trio.Process | None = None, + id: str, + sync_gateway: BaseGateway, host: TrioHost, ) -> None: - self.gateway = gateway - self.io = io - self.process = process + super().__init__(stream, id=id) + self.sync_gateway = sync_gateway self.host = host - self._outbound_send: trio.MemorySendChannel[object] - self._outbound_recv: trio.MemoryReceiveChannel[object] - self._outbound_send, self._outbound_recv = trio.open_memory_channel(math.inf) - self._done = threading.Event() - self._process_exitcode: int | None = None - self._process_done = threading.Event() + self._done_sync = threading.Event() self._send_closed = False - self._lock = threading.Lock() + self._send_lock = threading.Lock() - def _post_outbound(self, item: object) -> None: - """Schedule ``item`` onto the outbound channel. + # -- engine hooks (run on the host loop) -- - Goes through the portal even from the host thread so every send — - loop callbacks and foreign threads alike — lands in one global FIFO - order. Raises ``trio.RunFinishedError`` after loop shutdown. - """ - self.host.portal.post(self._outbound_send.send_nowait, item) + def _dispatch(self, message: Message) -> None: + gateway = self.sync_gateway + try: + with gateway._receivelock: + message.received(gateway) + except (GatewayReceivedTerminate, EOFError): + raise + except Exception as exc: + gateway._trace("dispatch failed:", gateway._geterrortext(exc)) + raise EOFError("error dispatching message") from exc + + async def _finalize(self) -> None: + gateway = self.sync_gateway + with self._send_lock: + self._send_closed = True + if gateway._error is None: + gateway._error = self._error + gateway._trace("[trio-bridge] finishing channels") + gateway._channelfactory._finished_receiving() + # Unblock the worker's join() before heavy exec-pool shutdown + # so the primary thread is not waiting on _done while terminate waits + # on the primary thread draining work. + self._done_sync.set() + gateway._trace("[trio-bridge] terminating execution") + # May sleep/SIGINT; keep it off the Trio scheduling thread. + await trio.to_thread.run_sync( + gateway._terminate_execution, abandon_on_cancel=True + ) + + # -- sync session interface (any thread) -- def enqueue_message(self, message: Message) -> None: """Enqueue a frame; wait until written when safe to block. - Non-host threads wait for the OS write so abrupt ``os._exit`` (xdist - crash tests) cannot drop already-"sent" data still in the queue. - - The Trio host thread (receiver callbacks) must not wait — that would - deadlock the writer task on the same event loop. + The Trio host thread (receiver callbacks) must not wait — that + would deadlock the writer task on the same event loop. """ + frame = message.pack() wait = not self.host.portal.is_loop_thread() done = threading.Event() if wait else None errors: list[BaseException] = [] - with self._lock: - if self._send_closed or self._done.is_set(): + + def on_written(error: BaseException | None) -> None: + if error is not None: + errors.append(error) + if done is not None: + done.set() + + def post() -> None: + try: + self.enqueue_frame(frame, on_written) + except OSError as exc: + on_written(exc) + + with self._send_lock: + if self._send_closed: raise OSError("cannot send (already closed?)") try: - self._post_outbound((message.pack(), done, errors)) + # Through the portal even from the host thread so every + # send lands in one global FIFO order. + self.host.portal.post(post) except trio.RunFinishedError: raise OSError("cannot send (already closed?)") from None if done is None: @@ -219,155 +293,33 @@ def enqueue_message(self, message: Message) -> None: if errors: raise OSError("cannot send (already closed?)") from errors[0] + def post_message(self, message: Message) -> None: + """Best-effort non-waiting send (Channel.__del__ during GC).""" + frame = message.pack() + + def post() -> None: + with suppress(OSError): + self.enqueue_frame(frame) + + try: + self.host.portal.post(post) + except trio.RunFinishedError: + raise OSError("cannot send (already closed?)") from None + def request_close_write(self) -> None: - with self._lock: + with self._send_lock: if self._send_closed: return self._send_closed = True with suppress(trio.RunFinishedError): - self._post_outbound(_CLOSE_WRITE) - - def request_close_read(self) -> None: - # Reader observes EOF / cancel; nothing required from callers. - return + # The writer drains queued frames, then signals write-EOF. + self.host.portal.post(self._outbound_send.close) def wait_done(self, timeout: float | None = None) -> bool: - return self._done.wait(timeout) - - def wait_process(self) -> int | None: - if self.process is None: - return None - - async def _wait() -> int | None: - assert self.process is not None - # Always await wait() so the child is reaped (no zombies). - code: int | None = await self.process.wait() - self._process_exitcode = code - self._process_done.set() - return code - - try: - return self.host.call(_wait) - except Exception: - self._process_done.wait() - return self._process_exitcode - - def kill_process(self) -> None: - if self.process is None: - return - - async def _kill() -> None: - assert self.process is not None - with trio.move_on_after(5): - self.process.kill() - self._process_exitcode = await self.process.wait() - self._process_done.set() - - try: - self.host.call(_kill) - except Exception as exc: - trace("ERROR killing trio process:", exc) + return self._done_sync.wait(timeout) def is_alive(self) -> bool: - return not self._done.is_set() - - async def task( - self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED - ) -> None: - try: - async with trio.open_nursery() as nursery: - nursery.start_soon(self._writer) - if self.process is not None: - nursery.start_soon(self._supervisor) - task_status.started() - await self._reader() - nursery.cancel_scope.cancel() - finally: - await self._finish() - - async def _reader(self) -> None: - gateway = self.gateway - - def log(*msg: object) -> None: - gateway._trace("[trio-receiver]", *msg) - - log("RECEIVER: starting") - decoder = FrameDecoder() - try: - while True: - data = await self.io.receive_some(RECEIVE_CHUNK) - if not data: - decoder.close() # raises EOFError on a mid-frame EOF - raise EOFError("clean EOF") - for msg in decoder.feed(data): - log("received", msg) - with gateway._receivelock: - msg.received(gateway) - except GatewayReceivedTerminate: - log("GATEWAY_TERMINATE") - except EOFError as exc: - log("EOF without prior gateway termination message") - gateway._error = exc - except Exception as exc: - log(gateway._geterrortext(exc)) - log("finishing receiver") - - async def _writer(self) -> None: - async for item in self._outbound_recv: - if not await self._writer_handle_item(item): - return - - async def _writer_handle_item(self, item: object) -> bool: - """Handle one outbound queue item. Return False when writer should stop.""" - if item is _CLOSE_WRITE: - try: - await self.io.send_eof() - except Exception as exc: - self.gateway._trace("send_eof failed", exc) - return False - assert isinstance(item, tuple) - blob, done, errors = item - assert isinstance(blob, bytes) - try: - await self.io.send_all(blob) - except Exception as exc: - self.gateway._trace("write failed", exc) - errors.append(exc) - with self._lock: - self._send_closed = True - finally: - if done is not None: - done.set() - return True - - async def _supervisor(self) -> None: - assert self.process is not None - try: - self._process_exitcode = await self.process.wait() - finally: - self._process_done.set() - - async def _finish(self) -> None: - gateway = self.gateway - with self._lock: - self._send_closed = True - with suppress(trio.RunFinishedError): - self._post_outbound(_CLOSE_WRITE) - gateway._trace("[trio-receiver] finishing channels") - gateway._channelfactory._finished_receiving() - # Unblock the worker's join() before heavy exec-pool shutdown - # so the primary thread is not waiting on _done while terminate waits - # on the primary thread draining work. - self._done.set() - gateway._trace("[trio-receiver] terminating execution") - # May sleep/SIGINT; keep it off the Trio scheduling thread. - await trio.to_thread.run_sync( - gateway._terminate_execution, abandon_on_cancel=True - ) - try: - await self.io.aclose() - except Exception: - pass + return not self._done_sync.is_set() class TrioHost: @@ -430,16 +382,15 @@ def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: self._nursery.start_soon(async_fn, *args) async def start_session( - self, - gateway: BaseGateway, - io: ByteStream, - *, - process: trio.Process | None = None, - ) -> ProtocolSession: + self, gateway: BaseGateway, io: ByteStream + ) -> SyncBridgeGateway: + """Serve ``gateway`` over ``io`` as a task on the root nursery.""" if self._nursery is None: raise RuntimeError("TrioHost nursery is not available") - session = ProtocolSession(gateway, io, process=process, host=self) - await self._nursery.start(session.task) + session = SyncBridgeGateway( + io, id=str(gateway.id), sync_gateway=gateway, host=self + ) + await self._nursery.start(session._serve) return session def stop(self, timeout: float | None = 5.0) -> None: @@ -484,145 +435,79 @@ def kill(self) -> None: return -def _open_trio_gateway( - group: Any, - spec: Any, - args: list[str], - *, - remoteaddress: str | None = None, - preamble: bytes = b"", -) -> Gateway: - """Spawn ``args``, do the worker handshake, and attach a Trio session. - - Shared by the popen and ssh factories; ``args`` already encodes how the - worker is launched (direct module, uv-provisioned, or wrapped in ssh). - ``preamble`` is streamed to the worker's stdin before the handshake (used to - ship a wheel to a remote that receives it with ``head -c``). - """ - import execnet - - from .gateway_base import HostNotFound - - host: TrioHost = group._ensure_trio_host() - - async def _create_and_attach() -> Gateway: - process = await open_popen_process(args) - try: - async_io = staple_process_stream(process) - if preamble: - await async_io.send_all(preamble) - await read_handshake_ack(async_io, "bootstrap") - except EOFError: - with trio.move_on_after(5): - code = await process.wait() - # ssh exits 255 when it cannot reach/authenticate the host. - if remoteaddress is not None and code == 255: - raise HostNotFound(remoteaddress) from None - with trio.move_on_after(5): - process.kill() - await process.wait() - raise - except BaseException: - with trio.move_on_after(5): - process.kill() - await process.wait() - raise - - gw = execnet.Gateway(_TempIO(group.execmodel), spec) - session = await host.start_session(gw, async_io, process=process) - gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) - return gw - - return host.call(_create_and_attach) - - -def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: - """Create a popen Gateway on the Trio IO path. +class FacadeAsyncGroup(AsyncGroup): + """AsyncGroup owning the async side of a sync ``Group``. - Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; - a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the - worker imports execnet + trio; nothing is sent over the wire to bootstrap it. + Runs on the group's :class:`TrioHost`. Gateways come out as + :class:`SyncBridgeGateway` objects bound to freshly built sync + ``Gateway`` facades, and the via / installvia flows go through the sync + master gateway (its dispatch is sync, so async channels cannot be used + on it). """ - return _open_trio_gateway(group, spec, popen_worker_argv(spec)) - - -def ssh_trio_args(spec: Any) -> tuple[list[str], bytes]: - """``(ssh argv, stdin preamble)`` for the Trio ssh path. - - The remote runs the uv-provisioned worker; for a dev coordinator the - preamble carries the wheel bytes that the remote command receives. - """ - from . import _provision - - remote_command, preamble = _provision.ssh_remote_command(spec) - assert spec.ssh is not None - return _provision.ssh_argv(spec.ssh, spec.ssh_config, remote_command), preamble - - -def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: - """Create an ssh Gateway on the Trio IO path (uv-provisioned worker).""" - args, preamble = ssh_trio_args(spec) - return _open_trio_gateway( - group, spec, args, remoteaddress=spec.ssh, preamble=preamble - ) - - -def makegateway_vagrant_trio(group: Any, spec: Any) -> Gateway: - """Create a ``vagrant ssh``-wrapped Gateway on the Trio IO path.""" - from . import _provision - - remote_command, preamble = _provision.ssh_remote_command(spec) - assert spec.vagrant_ssh is not None - args = _provision.vagrant_ssh_argv( - spec.vagrant_ssh, spec.ssh_config, remote_command - ) - return _open_trio_gateway( - group, spec, args, remoteaddress=spec.vagrant_ssh, preamble=preamble - ) + def __init__(self, group: Group, host: TrioHost) -> None: + super().__init__() + self.group = group + self.host = host + self.shutdown = trio.Event() -def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: - """Connect a Trio TCP stream to a running ``execnet-socketserver``. + def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: + import execnet - The server spawns the worker and synthesises its config, so the coordinator - just connects, waits for the worker's ``b"1"`` handshake, and attaches a Trio - session (no local process). - """ - import execnet + sync_gw = execnet.Gateway(_TempIO(self.group.execmodel), spec) + bridge = SyncBridgeGateway( + stream, id=spec.id, sync_gateway=sync_gw, host=self.host + ) + sync_gw._attach_trio_session(bridge) + return bridge - from .gateway_base import HostNotFound + async def _open_via_stream(self, spec: Any) -> ByteStream: + from . import _provision - host: TrioHost = group._ensure_trio_host() - if getattr(spec, "installvia", None): - realhost, realport = start_socketserver_via(group[spec.installvia]) - address = (realhost, realport) - remoteaddress = "%s:%d" % (realhost, realport) - else: + master = self.group[spec.via] + channelid = master._channelfactory.allocate_id() + request = _provision.spawn_request(spec) + # Register the raw receiver before the request goes out so no + # relayed frame can arrive unrouted. + io = RawTunnelStream(master, channelid) + master._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) + await read_handshake_ack(io, "via") + return io + + async def _resolve_socket_address(self, spec: Any) -> tuple[tuple[str, int], str]: + if getattr(spec, "installvia", None): + master = self.group[spec.installvia] + # Blocking sync channel receive on the master: run in a thread + # while this loop keeps dispatching the master's messages. + realhost, realport = await trio.to_thread.run_sync( + start_socketserver_via, master, abandon_on_cancel=True + ) + return (realhost, realport), "%s:%d" % (realhost, realport) assert spec.socket is not None host_str, _, port_str = spec.socket.rpartition(":") - address = (host_str, int(port_str)) - remoteaddress = spec.socket + return (host_str, int(port_str)), spec.socket - async def _create_and_attach() -> Gateway: - try: - stream = await trio.open_tcp_stream(*address) - except OSError as exc: - raise HostNotFound(remoteaddress) from exc - try: - await read_handshake_ack(stream, "socket") - except BaseException: - with trio.move_on_after(5): - await stream.aclose() - raise + async def run(self, task_status: trio.TaskStatus[FacadeAsyncGroup]) -> None: + """Own the group nursery as a host task until :attr:`shutdown`.""" + async with self: + task_status.started(self) + await self.shutdown.wait() - gw = execnet.Gateway(_TempIO(group.execmodel), spec) - session = await host.start_session(gw, stream) - gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) - return gw - return host.call(_create_and_attach) +def makegateway_trio(group: Group, spec: Any) -> Gateway: + """Create a sync-facade Gateway for ``spec`` on the group's Trio host.""" + host: TrioHost = group._ensure_trio_host() + async_group: FacadeAsyncGroup = group._ensure_async_group() + bridge = host.call(async_group.makegateway, spec) + assert isinstance(bridge, SyncBridgeGateway) + gw: Gateway = bridge.sync_gateway # type: ignore[assignment] + gw._io = SyncIOHandle( + group.execmodel, + bridge, + process=async_group._processes.get(bridge), + remoteaddress=bridge.remoteaddress, + ) + return gw _socket_worker_counter = itertools.count() @@ -784,31 +669,3 @@ def handle_start_sub(gateway: BaseGateway, channelid: int, data: bytes) -> None: assert isinstance(request, dict) host: TrioHost = gateway._trio_exec.host # type: ignore[attr-defined] host.start_soon(_start_sub_and_relay, gateway, channelid, request) - - -def makegateway_via_trio(group: Any, spec: Any) -> Gateway: - """Create a ``via`` gateway: a sub-worker spawned by and relayed through the master.""" - import execnet - - from . import _provision - - master = group[spec.via] - host: TrioHost = group._ensure_trio_host() - channelid = master._channelfactory.allocate_id() - request = _provision.spawn_request(spec) - remote = spec.ssh or spec.vagrant_ssh - remoteaddress = f"{remote}[via {spec.via}]" if remote else None - - async def _create_and_attach() -> Gateway: - # Register the raw receiver before the request goes out so no - # relayed frame can arrive unrouted. - io = RawTunnelStream(master, channelid) - master._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) - await read_handshake_ack(io, "via") - gw = execnet.Gateway(_TempIO(group.execmodel), spec) - session = await host.start_session(gw, io) - gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) - return gw - - return host.call(_create_and_attach) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 01b0186c..6a2d04f1 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -224,7 +224,7 @@ def _run_worker(host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel) - """Attach ``io`` as the gateway session and serve until shutdown.""" gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model) - async def _start() -> _trio_host.ProtocolSession: + async def _start() -> _trio_host.SyncBridgeGateway: return await host.start_session(gateway, io) session = host.call(_start) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 22afa46e..2923eaa3 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -706,7 +706,11 @@ def __del__(self) -> None: else: msgcode = Message.CHANNEL_CLOSE with suppress(OSError, ValueError): # ignore problems with sending - self.gateway._send(msgcode, self.id) + # Never wait during GC: post the close best-effort. + send = getattr( + self.gateway, "_send_nonblocking", self.gateway._send + ) + send(msgcode, self.id) def _getremoteerror(self): try: @@ -1130,6 +1134,19 @@ def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: # ValueError might be because the IO is already closed raise OSError("cannot send (already closed?)") from e + def _send_nonblocking(self, msgcode: int, channelid: int = 0) -> None: + """Best-effort send that never waits (used during GC). + + Safe to call from any thread, including while the interpreter or + the IO loop is shutting down. + """ + message = Message(msgcode, channelid) + session = self._trio_session + if session is not None: + session.post_message(message) + return + message.to_io(self._io) + def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: channel.close("execution disallowed") diff --git a/src/execnet/multi.py b/src/execnet/multi.py index c4892995..ca3b69dd 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -12,7 +12,7 @@ from collections.abc import Iterable from collections.abc import Iterator from collections.abc import Sequence -from functools import partial +from contextlib import suppress from threading import Lock from typing import TYPE_CHECKING from typing import Any @@ -51,6 +51,7 @@ def __init__( self._autoidlock = Lock() self._gateways_to_join: list[Gateway] = [] self._trio_host: Any = None + self._async_group: Any = None # we use the same execmodel for all of the Gateway objects # we spawn on our side. Probably we should not allow different # execmodels between different groups but not clear. @@ -69,6 +70,20 @@ def _ensure_trio_host(self) -> Any: self._trio_host.start() return self._trio_host + def _ensure_async_group(self) -> Any: + """The FacadeAsyncGroup owning the async side, running on the host.""" + if self._async_group is None: + from . import _trio_host + + host = self._ensure_trio_host() + + async def _start() -> Any: + async_group = _trio_host.FacadeAsyncGroup(self, host) + return await host._nursery.start(async_group.run) + + self._async_group = host.call(_start) + return self._async_group + @property def execmodel(self) -> ExecModel: return self._execmodel @@ -152,18 +167,9 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: spec.execmodel = self.remote_execmodel.backend from . import _trio_host - if spec.socket: - gw = _trio_host.makegateway_socket_trio(self, spec) - elif spec.via: - gw = _trio_host.makegateway_via_trio(self, spec) - elif spec.ssh: - gw = _trio_host.makegateway_ssh_trio(self, spec) - elif spec.vagrant_ssh: - gw = _trio_host.makegateway_vagrant_trio(self, spec) - elif spec.popen: - gw = _trio_host.makegateway_popen_trio(self, spec) - else: + if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): raise ValueError(f"no gateway type found for {spec._spec!r}") + gw = _trio_host.makegateway_trio(self, spec) gw.spec = spec self._register(gw) if spec.chdir or spec.nice or spec.env: @@ -211,6 +217,10 @@ def _unregister(self, gateway: Gateway) -> None: def _cleanup_atexit(self) -> None: trace(f"=== atexit cleanup {self!r} ===") self.terminate(timeout=1.0) + if self._async_group is not None: + with suppress(Exception): + self._trio_host.call_sync(self._async_group.shutdown.set) + self._async_group = None if self._trio_host is not None: self._trio_host.stop(timeout=1.0) self._trio_host = None @@ -225,7 +235,7 @@ def terminate(self, timeout: float | None = None) -> None: Timeout defaults to None meaning open-ended waiting and no kill attempts. """ - while self: + while self or self._gateways_to_join: vias: set[str] = set() for gw in self: if gw.spec.via: @@ -233,23 +243,16 @@ def terminate(self, timeout: float | None = None) -> None: for gw in self: if gw.id not in vias: gw.exit() - - def join_wait(gw: Gateway) -> None: + if self._async_group is not None: + # Tunneled (via) gateways terminate before their masters, + # each with a GATEWAY_TERMINATE + timeout grace, then kill; + # bounded at roughly twice the timeout (issues #43 / #221). + try: + self._trio_host.call(self._async_group.terminate, timeout) + except Exception as exc: + trace("group terminate error:", exc) + for gw in self._gateways_to_join: gw.join() - gw._io.wait() - - def kill(gw: Gateway) -> None: - trace("Gateways did not come down after timeout: %r" % gw) - gw._io.kill() - - safe_terminate( - self.execmodel, - timeout, - [ - (partial(join_wait, gw), partial(kill, gw)) - for gw in self._gateways_to_join - ], - ) self._gateways_to_join[:] = [] def remote_exec( diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 630d3e7d..a6f27981 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -400,7 +400,7 @@ def test_unsupported_spec_is_rejected(self) -> None: async def main() -> None: async with AsyncGroup() as group: with pytest.raises(ValueError, match="unsupported spec"): - await group.makegateway("ssh=nowhere.example.invalid") + await group.makegateway("id=notype") trio.run(main) From 80e237ada130a0379dc35681bb661fd67ff9c322 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 23:51:32 +0200 Subject: [PATCH 27/59] docs: record B.5 completion in the phase handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-b-async-core.md | 88 +++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/handoff-phase-b-async-core.md b/handoff-phase-b-async-core.md index 8ba250ec..9379ca6d 100644 --- a/handoff-phase-b-async-core.md +++ b/handoff-phase-b-async-core.md @@ -1,7 +1,7 @@ # Handoff: Phase B — invert execnet onto an async-native Trio core -For a fresh session on branch `feat/trio-host-thread-io`. B.1–B.4 are done; -work continues at **B.5**. Plan context lives in session memory +For a fresh session on branch `feat/trio-host-thread-io`. B.1–B.5 are done; +work continues at **B.6**. Plan context lives in session memory (`trio-port-plan`), but everything needed is restated here. Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` @@ -9,28 +9,38 @@ Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` in `testing/test_ssh_local.py` (asyncssh server; system ssh client needed). Known flake: `test_socket_installvia` EOFs rarely under load. -## Where the repo stands (2026-07-24, after commit `8286b77`) +## Where the repo stands (2026-07-24, after B.5) Trio is the only IO path; no source is shipped over the wire (workers run `python -m execnet._trio_worker `, foreign/remote interpreters are uv-provisioned via `_provision.py`, dev coordinators ship a wheel). All transports work: popen, `python=`, ssh, vagrant_ssh, socket, and `via` sub-gateways (`GATEWAY_START_SUB` spawn-request + byte relay on the master). -The architecture is still sync-first: Trio hides behind the sync -`Channel`/`Gateway`/`Group`. Phase B inverts this. + +**The inversion landed (B.5)**: there is one protocol engine — +`AsyncGateway` — and the sync API is a facade over it. `ProtocolSession` +is gone. Coordinator and worker sessions are `SyncBridgeGateway` +(an `AsyncGateway` subclass) whose `_dispatch` runs the classic sync +`Message.received` handlers under `_receivelock`; the sync +`Channel`/`ChannelFactory` state machine in `gateway_base` is unchanged +and remains the semantic layer for blocking users (and the worker's +exec'd code). `multi.Group` owns a `FacadeAsyncGroup` task on its +`TrioHost`; `makegateway` and the bounded process shutdown delegate to +`AsyncGroup`, while `terminate` still calls each member's +`exit()`/`join()` (pinned by `test_basic_group`). File map (src/execnet/): | file | role | |---|---| -| `gateway_base.py` | `Message` wire protocol + **`FrameDecoder`** (sans-IO), serializer (incl. duck-typed `save_AsyncChannel` → CHANNEL opcode), sync `Channel`/`ChannelFactory` (now with **raw-receiver registry**: `register_raw_receiver`/`allocate_id`, verbatim CHANNEL_DATA routing), `BaseGateway`/`WorkerGateway`, `ExecModel`, `WorkerPool`, `HostNotFound` | -| `_trio_gateway.py` | **the async-native core (B.4)**: `ByteStream` Protocol, `RawChannel` (id-routed verbatim byte payloads, sync-Channel close semantics: OSError on closed sends, EOFError/RemoteError on receive, `send_eof` = LAST_MESSAGE/sendonly), `AsyncChannel` (dumps/loads per item, strconfig/RECONFIGURE, timeout via `fail_after`, `wait_closed`, channel-passing), `AsyncGateway` (single serve task: reader dispatches inline — no locks; writer drains unbounded queue, one `send_all` per frame), `AsyncGroup` (async CM owning the nursery; `makegateway` popen/`python=`/`via=`; terminate = tunneled first, then GATEWAY_TERMINATE + timeout + kill, ~2×timeout bound), `RawChannelStream` (RawChannel→ByteStream adapter), `open_popen_gateway`, transport helpers (`staple_*`, `read_handshake_ack`, `popen_worker_argv`) | +| `gateway_base.py` | `Message` wire protocol + **`FrameDecoder`** (sans-IO), serializer (incl. duck-typed `save_AsyncChannel` → CHANNEL opcode), sync `Channel`/`ChannelFactory` (raw-receiver registry: `register_raw_receiver`/`allocate_id`, verbatim CHANNEL_DATA routing), `BaseGateway`/`WorkerGateway` (`_send` via bridge session, `_send_nonblocking` for GC), `ExecModel`, `WorkerPool`, `HostNotFound` | +| `_trio_gateway.py` | **the async-native core**: `ByteStream` Protocol, `RawChannel`, `AsyncChannel`, `AsyncGateway` (single serve task; outbound queue items are `(frame, on_written)` so sync senders can wait for the OS write; writer fails pending frames on shutdown; `_finalize` hook for subclass shutdown), `AsyncGroup` (async CM owning the nursery; **all transports**: popen/`python=`/ssh/vagrant_ssh/socket(+installvia)/`via=`; overridable `_make_gateway`/`_open_via_stream`/`_resolve_socket_address`; per-process reaper tasks; terminate = tunneled first, GATEWAY_TERMINATE + timeout + kill, ~2×timeout bound), `RawChannelStream`, `open_popen_gateway`, transport helpers (`connect_command_worker` incl. ssh-255→HostNotFound, `connect_socket_worker`, `start_socketserver_via` (async), `ssh_transport_args`, `popen_worker_argv`) | | `portal.py` | **`LoopPortal`** (trio-token holder) and **`SyncReceiver`** (loop→plain-thread queue, KI-interruptible `get()`) | -| `_trio_host.py` | sync-facade host: `TrioHost` (loop thread), `ProtocolSession`, `RawTunnelStream` (via tunnel over sync master, raw registry based), gateway factories, `GATEWAY_START_*` handlers (`_start_sub_and_relay` is frame-native: ready byte alone, then one whole frame per CHANNEL_DATA) | -| `_trio_worker.py` | worker entry, `TrioWorkerExec` (FIFO `_pump` admission), `_prepare_protocol_fds` | +| `_trio_host.py` | sync-facade host: `TrioHost` (loop thread; `start_session` returns a bridge), **`SyncBridgeGateway`** (sync dispatch + portal-FIFO `enqueue_message`/`post_message`/`request_close_write`, threading-`Event` `wait_done`), **`FacadeAsyncGroup`** (bridge-building AsyncGroup; via/installvia through the sync master), `makegateway_trio`, `SyncIOHandle` (remoteaddress/wait/kill/close_write), `RawTunnelStream` (via tunnel; `aclose` feeds own reader EOF), `GATEWAY_START_*` handlers (`_start_sub_and_relay` frame-native) | +| `_trio_worker.py` | worker entry, `TrioWorkerExec` (FIFO `_pump` admission), `_prepare_protocol_fds`; serves on a `SyncBridgeGateway` | | `gateway.py` | sync coordinator `Gateway` (remote_exec, exit, rinfo) | | `_exec_source.py` | remote_exec source normalization shared by sync + async coordinators | -| `multi.py` | sync `Group`, `MultiChannel`, `safe_terminate` (WorkerPool-based) | +| `multi.py` | sync `Group` facade (`_ensure_async_group`, terminate via `AsyncGroup`), `MultiChannel`, `safe_terminate` (WorkerPool-based, kept for tests) | | `_provision.py` | uv provisioning, wheel build/ship/materialize, argv builders | Async-core tests live in `testing/test_trio_gateway.py` (memory_stream_pair @@ -87,32 +97,38 @@ Leftovers deliberately deferred: RemoteError ("unsupported message") — async exec is Phase C (`exec=task`). -### B.5 Rebuild the sync API as a facade - -Public surface stays byte-for-byte: `Group`, `makegateway` spec strings, -`Gateway.remote_exec/exit/reconfigure/remote_status/hasreceiver`, `Channel` -send/receive/setcallback/makefile/waitclose/reconfigure, `MultiChannel`, -`RSync`, `execnet.dumps/loads/dump/load`, `HostNotFound`, `TimeoutError`, -`RemoteError`, `DataFormatError`. Existing tests keep running against the -facade unchanged — that is the acceptance bar. - -Known tricky spots: - -- `Channel.__del__` sends CHANNEL_CLOSE/LAST_MESSAGE during GC — must go - through the portal as a non-waiting post, and tolerate interpreter - shutdown (today: `suppress(OSError, ValueError)` + `Message is not None` - guard). -- KeyboardInterrupt lands on the main thread while blocked inside a portal - call — decide propagation (today the sync side blocks in - `threading.Event.wait`, which KI can interrupt). -- `group.execmodel` / `set_execmodel` / `spec.execmodel` stay as deprecated - shims. `main_thread_only` must keep working throughout (pytest-xdist runs - GUI-bound code on the worker main thread via - `TrioWorkerExec.integrate_as_primary_thread`). -- Retire `ExecModel` internals and `WorkerPool` at the END of the phase - (WorkerPool is still the worker exec duck-type target for STATUS and is - used by `multi.safe_terminate` + `testing/test_threadpool.py` + - `testing/conftest.py`'s `pool` fixture). +### B.5 Rebuild the sync API as a facade — DONE (commit `f932084`) + +Public surface stayed byte-for-byte; the whole sync suite passes +unchanged (508 passed). What landed, and the decisions taken: + +- One engine: `SyncBridgeGateway(AsyncGateway)` serves both coordinator + and worker; `ProtocolSession` deleted. Sync dispatch still runs the + `gateway_base` Message handlers, so the sync `Channel`/`ChannelFactory` + semantics (queues, callbacks, ENDMARKER, weakref drop) are untouched — + and remain what the worker's exec'd code sees. +- Send invariant kept: `enqueue_message` posts every frame through the + portal (one global FIFO); non-loop threads wait on the frame's + `on_written` ack (120s → OSError), the loop thread only enqueues. +- `Channel.__del__` now goes through `_send_nonblocking` → + `SyncBridgeGateway.post_message` (portal post, never waits; falls back + to `_send` for duck-typed test gateways). +- KI decision: blocking data paths (send-ack, receive, waitclose, join) + wait on `threading.Event`/`queue.Queue` — KI-interruptible as before. + Management ops (makegateway, terminate) run inside `portal.run` + (`trio.from_thread.run`) where KI is deferred; accepted for now. +- `Group.terminate` still calls member `exit()`/`join()` (contract pinned + by `test_basic_group`), with process wait/kill delegated to + `AsyncGroup.terminate` (tunneled-first, ~2×timeout bound). +- Gotcha fixed on the way: a stream-closing `aclose` on the via tunnel + (`RawTunnelStream`) must feed EOF to its own reader, else the bridge's + serve task never finishes and terminate hangs. +- `ChannelFile`/`makefile`: kept the existing str-based `gateway_base` + classes as-is for backward compat (they only use channel send/receive). +- NOT yet retired: `ExecModel` internals and `WorkerPool` (still the + worker STATUS duck-type shape and used by `multi.safe_terminate`, + `testing/test_threadpool.py`, conftest's `pool` fixture). Do this at + the end of the phase (B.6 or later) if the tests pinning them move. ### B.6 Namespace split From 4f84335d5a465c8971376663ef47e0cc3f6ba648 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 16:53:36 +0200 Subject: [PATCH 28/59] feat: split the public surface into execnet.sync / execnet.trio / execnet.portal execnet.sync re-exports the blocking facade; the top-level execnet.* names now alias into it. execnet.trio exposes the async-native core (AsyncGroup/AsyncGateway/AsyncChannel, raw channels, stream helpers, shared serialization + errors) for use inside the caller's own trio.run. execnet.portal (LoopPortal/SyncReceiver) is the communicating layer both build on. The trio and portal modules load lazily via module __getattr__ so plain `import execnet` still does not import the trio event loop. Co-Authored-By: Claude Fable 5 --- src/execnet/__init__.py | 59 +++++++++++++++++--------- src/execnet/portal.py | 2 + src/execnet/sync.py | 49 ++++++++++++++++++++++ src/execnet/trio.py | 61 +++++++++++++++++++++++++++ testing/test_namespaces.py | 85 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 237 insertions(+), 19 deletions(-) create mode 100644 src/execnet/sync.py create mode 100644 src/execnet/trio.py create mode 100644 testing/test_namespaces.py diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index cb91cb66..2108f4b2 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -4,29 +4,40 @@ pure python lib for connecting to local and remote Python Interpreters. +Three public namespaces: + +* :mod:`execnet.sync` — the blocking API; the top-level ``execnet.*`` + names below are aliases into it. +* :mod:`execnet.trio` — the trio-native API, awaited inside your own + ``trio.run``. +* :mod:`execnet.portal` — cross-thread / cross-loop communication + primitives shared by both. + (c) 2012, Holger Krekel and others """ +from typing import Any + from ._version import version as __version__ -from .gateway import Gateway -from .gateway_base import Channel -from .gateway_base import DataFormatError -from .gateway_base import DumpError -from .gateway_base import HostNotFound -from .gateway_base import LoadError -from .gateway_base import RemoteError -from .gateway_base import TimeoutError -from .gateway_base import dump -from .gateway_base import dumps -from .gateway_base import load -from .gateway_base import loads -from .multi import Group -from .multi import MultiChannel -from .multi import default_group -from .multi import makegateway -from .multi import set_execmodel -from .rsync import RSync -from .xspec import XSpec +from .sync import Channel +from .sync import DataFormatError +from .sync import DumpError +from .sync import Gateway +from .sync import Group +from .sync import HostNotFound +from .sync import LoadError +from .sync import MultiChannel +from .sync import RemoteError +from .sync import RSync +from .sync import TimeoutError +from .sync import XSpec +from .sync import default_group +from .sync import dump +from .sync import dumps +from .sync import load +from .sync import loads +from .sync import makegateway +from .sync import set_execmodel __all__ = [ "Channel", @@ -50,3 +61,13 @@ "makegateway", "set_execmodel", ] + + +def __getattr__(name: str) -> Any: + # Lazy namespace modules: keep ``import execnet`` from loading the + # trio event loop machinery until it is actually used. + if name in ("trio", "portal"): + import importlib + + return importlib.import_module(f".{name}", __name__) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/execnet/portal.py b/src/execnet/portal.py index 9804ca45..a204cc5a 100644 --- a/src/execnet/portal.py +++ b/src/execnet/portal.py @@ -24,6 +24,8 @@ import trio +__all__ = ["LoopPortal", "SyncReceiver"] + T = TypeVar("T") diff --git a/src/execnet/sync.py b/src/execnet/sync.py new file mode 100644 index 00000000..0d3c835f --- /dev/null +++ b/src/execnet/sync.py @@ -0,0 +1,49 @@ +"""The blocking execnet API. + +A facade over the trio-native core in :mod:`execnet.trio`: gateways run +their protocol IO on a dedicated Trio host thread while this surface +blocks the calling thread. The top-level ``execnet.*`` names are aliases +into this module. +""" + +from .gateway import Gateway +from .gateway_base import Channel +from .gateway_base import DataFormatError +from .gateway_base import DumpError +from .gateway_base import HostNotFound +from .gateway_base import LoadError +from .gateway_base import RemoteError +from .gateway_base import TimeoutError +from .gateway_base import dump +from .gateway_base import dumps +from .gateway_base import load +from .gateway_base import loads +from .multi import Group +from .multi import MultiChannel +from .multi import default_group +from .multi import makegateway +from .multi import set_execmodel +from .rsync import RSync +from .xspec import XSpec + +__all__ = [ + "Channel", + "DataFormatError", + "DumpError", + "Gateway", + "Group", + "HostNotFound", + "LoadError", + "MultiChannel", + "RSync", + "RemoteError", + "TimeoutError", + "XSpec", + "default_group", + "dump", + "dumps", + "load", + "loads", + "makegateway", + "set_execmodel", +] diff --git a/src/execnet/trio.py b/src/execnet/trio.py new file mode 100644 index 00000000..df0ab493 --- /dev/null +++ b/src/execnet/trio.py @@ -0,0 +1,61 @@ +"""The trio-native execnet API. + +Everything here is awaited directly inside your own ``trio.run`` — no +host thread, no blocking calls:: + + import trio + import execnet.trio + + async def main(): + async with execnet.trio.AsyncGroup() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(6 * 7)") + print(await channel.receive()) + + trio.run(main) + +The serialization helpers and error types are shared with the blocking +API in :mod:`execnet.sync`. +""" + +from ._trio_gateway import AsyncChannel +from ._trio_gateway import AsyncGateway +from ._trio_gateway import AsyncGroup +from ._trio_gateway import ByteStream +from ._trio_gateway import RawChannel +from ._trio_gateway import RawChannelStream +from ._trio_gateway import open_popen_gateway +from ._trio_gateway import serve_gateway +from .gateway_base import DataFormatError +from .gateway_base import DumpError +from .gateway_base import HostNotFound +from .gateway_base import LoadError +from .gateway_base import RemoteError +from .gateway_base import TimeoutError +from .gateway_base import dump +from .gateway_base import dumps +from .gateway_base import load +from .gateway_base import loads +from .xspec import XSpec + +__all__ = [ + "AsyncChannel", + "AsyncGateway", + "AsyncGroup", + "ByteStream", + "DataFormatError", + "DumpError", + "HostNotFound", + "LoadError", + "RawChannel", + "RawChannelStream", + "RemoteError", + "TimeoutError", + "XSpec", + "dump", + "dumps", + "load", + "loads", + "open_popen_gateway", + "serve_gateway", +] diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py new file mode 100644 index 00000000..dd9b3777 --- /dev/null +++ b/testing/test_namespaces.py @@ -0,0 +1,85 @@ +"""The three public namespaces: execnet.sync / execnet.trio / execnet.portal. + +Top-level ``execnet.*`` is an alias surface over ``execnet.sync``; the +trio namespace exposes the async-native core; the portal namespace the +cross-thread primitives. +""" + +from __future__ import annotations + +import subprocess +import sys + +import execnet +import execnet.portal +import execnet.sync +import execnet.trio + + +def test_top_level_names_alias_execnet_sync() -> None: + for name in execnet.sync.__all__: + assert getattr(execnet, name) is getattr(execnet.sync, name), name + + +def test_top_level_all_matches_sync_surface() -> None: + assert set(execnet.__all__) == set(execnet.sync.__all__) | {"__version__"} + + +def test_trio_namespace_exposes_async_core() -> None: + from execnet import _trio_gateway + + assert execnet.trio.AsyncGroup is _trio_gateway.AsyncGroup + assert execnet.trio.AsyncGateway is _trio_gateway.AsyncGateway + assert execnet.trio.AsyncChannel is _trio_gateway.AsyncChannel + assert execnet.trio.open_popen_gateway is _trio_gateway.open_popen_gateway + # serialization + errors are shared with the sync surface + assert execnet.trio.RemoteError is execnet.RemoteError + assert execnet.trio.dumps is execnet.dumps + + +def test_portal_namespace() -> None: + assert execnet.portal.__all__ == ["LoopPortal", "SyncReceiver"] + assert execnet.portal.LoopPortal is not None + + +def test_lazy_submodule_attribute_access() -> None: + # After ``import execnet`` alone, execnet.trio / execnet.portal are + # reachable as attributes (PEP 562) without having been imported. + out = subprocess.run( + [ + sys.executable, + "-c", + "import execnet; print(execnet.trio.AsyncGroup.__name__);" + " print(execnet.portal.LoopPortal.__name__)", + ], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.split() == ["AsyncGroup", "LoopPortal"] + + +def test_import_execnet_does_not_import_trio() -> None: + # The blocking surface must stay importable without loading the trio + # event loop machinery (it loads lazily on first gateway / namespace + # use). + out = subprocess.run( + [ + sys.executable, + "-c", + "import sys, execnet; print('trio' in sys.modules)", + ], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.strip() == "False" + + +def test_unknown_attribute_raises() -> None: + try: + execnet.does_not_exist + except AttributeError as exc: + assert "does_not_exist" in str(exc) + else: + raise AssertionError("expected AttributeError") From 33decae92ad469da1e6568aa9b20c2c9f002590c Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 16:55:44 +0200 Subject: [PATCH 29/59] docs: record phase B completion in the handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-b-async-core.md | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/handoff-phase-b-async-core.md b/handoff-phase-b-async-core.md index 9379ca6d..ab078740 100644 --- a/handoff-phase-b-async-core.md +++ b/handoff-phase-b-async-core.md @@ -1,8 +1,8 @@ # Handoff: Phase B — invert execnet onto an async-native Trio core -For a fresh session on branch `feat/trio-host-thread-io`. B.1–B.5 are done; -work continues at **B.6**. Plan context lives in session memory -(`trio-port-plan`), but everything needed is restated here. +For a fresh session on branch `feat/trio-host-thread-io`. **Phase B is +complete (B.1–B.6)**; work continues at **Phase C**. Plan context lives in +session memory (`trio-port-plan`), but everything needed is restated here. Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` (never grep-filter pre-commit output). ssh paths have a real local harness @@ -130,12 +130,28 @@ unchanged (508 passed). What landed, and the decisions taken: `testing/test_threadpool.py`, conftest's `pool` fixture). Do this at the end of the phase (B.6 or later) if the tests pinning them move. -### B.6 Namespace split - -Introduce `execnet.sync` / `execnet.trio` / `execnet.portal`; top-level -`execnet.*` aliases into `execnet.sync`. Existing suite pinned to the -facade; add trio-native tests (memory_stream_pair + FrameDecoder for -protocol-level, real popen gateways inside `trio.run` for integration). +### B.6 Namespace split — DONE (commit `4f84335`) + +`execnet.sync` (blocking facade re-exports; top-level `execnet.*` aliases +into it), `execnet.trio` (AsyncGroup/AsyncGateway/AsyncChannel, raw +channels, stream helpers, shared serialization + errors), and +`execnet.portal` (LoopPortal/SyncReceiver). `trio`/`portal` load lazily +via module `__getattr__`, so `import execnet` still does not import the +trio event loop (pinned by `testing/test_namespaces.py`). Trio-native +protocol/integration tests already live in `testing/test_trio_gateway.py` +(memory_stream_pair + real popen gateways inside `trio.run`). + +### End-of-phase leftovers (deferred into C/D) + +- Retire `ExecModel` internals and `WorkerPool`: still pinned by + `multi.safe_terminate`, `testing/test_threadpool.py`, and conftest's + `pool` fixture, and `get_execmodel`/backend names feed Phase C's + `execmodel=` compat mapping — retire once C lands the new worker + config axes. +- ChannelFile / makefile over RawChannel (async makefile) — untouched; + the sync str-based wrappers stay for backward compat. +- rsync data plane / wheel shipping over raw channels — later. +- Async remote_exec on the worker (`exec=task`) — Phase C. ## Invariants (do not regress) From 51b9053a7663e088532e7b791017a24a7ec92666 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 20:38:41 +0200 Subject: [PATCH 30/59] fix: attach the bridge session before it can serve messages The worker created its SyncBridgeGateway inside host.call and only attached it to the sync gateway afterwards on the main thread. A coordinator message arriving in that window (STATUS, CHANNEL_EXEC as the first action on a fresh gateway) dispatched into a gateway whose _trio_session was still None, so the reply fell through to the sync IO stub and killed the session -- the whole test suite under pytest -n 12 showed this as freshly created gateways being dead on arrival. The race predates the B.5 facade (ProtocolSession had the same window). SyncBridgeGateway now attaches itself in __init__, before its serve task can dispatch anything, and the redundant attach calls at the two construction sites are gone. Also drop the apipkg-era unknown-attribute test from the namespace tests -- unknown attributes are plain Python module behaviour now. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_host.py | 8 +++++--- src/execnet/_trio_worker.py | 5 +++-- testing/test_namespaces.py | 9 --------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index bf0e3a7e..4d05690d 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -220,6 +220,10 @@ def __init__( self._done_sync = threading.Event() self._send_closed = False self._send_lock = threading.Lock() + # Attach before any serving can happen: the first inbound message + # may need to reply through gateway._send, which must already + # route to this session (not the sync IO stub). + sync_gateway._attach_trio_session(self) # -- engine hooks (run on the host loop) -- @@ -455,11 +459,9 @@ def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: import execnet sync_gw = execnet.Gateway(_TempIO(self.group.execmodel), spec) - bridge = SyncBridgeGateway( + return SyncBridgeGateway( stream, id=spec.id, sync_gateway=sync_gw, host=self.host ) - sync_gw._attach_trio_session(bridge) - return bridge async def _open_via_stream(self, spec: Any) -> ByteStream: from . import _provision diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 6a2d04f1..bfaeea43 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -225,10 +225,11 @@ def _run_worker(host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel) - gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model) async def _start() -> _trio_host.SyncBridgeGateway: + # The bridge attaches itself to the gateway before serving starts, + # so inbound messages can reply through gateway._send right away. return await host.start_session(gateway, io) - session = host.call(_start) - gateway._attach_trio_session(session) + host.call(_start) try: if main_thread_only: diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index dd9b3777..5f44f8a9 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -74,12 +74,3 @@ def test_import_execnet_does_not_import_trio() -> None: check=True, ) assert out.stdout.strip() == "False" - - -def test_unknown_attribute_raises() -> None: - try: - execnet.does_not_exist - except AttributeError as exc: - assert "does_not_exist" in str(exc) - else: - raise AssertionError("expected AttributeError") From a69b844d9e5f0d80546147218f18da56d59bb7cf Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 20:40:39 +0200 Subject: [PATCH 31/59] chore: add pytest-xdist to the dev group, ignore claude local settings Co-Authored-By: Claude Fable 5 --- .gitignore | 1 + pyproject.toml | 3 +++ uv.lock | 19 ++++++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d734b32f..9f676ac3 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ include/ .vagrant.d/ .config/ .local/ +.claude/settings.local.json diff --git a/pyproject.toml b/pyproject.toml index f419c31d..4e94c938 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,9 @@ testing = [ ] [dependency-groups] +dev = [ + "pytest-xdist>=3.8.0", +] testing = [ "execnet[testing]", ] diff --git a/uv.lock b/uv.lock index e4657f7b..15185f2a 100644 --- a/uv.lock +++ b/uv.lock @@ -372,7 +372,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -398,6 +398,9 @@ testing = [ ] [package.dev-dependencies] +dev = [ + { name = "pytest-xdist" }, +] testing = [ { name = "execnet", extra = ["testing"] }, ] @@ -416,6 +419,7 @@ requires-dist = [ provides-extras = ["testing"] [package.metadata.requires-dev] +dev = [{ name = "pytest-xdist", specifier = ">=3.8.0" }] testing = [{ name = "execnet", extras = ["testing"] }] [[package]] @@ -819,6 +823,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-discovery" version = "1.5.0" From 72b0eefd392f7f97f4693751091c574bf10b918e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 20:46:41 +0200 Subject: [PATCH 32/59] =?UTF-8?q?docs:=20hand=20off=20phase=20C/D=20?= =?UTF-8?q?=E2=80=94=20worker=20axes=20gap=20analysis,=20supersede=20the?= =?UTF-8?q?=20B=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- handoff-phase-b-async-core.md | 11 ++- handoff-phase-c-worker-axes.md | 139 +++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 handoff-phase-c-worker-axes.md diff --git a/handoff-phase-b-async-core.md b/handoff-phase-b-async-core.md index ab078740..d3a69ff6 100644 --- a/handoff-phase-b-async-core.md +++ b/handoff-phase-b-async-core.md @@ -1,8 +1,8 @@ # Handoff: Phase B — invert execnet onto an async-native Trio core -For a fresh session on branch `feat/trio-host-thread-io`. **Phase B is -complete (B.1–B.6)**; work continues at **Phase C**. Plan context lives in -session memory (`trio-port-plan`), but everything needed is restated here. +**SUPERSEDED by `handoff-phase-c-worker-axes.md`** — kept as the Phase B +record. **Phase B is complete (B.1–B.6)** on branch +`feat/trio-host-thread-io`; work continues at **Phase C**. Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` (never grep-filter pre-commit output). ssh paths have a real local harness @@ -123,6 +123,11 @@ unchanged (508 passed). What landed, and the decisions taken: - Gotcha fixed on the way: a stream-closing `aclose` on the via tunnel (`RawTunnelStream`) must feed EOF to its own reader, else the bridge's serve task never finishes and terminate hangs. +- Post-B fix (`51b9053`): the session-attach startup race — the bridge + must attach itself to the sync gateway in `__init__`, before serving + starts, or a first-message reply goes through the IO stub and kills + the fresh gateway (predated B.5; surfaced by `pytest -n 12`). The + suite now runs xdist-clean. - `ChannelFile`/`makefile`: kept the existing str-based `gateway_base` classes as-is for backward compat (they only use channel send/receive). - NOT yet retired: `ExecModel` internals and `WorkerPool` (still the diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md new file mode 100644 index 00000000..ced2d895 --- /dev/null +++ b/handoff-phase-c-worker-axes.md @@ -0,0 +1,139 @@ +# Handoff: Phase C — worker config axes (`loop=` × `exec=`) + Phase D + +For a fresh session on branch `feat/trio-host-thread-io`. Phase B (the +inversion onto the async-native Trio core) is **complete**; this doc +supersedes `handoff-phase-b-async-core.md` (kept for the B record). +Plan context lives in session memory (`trio-port-plan`), but everything +needed is restated here. + +Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` +(never grep-filter pre-commit output). The suite is xdist-clean: `uv run +pytest testing/ -n 12` passes (~7s) since the session-attach race fix +(`51b9053`) — keep it that way. ssh paths have a real local harness in +`testing/test_ssh_local.py` (asyncssh server; system ssh client needed). +Known flake: `test_socket_installvia` EOFs rarely under load. + +## Where the repo stands (2026-07-25, after `a69b844`) + +One protocol engine: `AsyncGateway` (`_trio_gateway.py`). The sync API +is a facade over it — coordinator and worker sessions are +`SyncBridgeGateway` (an `AsyncGateway` subclass in `_trio_host.py`) whose +`_dispatch` runs the classic sync `Message.received` handlers under +`_receivelock`; `SyncBridgeGateway.__init__` attaches itself to the sync +gateway *before* serving starts (the fix for the startup race where a +STATUS/CHANNEL_EXEC arriving first replied through the IO stub and killed +the session). `multi.Group` owns a `FacadeAsyncGroup` task on its +`TrioHost`; makegateway and bounded process shutdown delegate to +`AsyncGroup`, while `terminate` keeps the member `exit()`/`join()` +contract (pinned by `test_basic_group`). `AsyncGroup` supports every +transport (popen, `python=`, ssh, vagrant_ssh, socket with installvia, +`via=`). Public namespaces: `execnet.sync` (top-level `execnet.*` +aliases into it), `execnet.trio`, `execnet.portal` (trio/portal lazy via +module `__getattr__`; `import execnet` must not import trio — pinned by +`testing/test_namespaces.py`). + +No source is shipped over the wire: workers run +`python -m execnet._trio_worker `; foreign/remote +interpreters are uv-provisioned via `_provision.py`; dev coordinators +ship a wheel. + +File map (src/execnet/): + +| file | role | +|---|---| +| `gateway_base.py` | `Message` wire protocol + sans-IO `FrameDecoder`, serializer (CHANNEL opcode incl. duck-typed `save_AsyncChannel`), sync `Channel`/`ChannelFactory` (raw-receiver registry), `BaseGateway`/`WorkerGateway` (`_send` via bridge, `_send_nonblocking` for GC), `ExecModel`, `WorkerPool`, `HostNotFound` | +| `_trio_gateway.py` | async core: `ByteStream` Protocol, `RawChannel`/`AsyncChannel`, `AsyncGateway` (outbound queue of `(frame, on_written)`; `_finalize` hook), `AsyncGroup` (all transports, overridable `_make_gateway`/`_open_via_stream`/`_resolve_socket_address`, reapers, bounded terminate), stream/argv helpers | +| `_trio_host.py` | `TrioHost` (loop thread), `SyncBridgeGateway`, `FacadeAsyncGroup`, `makegateway_trio`, `SyncIOHandle`, `RawTunnelStream` (via tunnel; `aclose` feeds own reader EOF), `GATEWAY_START_*` handlers | +| `_trio_worker.py` | worker entry (`_main`/`serve_popen_trio`/`serve_socket_trio`), `TrioWorkerExec` (FIFO `_pump` admission; `integrate_as_primary_thread`), `_prepare_protocol_fds`, `_check_version` | +| `gateway.py` / `multi.py` | sync `Gateway` / sync `Group` facade, `MultiChannel`, `safe_terminate` (WorkerPool-based, kept for tests) | +| `sync.py` / `trio.py` / `portal.py` | the three public namespaces | +| `_exec_source.py`, `_provision.py`, `xspec.py`, `rsync.py` | source normalization, uv provisioning + argv builders, spec parsing, rsync | + +## Semantics that MUST survive (xdist depends on them) + +- Sends from non-loop threads block until the frame hit the OS write + (120s → OSError) so abrupt `os._exit` cannot drop "sent" data; loop + thread sends only enqueue. All sends go through one portal-posted FIFO. +- After close: `OSError("cannot send (already closed?)")`; + `trio.RunFinishedError` maps to the same. +- exec admission order = message arrival order (`TrioWorkerExec._pump`; + `test_main_thread_only_concurrent_remote_exec_deadlock` guards this — + trio shuffles its run batch, so never rely on task-spawn order). +- Channel callbacks run on the receiver (loop) thread — keep for now + (open decision: revisit in D). +- `Group.terminate(timeout)` never hangs (~2×timeout bound, issues + #43/#221). +- Sync blocking waits (send-ack, receive, waitclose, join) stay on + `threading.Event`/`queue.Queue` so KeyboardInterrupt can interrupt + them; `portal.run` (KI-deferred) is only for management ops. + +## Phase C — what is missing (nothing of C exists yet) + +Target axes: `loop=thread|main` × `exec=thread|main|task`. +Compat mapping: `execmodel=thread` → `loop=main` + `exec=thread`; +`execmodel=main_thread_only` → `loop=thread` + `exec=main`. + +1. **Spec surface**: `xspec.py` only knows `execmodel=`. Add `loop=` / + `exec=` keys, combination validation, and the compat mapping. Same + for the worker CLI config JSON (`_provision.worker_cli_arg` still + ships `{"id", "execmodel", "coordinator_version"}`). +2. **`loop=main` does not exist**: the worker always starts `TrioHost` + on a dedicated thread (`serve_popen_trio`), main thread parked in + `gateway.join()` or integrated as exec primary. The compat mapping + means plain `execmodel=thread` workers should end up with `trio.run` + on the main thread — a structural change to `_run_worker`, not a + flag. +3. **`exec=main`** is today's `main_thread_only` machinery + (`integrate_as_primary_thread` + `_executetask_complete` deadlock + guard) — keep behavior, re-home under the new naming. +4. **`exec=task` (async remote_exec)**: `CHANNEL_EXEC` on a pure + `AsyncGateway` is still rejected ("unsupported message" in + `_trio_gateway._dispatch`). Needs a task-based exec scheduler + handing exec'd code an `AsyncChannel`, preserving FIFO admission, + plus explicit cancellation semantics. +5. **STATUS / `remote_status()`** reports `execmodel`; must speak the + new axes compatibly. +6. **Retire `ExecModel`/`WorkerPool`** (deferred from B): replace + internals with plain `threading`/`queue`; deprecated shims for + `group.execmodel`/`set_execmodel`/`spec.execmodel`. Pinned today by + `multi.safe_terminate`, `testing/test_threadpool.py`, conftest's + `pool` fixture, `Channel._items` (`execmodel.queue`), and the worker + STATUS duck-type — gated on the compat mapping landing first. +7. **Wheel-on-demand for `GATEWAY_START_SUB`**: recorded TODO in + `_provision.spawn_request` (currently ships eagerly whenever the + sub-spec has `python=`/`ssh=`). + +## Phase D — what is missing + +1. **Docs are entirely stale**: `doc/` still describes source + bootstrapping and execmodels; nothing on uv provisioning, the + no-source-shipping/version-compat policy, the three namespaces, or + `execnet.trio`/`execnet.portal` APIs. `doc/implnotes.rst` references + pre-inversion internals. Changelog entries for the whole branch. +2. **Trio-surface test gaps**: async ssh/socket/vagrant transport tests + (the asyncssh harness only exercises the sync facade; both share + `connect_command_worker`, so a direct `AsyncGroup` ssh test is + cheap), cancellation mid-`remote_exec`/`receive`, `send_eof` and + reconfigure against real workers, B.5 engine paths (write-ack + failure, `enqueue_message` after close). +3. **pytest-xdist verification**: run pytest-xdist's own suite against + this branch plus real `-n` smoke runs (crash/endmarker tests, + `main_thread_only` GUI case). Our own suite running `-n 12` green is + necessary but not sufficient. +4. **Close the open decision**: channel callbacks on the loop thread — + keep or move. + +Recommended order: C.1+C.2 first (spec axes + loop placement) since the +compat mapping unblocks the ExecModel retirement, then `exec=task`; run +the xdist verification (D.3) mid-C as an early canary rather than only +at the end. + +## Invariants (do not regress) + +- No source shipping, ever. Workers import installed execnet+trio; + rough major/minor version check (`_trio_worker._check_version`) warns. +- Worker teardown: `_trio_worker._run_worker` ends with `os._exit(0)` + because trio's `to_thread` cache uses non-daemon threads. +- `import execnet` must not import the trio event loop. +- Keep async-core idioms anyio-portable (neutral `ByteStream`, sans-IO + `FrameDecoder`) — `execnet.anyio` is deferred Phase E. From 2742eb0aee4eb45fe5f39cfd27847146681ab6ed Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 21:05:43 +0200 Subject: [PATCH 33/59] fix: drain asyncssh redirect cleanup in the local ssh harness asyncssh stores its redirect cleanup coroutines (Queue.join et al) un-awaited and only runs them inside process.wait_closed(); the server handler never called it, so every ssh test leaked coroutines that pytest's unraisable hook surfaced as RuntimeWarnings during GC -- visible as stray noise in xdist runs. Await wait_closed() in the handler and drop the filterwarnings mark, which never covered the GC-time warning anyway. Co-Authored-By: Claude Fable 5 --- testing/test_ssh_local.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index 086ce877..443b5da2 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -20,14 +20,9 @@ import execnet -pytestmark = [ - pytest.mark.skipif( - shutil.which("ssh") is None, reason="system ssh client required" - ), - # asyncssh leaves an un-awaited internal Queue.join coroutine on shutdown, - # surfaced by pytest's unraisable-exception hook during GC; harmless here. - pytest.mark.filterwarnings("ignore:coroutine 'Queue.join' was never awaited"), -] +pytestmark = pytest.mark.skipif( + shutil.which("ssh") is None, reason="system ssh client required" +) # Committed, intentionally-insecure test keys (see sshkeys/README.md). SSHKEYS = Path(__file__).parent / "sshkeys" @@ -58,6 +53,10 @@ async def _handle(self, process: asyncssh.SSHServerProcess) -> None: ) await process.redirect(stdin=proc.stdin, stdout=proc.stdout, stderr=proc.stderr) process.exit(await proc.wait()) + # Drain asyncssh's redirect cleanup coroutines (Queue.join et al): + # they are stored un-awaited and only run inside wait_closed(); + # skipping this leaks them and GC prints RuntimeWarnings. + await process.wait_closed() async def _serve(self) -> None: self._server = await asyncssh.listen( From 0d123bdbd2476dcee227eb150c22ae8f30c97ef2 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 08:50:57 +0200 Subject: [PATCH 34/59] docs: record PR #422 and the xfail audit in the phase C handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-c-worker-axes.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index ced2d895..351f6c16 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -6,6 +6,10 @@ supersedes `handoff-phase-b-async-core.md` (kept for the B record). Plan context lives in session memory (`trio-port-plan`), but everything needed is restated here. +The rework is up as **draft PR pytest-dev/execnet#422** (branch pushed +to the `origin` fork). Keep it updated as C/D land; undraft once the +docs overhaul (D.1) at least covers migration notes. + Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` (never grep-filter pre-commit output). The suite is xdist-clean: `uv run pytest testing/ -n 12` passes (~7s) since the session-attach race fix @@ -122,6 +126,16 @@ Compat mapping: `execmodel=thread` → `loop=main` + `exec=thread`; necessary but not sufficient. 4. **Close the open decision**: channel callbacks on the loop thread — keep or move. +5. **xfail markers audit (done 2026-07-26, keep as-is)**: the 11 + consistent XPASSes were investigated — trio's single-loop dispatch + + FIFO admission makes them pass reliably when idle, but under + sustained load `test_gateway_status_busy` (numexecuting race: + `_track_start` runs in a separately scheduled task) and + `test_popen_stderr_tracing` (capfd race) still fail, so the + `flakytest` marks stay. `test_safe_terminate2`'s xpass is CPython + dummy-thread accounting, unrelated to trio. To retire the status + marks for real: retry-poll for `numexecuting == 2` like the tests + already do for `== 0`. Recommended order: C.1+C.2 first (spec axes + loop placement) since the compat mapping unblocks the ExecModel retirement, then `exec=task`; run From 2c7841678694539da379266560313049e4de4522 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 10:02:11 +0200 Subject: [PATCH 35/59] feat: introduce the boundary kit (Wakener/Mailbox/OneShot) in execnet.portal P1 of the host-boundary rethink (handoff-boundary-protocol-rethink.md): loop->consumer egress unifies on a pluggable thread-safe Wakener with two carriers on top -- Mailbox (item streams, generalizing the KI-safe SyncReceiver drain pattern with get_nowait/timeouts) and OneShot (single results). SyncReceiver is replaced by Mailbox; the SyncBridgeGateway write-ack and _done_sync move off ad-hoc threading.Events onto OneShot. Pure refactor: no behavior change, suite stays xdist-clean. Co-Authored-By: Claude Fable 5 --- handoff-boundary-protocol-rethink.md | 129 +++++++++++++++++++++ src/execnet/_trio_host.py | 40 ++++--- src/execnet/_trio_worker.py | 8 +- src/execnet/portal.py | 166 +++++++++++++++++++++++---- testing/test_namespaces.py | 8 +- testing/test_portal.py | 76 ++++++++++++ 6 files changed, 379 insertions(+), 48 deletions(-) create mode 100644 handoff-boundary-protocol-rethink.md create mode 100644 testing/test_portal.py diff --git a/handoff-boundary-protocol-rethink.md b/handoff-boundary-protocol-rethink.md new file mode 100644 index 00000000..52336693 --- /dev/null +++ b/handoff-boundary-protocol-rethink.md @@ -0,0 +1,129 @@ +# Handoff: host boundary protocol rethink (P1–P5, precedes Phase C proper) + +Decided with Ronny 2026-07-26. This supersedes the *ordering* of +`handoff-phase-c-worker-axes.md` (still valid for C content): the boundary +rethink lands first, then C's `loop=`/`exec=` work continues on the +smaller core. The wire protocol (Message framing) is untouched — this is +about how things cross between the trio host loop and its consumers. + +## Rationale + +Consumer→loop is already universal: `LoopPortal.post` rides +`TrioToken.run_sync_soon`, thread-safe from any foreign context (plain +thread, gevent greenlet, asyncio callback, another trio loop). One +ingress, strict FIFO — unchanged. + +Loop→consumer is hardwired to threading primitives (`Channel._items` +execmodel queue, `threading.Event` for waitclose/join/write-acks, +`SyncReceiver`), which is the only reason `ExecModel` still exists. The +fix: the loop never blocks and never knows who listens — it fires a +thread-safe wakeup callable the consumer supplied. Every event loop has +exactly one such primitive: + +| consumer | wakeup primitive | +|---|---| +| plain thread | `threading.Event.set` | +| asyncio | `loop.call_soon_threadsafe` | +| gevent | `hub.loop.async_()` watcher — `send()` is gevent's one documented thread-safe op | +| foreign trio loop | `TrioToken.run_sync_soon` | + +## The boundary kit (grows in `execnet.portal`) + +1. **`Wakener`** — protocol, one thread-safe method `notify()`. The + entire integration surface for a new event loop (eventlet, Qt, …) + is implementing this; no more ExecModel. +2. **`Mailbox[T]`** — unbounded deque + Wakener. `put()` from the loop + (append + notify, never blocks the loop); `get(timeout)` blocking for + sync backends — generalizes the KI-safe `SyncReceiver` drain pattern + (Event.wait + drain, re-check after clear) — and `await`-able for + asyncio/trio wakeners. Replaces `Channel._items`, `SyncReceiver`, + `MultiChannel`'s queue. +3. **`OneShot[T]`** — single-result future on the same wakener. + Replaces the per-send write-ack `threading.Event`; adds + `portal.start(async_fn) -> OneShot` so asyncio callers await + management ops instead of blocking a thread in `portal.run`. + +## Channel unification (decided: full rebase) + +`AsyncGateway._dispatch` becomes the ONLY message router. `RawChannel` +grows a consumer hook: a bound facade diverts payloads into a `Mailbox` +instead of the internal memory channel (callbacks: invoked on the loop +thread, preserving current semantics; moving them later is just posting +via the wakener). + +Sync `Channel` becomes a genuine facade over (raw id, mailbox, portal): + +- `receive()` = `mailbox.get(timeout)` + `loads_internal` at the call + site — deserialization moves off the loop thread; `DataFormatError` + now surfaces at `receive()`. +- `send()` = portal-posted frame + `OneShot` write-ack (non-loop threads + keep block-until-written, 120s → OSError, KI-safe). +- `waitclose()`/`join()` = closed-events on the wakener. +- `__del__` best-effort close keeps `portal.post` (post_message path). +- Unknown inbound ids still auto-create (`_channel_for`). + +Deleted outright: `_receivelock`, `ChannelFactory` dispatch paths +(`_local_receive`/`_local_close`), `SyncBridgeGateway._dispatch`, the +`Message.received` handler-table use, the duplicate close state machine. +`CHANNEL_EXEC`/`GATEWAY_START_*` become per-gateway hooks on the async +core. `TrioWorkerExec` FIFO pump survives unchanged; exec'd code gets +facade channels. + +## Execmodels: retired as machinery, preserved as presets + +execmodel names become presets over three orthogonal axes (spec + worker +CLI config; decided: **both** coordinator and worker side): + +| axis | values | meaning | +|---|---|---| +| `loop` | `main` \| `thread` | where the trio host runs (Phase C) | +| `exec` | `thread` \| `main` \| `task` | where remote_exec code runs (Phase C) | +| `wait` | `thread` \| `gevent` \| `asyncio` | which Wakener blocking facade waits park on (new) | + +Presets: `execmodel=thread` → `loop=main, exec=thread, wait=thread`; +`main_thread_only` → `loop=thread, exec=main, wait=thread`; `gevent` +returns as `loop=thread, exec=thread, wait=gevent` (greenlet channel +waits park the greenlet, not the hub; sends already greenlet-safe via +the portal). eventlet stays dead (third parties can implement Wakener). + +Dies: `ExecModel` ABC, `WorkerPool`, `Reply` (`multi.safe_terminate` +moves to plain threads or delegates to `FacadeAsyncGroup.terminate`). +Stays as deprecated preset-mappers: `get_execmodel`, `set_execmodel`, +`group.execmodel`, `spec.execmodel` (xdist compat). + +## asyncio (decided: land now) + +`execnet.aio` namespace mirroring `execnet.trio`'s surface; every op is +portal ingress + `OneShot`/`Mailbox` on an `AsyncioWakener`. Full +asyncio-native API over the trio host loop, all transports, no anyio +port. Phase E (anyio-native core) becomes optional purity/perf work. + +## Staging + +1. **P1 — boundary kit**: Wakener/Mailbox/OneShot + ThreadWakener; port + SyncReceiver, write-acks, `_done_sync` onto it. Pure refactor. +2. **P2 — channel unification** (big one): sync Channel onto + RawChannel+Mailbox, delete classic dispatch. Straight to the rebase, + no intermediate primitive-swap (suite pins semantics either way). + Canary: `uv run pytest testing/ -n 12` + xdist suite mid-step. +3. **P3 — retirement**: ExecModel/WorkerPool out, preset shims in, + `wait=` joins the C.1 spec axes + worker CLI config. +4. **P4 — `execnet.aio`**. +5. **P5 — gevent wakener** + opt-in CI job (gevent in a dev extra). + +Then Phase C `loop=main` / `exec=task` on the smaller core, and Phase D +docs cover the three namespaces + `execnet.aio` + the axes/presets. + +## Invariants (unchanged, re-mapped) + +- Non-loop-thread sends block until the frame hit the OS write (OneShot + write-ack; 120s → OSError); loop-thread sends only enqueue; all sends + one portal FIFO. +- After close: `OSError("cannot send (already closed?)")`. +- exec admission order = message arrival order (TrioWorkerExec._pump). +- Channel callbacks run on the loop thread (revisit-later stands; the + kit makes moving them cheap). +- `Group.terminate(timeout)` never hangs (~2×timeout). +- Sync blocking waits stay KeyboardInterrupt-interruptible on the main + thread (thread Wakener keeps the Event.wait + drain pattern). +- No source shipping; `import execnet` must not import trio. diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 4d05690d..68b278d7 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -37,6 +37,7 @@ from .gateway_base import loads_internal from .gateway_base import trace from .portal import LoopPortal +from .portal import OneShot if TYPE_CHECKING: from .gateway import Gateway @@ -217,7 +218,7 @@ def __init__( super().__init__(stream, id=id) self.sync_gateway = sync_gateway self.host = host - self._done_sync = threading.Event() + self._done_sync: OneShot[None] = OneShot() self._send_closed = False self._send_lock = threading.Lock() # Attach before any serving can happen: the first inbound message @@ -249,7 +250,7 @@ async def _finalize(self) -> None: # Unblock the worker's join() before heavy exec-pool shutdown # so the primary thread is not waiting on _done while terminate waits # on the primary thread draining work. - self._done_sync.set() + self._done_sync.set(None) gateway._trace("[trio-bridge] terminating execution") # May sleep/SIGINT; keep it off the Trio scheduling thread. await trio.to_thread.run_sync( @@ -266,20 +267,17 @@ def enqueue_message(self, message: Message) -> None: """ frame = message.pack() wait = not self.host.portal.is_loop_thread() - done = threading.Event() if wait else None - errors: list[BaseException] = [] - - def on_written(error: BaseException | None) -> None: - if error is not None: - errors.append(error) - if done is not None: - done.set() + # The ack carries the write failure as a value (never raised into + # the OneShot) so a KeyboardInterrupt in wait() stays distinguishable + # from a stream error. + ack: OneShot[BaseException | None] | None = OneShot() if wait else None def post() -> None: try: - self.enqueue_frame(frame, on_written) + self.enqueue_frame(frame, ack.set if ack is not None else None) except OSError as exc: - on_written(exc) + if ack is not None: + ack.set(exc) with self._send_lock: if self._send_closed: @@ -290,12 +288,14 @@ def post() -> None: self.host.portal.post(post) except trio.RunFinishedError: raise OSError("cannot send (already closed?)") from None - if done is None: + if ack is None: return - if not done.wait(timeout=120.0): - raise OSError("cannot send (write timed out)") - if errors: - raise OSError("cannot send (already closed?)") from errors[0] + try: + error = ack.wait(timeout=120.0) + except TimeoutError: + raise OSError("cannot send (write timed out)") from None + if error is not None: + raise OSError("cannot send (already closed?)") from error def post_message(self, message: Message) -> None: """Best-effort non-waiting send (Channel.__del__ during GC).""" @@ -320,7 +320,11 @@ def request_close_write(self) -> None: self.host.portal.post(self._outbound_send.close) def wait_done(self, timeout: float | None = None) -> bool: - return self._done_sync.wait(timeout) + try: + self._done_sync.wait(timeout) + except TimeoutError: + return False + return True def is_alive(self) -> bool: return not self._done_sync.is_set() diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index bfaeea43..aa3e68d9 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -16,7 +16,7 @@ from .gateway_base import get_execmodel from .gateway_base import loads_internal from .gateway_base import trace -from .portal import SyncReceiver +from .portal import Mailbox if TYPE_CHECKING: from . import _trio_host @@ -48,9 +48,9 @@ def __init__( self._shutting_down = False self._idle = threading.Event() self._idle.set() - self._primary: SyncReceiver[ - tuple[Channel, ExecItem, threading.Event] | None - ] = SyncReceiver() + self._primary: Mailbox[tuple[Channel, ExecItem, threading.Event] | None] = ( + Mailbox() + ) # Exec requests flow through a single pump task so admission happens # strictly in message-arrival order (trio task scheduling order is # deliberately unordered, so per-request tasks would race for the diff --git a/src/execnet/portal.py b/src/execnet/portal.py index a204cc5a..223fdd97 100644 --- a/src/execnet/portal.py +++ b/src/execnet/portal.py @@ -1,30 +1,39 @@ -"""Cross-thread / cross-loop communication primitives. +"""Cross-thread / cross-loop communication primitives — the boundary kit. A :class:`LoopPortal` is a handle to a running trio loop that foreign -threads use to run functions on the loop or push work into it. Because -each direction only needs the *receiving* loop's token, two trio loops in -two threads can communicate by holding each other's portal (the sync -facade's host loop and a user loop; the worker loop and the process main -thread). - -:class:`SyncReceiver` covers the opposite direction: a plain thread -(typically the process main thread) receiving items produced on a loop, -in a way that stays interruptible by KeyboardInterrupt. +threads use to run functions on the loop or push work into it (the +consumer -> loop direction). Because each direction only needs the +*receiving* loop's token, two trio loops in two threads can communicate +by holding each other's portal. + +The loop -> consumer direction never blocks the loop and never knows who +is listening: the loop fires a :class:`Wakener`, a single thread-safe +``notify()`` supplied by the consumer. Implementing that one method is +the entire integration surface for an event loop backend (threads here; +asyncio/gevent wakeners come with their facades). On top of it sit the +two carriers: + +* :class:`Mailbox` -- an item stream (channel payloads, exec requests), +* :class:`OneShot` -- a single result (write acknowledgements, call + results a consumer wants to await instead of block on). """ from __future__ import annotations import queue import threading +import time from collections.abc import Awaitable from collections.abc import Callable from typing import Any from typing import Generic +from typing import Protocol from typing import TypeVar +from typing import cast import trio -__all__ = ["LoopPortal", "SyncReceiver"] +__all__ = ["LoopPortal", "Mailbox", "OneShot", "ThreadWakener", "Wakener"] T = TypeVar("T") @@ -64,35 +73,142 @@ def post(self, sync_fn: Callable[..., object], *args: Any) -> None: self._token.run_sync_soon(sync_fn, *args) -class SyncReceiver(Generic[T]): - """Receive items on a plain thread, KeyboardInterrupt-friendly. +class Wakener(Protocol): + """Thread-safe consumer wakeup fired by the loop. - ``queue.SimpleQueue.get`` parks in a C-level lock acquire that shields - KeyboardInterrupt on the main thread; ``threading.Event.wait`` does not. - So producers put into an unbounded queue and set a wake event, and - :meth:`get` waits on the event and drains the queue. + ``notify()`` must never block and must be safe from any thread; it is + the only thing the loop side ever calls. The blocking mailbox/oneshot + waits additionally need :meth:`wait`/:meth:`clear` executed in the + consumer's own context (a thread here, a greenlet for a gevent + wakener); loop-native consumers such as asyncio wait on their own + side of ``notify`` instead. + """ + + def notify(self) -> None: ... + + def wait(self, timeout: float | None = None) -> bool: ... + + def clear(self) -> None: ... + + +class ThreadWakener: + """Plain-thread wakener on a ``threading.Event``. + + ``threading.Event.wait`` stays interruptible by KeyboardInterrupt on + the main thread (a C-level ``queue.SimpleQueue.get`` does not), which + is why the carriers wait on the wakener and drain a queue instead of + blocking in the queue itself. """ def __init__(self) -> None: + self._event = threading.Event() + + def notify(self) -> None: + self._event.set() + + def wait(self, timeout: float | None = None) -> bool: + return self._event.wait(timeout) + + def clear(self) -> None: + self._event.clear() + + +class Mailbox(Generic[T]): + """Loop -> consumer item stream: an unbounded queue plus a wakener. + + :meth:`put` never blocks and is safe from any thread (including the + loop thread). :meth:`get` blocks in the consumer's context via the + wakener; after draining to empty it clears and re-checks so a ``put`` + racing the clear cannot be lost. Multiple consumers are allowed. + """ + + def __init__(self, wakener: Wakener | None = None) -> None: self._items: queue.SimpleQueue[T] = queue.SimpleQueue() - self._wake = threading.Event() + self._wakener = ThreadWakener() if wakener is None else wakener def put(self, item: T) -> None: """Thread-safe; usable from a loop thread (never blocks).""" self._items.put(item) - self._wake.set() + self._wakener.notify() + + def get_nowait(self) -> T: + """Return the next item or raise ``queue.Empty``.""" + return self._items.get_nowait() - def get(self) -> T: - """Block until an item is available (interruptible on main thread).""" + def get(self, timeout: float | None = None) -> T: + """Block until an item is available; TimeoutError after ``timeout``.""" + deadline = None if timeout is None else time.monotonic() + timeout while True: - self._wake.wait() + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if remaining <= 0 or not self._wakener.wait(remaining): + # Final non-blocking check: a put may have raced the + # timeout (its notify landing after our last wait). + try: + return self._items.get_nowait() + except queue.Empty: + raise TimeoutError( + "no item after %r seconds" % timeout + ) from None try: return self._items.get_nowait() except queue.Empty: - # Re-check after clearing so a put between get_nowait and - # clear cannot be lost. - self._wake.clear() + # Empty: clear, then re-check so a put between get_nowait + # and clear cannot be lost. + self._wakener.clear() try: return self._items.get_nowait() except queue.Empty: continue + + +class OneShot(Generic[T]): + """A single result crossing loop -> consumer exactly once. + + The loop side calls :meth:`set` (or :meth:`set_error`) at most once; + the consumer blocks in :meth:`wait`, which returns the value, + re-raises the stored error, or raises ``TimeoutError``. + """ + + _NOTSET = object() + + def __init__(self, wakener: Wakener | None = None) -> None: + self._wakener = ThreadWakener() if wakener is None else wakener + self._value: Any = self._NOTSET + self._error: BaseException | None = None + self._done = False + + def is_set(self) -> bool: + return self._done + + def set(self, value: T) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + assert not self._done, "OneShot already resolved" + self._value = value + self._done = True + self._wakener.notify() + + def set_error(self, error: BaseException) -> None: + """Resolve with an error that :meth:`wait` will re-raise.""" + assert not self._done, "OneShot already resolved" + self._error = error + self._done = True + self._wakener.notify() + + def wait(self, timeout: float | None = None) -> T: + """Block until resolved; TimeoutError after ``timeout`` seconds.""" + deadline = None if timeout is None else time.monotonic() + timeout + while not self._done: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if (remaining <= 0 or not self._wakener.wait(remaining)) and ( + not self._done + ): + raise TimeoutError("not resolved after %r seconds" % timeout) + if self._error is not None: + raise self._error + return cast("T", self._value) diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index 5f44f8a9..a8bfef0c 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -38,7 +38,13 @@ def test_trio_namespace_exposes_async_core() -> None: def test_portal_namespace() -> None: - assert execnet.portal.__all__ == ["LoopPortal", "SyncReceiver"] + assert execnet.portal.__all__ == [ + "LoopPortal", + "Mailbox", + "OneShot", + "ThreadWakener", + "Wakener", + ] assert execnet.portal.LoopPortal is not None diff --git a/testing/test_portal.py b/testing/test_portal.py new file mode 100644 index 00000000..1a3759af --- /dev/null +++ b/testing/test_portal.py @@ -0,0 +1,76 @@ +"""The boundary kit: Mailbox and OneShot on the thread wakener.""" + +from __future__ import annotations + +import queue +import threading + +import pytest + +from execnet.portal import Mailbox +from execnet.portal import OneShot + + +class TestMailbox: + def test_put_get_fifo(self) -> None: + box: Mailbox[int] = Mailbox() + for n in range(3): + box.put(n) + assert [box.get(), box.get(), box.get()] == [0, 1, 2] + + def test_get_nowait_empty(self) -> None: + box: Mailbox[int] = Mailbox() + with pytest.raises(queue.Empty): + box.get_nowait() + + def test_get_timeout(self) -> None: + box: Mailbox[int] = Mailbox() + with pytest.raises(TimeoutError): + box.get(timeout=0.01) + + def test_get_blocks_until_put_from_other_thread(self) -> None: + box: Mailbox[str] = Mailbox() + threading.Timer(0.05, box.put, args=["item"]).start() + assert box.get(timeout=5.0) == "item" + + def test_get_after_stale_wakeup(self) -> None: + # A consumed notify leaves the wakener set; the drain pattern must + # still block (and then receive) rather than spin or miss items. + box: Mailbox[int] = Mailbox() + box.put(1) + assert box.get() == 1 + threading.Timer(0.05, box.put, args=[2]).start() + assert box.get(timeout=5.0) == 2 + + +class TestOneShot: + def test_set_then_wait(self) -> None: + shot: OneShot[int] = OneShot() + shot.set(42) + assert shot.is_set() + assert shot.wait() == 42 + # a resolved OneShot stays readable + assert shot.wait(timeout=0.01) == 42 + + def test_wait_timeout(self) -> None: + shot: OneShot[int] = OneShot() + with pytest.raises(TimeoutError): + shot.wait(timeout=0.01) + assert not shot.is_set() + + def test_set_error_reraises(self) -> None: + shot: OneShot[None] = OneShot() + shot.set_error(RuntimeError("boom")) + with pytest.raises(RuntimeError, match="boom"): + shot.wait() + + def test_wait_blocks_until_set_from_other_thread(self) -> None: + shot: OneShot[str] = OneShot() + threading.Timer(0.05, shot.set, args=["done"]).start() + assert shot.wait(timeout=5.0) == "done" + + def test_single_resolution_asserted(self) -> None: + shot: OneShot[int] = OneShot() + shot.set(1) + with pytest.raises(AssertionError): + shot.set(2) From 2a40a559c68170f53751d3eef4116b659c70e583 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:05:02 +0200 Subject: [PATCH 36/59] feat!: rebase the sync Channel onto the async core (single dispatch) P2 of the boundary rethink: AsyncGateway._dispatch is now the only Message router. RawChannel grows a consumer hook (set_consumer) that diverts inbound payloads/closes to a bound facade and replays anything that arrived first; the sync Channel becomes a genuine facade -- a Mailbox fed from the loop (deserialization at the receive() call site), callbacks invoked on the loop thread with the switch-over running there too, sends unchanged through gateway._send. Deleted: the classic Message.received handler table, ChannelFactory's dispatch paths (_local_receive, callback/raw-receiver registries) and BaseGateway._receivelock; RawTunnelStream (the via tunnel now uses RawChannelStream over the session's raw channel on both master and worker relay sides). ChannelFactory remains as registry + id allocator, keeping callback channels strongly alive while registered. An unconsumed remotely-closed RawChannel now stays registered until a consumer claims it: with call-site deserialization a passed channel can bind after its data AND close already arrived, and previously the close forgot the raw channel so the late binder found a fresh empty one and lost the payloads (latent race for async open_channel(id) too). The boundary kit moved to the trio-free execnet._boundary so gateway_base can use Mailbox without `import execnet` importing trio; the standalone-source tests inline it / import the package instead. Co-Authored-By: Claude Fable 5 --- src/execnet/_boundary.py | 161 +++++++++++++++ src/execnet/_trio_gateway.py | 48 ++++- src/execnet/_trio_host.py | 206 +++++++++++-------- src/execnet/_trio_worker.py | 2 +- src/execnet/gateway_base.py | 387 ++++++++++++++++------------------- src/execnet/portal.py | 152 +------------- testing/test_basics.py | 28 ++- testing/test_serializer.py | 9 +- 8 files changed, 542 insertions(+), 451 deletions(-) create mode 100644 src/execnet/_boundary.py diff --git a/src/execnet/_boundary.py b/src/execnet/_boundary.py new file mode 100644 index 00000000..f80f6998 --- /dev/null +++ b/src/execnet/_boundary.py @@ -0,0 +1,161 @@ +"""Trio-free half of the boundary kit: Wakener, Mailbox, OneShot. + +Importable without loading any event loop (``import execnet`` must not +import trio); ``execnet.portal`` re-exports these next to LoopPortal. +""" + +from __future__ import annotations + +import queue +import threading +import time +from typing import Any +from typing import Generic +from typing import Protocol +from typing import TypeVar +from typing import cast + +__all__ = ["Mailbox", "OneShot", "ThreadWakener", "Wakener"] + +T = TypeVar("T") + + +class Wakener(Protocol): + """Thread-safe consumer wakeup fired by the loop. + + ``notify()`` must never block and must be safe from any thread; it is + the only thing the loop side ever calls. The blocking mailbox/oneshot + waits additionally need :meth:`wait`/:meth:`clear` executed in the + consumer's own context (a thread here, a greenlet for a gevent + wakener); loop-native consumers such as asyncio wait on their own + side of ``notify`` instead. + """ + + def notify(self) -> None: ... + + def wait(self, timeout: float | None = None) -> bool: ... + + def clear(self) -> None: ... + + +class ThreadWakener: + """Plain-thread wakener on a ``threading.Event``. + + ``threading.Event.wait`` stays interruptible by KeyboardInterrupt on + the main thread (a C-level ``queue.SimpleQueue.get`` does not), which + is why the carriers wait on the wakener and drain a queue instead of + blocking in the queue itself. + """ + + def __init__(self) -> None: + self._event = threading.Event() + + def notify(self) -> None: + self._event.set() + + def wait(self, timeout: float | None = None) -> bool: + return self._event.wait(timeout) + + def clear(self) -> None: + self._event.clear() + + +class Mailbox(Generic[T]): + """Loop -> consumer item stream: an unbounded queue plus a wakener. + + :meth:`put` never blocks and is safe from any thread (including the + loop thread). :meth:`get` blocks in the consumer's context via the + wakener; after draining to empty it clears and re-checks so a ``put`` + racing the clear cannot be lost. Multiple consumers are allowed. + """ + + def __init__(self, wakener: Wakener | None = None) -> None: + self._items: queue.SimpleQueue[T] = queue.SimpleQueue() + self._wakener = ThreadWakener() if wakener is None else wakener + + def put(self, item: T) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + self._items.put(item) + self._wakener.notify() + + def get_nowait(self) -> T: + """Return the next item or raise ``queue.Empty``.""" + return self._items.get_nowait() + + def get(self, timeout: float | None = None) -> T: + """Block until an item is available; TimeoutError after ``timeout``.""" + deadline = None if timeout is None else time.monotonic() + timeout + while True: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if remaining <= 0 or not self._wakener.wait(remaining): + # Final non-blocking check: a put may have raced the + # timeout (its notify landing after our last wait). + try: + return self._items.get_nowait() + except queue.Empty: + raise TimeoutError( + "no item after %r seconds" % timeout + ) from None + try: + return self._items.get_nowait() + except queue.Empty: + # Empty: clear, then re-check so a put between get_nowait + # and clear cannot be lost. + self._wakener.clear() + try: + return self._items.get_nowait() + except queue.Empty: + continue + + +class OneShot(Generic[T]): + """A single result crossing loop -> consumer exactly once. + + The loop side calls :meth:`set` (or :meth:`set_error`) at most once; + the consumer blocks in :meth:`wait`, which returns the value, + re-raises the stored error, or raises ``TimeoutError``. + """ + + _NOTSET = object() + + def __init__(self, wakener: Wakener | None = None) -> None: + self._wakener = ThreadWakener() if wakener is None else wakener + self._value: Any = self._NOTSET + self._error: BaseException | None = None + self._done = False + + def is_set(self) -> bool: + return self._done + + def set(self, value: T) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + assert not self._done, "OneShot already resolved" + self._value = value + self._done = True + self._wakener.notify() + + def set_error(self, error: BaseException) -> None: + """Resolve with an error that :meth:`wait` will re-raise.""" + assert not self._done, "OneShot already resolved" + self._error = error + self._done = True + self._wakener.notify() + + def wait(self, timeout: float | None = None) -> T: + """Block until resolved; TimeoutError after ``timeout`` seconds.""" + deadline = None if timeout is None else time.monotonic() + timeout + while not self._done: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if (remaining <= 0 or not self._wakener.wait(remaining)) and ( + not self._done + ): + raise TimeoutError("not resolved after %r seconds" % timeout) + if self._error is not None: + raise self._error + return cast("T", self._value) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index fa4f2ba9..fc364772 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -159,6 +159,12 @@ def __init__(self, gateway: AsyncGateway, id: int) -> None: self._receive_closed = trio.Event() # no more payloads will arrive self._remote_error: RemoteError | None = None self._payload_send, self._payloads = trio.open_memory_channel[bytes](math.inf) + # Diversion hooks for a bound facade (sync channel): when set, + # inbound payloads/closes route out of the loop instead of + # buffering for receive_bytes. + self._consumer_payload: Callable[[bytes], None] | None = None + self._consumer_close: Callable[[RemoteError | None, bool], None] | None = None + self._pending_close: tuple[RemoteError | None, bool] | None = None def __repr__(self) -> str: state = "closed" if self._closed else "open" @@ -228,9 +234,40 @@ def _pending_error(self) -> BaseException: or EOFError(f"raw channel {self.id} closed") ) + def set_consumer( + self, + on_payload: Callable[[bytes], None], + on_close: Callable[[RemoteError | None, bool], None], + ) -> None: + """Divert inbound payloads and the close to callbacks (loop thread). + + Already-buffered payloads flush to ``on_payload`` first, and a close + that arrived before binding is replayed to ``on_close`` -- so a + consumer bound late (facade channels bind via a portal post) sees + the exact inbound order. + """ + self._consumer_payload = on_payload + self._consumer_close = on_close + while True: + try: + data = self._payloads.receive_nowait() + except (trio.WouldBlock, trio.EndOfChannel, trio.ClosedResourceError): + break + on_payload(data) + if self._pending_close is not None: + error, sendonly = self._pending_close + self._pending_close = None + on_close(error, sendonly) + if not sendonly: + # the late binder has now claimed the buffered close + self.gateway._forget_channel(self.id) + # dispatch-loop internals (inline on the gateway's serve task) def _feed(self, data: bytes) -> None: + if self._consumer_payload is not None: + self._consumer_payload(data) + return try: self._payload_send.send_nowait(data) except (trio.BrokenResourceError, trio.ClosedResourceError): @@ -243,8 +280,17 @@ def _close_from_remote(self, error: RemoteError | None, *, sendonly: bool) -> No self._receive_closed.set() if not sendonly: self._closed = True - self.gateway._forget_channel(self.id) + if self._consumer_close is not None: + # Without a consumer the closed channel stays registered: + # a passed-channel reference may still bind late and must + # find the buffered payloads and this close, not a fresh + # empty channel under the same id. + self.gateway._forget_channel(self.id) self._payload_send.close() + if self._consumer_close is not None: + self._consumer_close(error, sendonly) + else: + self._pending_close = (error, sendonly) class RawChannelStream: diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 68b278d7..ed4f876e 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -7,12 +7,13 @@ from __future__ import annotations +import functools import itertools import json -import math import subprocess import sys import threading +import weakref from collections.abc import Awaitable from collections.abc import Callable from contextlib import suppress @@ -26,6 +27,7 @@ from ._trio_gateway import AsyncGateway from ._trio_gateway import AsyncGroup from ._trio_gateway import ByteStream +from ._trio_gateway import RawChannelStream from ._trio_gateway import open_popen_process from ._trio_gateway import read_handshake_ack from ._trio_gateway import ssh_transport_args @@ -33,6 +35,7 @@ from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message +from .gateway_base import RemoteError from .gateway_base import dumps_internal from .gateway_base import loads_internal from .gateway_base import trace @@ -51,72 +54,6 @@ ssh_trio_args = ssh_transport_args -_CHANNEL_EOF = object() - - -class RawTunnelStream: - """``ByteStream`` over a raw channel id on a sync master ``Gateway``. - - The frame-native via tunnel: the master relays whole sub-protocol - frames as verbatim CHANNEL_DATA payloads (no serialization, no - double-framing), so this stream only buffers payloads; the - sub-session's FrameDecoder sees exact frame boundaries. - """ - - def __init__(self, gateway: BaseGateway, channelid: int) -> None: - self._gateway = gateway - self.channelid = channelid - self._send, self._recv = trio.open_memory_channel[Any](math.inf) - self._buf = bytearray() - self._eof = False - self._closed = False - gateway._channelfactory.register_raw_receiver( - channelid, self._on_data, self._on_close - ) - - def _on_data(self, data: bytes) -> None: - self._send.send_nowait(data) - - def _on_close(self, error: Any) -> None: - self._send.send_nowait(_CHANNEL_EOF if error is None else error) - - async def send_all(self, data: bytes) -> None: - self._gateway._send(Message.CHANNEL_DATA, self.channelid, data) - - async def receive_some(self, max_bytes: int | None = None) -> bytes: - if not self._buf and not self._eof: - item = await self._recv.receive() - if item is _CHANNEL_EOF: - self._eof = True - elif isinstance(item, Exception): - self._eof = True - raise EOFError(f"via tunnel closed: {item}") from None - else: - assert isinstance(item, bytes) - self._buf += item - if max_bytes is None: - max_bytes = len(self._buf) - out = bytes(self._buf[:max_bytes]) - del self._buf[:max_bytes] - return out - - async def send_eof(self) -> None: - self._close_tunnel() - - async def aclose(self) -> None: - self._close_tunnel() - - def _close_tunnel(self) -> None: - if self._closed: - return - self._closed = True - self._gateway._channelfactory.unregister_raw_receiver(self.channelid) - # Unblock our own reader: no more payloads will be routed here. - self._send.send_nowait(_CHANNEL_EOF) - with suppress(OSError): - self._gateway._send(Message.CHANNEL_CLOSE, self.channelid) - - async def adopt_socket(socket_fd: int) -> trio.SocketStream: """Worker side: wrap an inherited socket fd and send the handshake. @@ -229,16 +166,52 @@ def __init__( # -- engine hooks (run on the host loop) -- def _dispatch(self, message: Message) -> None: + """Route one message: sync-facade concerns here, the rest to the core. + + CHANNEL_DATA/CLOSE/CLOSE_ERROR/LAST_MESSAGE fall through to the + async core, which routes them to the RawChannel whose consumer is + the bound sync ``Channel``. + """ gateway = self.sync_gateway + code = message.msgcode try: - with gateway._receivelock: - message.received(gateway) + if code == Message.STATUS: + self._answer_status(message) + elif code == Message.CHANNEL_EXEC: + channel = gateway._channelfactory.new(message.channelid) + gateway._local_schedulexec(channel=channel, sourcetask=message.data) + elif code == Message.RECONFIGURE: + data = loads_internal(message.data, gateway) + assert isinstance(data, tuple) + if message.channelid == 0: + gateway._strconfig = data + else: + gateway._channelfactory.new(message.channelid)._strconfig = data + elif code == Message.GATEWAY_START_SOCKET: + handle_start_socket(gateway, message.channelid, message.data) + elif code == Message.GATEWAY_START_SUB: + handle_start_sub(gateway, message.channelid, message.data) + else: + super()._dispatch(message) except (GatewayReceivedTerminate, EOFError): raise except Exception as exc: gateway._trace("dispatch failed:", gateway._geterrortext(exc)) raise EOFError("error dispatching message") from exc + def _answer_status(self, message: Message) -> None: + # we use the channelid to send back information + # but don't instantiate a channel object + gateway = self.sync_gateway + execpool = getattr(gateway, "_execpool", None) + d = { + "numchannels": len(gateway._channelfactory._channels), + "numexecuting": execpool.active_count() if execpool is not None else 0, + "execmodel": gateway.execmodel.backend, + } + gateway._send(Message.CHANNEL_DATA, message.channelid, dumps_internal(d)) + gateway._send(Message.CHANNEL_CLOSE, message.channelid) + async def _finalize(self) -> None: gateway = self.sync_gateway with self._send_lock: @@ -247,6 +220,9 @@ async def _finalize(self) -> None: gateway._error = self._error gateway._trace("[trio-bridge] finishing channels") gateway._channelfactory._finished_receiving() + # EOF the loop-side raw channels that have no sync consumer + # (via-tunnel relays and readers blocked in receive_bytes). + self._finish_channels() # Unblock the worker's join() before heavy exec-pool shutdown # so the primary thread is not waiting on _done while terminate waits # on the primary thread draining work. @@ -259,6 +235,70 @@ async def _finalize(self) -> None: # -- sync session interface (any thread) -- + def bind_sync_channel(self, channel: Any) -> None: + """Route inbound data for ``channel.id`` to the sync channel. + + Callable from any thread: binding happens on the loop via the + portal so it cannot interleave with dispatch, and the raw channel + replays anything (payloads, a close) that arrived first. The loop + side only holds a weakref, preserving the factory's weak-registry + semantics (GC of the last user reference sends the close message); + channels with callbacks are kept alive by the factory instead. + """ + ref = weakref.ref(channel) + channelid = channel.id + + def bind() -> None: + raw = self._channel_for(channelid) + raw.set_consumer( + functools.partial(self._sync_payload, ref), + functools.partial(self._sync_close, ref, channelid), + ) + + with suppress(trio.RunFinishedError): + self.host.portal.post(bind) + + def _sync_payload(self, ref: weakref.ref[Any], data: bytes) -> None: + channel = ref() + if channel is not None: + channel._deliver_payload(data) + # dead ref: data for a deleted channel is dropped, like before + + def _sync_close( + self, + ref: weakref.ref[Any], + channelid: int, + error: RemoteError | None, + sendonly: bool, + ) -> None: + channel = ref() + if channel is None: + # channel already in "deleted" state + if error is not None: + error.warn() + self.sync_gateway._channelfactory._no_longer_opened(channelid) + return + channel._close_from_remote(error, sendonly=sendonly) + + def release_channel(self, channelid: int) -> None: + """Drop the loop-side raw channel for ``channelid`` (best-effort).""" + with suppress(trio.RunFinishedError): + self.host.portal.post(self._forget_channel, channelid) + + def run_on_loop(self, sync_fn: Callable[[], T]) -> T: + """Run ``sync_fn`` on the host loop, excluding dispatch interleaving. + + Falls back to running inline once the loop is gone (no more + deliveries can interleave then anyway). + """ + portal = self.host.portal + if portal.is_loop_thread(): + return sync_fn() + try: + return portal.run_sync(sync_fn) + except trio.RunFinishedError: + return sync_fn() + def enqueue_message(self, message: Message) -> None: """Enqueue a frame; wait until written when safe to block. @@ -471,11 +511,13 @@ async def _open_via_stream(self, spec: Any) -> ByteStream: from . import _provision master = self.group[spec.via] + session = master._trio_session + assert isinstance(session, SyncBridgeGateway) channelid = master._channelfactory.allocate_id() + # Create the raw channel before the request goes out so no relayed + # frame can arrive unrouted (we are on the loop: no dispatch races). + io = RawChannelStream(session._channel_for(channelid)) request = _provision.spawn_request(spec) - # Register the raw receiver before the request goes out so no - # relayed frame can arrive unrouted. - io = RawTunnelStream(master, channelid) master._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) await read_handshake_ack(io, "via") return io @@ -602,7 +644,7 @@ async def _start_sub_and_relay( Runs on the master's Trio host (the ``via`` transport). The tunnel is frame-native both ways: coordinator payloads arrive verbatim through the - raw-receiver registry and go to the sub's stdin unchanged (each payload + session's raw channel and go to the sub's stdin unchanged (each payload one whole frame), while the sub's stdout runs through a FrameDecoder so every CHANNEL_DATA sent back carries exactly one frame -- except the initial ready byte, which is forwarded on its own for the handshake. @@ -621,21 +663,17 @@ def send_close_error(text: str) -> None: except Exception as exc: send_close_error(f"could not spawn via sub-gateway: {exc}") return - send_ch, recv_ch = trio.open_memory_channel[Any](math.inf) - gateway._channelfactory.register_raw_receiver( - channelid, - send_ch.send_nowait, - lambda error: send_ch.send_nowait(_CHANNEL_EOF), - ) + session = gateway._trio_session + assert isinstance(session, SyncBridgeGateway) + raw = session._channel_for(channelid) async def coordinator_to_sub() -> None: assert process.stdin is not None if preamble: await process.stdin.send_all(preamble) - async for data in recv_ch: - if data is _CHANNEL_EOF: - break - await process.stdin.send_all(data) + with suppress(RemoteError): + async for data in raw: + await process.stdin.send_all(data) with trio.move_on_after(5): await process.stdin.aclose() @@ -664,7 +702,7 @@ async def sub_to_coordinator() -> None: gateway._trace("via sub relay failed:", exc) send_close_error(f"via sub-gateway relay failed: {exc}") finally: - gateway._channelfactory.unregister_raw_receiver(channelid) + session._forget_channel(channelid) with trio.move_on_after(5): await process.wait() diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index aa3e68d9..6506199d 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -76,7 +76,7 @@ def _track_finish(self) -> None: self._idle.set() def schedule(self, channel: Channel, sourcetask: bytes) -> None: - """Called from the Trio receiver while holding ``_receivelock``. + """Called from the session dispatch on the Trio host thread. Must not block: deadlock checks and exec run in a nursery task. """ diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 2923eaa3..47bf7d32 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -12,9 +12,12 @@ from __future__ import annotations import abc +import builtins import os +import queue as _queue import struct import sys +import threading import traceback import weakref from _thread import interrupt_main @@ -28,6 +31,8 @@ from typing import cast from typing import overload +from ._boundary import Mailbox + class WriteIO(Protocol): def write(self, data: bytes, /) -> None: ... @@ -390,10 +395,37 @@ def trace(*msg: object) -> None: class Message: - """Encapsulates Messages and their wire protocol.""" + """Encapsulates Messages and their wire protocol. + + Dispatch lives in the async core and the sync bridge session + (``AsyncGateway._dispatch`` / ``SyncBridgeGateway._dispatch``); this + class only carries the framing and the code constants. + """ + + STATUS = 0 + RECONFIGURE = 1 + GATEWAY_TERMINATE = 2 + CHANNEL_EXEC = 3 + CHANNEL_DATA = 4 + CHANNEL_CLOSE = 5 + CHANNEL_CLOSE_ERROR = 6 + CHANNEL_LAST_MESSAGE = 7 + GATEWAY_START_SOCKET = 8 + GATEWAY_START_SUB = 9 - # message code -> name, handler - _types: dict[int, tuple[str, Callable[[Message, BaseGateway], None]]] = {} + # message code -> name + _types: dict[int, str] = { + STATUS: "STATUS", + RECONFIGURE: "RECONFIGURE", + GATEWAY_TERMINATE: "GATEWAY_TERMINATE", + CHANNEL_EXEC: "CHANNEL_EXEC", + CHANNEL_DATA: "CHANNEL_DATA", + CHANNEL_CLOSE: "CHANNEL_CLOSE", + CHANNEL_CLOSE_ERROR: "CHANNEL_CLOSE_ERROR", + CHANNEL_LAST_MESSAGE: "CHANNEL_LAST_MESSAGE", + GATEWAY_START_SOCKET: "GATEWAY_START_SOCKET", + GATEWAY_START_SUB: "GATEWAY_START_SUB", + } def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: self.msgcode = msgcode @@ -431,103 +463,10 @@ def from_io(io: ReadIO) -> Message: def to_io(self, io: WriteIO) -> None: io.write(self.pack()) - def received(self, gateway: BaseGateway) -> None: - handler = self._types[self.msgcode][1] - handler(self, gateway) - def __repr__(self) -> str: - name = self._types[self.msgcode][0] + name = self._types[self.msgcode] return f"" - def _status(message: Message, gateway: BaseGateway) -> None: - # we use the channelid to send back information - # but don't instantiate a channel object - d = { - "numchannels": len(gateway._channelfactory._channels), - # TODO(typing): Attribute `_execpool` is only on WorkerGateway. - "numexecuting": gateway._execpool.active_count(), # type: ignore[attr-defined] - "execmodel": gateway.execmodel.backend, - } - gateway._send(Message.CHANNEL_DATA, message.channelid, dumps_internal(d)) - gateway._send(Message.CHANNEL_CLOSE, message.channelid) - - STATUS = 0 - _types[STATUS] = ("STATUS", _status) - - def _reconfigure(message: Message, gateway: BaseGateway) -> None: - data = loads_internal(message.data, gateway) - assert isinstance(data, tuple) - strconfig: tuple[bool, bool] = data - if message.channelid == 0: - gateway._strconfig = strconfig - else: - gateway._channelfactory.new(message.channelid)._strconfig = strconfig - - RECONFIGURE = 1 - _types[RECONFIGURE] = ("RECONFIGURE", _reconfigure) - - def _gateway_terminate(message: Message, gateway: BaseGateway) -> None: - raise GatewayReceivedTerminate(gateway) - - GATEWAY_TERMINATE = 2 - _types[GATEWAY_TERMINATE] = ("GATEWAY_TERMINATE", _gateway_terminate) - - def _channel_exec(message: Message, gateway: BaseGateway) -> None: - channel = gateway._channelfactory.new(message.channelid) - gateway._local_schedulexec(channel=channel, sourcetask=message.data) - - CHANNEL_EXEC = 3 - _types[CHANNEL_EXEC] = ("CHANNEL_EXEC", _channel_exec) - - def _channel_data(message: Message, gateway: BaseGateway) -> None: - gateway._channelfactory._local_receive(message.channelid, message.data) - - CHANNEL_DATA = 4 - _types[CHANNEL_DATA] = ("CHANNEL_DATA", _channel_data) - - def _channel_close(message: Message, gateway: BaseGateway) -> None: - gateway._channelfactory._local_close(message.channelid) - - CHANNEL_CLOSE = 5 - _types[CHANNEL_CLOSE] = ("CHANNEL_CLOSE", _channel_close) - - def _channel_close_error(message: Message, gateway: BaseGateway) -> None: - error_message = loads_internal(message.data) - assert isinstance(error_message, str) - remote_error = RemoteError(error_message) - gateway._channelfactory._local_close(message.channelid, remote_error) - - CHANNEL_CLOSE_ERROR = 6 - _types[CHANNEL_CLOSE_ERROR] = ("CHANNEL_CLOSE_ERROR", _channel_close_error) - - def _channel_last_message(message: Message, gateway: BaseGateway) -> None: - gateway._channelfactory._local_close(message.channelid, sendonly=True) - - CHANNEL_LAST_MESSAGE = 7 - _types[CHANNEL_LAST_MESSAGE] = ("CHANNEL_LAST_MESSAGE", _channel_last_message) - - def _gateway_start_socket(message: Message, gateway: BaseGateway) -> None: - # Start a one-shot socketserver on this (Trio) gateway's host and reply - # with the bound (host, port) on the request channel. Handled natively - # instead of shipping source via remote_exec. - from . import _trio_host - - _trio_host.handle_start_socket(gateway, message.channelid, message.data) - - GATEWAY_START_SOCKET = 8 - _types[GATEWAY_START_SOCKET] = ("GATEWAY_START_SOCKET", _gateway_start_socket) - - def _gateway_start_sub(message: Message, gateway: BaseGateway) -> None: - # Spawn a sub-gateway worker (popen, foreign python, or ssh) on this - # (Trio) gateway's host and relay its Message protocol over the request - # channel (the ``via`` transport). - from . import _trio_host - - _trio_host.handle_start_sub(gateway, message.channelid, message.data) - - GATEWAY_START_SUB = 9 - _types[GATEWAY_START_SUB] = ("GATEWAY_START_SUB", _gateway_start_sub) - class FrameDecoder: """Incremental decoder for the 9-byte-header Message framing. @@ -617,7 +556,13 @@ class TimeoutError(IOError): class Channel: - """Communication channel between two Python Interpreter execution points.""" + """Communication channel between two Python Interpreter execution points. + + A facade over the async core: the gateway's Trio session diverts + inbound payloads for this id into a :class:`Mailbox` (or a registered + callback, invoked on the loop thread); ``receive()`` deserializes at + the call site. Sends go through ``gateway._send``. + """ RemoteError = RemoteError TimeoutError = TimeoutError @@ -632,9 +577,12 @@ def __init__(self, gateway: BaseGateway, id: int) -> None: # XXX: defaults copied from Unserializer self._strconfig = getattr(gateway, "_strconfig", (True, False)) self.id = id - self._items = self.gateway.execmodel.queue.Queue() + # serialized payloads (or ENDMARKER); None once a callback is set + self._mailbox: Mailbox[Any] | None = Mailbox() + self._callback: Callable[[Any], Any] | None = None + self._endmarker: object = NO_ENDMARKER_WANTED self._closed = False - self._receiveclosed = self.gateway.execmodel.Event() + self._receiveclosed = threading.Event() self._remoteerrors: list[RemoteError] = [] def _trace(self, *msg: object) -> None: @@ -648,33 +596,36 @@ def setcallback( """Set a callback function for receiving items. All already-queued items will immediately trigger the callback. - Afterwards the callback will execute in the receiver thread + Afterwards the callback will execute in the receiver (loop) thread for each received data item and calls to ``receive()`` will raise an error. If an endmarker is specified the callback will eventually be called with the endmarker when the channel closes. """ - _callbacks = self.gateway._channelfactory._callbacks - with self.gateway._receivelock: - if self._items is None: + + def switch() -> None: + # Runs on the loop thread (inline without a session), so the + # switch-over cannot interleave with payload delivery. + mailbox = self._mailbox + if mailbox is None: raise OSError(f"{self!r} has callback already registered") - items = self._items - self._items = None + self._mailbox = None while 1: try: - olditem = items.get(block=False) - except self.gateway.execmodel.queue.Empty: + olditem = mailbox.get_nowait() + except _queue.Empty: if not (self._closed or self._receiveclosed.is_set()): - _callbacks[self.id] = (callback, endmarker, self._strconfig) + self._callback = callback + self._endmarker = endmarker + self.gateway._channelfactory._register_callback_channel(self) break - else: - if olditem is ENDMARKER: - items.put(olditem) # for other receivers - if endmarker is not NO_ENDMARKER_WANTED: - callback(endmarker) - break - else: - callback(olditem) + if olditem is ENDMARKER: + if endmarker is not NO_ENDMARKER_WANTED: + callback(endmarker) + break + callback(loads_internal(olditem, self)) + + self.gateway._run_on_loop(switch) def __repr__(self) -> str: flag = (self.isclosed() and "closed") or "open" @@ -701,7 +652,7 @@ def __del__(self) -> None: # don't need to try to send a closing or last message # (and often it won't work anymore to send things out) if Message is not None: - if self._items is None: # has_callback + if self._mailbox is None: # has_callback msgcode = Message.CHANNEL_LAST_MESSAGE else: msgcode = Message.CHANNEL_CLOSE @@ -711,6 +662,8 @@ def __del__(self) -> None: self.gateway, "_send_nonblocking", self.gateway._send ) send(msgcode, self.id) + with suppress(Exception): + self.gateway._release_channel(self.id) def _getremoteerror(self): try: @@ -722,6 +675,50 @@ def _getremoteerror(self): pass return None + # + # loop-side delivery (called by the session's raw-channel consumer) + # + def _deliver_payload(self, data: bytes) -> None: + """Route one inbound serialized payload (loop thread).""" + if self._closed: + return # late data for a locally closed channel: drop + callback = self._callback + if callback is None: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(data) + # no mailbox and no callback: closed for receiving -- drop + else: + try: + callback(loads_internal(data, self)) + except Exception as exc: + self.gateway._trace("exception during callback: %s" % exc) + errortext = self.gateway._geterrortext(exc) + self.gateway._send( + Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(errortext) + ) + self._close_from_remote(RemoteError(errortext)) + + def _close_from_remote(self, remoteerror=None, *, sendonly: bool = False) -> None: + """Close initiated by the peer or session shutdown (loop thread).""" + if remoteerror: + self._remoteerrors.append(remoteerror) + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) + self._fire_endmarker() + self.gateway._channelfactory._no_longer_opened(self.id) + if not sendonly: # otherwise #--> "sendonly" + self._closed = True # --> "closed" + self._receiveclosed.set() + + def _fire_endmarker(self) -> None: + callback = self._callback + if callback is not None: + self._callback = None + if self._endmarker is not NO_ENDMARKER_WANTED: + callback(self._endmarker) + # # public API for channel objects # @@ -788,10 +785,12 @@ def close(self, error=None) -> None: self._remoteerrors.append(error) self._closed = True # --> "closed" self._receiveclosed.set() - queue = self._items - if queue is not None: - queue.put(ENDMARKER) + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) + self._fire_endmarker() self.gateway._channelfactory._no_longer_opened(self.id) + self.gateway._release_channel(self.id) def waitclose(self, timeout: float | None = None) -> None: """Wait until this channel is closed (or the remote side @@ -841,18 +840,18 @@ def receive(self, timeout: float | None = None) -> Any: reraised as channel.RemoteError exceptions containing a textual representation of the remote traceback. """ - itemqueue = self._items - if itemqueue is None: + mailbox = self._mailbox + if mailbox is None: raise OSError("cannot receive(), channel has receiver callback") try: - x = itemqueue.get(timeout=timeout) - except self.gateway.execmodel.queue.Empty: + x = mailbox.get(timeout) + except builtins.TimeoutError: raise self.TimeoutError("no item after %r seconds" % timeout) from None if x is ENDMARKER: - itemqueue.put(x) # for other receivers + mailbox.put(x) # for other receivers raise self._getremoteerror() or EOFError() else: - return x + return loads_internal(x, self) def __iter__(self) -> Iterator[Any]: return self @@ -886,21 +885,22 @@ def reconfigure( class ChannelFactory: + """Registry and id allocator for a gateway's sync channels. + + Message routing lives in the Trio session (the sync channel binds a + consumer on the session's raw channel); the factory only tracks live + channels -- weakly, so dropping the last user reference triggers + ``Channel.__del__``'s close message -- and keeps channels with a + registered callback strongly alive until they close. + """ + def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: self._channels: weakref.WeakValueDictionary[int, Channel] = ( weakref.WeakValueDictionary() ) - # Channel ID => (callback, end marker, strconfig) - self._callbacks: dict[ - int, tuple[Callable[[Any], Any], object, tuple[bool, bool]] - ] = {} - # Channel ID => (feed, on_close) receiving CHANNEL_DATA verbatim - # (no serialization) -- the low-level raw channel routing used by - # byte-shaped consumers such as the via tunnel. - self._raw_receivers: dict[ - int, tuple[Callable[[bytes], None], Callable[[RemoteError | None], None]] - ] = {} - self._writelock = gateway.execmodel.Lock() + # channels kept strongly alive while their callback is registered + self._callback_channels: dict[int, Channel] = {} + self._writelock = threading.Lock() self.gateway = gateway self.count = startcount self.finished = False @@ -918,6 +918,7 @@ def new(self, id: int | None = None) -> Channel: channel = self._channels[id] except KeyError: channel = self._channels[id] = Channel(self.gateway, id) + self.gateway._bind_channel(channel) return channel def allocate_id(self) -> int: @@ -929,42 +930,21 @@ def allocate_id(self) -> int: self.count += 2 return id - def register_raw_receiver( - self, - id: int, - feed: Callable[[bytes], None], - on_close: Callable[[RemoteError | None], None], - ) -> None: - """Route CHANNEL_DATA payloads for ``id`` verbatim to ``feed``. - - ``on_close`` fires once when the channel closes (with the peer's - RemoteError, if any) or when receiving finishes. - """ - self._raw_receivers[id] = (feed, on_close) - - def unregister_raw_receiver(self, id: int) -> None: - self._raw_receivers.pop(id, None) - def channels(self) -> list[Channel]: return self._list(self._channels.values()) # - # internal methods, called from the receiver thread + # internal methods, called from the loop thread (or local close paths) # + def _register_callback_channel(self, channel: Channel) -> None: + self._callback_channels[channel.id] = channel + def _no_longer_opened(self, id: int) -> None: self._channels.pop(id, None) - item = self._callbacks.pop(id, None) - if item is not None: - callback, endmarker, _strconfig = item - if endmarker is not NO_ENDMARKER_WANTED: - callback(endmarker) + self._callback_channels.pop(id, None) def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: - raw = self._raw_receivers.pop(id, None) - if raw is not None: - _feed, on_close = raw - on_close(remoteerror) - return + """Close ``id`` as if the peer had closed it (no message is sent).""" channel = self._channels.get(id) if channel is None: # channel already in "deleted" state @@ -972,57 +952,13 @@ def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> Non remoteerror.warn() self._no_longer_opened(id) else: - # state transition to "closed" state - if remoteerror: - channel._remoteerrors.append(remoteerror) - queue = channel._items - if queue is not None: - queue.put(ENDMARKER) - self._no_longer_opened(id) - if not sendonly: # otherwise #--> "sendonly" - channel._closed = True # --> "closed" - channel._receiveclosed.set() - - def _local_receive(self, id: int, data) -> None: - # executes in receiver thread - raw = self._raw_receivers.get(id) - if raw is not None: - feed, _on_close = raw - feed(data) - return - channel = self._channels.get(id) - try: - callback, _endmarker, strconfig = self._callbacks[id] - except KeyError: - queue = channel._items if channel is not None else None - if queue is None: - pass # drop data - else: - item = loads_internal(data, channel) - queue.put(item) - else: - try: - data = loads_internal(data, channel, strconfig) - callback(data) # even if channel may be already closed - except Exception as exc: - self.gateway._trace("exception during callback: %s" % exc) - errortext = self.gateway._geterrortext(exc) - self.gateway._send( - Message.CHANNEL_CLOSE_ERROR, id, dumps_internal(errortext) - ) - self._local_close(id, errortext) + channel._close_from_remote(remoteerror, sendonly=sendonly) def _finished_receiving(self) -> None: with self._writelock: self.finished = True for id in self._list(self._channels): self._local_close(id, sendonly=True) - for id in self._list(self._callbacks): - self._no_longer_opened(id) - for id in self._list(self._raw_receivers): - item = self._raw_receivers.pop(id, None) - if item is not None: - item[1](None) class ChannelFile: @@ -1099,7 +1035,6 @@ def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.id = id self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) self._channelfactory = ChannelFactory(self, _startcount) - self._receivelock = self.execmodel.RLock() # globals may be NONE at process-termination self.__trace = trace self._geterrortext = geterrortext @@ -1109,8 +1044,36 @@ def _trace(self, *msg: object) -> None: self.__trace(self.id, *msg) def _attach_trio_session(self, session: Any) -> None: - """Attach a Trio ProtocolSession for Message IO (no receiver thread).""" + """Attach the Trio bridge session doing the Message IO.""" self._trio_session = session + # Defensive: channels created before the session existed still + # need their inbound routing diverted to them. + for channel in self._channelfactory.channels(): + session.bind_sync_channel(channel) + + def _bind_channel(self, channel: Channel) -> None: + """Divert the session's inbound routing for ``channel.id`` to it.""" + session = self._trio_session + if session is not None: + session.bind_sync_channel(channel) + + def _release_channel(self, id: int) -> None: + """Drop the session's loop-side state for ``id`` (best-effort).""" + session = self._trio_session + if session is not None: + with suppress(Exception): + session.release_channel(id) + + def _run_on_loop(self, sync_fn: Callable[[], Any]) -> Any: + """Run ``sync_fn`` on the session's loop thread (inline without one). + + Payload dispatch happens on the loop thread, so state switches run + there to exclude interleaving with deliveries. + """ + session = self._trio_session + if session is None: + return sync_fn() + return session.run_on_loop(sync_fn) def _terminate_execution(self) -> None: pass diff --git a/src/execnet/portal.py b/src/execnet/portal.py index 223fdd97..5cc3e30e 100644 --- a/src/execnet/portal.py +++ b/src/execnet/portal.py @@ -20,19 +20,18 @@ from __future__ import annotations -import queue -import threading -import time from collections.abc import Awaitable from collections.abc import Callable from typing import Any -from typing import Generic -from typing import Protocol from typing import TypeVar -from typing import cast import trio +from ._boundary import Mailbox +from ._boundary import OneShot +from ._boundary import ThreadWakener +from ._boundary import Wakener + __all__ = ["LoopPortal", "Mailbox", "OneShot", "ThreadWakener", "Wakener"] T = TypeVar("T") @@ -71,144 +70,3 @@ def post(self, sync_fn: Callable[..., object], *args: Any) -> None: ``trio.RunFinishedError`` once the loop has shut down. """ self._token.run_sync_soon(sync_fn, *args) - - -class Wakener(Protocol): - """Thread-safe consumer wakeup fired by the loop. - - ``notify()`` must never block and must be safe from any thread; it is - the only thing the loop side ever calls. The blocking mailbox/oneshot - waits additionally need :meth:`wait`/:meth:`clear` executed in the - consumer's own context (a thread here, a greenlet for a gevent - wakener); loop-native consumers such as asyncio wait on their own - side of ``notify`` instead. - """ - - def notify(self) -> None: ... - - def wait(self, timeout: float | None = None) -> bool: ... - - def clear(self) -> None: ... - - -class ThreadWakener: - """Plain-thread wakener on a ``threading.Event``. - - ``threading.Event.wait`` stays interruptible by KeyboardInterrupt on - the main thread (a C-level ``queue.SimpleQueue.get`` does not), which - is why the carriers wait on the wakener and drain a queue instead of - blocking in the queue itself. - """ - - def __init__(self) -> None: - self._event = threading.Event() - - def notify(self) -> None: - self._event.set() - - def wait(self, timeout: float | None = None) -> bool: - return self._event.wait(timeout) - - def clear(self) -> None: - self._event.clear() - - -class Mailbox(Generic[T]): - """Loop -> consumer item stream: an unbounded queue plus a wakener. - - :meth:`put` never blocks and is safe from any thread (including the - loop thread). :meth:`get` blocks in the consumer's context via the - wakener; after draining to empty it clears and re-checks so a ``put`` - racing the clear cannot be lost. Multiple consumers are allowed. - """ - - def __init__(self, wakener: Wakener | None = None) -> None: - self._items: queue.SimpleQueue[T] = queue.SimpleQueue() - self._wakener = ThreadWakener() if wakener is None else wakener - - def put(self, item: T) -> None: - """Thread-safe; usable from a loop thread (never blocks).""" - self._items.put(item) - self._wakener.notify() - - def get_nowait(self) -> T: - """Return the next item or raise ``queue.Empty``.""" - return self._items.get_nowait() - - def get(self, timeout: float | None = None) -> T: - """Block until an item is available; TimeoutError after ``timeout``.""" - deadline = None if timeout is None else time.monotonic() + timeout - while True: - if deadline is None: - self._wakener.wait() - else: - remaining = deadline - time.monotonic() - if remaining <= 0 or not self._wakener.wait(remaining): - # Final non-blocking check: a put may have raced the - # timeout (its notify landing after our last wait). - try: - return self._items.get_nowait() - except queue.Empty: - raise TimeoutError( - "no item after %r seconds" % timeout - ) from None - try: - return self._items.get_nowait() - except queue.Empty: - # Empty: clear, then re-check so a put between get_nowait - # and clear cannot be lost. - self._wakener.clear() - try: - return self._items.get_nowait() - except queue.Empty: - continue - - -class OneShot(Generic[T]): - """A single result crossing loop -> consumer exactly once. - - The loop side calls :meth:`set` (or :meth:`set_error`) at most once; - the consumer blocks in :meth:`wait`, which returns the value, - re-raises the stored error, or raises ``TimeoutError``. - """ - - _NOTSET = object() - - def __init__(self, wakener: Wakener | None = None) -> None: - self._wakener = ThreadWakener() if wakener is None else wakener - self._value: Any = self._NOTSET - self._error: BaseException | None = None - self._done = False - - def is_set(self) -> bool: - return self._done - - def set(self, value: T) -> None: - """Thread-safe; usable from a loop thread (never blocks).""" - assert not self._done, "OneShot already resolved" - self._value = value - self._done = True - self._wakener.notify() - - def set_error(self, error: BaseException) -> None: - """Resolve with an error that :meth:`wait` will re-raise.""" - assert not self._done, "OneShot already resolved" - self._error = error - self._done = True - self._wakener.notify() - - def wait(self, timeout: float | None = None) -> T: - """Block until resolved; TimeoutError after ``timeout`` seconds.""" - deadline = None if timeout is None else time.monotonic() + timeout - while not self._done: - if deadline is None: - self._wakener.wait() - else: - remaining = deadline - time.monotonic() - if (remaining <= 0 or not self._wakener.wait(remaining)) and ( - not self._done - ): - raise TimeoutError("not resolved after %r seconds" % timeout) - if self._error is not None: - raise self._error - return cast("T", self._value) diff --git a/testing/test_basics.py b/testing/test_basics.py index 473cd741..29ceef19 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -15,6 +15,7 @@ import pytest import execnet +from execnet import _boundary from execnet import gateway from execnet import gateway_base from execnet.gateway_base import ChannelFactory @@ -66,6 +67,21 @@ def test_errors_on_execnet() -> None: assert hasattr(execnet, "DataFormatError") +def standalone_gateway_base_source() -> str: + """gateway_base's source as a self-contained script. + + The module imports the trio-free boundary kit relatively; for the + run-on-any-python checks the kit source is inlined at the import site + (minus its own future import, which must stay file-leading). + """ + boundary_source = inspect.getsource(_boundary).replace( + "from __future__ import annotations\n", "" + ) + return inspect.getsource(gateway_base).replace( + "from ._boundary import Mailbox\n", boundary_source + ) + + IO_MESSAGE_EXTRA_SOURCE = """ from io import BytesIO @@ -124,7 +140,7 @@ def checker(anypython: str, tmp_path: Path) -> Checker: def test_io_message(checker: Checker) -> None: - out = checker.run_check(inspect.getsource(gateway_base) + IO_MESSAGE_EXTRA_SOURCE) + out = checker.run_check(standalone_gateway_base_source() + IO_MESSAGE_EXTRA_SOURCE) print(out.stdout) assert "all passed" in out.stdout @@ -147,7 +163,7 @@ def send(self, data): def test_geterrortext(checker: Checker) -> None: out = checker.run_check( - inspect.getsource(gateway_base) + standalone_gateway_base_source() + """ class Arg(Exception): pass @@ -324,12 +340,20 @@ class TestPureChannel: @pytest.fixture def fac(self, execmodel: ExecModel) -> ChannelFactory: class FakeGateway: + _trio_session = None + def _trace(self, *args) -> None: pass def _send(self, *k) -> None: pass + def _bind_channel(self, channel) -> None: + pass + + def _release_channel(self, id) -> None: + pass + FakeGateway.execmodel = execmodel # type: ignore[attr-defined] return ChannelFactory(FakeGateway()) # type: ignore[arg-type] diff --git a/testing/test_serializer.py b/testing/test_serializer.py index 06a84cd2..cb2d422c 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -9,8 +9,9 @@ import execnet -# We use the execnet folder in order to avoid triggering a missing apipkg. -pyimportdir = os.fspath(Path(execnet.__file__).parent) +# The package parent: the serializer is imported as execnet.gateway_base +# (it needs its trio-free sibling execnet._boundary; only stdlib beyond that). +pyimportdir = os.fspath(Path(execnet.__file__).parent.parent) class PythonWrapper: @@ -24,7 +25,7 @@ def dump(self, obj_rep: str) -> bytes: f""" import sys sys.path.insert(0, {pyimportdir!r}) -import gateway_base as serializer +import execnet.gateway_base as serializer sys.stdout = sys.stdout.detach() sys.stdout.write(serializer.dumps_internal({obj_rep})) """ @@ -40,7 +41,7 @@ def load(self, data: bytes) -> list[str]: rf""" import sys sys.path.insert(0, {pyimportdir!r}) -import gateway_base as serializer +import execnet.gateway_base as serializer from io import BytesIO data = {data!r} io = BytesIO(data) From 811fd9956374b2cf2685094d3aca2f30bcec26cd Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:15:41 +0200 Subject: [PATCH 37/59] feat!: retire the ExecModel machinery; add the wait= spec axis P3 of the boundary rethink. The ExecModel ABC, ThreadExecModel, MainThreadOnlyExecModel, WorkerPool and Reply are gone: protocol IO always runs on the Trio host, blocking waits go through the boundary kit, and execution placement is the worker config axes' job. What survives is a single deprecated ExecModel preset carrying the backend name plus the stdlib-delegating members (pytest-xdist builds its remote test queue on execmodel.RLock/Event, so the duck-type stays live); get_execmodel/set_execmodel/spec.execmodel keep working on top of it. safe_terminate now runs its term/kill pairs on daemon threads with the same bounded-wait contract; MultiChannel.make_receive_queue hands out a plain stdlib queue. New wait= axis end to end: the boundary kit gains Flag (idempotent event on a wakener) and a wakener-factory registry ("thread" built in; gevent/asyncio register later); specs accept wait= (validated in makegateway), the worker CLI config ships it, and every blocking carrier (channel mailbox, receive-closed flag, write-acks, join) is created through gateway._new_wakener() on both sides of the wire. Test suite: test_threadpool.py removed with WorkerPool; the pool fixture became a ThreadPoolExecutor; direct execmodel.queue/Event/sleep uses in tests moved to the stdlib equivalents. Co-Authored-By: Claude Fable 5 --- src/execnet/_boundary.py | 70 ++++++++- src/execnet/_provision.py | 1 + src/execnet/_trio_host.py | 6 +- src/execnet/_trio_worker.py | 34 +++-- src/execnet/gateway.py | 2 + src/execnet/gateway_base.py | 274 ++++-------------------------------- src/execnet/multi.py | 63 ++++++--- src/execnet/xspec.py | 1 + testing/conftest.py | 7 +- testing/test_basics.py | 19 ++- testing/test_channel.py | 5 +- testing/test_gateway.py | 2 +- testing/test_multi.py | 7 +- testing/test_termination.py | 19 ++- testing/test_threadpool.py | 222 ----------------------------- testing/test_xspec.py | 8 ++ 16 files changed, 213 insertions(+), 527 deletions(-) delete mode 100644 testing/test_threadpool.py diff --git a/src/execnet/_boundary.py b/src/execnet/_boundary.py index f80f6998..ce211ae1 100644 --- a/src/execnet/_boundary.py +++ b/src/execnet/_boundary.py @@ -9,13 +9,22 @@ import queue import threading import time +from collections.abc import Callable from typing import Any from typing import Generic from typing import Protocol from typing import TypeVar from typing import cast -__all__ = ["Mailbox", "OneShot", "ThreadWakener", "Wakener"] +__all__ = [ + "Flag", + "Mailbox", + "OneShot", + "ThreadWakener", + "Wakener", + "make_wakener", + "register_wakener", +] T = TypeVar("T") @@ -159,3 +168,62 @@ def wait(self, timeout: float | None = None) -> T: if self._error is not None: raise self._error return cast("T", self._value) + + +class Flag: + """An idempotent event on a wakener: may be set any number of times. + + Each Flag owns its wakener exclusively -- sharing one wakener between + carriers would lose wakeups (another carrier's ``clear`` can swallow + this one's ``notify``). + """ + + def __init__(self, wakener: Wakener | None = None) -> None: + self._wakener = ThreadWakener() if wakener is None else wakener + self._flag = False + + def is_set(self) -> bool: + return self._flag + + def set(self) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + self._flag = True + self._wakener.notify() + + def wait(self, timeout: float | None = None) -> bool: + """Block until set; returns whether the flag is set.""" + deadline = None if timeout is None else time.monotonic() + timeout + while not self._flag: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if remaining <= 0 or not self._wakener.wait(remaining): + break + return self._flag + + +# wait= axis: named wakener factories; each call returns a fresh instance +# (carriers own their wakener exclusively, see Flag). Backends register +# here ("gevent", "asyncio") next to the built-in "thread". +_WAKENER_FACTORIES: dict[str, Callable[[], Wakener]] = { + "thread": ThreadWakener, +} + + +def register_wakener(name: str, factory: Callable[[], Wakener]) -> None: + """Register a wakener factory for the ``wait=`` spec axis.""" + _WAKENER_FACTORIES[name] = factory + + +def wakener_names() -> list[str]: + return list(_WAKENER_FACTORIES) + + +def make_wakener(name: str) -> Wakener: + """Create a fresh wakener for the named wait backend.""" + try: + factory = _WAKENER_FACTORIES[name] + except KeyError: + raise ValueError(f"unknown wait backend {name!r}") from None + return factory() diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index b1c16335..f5114ad8 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -168,6 +168,7 @@ def worker_cli_arg(spec: Any) -> str: { "id": f"{spec.id}-worker", "execmodel": spec.execmodel, + "wait": spec.wait or "thread", "coordinator_version": execnet.__version__, } ) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index ed4f876e..e29af4d2 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -155,7 +155,7 @@ def __init__( super().__init__(stream, id=id) self.sync_gateway = sync_gateway self.host = host - self._done_sync: OneShot[None] = OneShot() + self._done_sync: OneShot[None] = OneShot(sync_gateway._new_wakener()) self._send_closed = False self._send_lock = threading.Lock() # Attach before any serving can happen: the first inbound message @@ -310,7 +310,9 @@ def enqueue_message(self, message: Message) -> None: # The ack carries the write failure as a value (never raised into # the OneShot) so a KeyboardInterrupt in wait() stays distinguishable # from a stream error. - ack: OneShot[BaseException | None] | None = OneShot() if wait else None + ack: OneShot[BaseException | None] | None = ( + OneShot(self.sync_gateway._new_wakener()) if wait else None + ) def post() -> None: try: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 6506199d..89aad122 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -201,12 +201,13 @@ def kill(self) -> None: def _build_worker_gateway( - host: _trio_host.TrioHost, id: str, model: ExecModel + host: _trio_host.TrioHost, id: str, model: ExecModel, wait: str = "thread" ) -> tuple[WorkerGateway, TrioWorkerExec, bool]: """Construct the WorkerGateway + Trio exec pool (no IO yet).""" trace(f"creating workergateway on trio id={id!r}") io_stub = _WorkerIOStub(model) gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) + gateway._wait_backend = wait main_thread_only = model.backend == "main_thread_only" trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) @@ -215,14 +216,16 @@ def _build_worker_gateway( gateway._trio_exec = trio_exec gateway._executetask_complete = None if main_thread_only: - gateway._executetask_complete = model.Event() + gateway._executetask_complete = threading.Event() gateway._executetask_complete.set() return gateway, trio_exec, main_thread_only -def _run_worker(host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel) -> None: +def _run_worker( + host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel, wait: str = "thread" +) -> None: """Attach ``io`` as the gateway session and serve until shutdown.""" - gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model) + gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model, wait) async def _start() -> _trio_host.SyncBridgeGateway: # The bridge attaches itself to the gateway before serving starts, @@ -252,7 +255,7 @@ async def _make_fd_io(read_fd: int, write_fd: int) -> Any: return _trio_gateway.staple_fd_stream(read_fd, write_fd) -def serve_popen_trio(id: str, execmodel: str = "thread") -> None: +def serve_popen_trio(id: str, execmodel: str = "thread", wait: str = "thread") -> None: """Serve a WorkerGateway over the stdio pipes (popen / ssh worker).""" from . import _trio_host @@ -266,10 +269,12 @@ def serve_popen_trio(id: str, execmodel: str = "thread") -> None: host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") host.start() io = host.call(_make_fd_io, read_fd, write_fd) - _run_worker(host, io, id, model) + _run_worker(host, io, id, model, wait) -def serve_socket_trio(id: str, execmodel: str, socket_fd: int) -> None: +def serve_socket_trio( + id: str, execmodel: str, socket_fd: int, wait: str = "thread" +) -> None: """Serve a WorkerGateway over an inherited socket fd. Used for the socketserver (an accepted TCP connection) and, in future, a @@ -282,7 +287,7 @@ def serve_socket_trio(id: str, execmodel: str, socket_fd: int) -> None: host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") host.start() io = host.call(_trio_host.adopt_socket, socket_fd) - _run_worker(host, io, id, model) + _run_worker(host, io, id, model, wait) def _rough_version(version: str) -> tuple[int, ...]: @@ -341,9 +346,18 @@ def _main() -> None: config = json.loads(ns.config) _check_version(config["coordinator_version"]) if ns.socket_fd is not None: - serve_socket_trio(config["id"], config["execmodel"], ns.socket_fd) + serve_socket_trio( + config["id"], + config["execmodel"], + ns.socket_fd, + wait=config.get("wait", "thread"), + ) else: - serve_popen_trio(id=config["id"], execmodel=config["execmodel"]) + serve_popen_trio( + id=config["id"], + execmodel=config["execmodel"], + wait=config.get("wait", "thread"), + ) if __name__ == "__main__": diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index b1d3ed46..bec4c778 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -43,6 +43,8 @@ def __init__(self, io: IO, spec: XSpec) -> None: """ super().__init__(io=io, id=spec.id, _startcount=1) self.spec = spec + if spec.wait: + self._wait_backend = spec.wait @property def remoteaddress(self) -> str: diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 47bf7d32..6a286302 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -11,7 +11,6 @@ from __future__ import annotations -import abc import builtins import os import queue as _queue @@ -31,7 +30,10 @@ from typing import cast from typing import overload +from ._boundary import Flag from ._boundary import Mailbox +from ._boundary import Wakener +from ._boundary import make_wakener class WriteIO(Protocol): @@ -58,74 +60,23 @@ def wait(self) -> int | None: ... def kill(self) -> None: ... -class Event(Protocol): - """Protocol for types which look like threading.Event.""" - - def is_set(self) -> bool: ... - - def set(self) -> None: ... - - def clear(self) -> None: ... - - def wait(self, timeout: float | None = None) -> bool: ... +class ExecModel: + """Deprecated preset name for an execution model. + The machinery behind execution models was retired: protocol IO always + runs on the Trio host and blocking waits go through the boundary kit's + wakeners (``execnet._boundary``); the name maps onto the worker config + axes (``loop=`` / ``exec=`` / ``wait=``). The stdlib-delegating + members stay for API compatibility (pytest-xdist builds its test queue + on ``execmodel.RLock``/``Event``) -- every preset is thread-shaped. + """ -class ExecModel(metaclass=abc.ABCMeta): - @property - @abc.abstractmethod - def backend(self) -> str: - raise NotImplementedError() + def __init__(self, backend: str) -> None: + self.backend = backend def __repr__(self) -> str: return "" % self.backend - @property - @abc.abstractmethod - def queue(self): - raise NotImplementedError() - - @property - @abc.abstractmethod - def subprocess(self): - raise NotImplementedError() - - @property - @abc.abstractmethod - def socket(self): - raise NotImplementedError() - - @abc.abstractmethod - def start(self, func, args=()) -> None: - raise NotImplementedError() - - @abc.abstractmethod - def get_ident(self) -> int: - raise NotImplementedError() - - @abc.abstractmethod - def sleep(self, delay: float) -> None: - raise NotImplementedError() - - @abc.abstractmethod - def fdopen(self, fd, mode, bufsize=1, closefd=True): - raise NotImplementedError() - - @abc.abstractmethod - def Lock(self): - raise NotImplementedError() - - @abc.abstractmethod - def RLock(self): - raise NotImplementedError() - - @abc.abstractmethod - def Event(self) -> Event: - raise NotImplementedError() - - -class ThreadExecModel(ExecModel): - backend = "thread" - @property def queue(self): import queue @@ -160,200 +111,24 @@ def start(self, func, args=()) -> None: _thread.start_new_thread(func, args) def fdopen(self, fd, mode, bufsize=1, closefd=True): - import os - return os.fdopen(fd, mode, bufsize, encoding="utf-8", closefd=closefd) def Lock(self): - import threading - return threading.RLock() def RLock(self): - import threading - return threading.RLock() - def Event(self): - import threading - + def Event(self) -> threading.Event: return threading.Event() -class MainThreadOnlyExecModel(ThreadExecModel): - backend = "main_thread_only" - - def get_execmodel(backend: str | ExecModel) -> ExecModel: if isinstance(backend, ExecModel): return backend - if backend == "thread": - return ThreadExecModel() - elif backend == "main_thread_only": - return MainThreadOnlyExecModel() - else: - raise ValueError(f"unknown execmodel {backend!r}") - - -class Reply: - """Provide access to the result of a function execution that got dispatched - through WorkerPool.spawn().""" - - def __init__(self, task, threadmodel: ExecModel) -> None: - self.task = task - self._result_ready = threadmodel.Event() - self.running = True - - def get(self, timeout: float | None = None): - """get the result object from an asynchronous function execution. - if the function execution raised an exception, - then calling get() will reraise that exception - including its traceback. - """ - self.waitfinish(timeout) - try: - return self._result - except AttributeError: - raise self._exc from None - - def waitfinish(self, timeout: float | None = None) -> None: - if not self._result_ready.wait(timeout): - raise OSError(f"timeout waiting for {self.task!r}") - - def run(self) -> None: - func, args, kwargs = self.task - try: - try: - self._result = func(*args, **kwargs) - except BaseException as exc: - self._exc = exc - finally: - self._result_ready.set() - self.running = False - - -class WorkerPool: - """A WorkerPool allows to spawn function executions - to threads, returning a reply object on which you - can ask for the result (and get exceptions reraised). - - This implementation allows the main thread to integrate - itself into performing function execution through - calling integrate_as_primary_thread() which will return - when the pool received a trigger_shutdown(). - - By default allows unlimited number of spawns. - """ - - _primary_thread_task: Reply | None - - def __init__(self, execmodel: ExecModel, hasprimary: bool = False) -> None: - self.execmodel = execmodel - self._running_lock = self.execmodel.Lock() - self._running: set[Reply] = set() - self._shuttingdown = False - self._waitall_events: list[Event] = [] - if hasprimary: - if self.execmodel.backend not in ("thread", "main_thread_only"): - raise ValueError("hasprimary=True requires thread model") - self._primary_thread_task_ready: Event | None = self.execmodel.Event() - else: - self._primary_thread_task_ready = None - - def integrate_as_primary_thread(self) -> None: - """Integrate the thread with which we are called as a primary - thread for executing functions triggered with spawn().""" - assert self.execmodel.backend in ("thread", "main_thread_only"), self.execmodel - primary_thread_task_ready = self._primary_thread_task_ready - assert primary_thread_task_ready is not None - # interacts with code at REF1 - while 1: - primary_thread_task_ready.wait() - reply = self._primary_thread_task - if reply is None: # trigger_shutdown() woke us up - break - self._perform_spawn(reply) - # we are concurrent with trigger_shutdown and spawn - with self._running_lock: - if self._shuttingdown: - break - # Only clear if _try_send_to_primary_thread has not - # yet set the next self._primary_thread_task reply - # after waiting for this one to complete. - if reply is self._primary_thread_task: - primary_thread_task_ready.clear() - - def trigger_shutdown(self) -> None: - with self._running_lock: - self._shuttingdown = True - if self._primary_thread_task_ready is not None: - self._primary_thread_task = None - self._primary_thread_task_ready.set() - - def active_count(self) -> int: - return len(self._running) - - def _perform_spawn(self, reply: Reply) -> None: - reply.run() - with self._running_lock: - self._running.remove(reply) - if not self._running: - while self._waitall_events: - waitall_event = self._waitall_events.pop() - waitall_event.set() - - def _try_send_to_primary_thread(self, reply: Reply) -> bool: - # REF1 in 'thread' model we give priority to running in main thread - # note that we should be called with _running_lock hold - primary_thread_task_ready = self._primary_thread_task_ready - if primary_thread_task_ready is not None: - if not primary_thread_task_ready.is_set(): - self._primary_thread_task = reply - # wake up primary thread - primary_thread_task_ready.set() - return True - elif ( - self.execmodel.backend == "main_thread_only" - and self._primary_thread_task is not None - ): - self._primary_thread_task.waitfinish() - self._primary_thread_task = reply - # wake up primary thread (it's okay if this is already set - # because we waited for the previous task to finish above - # and integrate_as_primary_thread will not clear it when - # it enters self._running_lock if it detects that a new - # task is available) - primary_thread_task_ready.set() - return True - return False - - def spawn(self, func, *args, **kwargs) -> Reply: - """Asynchronously dispatch func(*args, **kwargs) and return a Reply.""" - reply = Reply((func, args, kwargs), self.execmodel) - with self._running_lock: - if self._shuttingdown: - raise ValueError("pool is shutting down") - self._running.add(reply) - if not self._try_send_to_primary_thread(reply): - self.execmodel.start(self._perform_spawn, (reply,)) - return reply - - def terminate(self, timeout: float | None = None) -> bool: - """Trigger shutdown and wait for completion of all executions.""" - self.trigger_shutdown() - return self.waitall(timeout=timeout) - - def waitall(self, timeout: float | None = None) -> bool: - """Wait until all active spawns have finished executing.""" - with self._running_lock: - if not self._running: - return True - # if a Reply still runs, we let run_and_release - # signal us -- note that we are still holding the - # _running_lock to avoid race conditions - my_waitall_event = self.execmodel.Event() - self._waitall_events.append(my_waitall_event) - return my_waitall_event.wait(timeout=timeout) + if backend in ("thread", "main_thread_only"): + return ExecModel(backend) + raise ValueError(f"unknown execmodel {backend!r}") sysex = (KeyboardInterrupt, SystemExit) @@ -578,11 +353,11 @@ def __init__(self, gateway: BaseGateway, id: int) -> None: self._strconfig = getattr(gateway, "_strconfig", (True, False)) self.id = id # serialized payloads (or ENDMARKER); None once a callback is set - self._mailbox: Mailbox[Any] | None = Mailbox() + self._mailbox: Mailbox[Any] | None = Mailbox(gateway._new_wakener()) self._callback: Callable[[Any], Any] | None = None self._endmarker: object = NO_ENDMARKER_WANTED self._closed = False - self._receiveclosed = threading.Event() + self._receiveclosed = Flag(gateway._new_wakener()) self._remoteerrors: list[RemoteError] = [] def _trace(self, *msg: object) -> None: @@ -1028,6 +803,9 @@ class BaseGateway: _trio_session: Any = None # Set by the receiver on EOF without a prior termination message. _error: BaseException | None = None + #: wait= axis: which wakener backend this gateway's blocking waits + #: park on (channels, write-acks, join) + _wait_backend: str = "thread" def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel @@ -1051,6 +829,10 @@ def _attach_trio_session(self, session: Any) -> None: for channel in self._channelfactory.channels(): session.bind_sync_channel(channel) + def _new_wakener(self) -> Wakener: + """A fresh wakener for one blocking-wait carrier (wait= axis).""" + return make_wakener(self._wait_backend) + def _bind_channel(self, channel: Channel) -> None: """Divert the session's inbound routing for ``channel.id`` to it.""" session = self._trio_session @@ -1134,7 +916,7 @@ class WorkerGateway(BaseGateway): _trio_exec: Any = None # The exec pool (a TrioWorkerExec duck-typed as WorkerPool for STATUS). _execpool: Any = None - _executetask_complete: Event | None = None + _executetask_complete: threading.Event | None = None def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: trio_exec = self._trio_exec diff --git a/src/execnet/multi.py b/src/execnet/multi.py index ca3b69dd..ba73bf62 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -7,6 +7,9 @@ from __future__ import annotations import atexit +import queue +import threading +import time import types from collections.abc import Callable from collections.abc import Iterable @@ -20,9 +23,9 @@ from typing import TypeAlias from typing import overload +from ._boundary import wakener_names from .gateway_base import Channel from .gateway_base import ExecModel -from .gateway_base import WorkerPool from .gateway_base import get_execmodel from .gateway_base import trace from .xspec import XSpec @@ -152,6 +155,7 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: id= specifies the gateway id python= specifies which python interpreter to execute execmodel=model 'thread' or 'main_thread_only' execution model + wait=backend wakener for blocking waits ('thread' default) chdir= specifies to which directory to change nice= specifies process priority of new process env:NAME=value specifies a remote environment variable setting. @@ -165,6 +169,10 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: self.allocate_id(spec) if spec.execmodel is None: spec.execmodel = self.remote_execmodel.backend + if spec.wait is not None and spec.wait not in wakener_names(): + raise ValueError( + f"unknown wait backend {spec.wait!r} (known: {wakener_names()})" + ) from . import _trio_host if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): @@ -313,10 +321,10 @@ def make_receive_queue(self, endmarker: object = NO_ENDMARKER_WANTED): try: return self._queue # type: ignore[has-type] except AttributeError: - self._queue = None + self._queue: queue.Queue[tuple[Channel, Any]] | None = None for ch in self._channels: if self._queue is None: - self._queue = ch.gateway.execmodel.queue.Queue() + self._queue = queue.Queue() def putreceived(obj, channel: Channel = ch) -> None: self._queue.put((channel, obj)) # type: ignore[union-attr] @@ -351,32 +359,45 @@ def safe_terminate( """Run terminate/kill pairs in parallel with a hard wait bound. Each termfunc is given ``timeout``. If it does not finish, killfunc runs. - Waiting for the worker pool is also bounded so a stuck kill cannot hang - the caller forever (see issues #43 / #221). + The final wait is also bounded so a stuck kill cannot hang the caller + forever (see issues #43 / #221). ``execmodel`` is accepted for + backward compatibility and unused (daemon threads do the waiting). """ - workerpool = WorkerPool(execmodel) + errors: list[BaseException] = [] def termkill(termfunc: TermKillFunc, killfunc: TermKillFunc) -> None: - termreply = workerpool.spawn(termfunc) - try: - termreply.get(timeout=timeout) - except OSError: + term_done = threading.Event() + term_errors: list[BaseException] = [] + + def run_term() -> None: + try: + termfunc() + except BaseException as exc: + term_errors.append(exc) + finally: + term_done.set() + + threading.Thread(target=run_term, daemon=True).start() + if not term_done.wait(timeout): killfunc() + return + if term_errors: + errors.append(term_errors[0]) - replylist = [ - workerpool.spawn(termkill, termfunc, killfunc) - for termfunc, killfunc in list_of_paired_functions + threads = [ + threading.Thread(target=termkill, args=pair, daemon=True) + for pair in list_of_paired_functions ] + for thread in threads: + thread.start() # Allow term timeout plus a kill attempt; never block indefinitely. wait_timeout = None if timeout is None else timeout * 2 - for reply in replylist: - try: - reply.waitfinish(timeout=wait_timeout) - except OSError: - # termkill still running (typically stuck in killfunc). - continue - reply.get() # propagate worker exceptions, if any - workerpool.waitall(timeout=wait_timeout) + deadline = None if wait_timeout is None else time.monotonic() + wait_timeout + for thread in threads: + remaining = None if deadline is None else max(0, deadline - time.monotonic()) + thread.join(remaining) + if errors: + raise errors[0] default_group = Group() diff --git a/src/execnet/xspec.py b/src/execnet/xspec.py index 0559ed8c..afc58f4c 100644 --- a/src/execnet/xspec.py +++ b/src/execnet/xspec.py @@ -29,6 +29,7 @@ class XSpec: ssh_config: str | None = None vagrant_ssh: str | None = None via: str | None = None + wait: str | None = None def __init__(self, string: str) -> None: self._spec = string diff --git a/testing/conftest.py b/testing/conftest.py index 4d97bf35..7c3ed7a6 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -5,6 +5,7 @@ from collections.abc import Callable from collections.abc import Generator from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor from functools import lru_cache import pytest @@ -12,7 +13,6 @@ import execnet from execnet.gateway import Gateway from execnet.gateway_base import ExecModel -from execnet.gateway_base import WorkerPool from execnet.gateway_base import get_execmodel collect_ignore = ["build", "doc/_build"] @@ -184,5 +184,6 @@ def execmodel(request: pytest.FixtureRequest) -> ExecModel: @pytest.fixture -def pool(execmodel: ExecModel) -> WorkerPool: - return WorkerPool(execmodel=execmodel) +def executor() -> Iterator[ThreadPoolExecutor]: + with ThreadPoolExecutor() as tpe: + yield tpe diff --git a/testing/test_basics.py b/testing/test_basics.py index 29ceef19..c58e4728 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -71,15 +71,23 @@ def standalone_gateway_base_source() -> str: """gateway_base's source as a self-contained script. The module imports the trio-free boundary kit relatively; for the - run-on-any-python checks the kit source is inlined at the import site - (minus its own future import, which must stay file-leading). + run-on-any-python checks the kit source is inlined in place of those + imports (minus its own future import, which must stay file-leading). """ boundary_source = inspect.getsource(_boundary).replace( "from __future__ import annotations\n", "" ) - return inspect.getsource(gateway_base).replace( - "from ._boundary import Mailbox\n", boundary_source - ) + lines = [] + inlined = False + for line in inspect.getsource(gateway_base).splitlines(keepends=True): + if line.startswith("from ._boundary import"): + if not inlined: + inlined = True + lines.append(boundary_source) + else: + lines.append(line) + assert inlined + return "".join(lines) IO_MESSAGE_EXTRA_SOURCE = """ @@ -341,6 +349,7 @@ class TestPureChannel: def fac(self, execmodel: ExecModel) -> ChannelFactory: class FakeGateway: _trio_session = None + _new_wakener = staticmethod(_boundary.ThreadWakener) def _trace(self, *args) -> None: pass diff --git a/testing/test_channel.py b/testing/test_channel.py index ba02b57b..f06b8632 100644 --- a/testing/test_channel.py +++ b/testing/test_channel.py @@ -4,6 +4,7 @@ from __future__ import annotations +import queue import time import pytest @@ -270,14 +271,14 @@ def test_channel_endmarker_callback(self, gw: Gateway) -> None: assert l[3] == 999 def test_channel_endmarker_callback_error(self, gw: Gateway) -> None: - q = gw.execmodel.queue.Queue() + q: queue.Queue[object] = queue.Queue() channel = gw.remote_exec( source=""" raise ValueError() """ ) channel.setcallback(q.put, endmarker=999) - val = q.get(TESTTIMEOUT) + val = q.get(timeout=TESTTIMEOUT) assert val == 999 err = channel._getremoteerror() assert err diff --git a/testing/test_gateway.py b/testing/test_gateway.py index b7098773..da519d4e 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -599,7 +599,7 @@ def test_main_thread_only_concurrent_remote_exec_deadlock( import threading channel.send(threading.current_thread() is threading.main_thread()) # Wait forever, ensuring that the deadlock case triggers. - channel.gateway.execmodel.Event().wait() + threading.Event().wait() """ ) ) diff --git a/testing/test_multi.py b/testing/test_multi.py index e0390298..2f1b634e 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -5,6 +5,7 @@ from __future__ import annotations import gc +import threading from collections.abc import Callable from time import sleep @@ -308,12 +309,12 @@ def test_safe_terminate_does_not_hang_when_kill_blocks( out, so a blocking killfunc made Group.terminate() hang indefinitely (seen via pytest-xdist teardown). """ - kill_started = execmodel.Event() - release_kill = execmodel.Event() + kill_started = threading.Event() + release_kill = threading.Event() other_killed: list[int] = [] def term_slow() -> None: - execmodel.sleep(10) + sleep(10) def kill_hang() -> None: kill_started.set() diff --git a/testing/test_termination.py b/testing/test_termination.py index ca119304..01c57427 100644 --- a/testing/test_termination.py +++ b/testing/test_termination.py @@ -1,10 +1,12 @@ import os import pathlib +import queue import shutil import signal import subprocess import sys from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor import pytest from test_gateway import TESTTIMEOUT @@ -12,7 +14,6 @@ import execnet from execnet.gateway import Gateway from execnet.gateway_base import ExecModel -from execnet.gateway_base import WorkerPool execnetdir = pathlib.Path(execnet.__file__).parent.parent @@ -23,7 +24,7 @@ def test_exit_blocked_worker_execution_gateway( - anypython: str, makegateway: Callable[[str], Gateway], pool: WorkerPool + anypython: str, makegateway: Callable[[str], Gateway], executor: ThreadPoolExecutor ) -> None: gateway = makegateway("popen//python=%s" % anypython) gateway.remote_exec( @@ -37,8 +38,7 @@ def doit() -> int: gateway.exit() return 17 - reply = pool.spawn(doit) - x = reply.get(timeout=5.0) + x = executor.submit(doit).result(timeout=5.0) assert x == 17 @@ -48,7 +48,7 @@ def test_endmarker_delivery_on_remote_killterm( if execmodel.backend not in ("thread", "main_thread_only"): pytest.xfail("test and execnet not compatible to greenlets yet") gw = makegateway("popen") - q = execmodel.queue.Queue() + q: queue.Queue[object] = queue.Queue() channel = gw.remote_exec( source=""" import os, time @@ -60,7 +60,7 @@ def test_endmarker_delivery_on_remote_killterm( assert isinstance(pid, int) os.kill(pid, signal.SIGTERM) channel.setcallback(q.put, endmarker=999) - val = q.get(TESTTIMEOUT) + val = q.get(timeout=TESTTIMEOUT) assert val == 999 err = channel._getremoteerror() assert isinstance(err, EOFError) @@ -113,10 +113,8 @@ def test_terminate_implicit_does_trykill( pytester: pytest.Pytester, anypython: str, capfd: pytest.CaptureFixture[str], - pool: WorkerPool, + executor: ThreadPoolExecutor, ) -> None: - if pool.execmodel.backend not in ("thread", "main_thread_only"): - pytest.xfail("only os threading model supported") if sys.version_info >= (3, 12): pytest.xfail( "since python3.12 this test triggers RuntimeError: can't create new thread at interpreter shutdown" @@ -149,8 +147,7 @@ def flush(self): # sync with start-up assert popen.stdout is not None popen.stdout.readline() - reply = pool.spawn(popen.communicate) - reply.get(timeout=50) + executor.submit(popen.communicate).result(timeout=50) _out, err = capfd.readouterr() lines = [x for x in err.splitlines() if "*sys-package" not in x] assert not lines or "Killed" in err diff --git a/testing/test_threadpool.py b/testing/test_threadpool.py deleted file mode 100644 index 510ae020..00000000 --- a/testing/test_threadpool.py +++ /dev/null @@ -1,222 +0,0 @@ -import os -from pathlib import Path - -import pytest - -from execnet.gateway_base import ExecModel -from execnet.gateway_base import WorkerPool - - -def test_execmodel(execmodel: ExecModel, tmp_path: Path) -> None: - assert execmodel.backend - p = tmp_path / "somefile" - p.write_text("content") - fd = os.open(p, os.O_RDONLY) - f = execmodel.fdopen(fd, "r") - assert f.read() == "content" - f.close() - - -def test_execmodel_basic_attrs(execmodel: ExecModel) -> None: - m = execmodel - assert callable(m.start) - assert m.get_ident() - - -def test_simple(pool: WorkerPool) -> None: - reply = pool.spawn(lambda: 42) - assert reply.get() == 42 - - -def test_some(pool: WorkerPool, execmodel: ExecModel) -> None: - q = execmodel.queue.Queue() - num = 4 - - def f(i: int) -> None: - q.put(i) - while q.qsize(): - execmodel.sleep(0.01) - - for i in range(num): - pool.spawn(f, i) - for i in range(num): - q.get() - # assert len(pool._running) == 4 - assert pool.waitall(timeout=1.0) - # execmodel.sleep(1) helps on windows? - assert len(pool._running) == 0 - - -def test_running_semnatics(pool: WorkerPool, execmodel: ExecModel) -> None: - q = execmodel.queue.Queue() - - def first() -> None: - q.get() - - reply = pool.spawn(first) - assert reply.running - assert pool.active_count() == 1 - q.put(1) - assert pool.waitall() - assert pool.active_count() == 0 - assert not reply.running - - -def test_waitfinish_on_reply(pool: WorkerPool) -> None: - l = [] - reply = pool.spawn(lambda: l.append(1)) - reply.waitfinish() - assert l == [1] - reply = pool.spawn(lambda: 0 / 0) - reply.waitfinish() # no exception raised - pytest.raises(ZeroDivisionError, reply.get) - - -@pytest.mark.xfail(reason="WorkerPool does not implement limited size") -def test_limited_size(execmodel: ExecModel) -> None: - pool = WorkerPool(execmodel, size=1) # type: ignore[call-arg] - q = execmodel.queue.Queue() - q2 = execmodel.queue.Queue() - q3 = execmodel.queue.Queue() - - def first() -> None: - q.put(1) - q2.get() - - pool.spawn(first) - assert q.get() == 1 - - def second() -> None: - q3.put(3) - - # we spawn a second pool to spawn the second function - # which should block - pool2 = WorkerPool(execmodel) - pool2.spawn(pool.spawn, second) - assert not pool2.waitall(1.0) - assert q3.qsize() == 0 - q2.put(2) - assert pool2.waitall() - assert pool.waitall() - - -def test_get(pool: WorkerPool) -> None: - def f() -> int: - return 42 - - reply = pool.spawn(f) - result = reply.get() - assert result == 42 - - -def test_get_timeout(execmodel: ExecModel, pool: WorkerPool) -> None: - def f() -> int: - execmodel.sleep(0.2) - return 42 - - reply = pool.spawn(f) - with pytest.raises(IOError): - reply.get(timeout=0.01) - - -def test_get_excinfo(pool: WorkerPool) -> None: - def f() -> None: - raise ValueError("42") - - reply = pool.spawn(f) - with pytest.raises(ValueError): - reply.get(1.0) - with pytest.raises(ValueError): - reply.get(1.0) - - -def test_waitall_timeout(pool: WorkerPool, execmodel: ExecModel) -> None: - q = execmodel.queue.Queue() - - def f() -> None: - q.get() - - reply = pool.spawn(f) - assert not pool.waitall(0.01) - q.put(None) - reply.get(timeout=1.0) - assert pool.waitall(timeout=0.1) - - -@pytest.mark.skipif(not hasattr(os, "dup"), reason="no os.dup") -def test_pool_clean_shutdown( - pool: WorkerPool, capfd: pytest.CaptureFixture[str] -) -> None: - q = pool.execmodel.queue.Queue() - - def f() -> None: - q.get() - - pool.spawn(f) - assert not pool.waitall(timeout=1.0) - pool.trigger_shutdown() - with pytest.raises(ValueError): - pool.spawn(f) - - def wait_then_put() -> None: - pool.execmodel.sleep(0.1) - q.put(1) - - pool.execmodel.start(wait_then_put) - assert pool.waitall() - _out, err = capfd.readouterr() - assert err == "" - - -def test_primary_thread_integration(execmodel: ExecModel) -> None: - if execmodel.backend not in ("thread", "main_thread_only"): - with pytest.raises(ValueError): - WorkerPool(execmodel=execmodel, hasprimary=True) - return - pool = WorkerPool(execmodel=execmodel, hasprimary=True) - queue = execmodel.queue.Queue() - - def do_integrate() -> None: - queue.put(execmodel.get_ident()) - pool.integrate_as_primary_thread() - - execmodel.start(do_integrate) - - def func() -> None: - queue.put(execmodel.get_ident()) - - pool.spawn(func) - ident1 = queue.get() - ident2 = queue.get() - assert ident1 == ident2 - pool.terminate() - - -def test_primary_thread_integration_shutdown(execmodel: ExecModel) -> None: - if execmodel.backend not in ("thread", "main_thread_only"): - pytest.skip("can only run with threading") - pool = WorkerPool(execmodel=execmodel, hasprimary=True) - queue = execmodel.queue.Queue() - - def do_integrate() -> None: - queue.put(execmodel.get_ident()) - pool.integrate_as_primary_thread() - - execmodel.start(do_integrate) - queue.get() - - queue2 = execmodel.queue.Queue() - - def get_two() -> None: - queue.put(execmodel.get_ident()) - queue2.get() - - reply = pool.spawn(get_two) - # make sure get_two is running and blocked on queue2 - queue.get() - # then shut down - pool.trigger_shutdown() - # and let get_two finish - queue2.put(1) - reply.get() - assert pool.waitall(5.0) diff --git a/testing/test_xspec.py b/testing/test_xspec.py index 661e50af..850acbe1 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -119,6 +119,14 @@ class TestMakegateway: def test_no_type(self, makegateway: Callable[[str], Gateway]) -> None: pytest.raises(ValueError, lambda: makegateway("hello")) + def test_wait_axis(self, makegateway: Callable[[str], Gateway]) -> None: + with pytest.raises(ValueError, match="unknown wait backend"): + makegateway("popen//wait=nope") + gw = makegateway("popen//wait=thread") + assert gw._wait_backend == "thread" + channel = gw.remote_exec("channel.send(channel.gateway._wait_backend)") + assert channel.receive() == "thread" + @skip_win_pypy def test_popen_default(self, makegateway: Callable[[str], Gateway]) -> None: gw = makegateway("") From 845510e64a8e9e8542cae7234f6197ec812527f7 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:19:08 +0200 Subject: [PATCH 38/59] feat: add execnet.aio -- the asyncio-native API over the Trio host P4 of the boundary rethink. A fourth public namespace mirroring execnet.trio's surface for asyncio applications: Group/Gateway/Channel are async context managers / awaitables whose operations run as tasks on a dedicated Trio host thread and resolve asyncio futures via loop.call_soon_threadsafe (a _HostBridge per group). All transports work because the protocol engine is the same AsyncGroup; no anyio port and no per-call executor threads. Received channel references arrive wrapped as aio Channels. Known v1 limitation (documented): cancelling a bridged await abandons the operation asyncio-side only; the host-side task runs to completion. `import execnet` stays free of trio/asyncio -- execnet.aio loads lazily like execnet.trio. Co-Authored-By: Claude Fable 5 --- src/execnet/__init__.py | 8 +- src/execnet/aio.py | 321 ++++++++++++++++++++++++++++++++++++++++ testing/test_aio.py | 116 +++++++++++++++ 3 files changed, 442 insertions(+), 3 deletions(-) create mode 100644 src/execnet/aio.py create mode 100644 testing/test_aio.py diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 2108f4b2..66c61821 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -4,14 +4,16 @@ pure python lib for connecting to local and remote Python Interpreters. -Three public namespaces: +Four public namespaces: * :mod:`execnet.sync` — the blocking API; the top-level ``execnet.*`` names below are aliases into it. * :mod:`execnet.trio` — the trio-native API, awaited inside your own ``trio.run``. +* :mod:`execnet.aio` — the asyncio-native API, bridged over a Trio + host thread. * :mod:`execnet.portal` — cross-thread / cross-loop communication - primitives shared by both. + primitives shared by all of them. (c) 2012, Holger Krekel and others """ @@ -66,7 +68,7 @@ def __getattr__(name: str) -> Any: # Lazy namespace modules: keep ``import execnet`` from loading the # trio event loop machinery until it is actually used. - if name in ("trio", "portal"): + if name in ("aio", "portal", "trio"): import importlib return importlib.import_module(f".{name}", __name__) diff --git a/src/execnet/aio.py b/src/execnet/aio.py new file mode 100644 index 00000000..e3004a28 --- /dev/null +++ b/src/execnet/aio.py @@ -0,0 +1,321 @@ +"""The asyncio-native execnet API, bridged over a Trio host thread. + +Everything here is awaited inside your own asyncio event loop:: + + import asyncio + import execnet.aio + + async def main(): + async with execnet.aio.Group() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(6 * 7)") + print(await channel.receive()) + + asyncio.run(main()) + +Protocol IO keeps running on a dedicated Trio host thread (the same +engine as the blocking and trio-native APIs, all transports included); +each awaited operation runs as a task on that host and resolves an +asyncio future via ``loop.call_soon_threadsafe``. No anyio port and no +executor threads per call. + +Note: cancelling a bridged await abandons the operation on the asyncio +side only -- the host-side task runs to completion (a cancelled +``receive`` may still consume the next item). + +The serialization helpers and error types are shared with +:mod:`execnet.sync` and :mod:`execnet.trio`. +""" + +from __future__ import annotations + +import asyncio +import functools +import types +from collections.abc import AsyncIterator +from collections.abc import Awaitable +from collections.abc import Callable +from contextlib import asynccontextmanager +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Any +from typing import TypeVar + +import trio + +from ._trio_gateway import AsyncChannel +from ._trio_gateway import AsyncGateway +from ._trio_gateway import AsyncGroup +from ._trio_host import TrioHost +from .gateway_base import DataFormatError +from .gateway_base import DumpError +from .gateway_base import HostNotFound +from .gateway_base import LoadError +from .gateway_base import RemoteError +from .gateway_base import TimeoutError +from .gateway_base import dump +from .gateway_base import dumps +from .gateway_base import load +from .gateway_base import loads +from .xspec import XSpec + +if TYPE_CHECKING: + from typing_extensions import Self + +__all__ = [ + "Channel", + "DataFormatError", + "DumpError", + "Gateway", + "Group", + "HostNotFound", + "LoadError", + "RemoteError", + "TimeoutError", + "XSpec", + "dump", + "dumps", + "load", + "loads", + "open_popen_gateway", +] + +T = TypeVar("T") + + +class _HostBridge: + """Await trio-native coroutines on a TrioHost from asyncio.""" + + def __init__(self, host: TrioHost) -> None: + self._host = host + + async def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: + loop = asyncio.get_running_loop() + future: asyncio.Future[T] = loop.create_future() + + def resolve(result: Any, error: BaseException | None) -> None: + if future.cancelled(): + return + if error is not None: + future.set_exception(error) + else: + future.set_result(result) + + def post_result(result: Any, error: BaseException | None) -> None: + # The asyncio loop may already be gone at interpreter/test + # teardown; the result is undeliverable then. + with suppress(RuntimeError): + loop.call_soon_threadsafe(resolve, result, error) + + async def runner() -> None: + try: + result = await async_fn(*args) + except trio.Cancelled: + # host shutdown: the nursery cancel must propagate + post_result(None, RuntimeError("execnet aio host was shut down")) + raise + except BaseException as exc: + post_result(None, exc) + else: + post_result(result, None) + + def spawn() -> None: + self._host.start_soon(runner) + + try: + self._host.portal.post(spawn) + except trio.RunFinishedError: + raise RuntimeError("execnet aio group is not running") from None + return await future + + +class _HostedGroup(AsyncGroup): + """Trio AsyncGroup living as a task on the host nursery.""" + + def __init__(self, termination_timeout: float) -> None: + super().__init__(termination_timeout) + self.shutdown = trio.Event() + self.finished = trio.Event() + + async def run(self, task_status: trio.TaskStatus[_HostedGroup]) -> None: + try: + async with self: + task_status.started(self) + await self.shutdown.wait() + finally: + self.finished.set() + + +class Channel: + """asyncio facade over a trio-native :class:`AsyncChannel`.""" + + RemoteError = RemoteError + TimeoutError = TimeoutError + + def __init__(self, bridge: _HostBridge, channel: AsyncChannel) -> None: + self._bridge = bridge + self._channel = channel + + @property + def id(self) -> int: + return self._channel.id + + def __repr__(self) -> str: + return f"" + + def isclosed(self) -> bool: + """Return True if the channel is closed for sending.""" + return self._channel.isclosed() + + async def send(self, item: object) -> None: + """Serialize ``item`` and send it to the other side.""" + await self._bridge.call(self._channel.send, item) + + async def receive(self, timeout: float | None = None) -> Any: + """Receive the next item sent from the other side. + + EOFError once the peer closed or sent EOF, RemoteError for a peer + close-with-error, TimeoutError after ``timeout`` seconds. A + received channel reference arrives as an :class:`~execnet.aio.Channel`. + """ + result = await self._bridge.call(self._channel.receive, timeout) + if isinstance(result, AsyncChannel): + return Channel(self._bridge, result) + return result + + async def send_eof(self) -> None: + """Signal that no more items follow (peer keeps its send side).""" + await self._bridge.call(self._channel.send_eof) + + async def aclose(self, error: str | None = None) -> None: + """Close the channel; ``error`` reaches the peer as a RemoteError.""" + await self._bridge.call(self._channel.aclose, error) + + async def wait_closed(self) -> None: + """Wait until the peer closed or sent EOF; reraise remote errors.""" + await self._bridge.call(self._channel.wait_closed) + + def __aiter__(self) -> Channel: + return self + + async def __anext__(self) -> Any: + try: + return await self.receive() + except EOFError: + raise StopAsyncIteration from None + + +class Gateway: + """asyncio facade over a trio-native :class:`AsyncGateway`.""" + + def __init__(self, bridge: _HostBridge, gateway: AsyncGateway) -> None: + self._bridge = bridge + self._gateway = gateway + + @property + def id(self) -> str: + return self._gateway.id + + @property + def remoteaddress(self) -> str | None: + return self._gateway.remoteaddress + + def __repr__(self) -> str: + return f"" + + async def remote_exec( + self, + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + **kwargs: object, + ) -> Channel: + """Connect a new channel to remote execution of ``source``. + + Accepts the same source kinds as ``Gateway.remote_exec``: a source + string, a pure function called with ``channel`` and ``**kwargs``, + or a module. + """ + channel = await self._bridge.call( + functools.partial(self._gateway.remote_exec, source, **kwargs) + ) + return Channel(self._bridge, channel) + + async def terminate(self) -> None: + """Send GATEWAY_TERMINATE to the peer, then close this side.""" + await self._bridge.call(self._gateway.terminate) + + +class Group: + """asyncio-native gateway group over a dedicated Trio host thread. + + An async context manager mirroring :class:`execnet.trio.AsyncGroup`; + leaving the ``async with`` block terminates every gateway with the + same bounded contract, then stops the host thread. + """ + + def __init__(self, termination_timeout: float = 10.0) -> None: + self._termination_timeout = termination_timeout + self._host: TrioHost | None = None + self._bridge: _HostBridge | None = None + self._group: _HostedGroup | None = None + + def __repr__(self) -> str: + state = "running" if self._group is not None else "idle" + return f"" + + async def __aenter__(self) -> Self: + assert self._host is None, "group already entered" + loop = asyncio.get_running_loop() + host = TrioHost(name="execnet-aio-group") + # start() blocks until the host loop is ready; keep it off the + # asyncio loop thread. + await loop.run_in_executor(None, host.start) + self._host = host + self._bridge = _HostBridge(host) + + async def start_group() -> _HostedGroup: + # runs on the host loop + group = _HostedGroup(self._termination_timeout) + started: _HostedGroup = await host._nursery.start(group.run) + return started + + self._group = await self._bridge.call(start_group) + return self + + async def __aexit__(self, *exc_info: object) -> None: + group, bridge, host = self._group, self._bridge, self._host + self._group = self._bridge = None + if group is not None and bridge is not None: + + async def stop_group() -> None: + group.shutdown.set() + await group.finished.wait() + + with suppress(RuntimeError): + await bridge.call(stop_group) + if host is not None: + self._host = None + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, functools.partial(host.stop, 5.0)) + + async def makegateway(self, spec: str | XSpec = "popen") -> Gateway: + """Create a gateway for ``spec`` served on the group's host. + + All transports are supported: popen (including uv-provisioned + ``python=``), ``ssh=``, ``vagrant_ssh=``, ``socket=`` (with + ``installvia=``), and ``via=`` sub-gateways. + """ + group, bridge = self._group, self._bridge + if group is None or bridge is None: + raise RuntimeError(f"{self!r} is not entered") + gateway = await bridge.call(group.makegateway, spec) + return Gateway(bridge, gateway) + + +@asynccontextmanager +async def open_popen_gateway(spec: str | XSpec = "popen") -> AsyncIterator[Gateway]: + """Spawn one popen worker and yield an asyncio Gateway to it. + + Convenience for a single-gateway :class:`Group`. + """ + async with Group() as group: + yield await group.makegateway(spec) diff --git a/testing/test_aio.py b/testing/test_aio.py new file mode 100644 index 00000000..6bdb7724 --- /dev/null +++ b/testing/test_aio.py @@ -0,0 +1,116 @@ +"""The asyncio-native API: execnet.aio bridged over the Trio host.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable +from typing import Any +from typing import TypeVar + +import pytest + +import execnet.aio + +T = TypeVar("T") + +TESTTIMEOUT = 30.0 + + +def run(main: Awaitable[T]) -> T: + return asyncio.run(asyncio.wait_for(main, TESTTIMEOUT)) + + +def test_popen_roundtrip() -> None: + async def main() -> None: + async with execnet.aio.Group() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(channel.receive() + 1)") + await channel.send(41) + assert await channel.receive() == 42 + await channel.wait_closed() + + run(main()) + + +def test_open_popen_gateway_iteration() -> None: + async def main() -> list[int]: + async with execnet.aio.open_popen_gateway() as gateway: + channel = await gateway.remote_exec( + "for i in range(4): channel.send(i * 2)" + ) + return [item async for item in channel] + + assert run(main()) == [0, 2, 4, 6] + + +def test_receive_timeout() -> None: + async def main() -> None: + async with execnet.aio.open_popen_gateway() as gateway: + channel = await gateway.remote_exec("channel.receive()") + with pytest.raises(channel.TimeoutError): + await channel.receive(timeout=0.05) + await channel.send(None) + + run(main()) + + +def test_remote_error() -> None: + async def main() -> None: + async with execnet.aio.open_popen_gateway() as gateway: + channel = await gateway.remote_exec("raise ValueError(17)") + with pytest.raises(execnet.aio.RemoteError, match="ValueError"): + await channel.receive() + + run(main()) + + +def test_channel_passing_wraps_aio() -> None: + async def main() -> None: + async with execnet.aio.open_popen_gateway() as gateway: + channel = await gateway.remote_exec( + """ + c = channel.gateway.newchannel() + channel.send(c) + c.send(42) + """ + ) + passed = await channel.receive() + assert isinstance(passed, execnet.aio.Channel) + assert await passed.receive() == 42 + + run(main()) + + +def test_multiple_gateways_and_send_each() -> None: + async def main() -> list[Any]: + async with execnet.aio.Group() as group: + gateways = [await group.makegateway("popen") for _ in range(2)] + channels = [ + await gw.remote_exec("channel.send(channel.receive() * 2)") + for gw in gateways + ] + for i, channel in enumerate(channels): + await channel.send(i + 1) + return [await channel.receive() for channel in channels] + + assert run(main()) == [2, 4] + + +def test_group_not_entered() -> None: + async def main() -> None: + group = execnet.aio.Group() + with pytest.raises(RuntimeError, match="not entered"): + await group.makegateway("popen") + + run(main()) + + +def test_terminate_gateway_explicitly() -> None: + async def main() -> None: + async with execnet.aio.Group() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(1)") + assert await channel.receive() == 1 + await gateway.terminate() + + run(main()) From b01d00937d0428794b736bb06c42b714ec852bd1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:22:37 +0200 Subject: [PATCH 39/59] feat: gevent wait backend (wait=gevent) with opt-in tests P5, closing out the boundary rethink. execnet._gevent_support provides a GeventWakener whose notify() crosses from the trio host thread via a hub loop.async_ watcher (the one gevent primitive documented as thread-safe) while waits park only the calling greenlet. The hub-bound event/watcher are created lazily in the first waiter's hub so carriers can be constructed on the host loop; a pre-watcher notify is replayed, never lost. The boundary kit resolves wait=gevent by lazily importing the support module, so specs validate without gevent installed. Tests live behind the new `gevent` dependency group (pytest importorskip): wakener unit coverage plus gateway-level proof that channel.receive()/send() park the greenlet, not the hub -- no monkey-patching required. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 3 + src/execnet/_boundary.py | 12 +- src/execnet/_gevent_support.py | 76 ++++++++++++ src/execnet/aio.py | 1 + testing/test_gevent.py | 86 ++++++++++++++ uv.lock | 208 ++++++++++++++++++++++++++++++++- 6 files changed, 383 insertions(+), 3 deletions(-) create mode 100644 src/execnet/_gevent_support.py create mode 100644 testing/test_gevent.py diff --git a/pyproject.toml b/pyproject.toml index 4e94c938..aedd6b59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,9 @@ testing = [ dev = [ "pytest-xdist>=3.8.0", ] +gevent = [ + "gevent>=26.7.0", +] testing = [ "execnet[testing]", ] diff --git a/src/execnet/_boundary.py b/src/execnet/_boundary.py index ce211ae1..9c7c69f4 100644 --- a/src/execnet/_boundary.py +++ b/src/execnet/_boundary.py @@ -205,10 +205,14 @@ def wait(self, timeout: float | None = None) -> bool: # wait= axis: named wakener factories; each call returns a fresh instance # (carriers own their wakener exclusively, see Flag). Backends register -# here ("gevent", "asyncio") next to the built-in "thread". +# here next to the built-in "thread"; entries in the lazy table import +# their module (which registers itself) on first use. _WAKENER_FACTORIES: dict[str, Callable[[], Wakener]] = { "thread": ThreadWakener, } +_LAZY_WAKENER_MODULES: dict[str, str] = { + "gevent": "execnet._gevent_support", +} def register_wakener(name: str, factory: Callable[[], Wakener]) -> None: @@ -217,11 +221,15 @@ def register_wakener(name: str, factory: Callable[[], Wakener]) -> None: def wakener_names() -> list[str]: - return list(_WAKENER_FACTORIES) + return sorted(set(_WAKENER_FACTORIES) | set(_LAZY_WAKENER_MODULES)) def make_wakener(name: str) -> Wakener: """Create a fresh wakener for the named wait backend.""" + if name not in _WAKENER_FACTORIES and name in _LAZY_WAKENER_MODULES: + import importlib + + importlib.import_module(_LAZY_WAKENER_MODULES[name]) try: factory = _WAKENER_FACTORIES[name] except KeyError: diff --git a/src/execnet/_gevent_support.py b/src/execnet/_gevent_support.py new file mode 100644 index 00000000..cd761571 --- /dev/null +++ b/src/execnet/_gevent_support.py @@ -0,0 +1,76 @@ +"""The gevent wait backend (``wait=gevent``). + +Importing this module registers the ``gevent`` wakener factory; the +boundary kit imports it lazily when a spec asks for ``wait=gevent``. + +A blocking wait then parks only the calling greenlet: the carrier's +event lives in the waiting greenlet's hub, and the loop side's +``notify()`` crosses threads through a ``loop.async_`` watcher -- the +one libev/libuv primitive gevent documents as safe to use from other +threads. +""" + +from __future__ import annotations + +import threading +from typing import Any + +import gevent.event +from gevent.hub import get_hub + +from ._boundary import register_wakener + + +class GeventWakener: + """Wakener parking greenlets instead of OS threads. + + ``notify()`` is thread-safe and non-blocking (called from the trio + host thread). The hub-bound pieces (event + async watcher) are + created lazily in the first waiter's hub, so construction is safe + from any thread -- including the host loop, which creates channels + for inbound ids. Waiters must share one hub (one gevent thread per + carrier), which is the ordinary gevent setup. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._notified = False + self._event: gevent.event.Event | None = None + self._watcher: Any = None + + def notify(self) -> None: + with self._lock: + self._notified = True + watcher = self._watcher + if watcher is not None: + watcher.send() + + def _ensure(self) -> gevent.event.Event: + """Create the hub-bound event/watcher in the waiter's context.""" + event = self._event + if event is None: + event = gevent.event.Event() + watcher = get_hub().loop.async_() + watcher.start(event.set) + with self._lock: + self._event = event + self._watcher = watcher + return event + + def wait(self, timeout: float | None = None) -> bool: + event = self._ensure() + with self._lock: + # a notify may have fired before the watcher existed + if self._notified and not event.is_set(): + event.set() + return event.wait(timeout) + + def clear(self) -> None: + with self._lock: + self._notified = False + event = self._event + if event is not None: + event.clear() + + +register_wakener("gevent", GeventWakener) diff --git a/src/execnet/aio.py b/src/execnet/aio.py index e3004a28..bbfe06e3 100644 --- a/src/execnet/aio.py +++ b/src/execnet/aio.py @@ -275,6 +275,7 @@ async def __aenter__(self) -> Self: async def start_group() -> _HostedGroup: # runs on the host loop group = _HostedGroup(self._termination_timeout) + assert host._nursery is not None started: _HostedGroup = await host._nursery.start(group.run) return started diff --git a/testing/test_gevent.py b/testing/test_gevent.py new file mode 100644 index 00000000..e95edfc7 --- /dev/null +++ b/testing/test_gevent.py @@ -0,0 +1,86 @@ +"""The gevent wait backend (wait=gevent): greenlet-parking blocking waits. + +Opt-in: requires the ``gevent`` dependency group (``uv sync --group +gevent``); skipped when gevent is not installed. No monkey-patching is +needed -- the wakener parks the waiting greenlet while the trio host +thread keeps running the protocol. +""" + +from __future__ import annotations + +import threading + +import pytest + +gevent = pytest.importorskip("gevent") + +import execnet # noqa: E402 +from execnet._boundary import Flag # noqa: E402 +from execnet._boundary import Mailbox # noqa: E402 +from execnet._boundary import make_wakener # noqa: E402 + +TESTTIMEOUT = 10.0 + + +@pytest.fixture +def gevent_gw(): + group = execnet.Group() + try: + yield group.makegateway("popen//wait=gevent") + finally: + group.terminate(timeout=5.0) + + +class TestGeventWakener: + def test_mailbox_wakes_greenlet_from_foreign_thread(self) -> None: + box: Mailbox[str] = Mailbox(make_wakener("gevent")) + threading.Timer(0.05, box.put, args=["item"]).start() + result = gevent.spawn(box.get, 5.0) + assert result.get(timeout=TESTTIMEOUT) == "item" + + def test_notify_before_first_wait_is_not_lost(self) -> None: + flag = Flag(make_wakener("gevent")) + flag.set() + assert flag.wait(timeout=1.0) + + def test_wait_parks_greenlet_not_hub(self) -> None: + box: Mailbox[str] = Mailbox(make_wakener("gevent")) + progressed: list[int] = [] + + def other() -> None: + for i in range(5): + progressed.append(i) + gevent.sleep(0.01) + box.put("done") + + waiter = gevent.spawn(box.get, 5.0) + gevent.spawn(other) + # if get() blocked the hub, other() could never run and put() + assert waiter.get(timeout=TESTTIMEOUT) == "done" + assert progressed == [0, 1, 2, 3, 4] + + +class TestGeventGateway: + def test_receive_parks_greenlet_not_hub(self, gevent_gw: execnet.Gateway) -> None: + channel = gevent_gw.remote_exec("channel.send(channel.receive())") + progressed: list[int] = [] + + def other() -> None: + for i in range(5): + progressed.append(i) + gevent.sleep(0.01) + # sending from a greenlet blocks-until-written on the + # gevent wakener, parking only this greenlet + channel.send("hello") + + waiter = gevent.spawn(channel.receive, TESTTIMEOUT) + gevent.spawn(other) + # if receive() blocked the hub, other() could never send and + # the remote echo could never arrive -> this would hang + assert waiter.get(timeout=TESTTIMEOUT) == "hello" + assert progressed == [0, 1, 2, 3, 4] + + def test_waitclose_and_endmarker(self, gevent_gw: execnet.Gateway) -> None: + channel = gevent_gw.remote_exec("channel.send(1)") + assert gevent.spawn(channel.receive, TESTTIMEOUT).get(timeout=TESTTIMEOUT) == 1 + gevent.spawn(channel.waitclose, TESTTIMEOUT).get(timeout=TESTTIMEOUT) diff --git a/uv.lock b/uv.lock index 15185f2a..b885a4b7 100644 --- a/uv.lock +++ b/uv.lock @@ -372,7 +372,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -401,6 +401,9 @@ testing = [ dev = [ { name = "pytest-xdist" }, ] +gevent = [ + { name = "gevent" }, +] testing = [ { name = "execnet", extra = ["testing"] }, ] @@ -420,6 +423,7 @@ provides-extras = ["testing"] [package.metadata.requires-dev] dev = [{ name = "pytest-xdist", specifier = ">=3.8.0" }] +gevent = [{ name = "gevent", specifier = ">=26.7.0" }] testing = [{ name = "execnet", extras = ["testing"] }] [[package]] @@ -431,6 +435,151 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z" }, ] +[[package]] +name = "gevent" +version = "26.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, + { name = "greenlet", marker = "platform_python_implementation == 'CPython'" }, + { name = "zope-event" }, + { name = "zope-interface" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/5c/92002455a57cb3634383e2b822e3bccf409f43cde34528e46428971475cf/gevent-26.7.0.tar.gz", hash = "sha256:5b333a556e38a302b1b8c80525bef16d437e16f1e7767947789406841856a102", size = 6729213, upload-time = "2026-07-22T20:16:04.713Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/60/878d0cdef05d952ac7f17ffe385143fb0f3720afce0f6ff5ddbf7aac0342/gevent-26.7.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:80e98fc808bd9cc5c911d78a443d214bf0c8f96c9fdd296893df7e40364d5f37", size = 2198781, upload-time = "2026-07-22T16:48:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/69/79/6ce781b60049060e9d89b3d0fe60940353adeb39856aaad5ee925fd127e9/gevent-26.7.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bf4b946b47cc6fdbdf9221f891db9a44df92166435c027760ee7dbdfb4039adc", size = 2229318, upload-time = "2026-07-22T17:02:11.713Z" }, + { url = "https://files.pythonhosted.org/packages/6d/38/48898f35c2092d699b755b01918551358db72b554c63f553c8f027d2bf31/gevent-26.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:55ce0b7f87f9befcc788d77eb039b1de89a35f37afc31942e12c7ae090a563b8", size = 1700480, upload-time = "2026-07-22T16:26:16.794Z" }, + { url = "https://files.pythonhosted.org/packages/93/51/53370896942523c333699394ccad379d186648dbfb913f42ce094ddfb4b3/gevent-26.7.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:7f7143823ef99bc657534a2b6e8cbadedc910750cc0b4f4b4438a58d9fe43ab2", size = 1783048, upload-time = "2026-07-22T18:11:25.996Z" }, + { url = "https://files.pythonhosted.org/packages/ba/56/5a2cb36d75d3b626d6ffa116673b34442bf5afd6f7ab4b98e512c1b008d6/gevent-26.7.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:ca4019899830471910129968251c795c8aee59e225fd16326ae01c1f93f3cfa6", size = 1880257, upload-time = "2026-07-22T18:10:40.919Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ed/ee7eb2f03a38a4f33b0f327bab16d3158b3a794588a84dba739cbf4a3e68/gevent-26.7.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:e4042da317a96d12110831cc404855f0c501a5a5aa476a7a18c3b480a5a59233", size = 1819378, upload-time = "2026-07-22T18:29:06.444Z" }, + { url = "https://files.pythonhosted.org/packages/5e/aa/e2a202c03cff4f49bba54cc321c3afc29eee9d6fc452f4aa94d2f00e7d3d/gevent-26.7.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:5c97ca98e1aae427a267eae0fbfe8d0884327e6b1cd51fc2ef6642b8b0b82701", size = 2136837, upload-time = "2026-07-22T16:48:29.939Z" }, + { url = "https://files.pythonhosted.org/packages/ac/64/4892fbca47aa4e06b86aef96d6146bd61175b23b7db431ec7937d73b2f72/gevent-26.7.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5d5d1864bc3db92d1f82d1790395eda99f98b47fd9f7ec02c4e182d7828a8251", size = 1794058, upload-time = "2026-07-22T18:07:10.644Z" }, + { url = "https://files.pythonhosted.org/packages/21/23/90bb7d0c6f59d2973bb8f4bd3164be00e466823dea01568e0cb36f2afb77/gevent-26.7.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:15fd2d88ed5370f8084079758758df91f26d2f68575e1ee76fce604ddba83e5e", size = 2159797, upload-time = "2026-07-22T17:02:13.229Z" }, + { url = "https://files.pythonhosted.org/packages/a3/67/4d1e315ee3052530fa8537e0cdf1f9c3a6606740f7372f420c7d12d5cf82/gevent-26.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:514bda3fff741d7e5ab108ee1d31550a7f4b2fd3dc6e3b6f38dfb8685efdafaa", size = 1682414, upload-time = "2026-07-22T16:26:22.18Z" }, + { url = "https://files.pythonhosted.org/packages/02/b1/d1b1de89677ee39e641ad8501ed72c2b99f1ddba1c14c462675f40b37a66/gevent-26.7.0-cp311-cp311-win_arm64.whl", hash = "sha256:0f26f9a8c32ac0a73f6084c59b63deeacb350e7f1fee5301d95c5e0683a390d4", size = 1562794, upload-time = "2026-07-22T16:27:53.205Z" }, + { url = "https://files.pythonhosted.org/packages/2b/66/104590ad3a9e671b3ef77ad19c1cc50e7f1c8c220b27ddfaf34e5f88bd9c/gevent-26.7.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:92f256285fb43a57f152bd2e51a59cde1cd0b20869ae1e6da583b6beab88ed8a", size = 2953977, upload-time = "2026-07-22T16:23:41.654Z" }, + { url = "https://files.pythonhosted.org/packages/64/07/350d87161378633714184828bdc57c66f9b525eca5249ad1294dc7f8cf58/gevent-26.7.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:0e4fea187c5df7168b9538b4f543fcb0fcbaeb93be3d6cd499c324652c740704", size = 1800960, upload-time = "2026-07-22T18:11:27.242Z" }, + { url = "https://files.pythonhosted.org/packages/c4/6c/ddfe298c2ecb1cfc72a03dd69ba751759f7e7835d3135f171b284eb09ed1/gevent-26.7.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:ce732fe08d0ea65de07eff6e46bade8ac6a6fdb65cc748c713f3d31ae122529e", size = 1900387, upload-time = "2026-07-22T18:10:42.973Z" }, + { url = "https://files.pythonhosted.org/packages/f9/89/2647bbbf1da35a1c271a54452e76065308cbaf684ee32a79f989ca4265f6/gevent-26.7.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:c25b3522072137aecf3389031039230190038f888e257f490b3897d0e0620f74", size = 1848046, upload-time = "2026-07-22T18:29:08.074Z" }, + { url = "https://files.pythonhosted.org/packages/31/52/4f9b4c536b5a0424e328d5a5466640d185020f1f0b08b2de0ca939cc0a0b/gevent-26.7.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:eaaa75c9014df3f8c310c64f53f1152af8c6be32e82734396bed91e1d0e6f35c", size = 2132200, upload-time = "2026-07-22T16:48:31.203Z" }, + { url = "https://files.pythonhosted.org/packages/42/88/1daffa63b257c381df68018e4d3da72b429955cf95a171a46d3220422c8d/gevent-26.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f1956032a9926ac9b4152b2a50bc5a2cc020722ec16928ccaf32e227ee0aae47", size = 1814237, upload-time = "2026-07-22T18:07:12.438Z" }, + { url = "https://files.pythonhosted.org/packages/73/4c/b996cdddca78eac435195cd97dff590c65a9e0052640bcb6f8b6f1e30b1d/gevent-26.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d989a1ad6cc54f5c69bb7304360f98b4fda80da2b773f1047db9fba61ae7379a", size = 2157938, upload-time = "2026-07-22T17:02:14.887Z" }, + { url = "https://files.pythonhosted.org/packages/08/fd/44419d7559a95e238ee45d29df30bad78a91d343cb27b13a6af552b907bb/gevent-26.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:e0c9ce2d80fc0f8894d748a1045ff26ad188e294bad656b29839271800827c85", size = 1685319, upload-time = "2026-07-22T16:26:13.954Z" }, + { url = "https://files.pythonhosted.org/packages/6f/7a/7df1762ccd9a40ce1ca626f50cb7a75758f2c930aabcc73c91cf00cb3448/gevent-26.7.0-cp312-cp312-win_arm64.whl", hash = "sha256:959effe0c56cdee0bf761e5c4e78ab62880be147a2f2aa31112ca2f7e5754e53", size = 1558945, upload-time = "2026-07-22T16:26:56.737Z" }, + { url = "https://files.pythonhosted.org/packages/75/63/0fcfbe3f5696e56424f331ec41e0e447cea79c384848d6019e3f7f340f4b/gevent-26.7.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:b1b89eb5566f75aa8b2bbdb0308e1ac8d9113ca7cff85b45366aea9faad639a1", size = 2976844, upload-time = "2026-07-22T16:24:39.275Z" }, + { url = "https://files.pythonhosted.org/packages/3c/6c/ea2d0afbe760c18df5bd1631dbe5a73d840d9b141cb71e6810157c2ae28a/gevent-26.7.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:449857ce058183442e2d71d83ff0c587a3ddff631e93c6d19a6dffb4814eccad", size = 1802332, upload-time = "2026-07-22T18:11:29.155Z" }, + { url = "https://files.pythonhosted.org/packages/59/90/36f2258f1bfe8601224f6159066103b832c31e1451ce8dc2cd2408b6ecf4/gevent-26.7.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:8260a3f38b05fcf3c283417b18617562dbec74f5784f748e4ba3866789d7f3a4", size = 1901253, upload-time = "2026-07-22T18:10:44.402Z" }, + { url = "https://files.pythonhosted.org/packages/dd/38/86dd67e5c2dfab016a9c935b4338d6cf8f9bfa72dc5a0f3fb879d993127a/gevent-26.7.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:30894398d06747b433c8923a6a77ede61259ce6822a99f6c6e7fa0216ccb73c3", size = 1850489, upload-time = "2026-07-22T18:29:09.694Z" }, + { url = "https://files.pythonhosted.org/packages/f1/33/f5651942a5967483298b6ce6f45572d33120dd0fd01c8991d3fca5b1e8ee/gevent-26.7.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0b753522498118c9489753de7c612d4baed0edf384d9df2bf9492233ba1c20ff", size = 2129813, upload-time = "2026-07-22T16:48:32.551Z" }, + { url = "https://files.pythonhosted.org/packages/c0/09/abe8217a8fcd3f0e94c9eec024a5499c0f267cb269ccea6c0e2e812319b9/gevent-26.7.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:055a643026dc28daff2be228555a2097937448cc9b58307edebcf81b9d78ff4b", size = 1815121, upload-time = "2026-07-22T18:07:13.988Z" }, + { url = "https://files.pythonhosted.org/packages/40/d6/dbae1cd2d27b62664cefa086035530eb21203d45b12f466272441c048c9c/gevent-26.7.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4e1dc6a2712de67fd210e1f1a408601f6908b042f6420e188106f2f37f94ec71", size = 2155913, upload-time = "2026-07-22T17:02:16.35Z" }, + { url = "https://files.pythonhosted.org/packages/fc/42/90b662f4eb27d7727d4619d5c6be872117f1a9f187b243ec7f6fef988ce7/gevent-26.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:44e5280296129c0915addaefdb37d6e9bc124a77a433b1b1c8ddf1853c53f4e7", size = 1682483, upload-time = "2026-07-22T16:26:30.492Z" }, + { url = "https://files.pythonhosted.org/packages/e1/84/7297c56b9fff463c4ba2f685dbb913a855df903046dc68d14e8655a29ffe/gevent-26.7.0-cp313-cp313-win_arm64.whl", hash = "sha256:9f08b1aa6729f794409ca137e25f671e0d9bbda4451200c5e28a769375365388", size = 1556053, upload-time = "2026-07-22T16:26:14.365Z" }, + { url = "https://files.pythonhosted.org/packages/c1/bb/ab60d496cbdc0293ebbd6c2070b34da0632bd7a2ca20163c17e18d2d2dc9/gevent-26.7.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:0e0e3bf7ae0f82dbc5c6be26b4781e86c97f1e28d516b7a9746ac8b04bcc6948", size = 2992503, upload-time = "2026-07-22T16:24:36.503Z" }, + { url = "https://files.pythonhosted.org/packages/5c/35/75f27c06a82a5b22600aaccbd9567d89bb4091be43e96c02981f10aff23d/gevent-26.7.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:740050b53048207b080a1e183a377c47809ad0b7b7b0cd7eab0dea1045f7e480", size = 1809173, upload-time = "2026-07-22T18:11:30.724Z" }, + { url = "https://files.pythonhosted.org/packages/5b/5f/a6b32b4db3fa76bd8a070f0f46f5306123bf6336e7a0ca0cd2f9b99473df/gevent-26.7.0-cp314-cp314-manylinux_2_28_ppc64le.whl", hash = "sha256:67983607eb6c7bafa362c5c43b69a27145b936c34a3d6441ed42413d62fae0a6", size = 1906630, upload-time = "2026-07-22T18:10:45.836Z" }, + { url = "https://files.pythonhosted.org/packages/e1/87/832495d8fcc05ff7432f038b7c4decbd5632425a2cd5da2ce73cb2d800c4/gevent-26.7.0-cp314-cp314-manylinux_2_28_s390x.whl", hash = "sha256:475848518d708e07d1987c3d94cb8ff53e2b3a69df32e39feda2779cafe400b0", size = 1855278, upload-time = "2026-07-22T18:29:11.517Z" }, + { url = "https://files.pythonhosted.org/packages/72/8b/2f36c0fa389fa2b7ceb5a8972b0e7da7bc770f9135315cf4246c607ca5fc/gevent-26.7.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:0f8ed457dd616bfe6682569f92730f9ab45aafb1aeca5e80eb2f6b9a2ce26d11", size = 2136155, upload-time = "2026-07-22T16:48:33.865Z" }, + { url = "https://files.pythonhosted.org/packages/b5/98/09f2cfaa23dbce48e3271e95b0d003f93acece6b5cfd40f4cebe3850d79b/gevent-26.7.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:15373c68cf1fa14114bec2f09b16e2c65374bd5309e897e0a28740b09ce329e0", size = 1822108, upload-time = "2026-07-22T18:07:15.397Z" }, + { url = "https://files.pythonhosted.org/packages/b0/d8/05a294165c17569f04284ad3c889684c8780544885b4cdf77b1432947d0c/gevent-26.7.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:73f3d53f2f390369e290c933b75bd87f1f2261f2f2f2175aa667c43ee3049bad", size = 2162814, upload-time = "2026-07-22T17:02:18.066Z" }, + { url = "https://files.pythonhosted.org/packages/59/89/58a545c4eda33e106d6887a0387adc2249abc14c779e3eb88bbfdf3768d6/gevent-26.7.0-cp314-cp314-win_amd64.whl", hash = "sha256:f11b558d544ad2249029ba023cd6519ec3a0eee54a3d027e6515c1eaa322422a", size = 1706971, upload-time = "2026-07-22T16:27:02.263Z" }, + { url = "https://files.pythonhosted.org/packages/bb/cd/413f293e54961e5c89c54235370e3603ec0f561e7ace8357980410efbf78/gevent-26.7.0-cp314-cp314-win_arm64.whl", hash = "sha256:3871f4ca59ec2328c3ef638a0fe01a28a825443a133368dc78eb5ceadcad7609", size = 1585078, upload-time = "2026-07-22T16:30:48.145Z" }, + { url = "https://files.pythonhosted.org/packages/21/3a/47f29f632aaa38aa12410f57f1732fc50bfd4d4006d2e7e022ce731cabc9/gevent-26.7.0-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:3e3d6e20a94239ad353b776e72b8ce18c35dbe4e98c279aef3932651553d8404", size = 2996208, upload-time = "2026-07-22T16:23:23.859Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b3/4620f1ce81ecec9890229806c73f07dd022e40f76a552bd430391e7316c4/gevent-26.7.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:ddbd3cc76b9bc69df651a216c2a62fc6415ad463b3ac9c6cbbbb8b7b8224af17", size = 1811545, upload-time = "2026-07-22T18:11:32.224Z" }, + { url = "https://files.pythonhosted.org/packages/97/fd/d285212ffd5585d511299e13e61d76262def8e826e9f20c92cb85df406f2/gevent-26.7.0-cp315-cp315-manylinux_2_28_ppc64le.whl", hash = "sha256:01ceab7e608dc1b9859d9511a0a29d7ce2e7d909ab19fddc860e70a2ed5b10ce", size = 1910418, upload-time = "2026-07-22T18:10:47.709Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e0/c5d666e6065652918cfb6e6a3cf8f721d0e152c57cec217ad81792a1323b/gevent-26.7.0-cp315-cp315-manylinux_2_28_s390x.whl", hash = "sha256:2e6c917b2b8baeb6080797a6b25e35e1fd784319a05bb92b87c53546e5578eb2", size = 1857891, upload-time = "2026-07-22T18:29:13.13Z" }, + { url = "https://files.pythonhosted.org/packages/a1/67/e945ed458fa98b34572876bfd0d35fe4fa3f1159f43660b71d982b7cb63e/gevent-26.7.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:df75a1748b26030f2f7f10042cc45640b22954d9d0dc6b4b6f0dbe0b6751a2d4", size = 2138121, upload-time = "2026-07-22T16:48:35.358Z" }, + { url = "https://files.pythonhosted.org/packages/99/55/622468fa1a3c4cf51f20e14813f5cc1592fe6e44a42ccca9195a0b18c769/gevent-26.7.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:ee1b389587e5d5c1eb19d0455b5b4d7a0fb5c5287af4e226ec66d9dfd2548107", size = 1825114, upload-time = "2026-07-22T18:07:16.667Z" }, + { url = "https://files.pythonhosted.org/packages/0f/7a/151a2afcacf487ca25faf8b1bdd6c5b4ace2f7c1e6b4eaffe0a5e6e1df61/gevent-26.7.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:c2918641ba756f46aa01ab9dd82d6dfceec403c77c2787298746b411dcf0288e", size = 2165990, upload-time = "2026-07-22T17:02:19.817Z" }, +] + +[[package]] +name = "greenlet" +version = "3.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/74/b13368064b09053253555d3f2839cc2684d22d5aed0d2ccffbf7a6736558/greenlet-3.5.4.tar.gz", hash = "sha256:0232ae1de90a8e07867bb127d7a6ba2301e859145489f25cda8a6096dabe1d20", size = 206538, upload-time = "2026-07-22T12:47:14.468Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/9d/58f80897f4121f5c218bb931cf6d3b6514873f02ad0b729f744352926b9f/greenlet-3.5.4-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:ac5bf81d79d2c8eeb2ef6359b2e1687a1e9ebf46c2b1f970da9a9255df51d190", size = 293072, upload-time = "2026-07-22T11:38:14.299Z" }, + { url = "https://files.pythonhosted.org/packages/dd/9f/b4bc9bbd6a7855cbd8ad8a83c874eeeca56c24de9132b3323f81c03a30ba/greenlet-3.5.4-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89f3738167bab8c1084b94e23023d41d247117ac149fa0fbcb5bd4cf6262b353", size = 609393, upload-time = "2026-07-22T12:26:37.964Z" }, + { url = "https://files.pythonhosted.org/packages/05/0e/744b5e063af127d2e3c74fe0f1aef15573064c83b6066883524f5b258b17/greenlet-3.5.4-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e9a5e3406e3ed8125ae1a3b37c12f3434e2b1f0fa053197c5557895b4fb09606", size = 622750, upload-time = "2026-07-22T12:28:59.546Z" }, + { url = "https://files.pythonhosted.org/packages/5d/5c/53d6b94742a6f1ee1877c7ff76262c909e137f3f7383ce96a8ab78e1ae31/greenlet-3.5.4-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a2d614cb2372c7101a12ea8b96dd56f81c986d247c5a73db67063f3ed1ca4a52", size = 629659, upload-time = "2026-07-22T12:43:40.451Z" }, + { url = "https://files.pythonhosted.org/packages/d6/6b/d78ea2908e8e08985348f28ac396c2950be7ab66321dfe0054c73bd1f456/greenlet-3.5.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ab9f0704bccf6d3b38e0d2130b7b33271cff11453690da074fa280c3aa8e8e7", size = 622920, upload-time = "2026-07-22T11:51:06.83Z" }, + { url = "https://files.pythonhosted.org/packages/f2/34/957fc5577180ef2f57be82580ee1f59fdefad4f6c623c7d5e1b6980a76fb/greenlet-3.5.4-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:188e4d142f243051d92a1f5c244a741da02dddc070a0620c842804d7b56d008c", size = 425580, upload-time = "2026-07-22T12:39:48.326Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e4/3ce7009c948920b01527f8d9da29f501a31ac3d98318829e981fd879b850/greenlet-3.5.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2cdaadc3d31445a8f782bde3cd37e49a2c2a9c6da6daf76a3e34c683b271a3c7", size = 1582262, upload-time = "2026-07-22T12:25:01.131Z" }, + { url = "https://files.pythonhosted.org/packages/f2/4c/0408366102a33829f7bdd6a992dad75abbf75e86cc1e76caf19e57311d29/greenlet-3.5.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:70bdfacdc183dac838b2a0aaff2dd6134a457c52fe68a9c6bbab435483d2b9df", size = 1648906, upload-time = "2026-07-22T11:51:08.627Z" }, + { url = "https://files.pythonhosted.org/packages/13/52/ebfe8f6a1aeb8e430540b406c844ecc4e3367072b0192f69dcb85eeeec2b/greenlet-3.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:69173331fbc5d64bfac0065d7e22c39cfcd089e9b18d125bdcd5079363b09616", size = 246036, upload-time = "2026-07-22T11:38:30.073Z" }, + { url = "https://files.pythonhosted.org/packages/61/16/71eefcf68267bbf06a9b6bff57d0b222e49432326e85d74348b67694b8d4/greenlet-3.5.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e883de250e299654b1f1680f72a1a9f9ba62c9bd1bce84099c90657349a8dfbb", size = 294266, upload-time = "2026-07-22T11:37:56.142Z" }, + { url = "https://files.pythonhosted.org/packages/36/ea/a0b19adfc35d07e10acb626e9d22a3893b95f1309c42c4a20161dec16800/greenlet-3.5.4-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32802705c2c1ff25e8237b3bdacf2594fa02be80af8a66703eb7853ea7e68686", size = 613712, upload-time = "2026-07-22T12:26:39.375Z" }, + { url = "https://files.pythonhosted.org/packages/54/76/a121978b3337407d05a1ce5f79b4aa5998a43a9d8422f9726029b90b4471/greenlet-3.5.4-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:57aa201b351f7c7c75627c60d29e4d5b97a07d37efeb62b903466fca42c097d7", size = 625582, upload-time = "2026-07-22T12:29:00.814Z" }, + { url = "https://files.pythonhosted.org/packages/d1/4a/f301f1d85c69a86b90b5d581a73e8927bba4e79450037e6e2cbca05eb4fd/greenlet-3.5.4-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9667862a2e38ad379f11b845daeda22c8989186def44f06962c9c4c05e556da7", size = 633429, upload-time = "2026-07-22T12:43:42.073Z" }, + { url = "https://files.pythonhosted.org/packages/34/c2/080f16cf870e929e592f55767f01d6c98d2ee83bfdc36c3b892f2d0459ab/greenlet-3.5.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c3fe76c2cac86b4f7a1e92865ac0a54384deb05c92986287c1a7110d9bd53071", size = 624663, upload-time = "2026-07-22T11:51:08.016Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2e/26884072b0eb343a4d5fee903341bfe5171b32b7f14553886e2b6349135a/greenlet-3.5.4-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:ae53534b5dec0f4c2ec26f898f538dc8ea1ca3ef2927d597a9439e40a09da937", size = 428238, upload-time = "2026-07-22T12:39:49.973Z" }, + { url = "https://files.pythonhosted.org/packages/9e/bb/8f3ca88370b817369008faeceeee85970adc16c92a70a3e5fe5fea495a57/greenlet-3.5.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1e1a4a684b16c45ba324e60b32a4386a87722bcb815d2a149d2182f9b401ca72", size = 1585010, upload-time = "2026-07-22T12:25:02.539Z" }, + { url = "https://files.pythonhosted.org/packages/51/c2/45877154689709ebce9a0b83c2235e6ca0f31577889b02af308c8cc5f8fb/greenlet-3.5.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e849e6e139b9671adeac505f72fc05f4af7fd1921faef40295e214fc3b361b59", size = 1651283, upload-time = "2026-07-22T11:51:10.408Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7d/8711a75cb61d85246277c07ff6e1a6504621ba473d808c11ad225ffca43f/greenlet-3.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:dc418cf4c873357964d6624445ed09472e50def990c65dd4e76fc3ba8cd9cef6", size = 246434, upload-time = "2026-07-22T11:43:15.557Z" }, + { url = "https://files.pythonhosted.org/packages/00/62/e290b3bce433da8f0324ac02da0b128d683482229f1a8b789fa47818a4cd/greenlet-3.5.4-cp311-cp311-win_arm64.whl", hash = "sha256:c38c902a0986eba1f6e7ba1ab39ad5195926abde90f3fe080e08212db62176da", size = 244990, upload-time = "2026-07-22T11:39:22.626Z" }, + { url = "https://files.pythonhosted.org/packages/f3/04/81bd731d6d1e3a469d9a4c36f5eb069bcf0cbb2d5d342c9fec22245b91fc/greenlet-3.5.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3d66250e8b09f182ede05490998c818b5961f7a3640332d44c4927caec7bbfe4", size = 295909, upload-time = "2026-07-22T11:38:09.261Z" }, + { url = "https://files.pythonhosted.org/packages/cc/dd/f5f22903a6ae70f5ea328ed0beaec92ad903f0e3b7d2845133b354abc4b8/greenlet-3.5.4-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c90e930c9c192e5b3ee9fb8bcd920ea3926155e2e3ded39fc697323addecee17", size = 612011, upload-time = "2026-07-22T12:26:40.69Z" }, + { url = "https://files.pythonhosted.org/packages/8e/10/92a4a88d12b915d74ea5b6d288e4afefda4771647caa34442c156f7a454f/greenlet-3.5.4-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:791fdfeeb9c6e0c7b10fa151bf110d2a6974866f13dcb5b1c7efae698245893a", size = 624299, upload-time = "2026-07-22T12:29:02.089Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f9/03e26be3487c5238e81f2b84714959a86ea8515a869828cf41f4fc54b34e/greenlet-3.5.4-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b7c895310363f310361e0fe2072af85269d2a2a285cd04c0c59e79a5e3670dcf", size = 629603, upload-time = "2026-07-22T12:43:43.456Z" }, + { url = "https://files.pythonhosted.org/packages/50/6d/0b14bb9db2989f32cd9fe7f76afedea01ee8bee3f87c07e69f24adfe7e63/greenlet-3.5.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f88193799d43dbf8c8a806d6405c9c52fe2af40bf75072a606357b33cc336c7f", size = 621541, upload-time = "2026-07-22T11:51:09.464Z" }, + { url = "https://files.pythonhosted.org/packages/57/6b/7c55ca72ef80d57c16c4a55210f82582622462dc4485799a30f4ec6f3372/greenlet-3.5.4-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:13b980043cb1b3134e81ea469da1250ddcc6bfe6d245bbaa59168d9cdc8f228f", size = 432554, upload-time = "2026-07-22T12:39:51.379Z" }, + { url = "https://files.pythonhosted.org/packages/48/3d/25e9a2d9eb6b2e8b7ca4e80a3a26cb887cce6c8e0a87c921164f11bc5574/greenlet-3.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7a5f095767c4493afcd06067f2bb3b8716e3f3f9e92b99c88e7e99f885b3d4d", size = 1581444, upload-time = "2026-07-22T12:25:03.818Z" }, + { url = "https://files.pythonhosted.org/packages/b9/96/4c9bf2e2c408dcc0556edce69efa9f802e82223573c53240136a086821f1/greenlet-3.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:42afdc1ab5f66da8c586c32af9224a74a706b4f0ea0dc3a4188a0860a09c65c9", size = 1645842, upload-time = "2026-07-22T11:51:12.295Z" }, + { url = "https://files.pythonhosted.org/packages/b5/41/303ecb26a3a56122c0f4d4073ee078881847bd6b6f463ae0ec57ec20223b/greenlet-3.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:60149df8f462d1b230038e6590c23c3b4768bb5d6c022b3b6e82532b34b0b8a3", size = 247169, upload-time = "2026-07-22T11:38:19.893Z" }, + { url = "https://files.pythonhosted.org/packages/a4/e3/ef56864b4c35fcb3eb3b41b869f6cc46f4cd3f5e2c68e74acde8ac433951/greenlet-3.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:77d6ce04fed0d9aeed42e0f37923cc43eba9b027bdd9c34546bb4ccd143d0fe0", size = 245565, upload-time = "2026-07-22T11:38:27.061Z" }, + { url = "https://files.pythonhosted.org/packages/c0/9a/e51225dcd58713f16ccbdcc501a8da21098ea14515b7870f1f94459e5ff5/greenlet-3.5.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:24e61b88cb7e1b1d794b32a10cc346ac779681d6d74ff137a3e0a444d2bf1f02", size = 294831, upload-time = "2026-07-22T11:38:53.389Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ea/de50a50fadf979713ab18b46f22ad5ff5f2dcfc637a3ebdecf669801e1a5/greenlet-3.5.4-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:870d730fec833f5a06906a32596cc099b9161594642a92a520b7a88911c95356", size = 614619, upload-time = "2026-07-22T12:26:42.282Z" }, + { url = "https://files.pythonhosted.org/packages/db/c7/2aae27fea41205b8650294c301f042a2a4bb6155eea48c995b890a92f2c1/greenlet-3.5.4-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ec5ff0d1878df6af3bf9b638a5a92a7d5693291de77c91bff10fa48519c604ef", size = 627021, upload-time = "2026-07-22T12:29:03.445Z" }, + { url = "https://files.pythonhosted.org/packages/1b/80/fb4d4788bbc8e54761f1fc88533af9523a6e86299fa113d6e8a8503ed9fc/greenlet-3.5.4-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:07bd44616608d873d06735b63ef1a88191d6ca57c8d291d6559c71bc14c0893c", size = 632845, upload-time = "2026-07-22T12:43:45.19Z" }, + { url = "https://files.pythonhosted.org/packages/eb/56/79fd826f9ccaae0b84e1b4ef68dabba5e105bb044ffcd448a0b782fcba9a/greenlet-3.5.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d84d993f6e575c950d91a23c1345d18fe1a4310d447bf630849d7809196b52f0", size = 624002, upload-time = "2026-07-22T11:51:11.391Z" }, + { url = "https://files.pythonhosted.org/packages/42/e3/6086fa578ebb72772722cdc4bcd628459814b42e0c2db1e3cbd6552b3271/greenlet-3.5.4-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:3529a8a933582ad19e224792cac7372489526576b75b4c124e8e4f29948f4861", size = 435053, upload-time = "2026-07-22T12:39:52.715Z" }, + { url = "https://files.pythonhosted.org/packages/0a/1a/27319f97e731298513dcba1a2e91b63e9d8811d9de22130f960b129b1bf1/greenlet-3.5.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:58023945f421093de5e6fa108c0985a8659d43f49e0216da25099369a121bcbd", size = 1581533, upload-time = "2026-07-22T12:25:05.322Z" }, + { url = "https://files.pythonhosted.org/packages/b1/6d/24240bf562e9786dd2799ee0a4a4dadb4ded22510f41b20245099159ac8c/greenlet-3.5.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bae2728e1897aa8df8cb1af38cd48b3a743aefe29372de7b8b7a9f532501e69f", size = 1645781, upload-time = "2026-07-22T11:51:14.805Z" }, + { url = "https://files.pythonhosted.org/packages/c1/5a/442ab1a9ef7ca6bf7210e5397a95972206a91a31033a03c8900866a10039/greenlet-3.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:ca5726c0b08ca35ae873557266a78b2c3f3b2b7d7401aa5ff886c2045dd0111c", size = 247133, upload-time = "2026-07-22T11:39:20.661Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e6/9160210222386b1a378ff94db846b9508ca24a121cf684991561fdb69280/greenlet-3.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:7c1303791d603080cac6fc3b34df51c3b75b723739c282c8029e48a0d241672f", size = 245500, upload-time = "2026-07-22T11:40:22.185Z" }, + { url = "https://files.pythonhosted.org/packages/a5/a7/6ab1d4f9cd548d15ab90da29947f2076100130bb179b0bde59f795a459e3/greenlet-3.5.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:7e8afa5eac028f8140ceafe5ceec66e6aa127ddcb21452d2a564dcd2900b5f22", size = 295410, upload-time = "2026-07-22T11:40:35.747Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7a/422f63b4715cbc0b24385305407adf38b48f6bb68b3e6b04090e994d0f5a/greenlet-3.5.4-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73b37afe369021423ea53dd3123e04bffa7e93ac64429b9f50835b2e4fcae7cf", size = 661286, upload-time = "2026-07-22T12:26:43.8Z" }, + { url = "https://files.pythonhosted.org/packages/d0/31/5a1cac663bf5582190c5a714ef81364f03cde232227f39748f8ae4c11da5/greenlet-3.5.4-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ef964f56dfcb6f9bbef2a190d9126795eac408716aeae47b5e7c73c32aafca9", size = 673517, upload-time = "2026-07-22T12:29:04.815Z" }, + { url = "https://files.pythonhosted.org/packages/9c/bf/250c2921c7b585dde12f5239e313ca2dcbc464d161ecca36e4e6ef21762d/greenlet-3.5.4-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cef589bc65fae02d10bca2ac341191c5b33acc2967892ebf4fcbd10eabb7a74c", size = 677968, upload-time = "2026-07-22T12:43:46.788Z" }, + { url = "https://files.pythonhosted.org/packages/15/4a/2a82a1e3f8aaca020853ac8d12211280ca2b231aa08ea39f636f1060c319/greenlet-3.5.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c53ff01a5c53a40f2c16820ebc56d7c61a77f5fbe009dadd96292d5682f80f8", size = 670917, upload-time = "2026-07-22T11:51:13.589Z" }, + { url = "https://files.pythonhosted.org/packages/18/40/10bfcf6513558d82f7b95dd728001c63bd388259fe27d3e30ae01f103430/greenlet-3.5.4-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:dfc41ae893d9ceaf22c824f2153a88b30651b20e8758c2cd9ac143f23640563c", size = 480643, upload-time = "2026-07-22T12:39:54.149Z" }, + { url = "https://files.pythonhosted.org/packages/68/b0/e379a152b17bfdfa95795af4049e37c0fd1b4d81f020d426db104ed07c77/greenlet-3.5.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ecca4d80d55a01ad6b23b33262662956149fbb7b2c6be2910f1705921958cbf3", size = 1628478, upload-time = "2026-07-22T12:25:06.678Z" }, + { url = "https://files.pythonhosted.org/packages/5e/43/bffdfa64f7317f954c5c1230b5dd5922676ce198689a68c1ac1ed4b1b1a5/greenlet-3.5.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ffbc533e0eaf8e80d8471411646ab88fe58f641d508c0b02b24494479f4d9ec", size = 1692021, upload-time = "2026-07-22T11:51:17.008Z" }, + { url = "https://files.pythonhosted.org/packages/d0/11/f799f9637e2c6e9b0b716015e339040598b058cf7654dfc0d67468b177ed/greenlet-3.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:305f69e6c4523d7f6979ed001cff4e5853c063e5da04880296603aa0227e544c", size = 248031, upload-time = "2026-07-22T11:40:11.007Z" }, + { url = "https://files.pythonhosted.org/packages/05/75/625bcdd74d5e6b2dca1ecba3c3ac77bcf8a026c21a649a46cef23e421f97/greenlet-3.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:f260930bbbbcf9caee661211235a5111c86dfe5832fdf6ae4570da1e0995320f", size = 246892, upload-time = "2026-07-22T11:40:27.357Z" }, + { url = "https://files.pythonhosted.org/packages/ec/69/35c62ed49c320cb4d98e14698ccca5467d3bfe683984172be9cb564d9ce3/greenlet-3.5.4-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:41ddab54e4b238f4a6c323f39b4e59e176affd5a94d461a9fb7583dac74240a3", size = 305571, upload-time = "2026-07-22T11:40:31.659Z" }, + { url = "https://files.pythonhosted.org/packages/5b/6c/64d60216b3640dcb0b62d913dd9e0d80030c09115bb2e4ba70c95d10ca45/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b3dabe3e2809013052c68bdf0b7fa5f5f2859c43a80803131ad61af9cabd7867", size = 672568, upload-time = "2026-07-22T12:26:45.298Z" }, + { url = "https://files.pythonhosted.org/packages/5c/de/ba3ab0a96292e53039530333b0d2ae18d9e508f3a325cd7bf15f8172944c/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:27d3f00718634d4520a3a150154ac5da36f257869d41321953375b90bfbbc72c", size = 680076, upload-time = "2026-07-22T12:29:06.125Z" }, + { url = "https://files.pythonhosted.org/packages/ae/db/24a10af12bf8e639cec46c38b9ce1a282543ba42ff4fb0b31a970f1ab603/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:39169a11d87a6a263afda3e9a27d1df16d0f919d40a4837cc73986c9884c0dd8", size = 681690, upload-time = "2026-07-22T12:43:48.109Z" }, + { url = "https://files.pythonhosted.org/packages/a5/be/aeada79083c6f1c15f45d77a332f9c441af263ee298e3eb17522cd337d22/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbd60b5763c6543c1827e48faaf14ea9bfbad245f52b1a4d76a2a2d8884c6c66", size = 676733, upload-time = "2026-07-22T11:51:16.027Z" }, + { url = "https://files.pythonhosted.org/packages/f4/60/44a2eca7b9fd71ae0fae7ff184da1cd3169d176652b97aa1cffcbb0ef961/greenlet-3.5.4-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:bd3d1145f603b2db19feb9078c2e6855eb7c67e15580c010ed815cee519b86fd", size = 510263, upload-time = "2026-07-22T12:39:55.678Z" }, + { url = "https://files.pythonhosted.org/packages/e9/10/2392fc3a98948652ef5fd1e7275c04f861dd13f74b78a2b4309f4ee4d090/greenlet-3.5.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f00f910f0e7b35416c63b23ad78b769aeccfc1775f712b43c4ee525624a2eef7", size = 1637327, upload-time = "2026-07-22T12:25:07.879Z" }, + { url = "https://files.pythonhosted.org/packages/55/c6/e7237a3dfa1f205ed0d9ea1e46d70bd2811b32d516266399fb59d28ab90a/greenlet-3.5.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:91c26423753b92caf41ab3f98fd547d7374d4d9fc2d85be041886c1579d9255e", size = 1697493, upload-time = "2026-07-22T11:51:19.214Z" }, + { url = "https://files.pythonhosted.org/packages/55/e3/4ba8154ba2a3d43729e499f72471b4b5c993f3826d3e24da81d5f06d6572/greenlet-3.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:ee032b91fd8ec29ec6c4cea2b8c561b178435134bd0752c7334b94e9c736c132", size = 251637, upload-time = "2026-07-22T11:40:37.44Z" }, + { url = "https://files.pythonhosted.org/packages/90/03/e3f96dfc100261a29545ddc8270cafe58f9195b6651466910e820910de77/greenlet-3.5.4-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:178111881dd7a6c946471fda85485ec796e1043c2b939f694b096e2ecf986809", size = 296076, upload-time = "2026-07-22T11:39:38.364Z" }, + { url = "https://files.pythonhosted.org/packages/a4/3d/da52d208e5c977bce8667e784729e584e38b5785f4c1ec0f4c836e9a1c42/greenlet-3.5.4-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d92df08dd65fede97fc37aad36c2e9dcda3b31c467f8e0c2c096456cb818e927", size = 666870, upload-time = "2026-07-22T12:26:46.691Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8a/7e6dee25cb8a8cf9b362c8e597cc269593378bd916f16c736c059a52e85a/greenlet-3.5.4-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:99e8f8c4ebc4fd80aa26c1280ae9ad43a0976e786349703a181cf0bae60413e5", size = 677678, upload-time = "2026-07-22T12:29:07.508Z" }, + { url = "https://files.pythonhosted.org/packages/51/a7/dafc7415d430b0a43a16396eb49ecb3b62fd720877fb259cc4dcfaf5f31e/greenlet-3.5.4-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1f17e362d78e37559e0506c5a7d066bdd45073c36a0127a543e8a0df27242ff3", size = 681428, upload-time = "2026-07-22T12:43:49.623Z" }, + { url = "https://files.pythonhosted.org/packages/6c/21/5a38699fa45de749e3857d93b8f07e4c20489e77c2d35d915a2e1c456606/greenlet-3.5.4-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:394de08dad5ffcb1f50c2159d93e398d9d2da3ed437645eaa54771fa720db9f0", size = 676067, upload-time = "2026-07-22T11:51:18.163Z" }, + { url = "https://files.pythonhosted.org/packages/2e/d9/6298f3432de301d4718766cf934bd73c418c73f81fbb77247319364b0d96/greenlet-3.5.4-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:cd320d998cbaa032932830448e39abf3c6a12901295e386e8114db926e10cffb", size = 487446, upload-time = "2026-07-22T12:39:57.044Z" }, + { url = "https://files.pythonhosted.org/packages/5e/ba/863116ab8ff1ca7a729e327800268939d182db47aa433db70e216e7d9194/greenlet-3.5.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:c883d61f2282d72c767a14936641b3efcbde9d82f1080712aaea0b1d3126cb88", size = 1633489, upload-time = "2026-07-22T12:25:09.605Z" }, + { url = "https://files.pythonhosted.org/packages/fa/06/7466ced82818d6132462d7f26b3f83c66ea15d2b193a6c0088d558ed7d95/greenlet-3.5.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2a924f15d17957e252a810acefcb5942f5ca712298e8b6fcaed9a307d357522c", size = 1696584, upload-time = "2026-07-22T11:51:21.304Z" }, + { url = "https://files.pythonhosted.org/packages/f9/4d/55b638489260065de9ffce606c8b5d04507bef705de4b212a0c3d6a1a0df/greenlet-3.5.4-cp315-cp315-win_amd64.whl", hash = "sha256:ed17e5f3420360d5b459de8462efb52060399a5326a613d4cde31cef63ef95da", size = 248297, upload-time = "2026-07-22T11:42:04.055Z" }, + { url = "https://files.pythonhosted.org/packages/bb/08/9dd4ae635da93d41dc268bc34bd62a9d711ed8b8825c5d22ac910c7d6e6d/greenlet-3.5.4-cp315-cp315-win_arm64.whl", hash = "sha256:f908898d6fa484ce4b6f447ce70ea99b52c503fee419e53cf74d60a16bc9e667", size = 247423, upload-time = "2026-07-22T11:44:00.764Z" }, + { url = "https://files.pythonhosted.org/packages/19/66/7c87ed9cdbf1d49c2c6cd1c7b9dd4d16c33b24235ca03972293a1876b30c/greenlet-3.5.4-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:1833637f17d5e7472548a48575c394fe39f1b1890d676d162d86593610f44d8c", size = 306487, upload-time = "2026-07-22T11:41:25.118Z" }, + { url = "https://files.pythonhosted.org/packages/5b/05/0a4201e7c0054866eefc05da234f236dd4c950d0fbf9ca0517141f01b269/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:12cda9122e03341f1cb6b8207a19d7a9d375e52f1b4e9243918375f40fd7b4b9", size = 676479, upload-time = "2026-07-22T12:26:48.129Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c0/4b6b8c5a3aec70f0649cd89662d120fdd6421e2bdc8e3b15c3ab5ec568d8/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d83ae0e32d14957ab7170785a20f582635c8474deab1bfbb552b17e769a6ce25", size = 684321, upload-time = "2026-07-22T12:29:08.925Z" }, + { url = "https://files.pythonhosted.org/packages/88/15/0b167aeea95285b0e654ddce651922f666c089363c2ec528ca8b9a9ba74f/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:123aa379c962ed5fe90a880327e0c3066124ac64ec99e12a238be9fd8eb3db3d", size = 685995, upload-time = "2026-07-22T12:43:50.993Z" }, + { url = "https://files.pythonhosted.org/packages/24/c9/b49c31c9a972eee91e260445770e922244a7efc542697f51a012ec046d0f/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f1467de1bb767f75db0aa34c195e3a496d8d1278c796e70c24ce205d3e99cde", size = 681293, upload-time = "2026-07-22T11:51:20.43Z" }, + { url = "https://files.pythonhosted.org/packages/de/90/c023ec337f32ff505be7db759c80d98f0532bb94d0c6fa13645efe9bee2e/greenlet-3.5.4-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:adf2244d7f69409925a8f22ed22cc5f93cdfe5c9dc87ff3476be2c2aaae61a05", size = 516928, upload-time = "2026-07-22T12:39:58.359Z" }, + { url = "https://files.pythonhosted.org/packages/95/6f/7f2d4653770500eee667866016d42d7a68e3d3462f80df6b8e3fcd48a0eb/greenlet-3.5.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:0fa53040b78b578120eecdc0265e3f1051487cc425d11a2b7c761daadf4feaa8", size = 1642474, upload-time = "2026-07-22T12:25:10.819Z" }, + { url = "https://files.pythonhosted.org/packages/5a/d8/8cba31036a4caae448087ba5d150660ab03b4a0f54d9150f6495a3be7262/greenlet-3.5.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:60e0bc961d367df506660e9ac0177a76bc6d81305300704b0977d1634f76efe2", size = 1701012, upload-time = "2026-07-22T11:51:23.17Z" }, + { url = "https://files.pythonhosted.org/packages/e3/cd/3f77a4cce3bae631b08eb52f53a82a976669600337e21dfdba811cb50267/greenlet-3.5.4-cp315-cp315t-win_amd64.whl", hash = "sha256:f680e549edb3eaf21eea4e7fe101e15ec180c74b7879ab46adc080f22d4015d2", size = 251977, upload-time = "2026-07-22T11:41:38.125Z" }, + { url = "https://files.pythonhosted.org/packages/93/e8/65e8707d00fe2a49bf12f609a9b2b39ba6dd23c2810eacad877c4fc94bfe/greenlet-3.5.4-cp315-cp315t-win_arm64.whl", hash = "sha256:08fc36de8442d5c3e95b044550dbea9bf144d31ec0cc58e36fb241cb6ef6a994", size = 250538, upload-time = "2026-07-22T11:40:17.985Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1158,3 +1307,60 @@ sdist = { url = "https://files.pythonhosted.org/packages/3f/50/bad581df71744867e wheels = [ { url = "https://files.pythonhosted.org/packages/b7/1a/7e4798e9339adc931158c9d69ecc34f5e6791489d469f5e50ec15e35f458/zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931", size = 9630, upload-time = "2024-11-10T15:05:19.275Z" }, ] + +[[package]] +name = "zope-event" +version = "6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/93/41/faa10af34d48d9cd6fa0249a1162943ad84a9590bd1a06939981e6640416/zope_event-6.2.tar.gz", hash = "sha256:b97d5d6327067ee6b9dfcbdf606ade9ade70991e19c162e808ea39e5fcf0f8d3", size = 18958, upload-time = "2026-04-28T06:24:10.578Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/33/848922889e946d4befc415c219fe516af75c49555d8e736e183bfd30db42/zope_event-6.2-py3-none-any.whl", hash = "sha256:5e755153ac4faf64c10a4b6dd3307680166a3edf65b38df22df592610f8fa874", size = 6525, upload-time = "2026-04-28T06:24:09.176Z" }, +] + +[[package]] +name = "zope-interface" +version = "8.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/dc/50550cfcbb2ea3cbca5f1d7ed05c8aa840f831a0f2d63aec0a953f7c590e/zope_interface-8.5.tar.gz", hash = "sha256:7a3ba1c5877f0f3e3906b02ddf793abed2becc2948116414ce0e1dd820b68d6d", size = 257957, upload-time = "2026-05-26T06:50:14.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/43/9cd98bee951d23848de690ba2809f87e3b22c67c370987acc960da15ad37/zope_interface-8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0c8aa2bf8f3911ef37b87deb1bbe225a310e6eb6522a16d77f5d8330c4f6fbe", size = 210951, upload-time = "2026-05-26T06:49:00.178Z" }, + { url = "https://files.pythonhosted.org/packages/17/0f/8f1a29966bcf863e3a2121edcafb81c55715de7886bcc9544749cc79e7da/zope_interface-8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:efe234a0fafb4b6b1602e9be9245b97c2bf06d67c07af5a4bc3c0438978b555c", size = 211309, upload-time = "2026-05-26T06:49:02.732Z" }, + { url = "https://files.pythonhosted.org/packages/9f/9f/37e564eaaf85e3abc1ada40a79fa43f2ab45bdb67431b0ec0fe29e4763e2/zope_interface-8.5-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:dabeb6fe1228d411994f300811edc6866fff0cdcbc9cef98a78f05ea0da42e37", size = 254881, upload-time = "2026-05-26T06:49:04.303Z" }, + { url = "https://files.pythonhosted.org/packages/06/61/e6501d8ea7a2cac3217e03f404e1f98c1df7191d83cfe86b1895fbba5dac/zope_interface-8.5-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:147a9442dcc2b7339ecdb1be2b3cdb098e90462e39425054053ebfb50d99125a", size = 259811, upload-time = "2026-05-26T06:49:06.373Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/bfa25ef480b02af6e9452c478483fec75e87c9e2b60c407fd0b1f6054b9c/zope_interface-8.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a17e681224267880707c9ec9e730ad9a1ad2d65c371256843efba6cf48711b58", size = 260358, upload-time = "2026-05-26T06:49:08.317Z" }, + { url = "https://files.pythonhosted.org/packages/64/51/2b518072fea76242da64451d501c69b7b5ccdef9b57fead584ccf1c180d5/zope_interface-8.5-cp310-cp310-win_amd64.whl", hash = "sha256:d178968a1a611df30549a717d1624cb38ca810347339e3e37b7baa6f6781a170", size = 214822, upload-time = "2026-05-26T06:49:10.441Z" }, + { url = "https://files.pythonhosted.org/packages/ce/f1/83ad110fb847413affe71609bb50e59e1aa082e1236030122227c7c283d3/zope_interface-8.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:afc66ccaef2a3c0bef6ca02aad40d29a39276389dad16a8eac36f9f385e4d057", size = 211426, upload-time = "2026-05-26T06:49:12.595Z" }, + { url = "https://files.pythonhosted.org/packages/bb/a7/6b6e0c31ac240cb9fc015ae9ed45ca54be886c18fcf7bfa2377a4d7a8785/zope_interface-8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c28044972187245d7a309e4699319bfdbd2ffcbf7176d1d4ddf5adffb2dea80f", size = 211850, upload-time = "2026-05-26T06:49:14.474Z" }, + { url = "https://files.pythonhosted.org/packages/37/36/7599ecabcf80ce4fef2e1ef3c5ac0d4696b61f03f724cc44022f4d226af9/zope_interface-8.5-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:03bbecc7982af713d7499d4084bc03916413d17ffd45f89009348cc0c1d9e376", size = 260711, upload-time = "2026-05-26T06:49:16.568Z" }, + { url = "https://files.pythonhosted.org/packages/03/3e/1774b0ee46ccbb5498ee3c33ece40315b6ef58bc71957be94bd345340bc1/zope_interface-8.5-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bf917009a4a7457c7290225a019f4a0aa706d96accd2cfdba2418d3bc1fcde2f", size = 265277, upload-time = "2026-05-26T06:49:18.656Z" }, + { url = "https://files.pythonhosted.org/packages/b6/09/e533b2ffabaae4e5d5730d6768a591cf335defe8e37bec2ad905d09be656/zope_interface-8.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:31cff25b2aaedb5267e6e77b1e9be6b0ec4f622032de8a069202b8ffacda7dc2", size = 266369, upload-time = "2026-05-26T06:49:20.174Z" }, + { url = "https://files.pythonhosted.org/packages/49/4a/3ebe6a4c122b2d5340db45cbe7e490663d3228b172710ec71060cd5d541e/zope_interface-8.5-cp311-cp311-win_amd64.whl", hash = "sha256:17a3114bbdddb5e75e5784cdf318944636190cbbc72d357ef9fb1a8b0351f955", size = 215161, upload-time = "2026-05-26T06:49:21.799Z" }, + { url = "https://files.pythonhosted.org/packages/d2/59/056ad97af5b16db1975ee98ec7ab03d2ce3f3355efad904ced1dbce0e39f/zope_interface-8.5-cp311-cp311-win_arm64.whl", hash = "sha256:aab6bb5bee10f38ea688b95ba054396b67f613552d2c8378be7fcb2d2fba7646", size = 213481, upload-time = "2026-05-26T06:49:25.085Z" }, + { url = "https://files.pythonhosted.org/packages/97/cc/b84123a948f3162a34623e188922827cd845244fdd043ed20f8d02228caa/zope_interface-8.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8e6ee90c2e6de7c37058d5fa41f123c8b13a312db8d1e0fb5840d7f4bcdff9c9", size = 212165, upload-time = "2026-05-26T06:49:26.566Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/cbceec44f1b27208a76c1a688c131302685852406a23df5aab68324109cc/zope_interface-8.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c1adc90d3576b3b4c4de4953e6002c37bef28b78d7fa54c1bbfd0c50f022fe7c", size = 212341, upload-time = "2026-05-26T06:49:28.182Z" }, + { url = "https://files.pythonhosted.org/packages/e1/c3/005032195ff3b210c139b7c560ed5c534e844b0907d8e44d2b3d8919305e/zope_interface-8.5-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:e6347b8d8d12c5eca6502450a92be30079b7acfade2c4f693efa0deb8871b06e", size = 265296, upload-time = "2026-05-26T06:49:29.741Z" }, + { url = "https://files.pythonhosted.org/packages/c5/66/1036543d6a66bc04c19df3cf650f3ad938a002ab0a443c24e23e8de5e8b9/zope_interface-8.5-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5e970dabea777a24b0b0bbf9dae3ab75ce8b2d8e948edf4875627034b21f3560", size = 270689, upload-time = "2026-05-26T06:49:31.767Z" }, + { url = "https://files.pythonhosted.org/packages/30/4c/8b56259558cace4414e753ca6740396a1f59d4a95ddb55b4658600408670/zope_interface-8.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f0b48ccadaa9839e09ff81e969703cecb3f402c813bfe8b958652e699bea69f5", size = 270280, upload-time = "2026-05-26T06:49:33.489Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ea/649908c83aa8fdb7faf2ddca4d3cf6fb8f2157121267dc56e8f72681e26c/zope_interface-8.5-cp312-cp312-win_amd64.whl", hash = "sha256:e0e311f1277468c08fd59a2b41f71b43d25dff639789d364747acd1705c0df6e", size = 215019, upload-time = "2026-05-26T06:49:35.607Z" }, + { url = "https://files.pythonhosted.org/packages/9f/97/da13037b4c563e4df32eedbc819f8c00b754af494f68211e3dffd48d52da/zope_interface-8.5-cp312-cp312-win_arm64.whl", hash = "sha256:652b73107a04159ec6c020db6c1543d4f1e8f4d069bd2aac88a947820923517b", size = 213569, upload-time = "2026-05-26T06:49:37.317Z" }, + { url = "https://files.pythonhosted.org/packages/f4/8c/4c15755d701f2ec0e80d64a18e1ebaf5be2c584c0ec153fd516f5d13eada/zope_interface-8.5-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:28e80457c134d1fa57a7d758004dece348654e1b1467ac22dcdc20fc1d127c52", size = 212512, upload-time = "2026-05-26T06:49:38.996Z" }, + { url = "https://files.pythonhosted.org/packages/9a/2e/4360c54c465db042cc8fbeeec92abac28b4cedbf6ba63c1f092fd08a190f/zope_interface-8.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:09495ce9d559c06b70f2d4855b3e4f48a822a9ddc8be1d30c5b4e5be14ae1ace", size = 212541, upload-time = "2026-05-26T06:49:41.186Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a5/692a2b8d70f78e848793231d5fae5fecbf8d0cccd73430fdc34802a6d3c1/zope_interface-8.5-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:7849ad8fa90763cc1087f4dda78ca3a233e950b3e08fac7079297c9cafbbd7bb", size = 265191, upload-time = "2026-05-26T06:49:43.449Z" }, + { url = "https://files.pythonhosted.org/packages/70/8d/454a9cfc7a050c394ab4f11b3371f7897828b7415e096afff724637e65e0/zope_interface-8.5-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5578c9421ca409a1f39f153d6f7803e4cde01da592ec75a9ac5e1b777d18d33b", size = 270626, upload-time = "2026-05-26T06:49:45.425Z" }, + { url = "https://files.pythonhosted.org/packages/51/8c/db8409cfa3575b8e9b4800babd7d49f8228433cd1f0c56814bd0ada49c33/zope_interface-8.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e1bd7d96b4ca5fa311f54c9eac16dce4886b428c1531dbe06067763ccdf123b4", size = 270444, upload-time = "2026-05-26T06:49:47.025Z" }, + { url = "https://files.pythonhosted.org/packages/4a/df/a386940e41469ef615e100a216d8b386521e9e598817147f87932ca203c4/zope_interface-8.5-cp313-cp313-win_amd64.whl", hash = "sha256:0c8123d2a4dfde2a613c7cb772605477724782c20bc2e0ad1d9435376a6a44a3", size = 215021, upload-time = "2026-05-26T06:49:48.478Z" }, + { url = "https://files.pythonhosted.org/packages/89/75/477eb5669b6b2a7a843decd1a075e9b1971a8720017654143a7183abd3d9/zope_interface-8.5-cp313-cp313-win_arm64.whl", hash = "sha256:6d02be14f3173c6c7288bc2fdf530090c01c3cf8764ad46c68024686f364278e", size = 213610, upload-time = "2026-05-26T06:49:50.01Z" }, + { url = "https://files.pythonhosted.org/packages/d4/19/5032e954827fdf02db2d2f49737ac4378bb9cfc2cd95a8f2e2a5ae2ec01a/zope_interface-8.5-cp314-cp314-macosx_10_9_x86_64.whl", hash = "sha256:ffaecf013251a89d0de6feb49a46eba48ad8cbbf8a40aeb6045e459e7bec6784", size = 212597, upload-time = "2026-05-26T06:49:51.63Z" }, + { url = "https://files.pythonhosted.org/packages/f1/53/3ef644012cf8a6a234a2d6134aab5a5c65ac5467c86296865501d4fbc406/zope_interface-8.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:126fa9d1c52295ae076d4cf968634f0a1826afa408a20808b57ff72877b8f69f", size = 212626, upload-time = "2026-05-26T06:49:53.236Z" }, + { url = "https://files.pythonhosted.org/packages/32/67/bc8b4f465d388039255003e230c284a175cedf1203c692f23cb7bff64efe/zope_interface-8.5-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:3090e3a663d20194756a59a272e0c8508b889341e31d5894223331fe6b4f9b21", size = 266827, upload-time = "2026-05-26T06:49:54.873Z" }, + { url = "https://files.pythonhosted.org/packages/a7/eb/37d05b935ede53d79690fecc8d201440084418e590bcfc05f384451c7593/zope_interface-8.5-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9342fb74e2afefdb081bf1df727d209ea56995c6e13f5a0540e6d7aff4beafb8", size = 270139, upload-time = "2026-05-26T06:49:57.116Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0b/fd0c54579e2ce8dc6cf1a757903f3374bc6fbda929a46af9e0f53cb0e5f0/zope_interface-8.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6c54725d818f1b57a7efb8b16528326e1f3c257b602b32393fd255c45af8799d", size = 270338, upload-time = "2026-05-26T06:49:58.698Z" }, + { url = "https://files.pythonhosted.org/packages/c1/1d/c420dcd777bb761067ea92879ac766694a5ca78608185f1aecea64cbfc11/zope_interface-8.5-cp314-cp314-win_amd64.whl", hash = "sha256:29d74febbae1afeb6834c4ccbf42e242a673c860060f09e53142825270456140", size = 215789, upload-time = "2026-05-26T06:50:00.405Z" }, + { url = "https://files.pythonhosted.org/packages/62/94/50b5eb8f94e527edceac14f9955e58917424ea79bb572ddc18548561cbc2/zope_interface-8.5-cp314-cp314-win_arm64.whl", hash = "sha256:633c8c49396f38df030340797c533e9fe460d1b5d1e42d88e55e938e525f548c", size = 213757, upload-time = "2026-05-26T06:50:01.973Z" }, + { url = "https://files.pythonhosted.org/packages/17/6f/5d5f32c4dfcdb16ce2ec5363da686840f13c13e1a1214cb70b49e1cd6d9f/zope_interface-8.5-cp314-cp314t-macosx_10_9_x86_64.whl", hash = "sha256:133999820fdbae513c36c03d6f29ef87317aaa3edef39112222b155083664714", size = 213591, upload-time = "2026-05-26T06:50:03.529Z" }, + { url = "https://files.pythonhosted.org/packages/f3/55/de0c3459ff717fce3342f9a29464c281fdeb0d36c3171ee88d119d5f0650/zope_interface-8.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8bd75c96966e573232f0599deaff717564828031c7f05563ccc1ac35c5ee0304", size = 213733, upload-time = "2026-05-26T06:50:05.101Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/d97430abd5ae9677e8b9295b58720c0064a5b557dbb6b8bf5928484cf0d8/zope_interface-8.5-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:14b0e9799351d4c34fe99afd67f0cdd76e55ba15c66a98699d5fc22ea8241e08", size = 294905, upload-time = "2026-05-26T06:50:07.384Z" }, + { url = "https://files.pythonhosted.org/packages/41/ec/a0f8f3dad6e74992f4654bdd94802be0929eabca7b871cac3b6fbb5e961b/zope_interface-8.5-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cd6a732ac84b94eb1ef9222a117347a27efd294ee16810ffdf7ecd307677ed5", size = 300885, upload-time = "2026-05-26T06:50:08.997Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/6881b48803a0ee8d23eb5efa30fce3ed218a2bd9de5758ce489d224fee81/zope_interface-8.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:798b7c87d0e59a7d5d086d642208d0d8700ff0d55c4029134b3c479c3bfb110f", size = 304672, upload-time = "2026-05-26T06:50:10.563Z" }, + { url = "https://files.pythonhosted.org/packages/2e/0e/b4c01320859ff1d585438bc231fd60bd258d096359bccf6654fecdf0cffb/zope_interface-8.5-cp314-cp314t-win_amd64.whl", hash = "sha256:0fc3a9d45f114d27eaa1e53beeb144533689edca8a9f66505b1e8e8b3f075e42", size = 217241, upload-time = "2026-05-26T06:50:12.171Z" }, +] From 5df051ce664f023213af9c36b1078748321972b8 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:23:28 +0200 Subject: [PATCH 40/59] docs: record the landed boundary rethink (P1-P5) in the handoff Co-Authored-By: Claude Fable 5 --- handoff-boundary-protocol-rethink.md | 48 +++++++++++++++++++--------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/handoff-boundary-protocol-rethink.md b/handoff-boundary-protocol-rethink.md index 52336693..59cd1141 100644 --- a/handoff-boundary-protocol-rethink.md +++ b/handoff-boundary-protocol-rethink.md @@ -98,21 +98,39 @@ portal ingress + `OneShot`/`Mailbox` on an `AsyncioWakener`. Full asyncio-native API over the trio host loop, all transports, no anyio port. Phase E (anyio-native core) becomes optional purity/perf work. -## Staging - -1. **P1 — boundary kit**: Wakener/Mailbox/OneShot + ThreadWakener; port - SyncReceiver, write-acks, `_done_sync` onto it. Pure refactor. -2. **P2 — channel unification** (big one): sync Channel onto - RawChannel+Mailbox, delete classic dispatch. Straight to the rebase, - no intermediate primitive-swap (suite pins semantics either way). - Canary: `uv run pytest testing/ -n 12` + xdist suite mid-step. -3. **P3 — retirement**: ExecModel/WorkerPool out, preset shims in, - `wait=` joins the C.1 spec axes + worker CLI config. -4. **P4 — `execnet.aio`**. -5. **P5 — gevent wakener** + opt-in CI job (gevent in a dev extra). - -Then Phase C `loop=main` / `exec=task` on the smaller core, and Phase D -docs cover the three namespaces + `execnet.aio` + the axes/presets. +## Staging — ALL LANDED 2026-07-26 (commits 2c78416..b01d009) + +1. **P1 — boundary kit** (`2c78416`): Wakener/Mailbox/OneShot + + ThreadWakener; SyncReceiver, write-acks, `_done_sync` ported. +2. **P2 — channel unification** (`2a40a55`): sync Channel onto + RawChannel(+set_consumer)+Mailbox, classic dispatch deleted. The kit + lives in trio-free `execnet._boundary` (portal re-exports) so + `import execnet` stays trio-free. Fix worth knowing: an unconsumed + remotely-closed RawChannel stays registered until a consumer claims + it — call-site deserialization means a passed channel can bind after + its data AND close arrived (was a payload-loss race, latent in the + async core's `open_channel(id)` too). +3. **P3 — retirement** (`811fd99`): ExecModel ABC/WorkerPool/Reply gone; + one deprecated ExecModel preset KEEPS the stdlib-delegate members + because pytest-xdist's remote worker calls + `channel.gateway.execmodel.RLock()/Event()`; `wait=` axis end to end + (spec validation, worker CLI config, `BaseGateway._new_wakener`, + `_boundary` registry + Flag); safe_terminate on daemon threads; + test_threadpool.py deleted. +4. **P4 — `execnet.aio`** (`845510e`): implemented as a per-call + _HostBridge (host task + `loop.call_soon_threadsafe` future), NOT the + originally sketched AsyncioWakener/Mailbox — real awaitables over the + trio-native AsyncGroup/AsyncChannel, simpler and semantically exact. + The "asyncio" wait= registry entry is therefore unused/unregistered. + v1 limitation: cancelling a bridged await abandons it aio-side only. +5. **P5 — gevent wakener** (`b01d009`): `execnet._gevent_support`, + lazily imported by `make_wakener("gevent")`; hub-bound pieces created + in the first waiter's hub (carriers may be constructed on the host + loop); tests behind the `gevent` dependency group. + +Next: Phase C `loop=main` / `exec=task` on the smaller core +(handoff-phase-c-worker-axes.md), and Phase D docs cover the four +namespaces + the axes/presets. ## Invariants (unchanged, re-mapped) From 7969f7e37daea12c5b6543c0ed2680db96d81275 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:47:43 +0200 Subject: [PATCH 41/59] docs: record the decided Phase C plan (axes matrix, trio-only core) Co-Authored-By: Claude Fable 5 --- handoff-phase-c-worker-axes.md | 41 +++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index 351f6c16..40c222b6 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -71,7 +71,46 @@ File map (src/execnet/): `threading.Event`/`queue.Queue` so KeyboardInterrupt can interrupt them; `portal.run` (KI-deferred) is only for management ops. -## Phase C — what is missing (nothing of C exists yet) +## Phase C — DECIDED PLAN (Ronny, 2026-07-26, after the boundary rethink) + +Predates below gap analysis; where they conflict, this section wins. +Context: the boundary rethink (`handoff-boundary-protocol-rethink.md`) +landed first — `wait=` axis exists, ExecModel is a preset, the sync +Channel is a facade over the async core. + +**Worker matrix** (`loop=` = main-thread role, `exec=` = placement). +The main thread is the scarce resource (signals, interrupt_main, GUI, +tests calling asyncio.run/trio.run themselves): + +| loop= | exec= | main thread | exec'd code | channel | consumer | +|---|---|---|---|---|---| +| main | thread | host loop | worker threads | sync | default preset for execmodel=thread; no parked thread | +| main | task | host loop | tasks on the loop | AsyncChannel | async-native sources (C4) | +| thread | main | exec'd code (serialized) | true main thread | sync | pytest/xdist (forces main_thread_only today), GUI, signals | +| thread | thread | parked | worker threads | sync | valid, non-default (today's shape) | +| thread | task | parked | tasks on side loop | AsyncChannel | valid, niche | +| main | main | — | — | — | invalid | + +Interaction rule: exec'd code may start its own event loop only when it +does not share a thread with ours (exec=thread / exec=main); exec=task +sources are ``async def`` and join our loop — a sync source under +exec=task is a hard error. ``wait=`` composes orthogonally. + +**Backend decisions**: the core STAYS trio-only and trio stays the +default and a hard dependency (pure-python wheels; deployment cost +accepted). The anyio/asyncio-core port ("C0") is REJECTED for now; +asyncio apps use the ``execnet.aio`` bridge. ``backend=`` still lands +as a per-gateway spec axis, but reserved: only ``trio`` validates +(exactly how ``wait=`` landed with only ``thread``), keeping an +asyncio-core door open without paying for it. Consequence: exec=task +sources are trio-typed. + +Sequencing: C1 axes (`backend=`/`loop=`/`exec=` + validity matrix + +presets) → C2 `loop=main` worker restructure → C3 `exec=main` re-home → +C4 `exec=task`. Presets: `execmodel=thread` → `loop=main, exec=thread`; +`execmodel=main_thread_only` → `loop=thread, exec=main`. + +## Phase C — original gap analysis (pre-decision record) Target axes: `loop=thread|main` × `exec=thread|main|task`. Compat mapping: `execmodel=thread` → `loop=main` + `exec=thread`; From b2f43c38d0bb53265d481064e36f6216c5542b5e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 16:35:31 +0200 Subject: [PATCH 42/59] refactor: extract worker exec placement strategies behind the FIFO pump C1 of the profile plan (approved 2026-07-26): TrioWorkerExec becomes a pure admission pump (message-arrival FIFO preserved) delegating placement to a strategy object -- PoolExec (worker threads, today's "thread") and MainExec (serialized main thread + deadlock guard, today's "main_thread_only"), selected via WORKER_EXEC_STRATEGIES by execmodel name. _run_worker keys main-thread integration off strategy.needs_primary_thread instead of a hardcoded flag. No behavior change; upcoming profiles (trio task exec, classic hybrid, gevent greenlets, later subinterpreters) slot in as new strategies. Co-Authored-By: Claude Fable 5 --- src/execnet/_gevent_support.py | 2 +- src/execnet/_trio_worker.py | 177 +++++++++++++++++++++++---------- src/execnet/multi.py | 5 +- 3 files changed, 129 insertions(+), 55 deletions(-) diff --git a/src/execnet/_gevent_support.py b/src/execnet/_gevent_support.py index cd761571..a999ab96 100644 --- a/src/execnet/_gevent_support.py +++ b/src/execnet/_gevent_support.py @@ -63,7 +63,7 @@ def wait(self, timeout: float | None = None) -> bool: # a notify may have fired before the watcher existed if self._notified and not event.is_set(): event.set() - return event.wait(timeout) + return bool(event.wait(timeout)) def clear(self) -> None: with self._lock: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 89aad122..c5152cc2 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -6,6 +6,7 @@ import os import sys import threading +from collections.abc import Callable from typing import TYPE_CHECKING from typing import Any @@ -26,40 +27,128 @@ ExecItem = tuple[Any, ...] +class PoolExec: + """Exec strategy: run each request on a worker thread (trio's pool). + + The building block of the ``thread`` profile; exec'd code may freely + start its own event loops (it never shares a thread with ours). + """ + + #: whether _run_worker must hand this strategy the process main thread + needs_primary_thread = False + + def __init__(self, gateway: WorkerGateway) -> None: + self.gateway = gateway + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + """FIFO admission gate; always open for pool placement.""" + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + await trio.to_thread.run_sync( + self.gateway.executetask, + (channel, item), + abandon_on_cancel=True, + ) + + def integrate_as_primary_thread(self) -> None: + raise RuntimeError("pool exec strategy does not use the main thread") + + def trigger_shutdown(self) -> None: + pass + + +class MainExec: + """Exec strategy: serialize each request onto the process main thread. + + The ``main_thread_only`` profile (GUI/signal-safe: pytest under xdist + runs this way). Admission waits for the previous request to finish and + closes the channel with the deadlock text when it cannot within a + second (a second concurrent remote_exec would deadlock the requester). + """ + + needs_primary_thread = True + + def __init__(self, gateway: WorkerGateway) -> None: + self.gateway = gateway + self._primary: Mailbox[tuple[Channel, ExecItem, threading.Event] | None] = ( + Mailbox() + ) + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + complete = self.gateway._executetask_complete + assert complete is not None + wait_slot = functools.partial(complete.wait, timeout=1) + if not await trio.to_thread.run_sync(wait_slot, abandon_on_cancel=True): + channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) + return False + complete.clear() + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + done = threading.Event() + self._primary.put((channel, item, done)) + await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) + + def integrate_as_primary_thread(self) -> None: + """Block the main thread running exec tasks until shutdown.""" + while True: + task = self._primary.get() + if task is None: + break + channel, item, done = task + try: + self.gateway.executetask((channel, item)) + finally: + done.set() + + def trigger_shutdown(self) -> None: + self._primary.put(None) + + +# execmodel profile -> exec strategy. Placement strategies to come: +# "trio" (tasks on a main-thread loop), classic hybrid for "thread" +# (primary main + pool overflow), "gevent" (greenlets on a main-thread +# hub), and eventually subinterpreters. +WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { + "thread": PoolExec, + "main_thread_only": MainExec, +} + + class TrioWorkerExec: - """Schedule ``remote_exec`` work from the Trio host nursery. + """FIFO admission pump feeding an exec placement strategy. - * ``thread``: run ``executetask`` via ``trio.to_thread`` (concurrent). - * ``main_thread_only``: hand off to the process main thread (GUI-safe). + Exec requests flow through a single pump task so admission happens + strictly in message-arrival order (trio task scheduling order is + deliberately unordered, so per-request tasks would race for e.g. the + main_thread_only slot). Where an admitted request runs is the + strategy's business (:data:`WORKER_EXEC_STRATEGIES`). """ def __init__( self, host: _trio_host.TrioHost, gateway: WorkerGateway, - *, - main_thread_only: bool, + strategy: Any, ) -> None: self.host = host self.gateway = gateway - self.main_thread_only = main_thread_only + self.strategy = strategy self._lock = threading.Lock() self._running = 0 self._shutting_down = False self._idle = threading.Event() self._idle.set() - self._primary: Mailbox[tuple[Channel, ExecItem, threading.Event] | None] = ( - Mailbox() - ) - # Exec requests flow through a single pump task so admission happens - # strictly in message-arrival order (trio task scheduling order is - # deliberately unordered, so per-request tasks would race for the - # main_thread_only slot). self._pending_send: trio.MemorySendChannel[tuple[Channel, ExecItem]] self._pending_recv: trio.MemoryReceiveChannel[tuple[Channel, ExecItem]] self._pending_send, self._pending_recv = trio.open_memory_channel(float("inf")) self._pump_started = False + @property + def needs_primary_thread(self) -> bool: + return bool(self.strategy.needs_primary_thread) + def active_count(self) -> int: with self._lock: return self._running @@ -78,7 +167,7 @@ def _track_finish(self) -> None: def schedule(self, channel: Channel, sourcetask: bytes) -> None: """Called from the session dispatch on the Trio host thread. - Must not block: deadlock checks and exec run in a nursery task. + Must not block: admission checks and exec run in a nursery task. """ item = loads_internal(sourcetask) assert isinstance(item, tuple) @@ -95,48 +184,23 @@ def schedule(self, channel: Channel, sourcetask: bytes) -> None: async def _pump(self) -> None: """Admit queued exec requests in FIFO order, then run each as a task.""" async for channel, item in self._pending_recv: - if self.main_thread_only: - complete = self.gateway._executetask_complete - assert complete is not None - wait_slot = functools.partial(complete.wait, timeout=1) - if not await trio.to_thread.run_sync(wait_slot, abandon_on_cancel=True): - channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) - continue - complete.clear() - self.host.start_soon(self._run_exec, channel, item) + if await self.strategy.admit(channel, item): + self.host.start_soon(self._run_exec, channel, item) async def _run_exec(self, channel: Channel, item: ExecItem) -> None: self._track_start() try: - if self.main_thread_only: - done = threading.Event() - self._primary.put((channel, item, done)) - await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) - else: - await trio.to_thread.run_sync( - self.gateway.executetask, - (channel, item), - abandon_on_cancel=True, - ) + await self.strategy.run(channel, item) finally: self._track_finish() def integrate_as_primary_thread(self) -> None: - """Block the main thread running main_thread_only exec tasks.""" - while True: - task = self._primary.get() - if task is None: - break - channel, item, done = task - try: - self.gateway.executetask((channel, item)) - finally: - done.set() + self.strategy.integrate_as_primary_thread() def trigger_shutdown(self) -> None: with self._lock: self._shutting_down = True - self._primary.put(None) + self.strategy.trigger_shutdown() def waitall(self, timeout: float | None = None) -> bool: return self._idle.wait(timeout) @@ -202,30 +266,37 @@ def kill(self) -> None: def _build_worker_gateway( host: _trio_host.TrioHost, id: str, model: ExecModel, wait: str = "thread" -) -> tuple[WorkerGateway, TrioWorkerExec, bool]: - """Construct the WorkerGateway + Trio exec pool (no IO yet).""" +) -> tuple[WorkerGateway, TrioWorkerExec]: + """Construct the WorkerGateway + exec pump/strategy (no IO yet).""" trace(f"creating workergateway on trio id={id!r}") io_stub = _WorkerIOStub(model) gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) gateway._wait_backend = wait - main_thread_only = model.backend == "main_thread_only" - trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) - # Duck-type as WorkerPool for STATUS / _terminate_execution. + try: + strategy_factory = WORKER_EXEC_STRATEGIES[model.backend] + except KeyError: + raise ValueError( + f"execmodel {model.backend!r} has no worker exec strategy " + f"(known: {sorted(WORKER_EXEC_STRATEGIES)})" + ) from None + strategy = strategy_factory(gateway) + trio_exec = TrioWorkerExec(host, gateway, strategy) + # Duck-type as the exec pool for STATUS / _terminate_execution. gateway._execpool = trio_exec gateway._trio_exec = trio_exec gateway._executetask_complete = None - if main_thread_only: + if strategy.needs_primary_thread: gateway._executetask_complete = threading.Event() gateway._executetask_complete.set() - return gateway, trio_exec, main_thread_only + return gateway, trio_exec def _run_worker( host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel, wait: str = "thread" ) -> None: """Attach ``io`` as the gateway session and serve until shutdown.""" - gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model, wait) + gateway, trio_exec = _build_worker_gateway(host, id, model, wait) async def _start() -> _trio_host.SyncBridgeGateway: # The bridge attaches itself to the gateway before serving starts, @@ -235,7 +306,7 @@ async def _start() -> _trio_host.SyncBridgeGateway: host.call(_start) try: - if main_thread_only: + if trio_exec.needs_primary_thread: trace("integrating as primary thread (trio worker)") trio_exec.integrate_as_primary_thread() gateway.join() diff --git a/src/execnet/multi.py b/src/execnet/multi.py index ba73bf62..4bbfa7cf 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -154,7 +154,10 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: id= specifies the gateway id python= specifies which python interpreter to execute - execmodel=model 'thread' or 'main_thread_only' execution model + execmodel=name worker profile: where exec'd code runs relative + to the worker's protocol loop. 'thread' (pool + threads) or 'main_thread_only' (serialized on + the worker main thread, GUI/signal-safe). wait=backend wakener for blocking waits ('thread' default) chdir= specifies to which directory to change nice= specifies process priority of new process From d8f22c5f269536bac36a8de2f2d503a150b870eb Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 16:44:20 +0200 Subject: [PATCH 43/59] feat!: answer info natively and apply chdir/nice/env from worker config C2 of the profile plan: coordinator bookkeeping no longer goes through remote_exec, so it can never claim an exec slot -- with main-thread profiles an info call used to occupy the main thread (or trip the main_thread_only deadlock guard) and push the real workload (pytest) onto a worker thread. - New Message.GATEWAY_INFO (code 10), answered by the dispatch loop like STATUS with the sys/env payload (gateway_base.gateway_info); Gateway._rinfo() sends it instead of executing rinfo_source, which is deleted. Handled in both the sync bridge and the async core. - chdir/nice/env ship in the worker config JSON (worker_cli_arg; the via spawn_request inherits it) and are applied by the worker before serving; the post-start remote_exec setup block in multi.makegateway is gone. This also makes setup valid for the upcoming pure-async profile, which cannot run sync sources at all. Regression test: _rinfo(update=True) while an exec occupies the worker, for both execmodels. Co-Authored-By: Claude Fable 5 --- src/execnet/_provision.py | 23 +++++++++++++++-------- src/execnet/_trio_gateway.py | 6 ++++++ src/execnet/_trio_host.py | 8 ++++++++ src/execnet/_trio_worker.py | 20 ++++++++++++++++++++ src/execnet/gateway.py | 34 ++++++++++++---------------------- src/execnet/gateway_base.py | 19 +++++++++++++++++++ src/execnet/multi.py | 21 ++------------------- testing/test_basics.py | 16 ---------------- testing/test_gateway.py | 13 +++++++++++++ 9 files changed, 95 insertions(+), 65 deletions(-) diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index f5114ad8..ffe9ffaa 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -164,14 +164,21 @@ def worker_cli_arg(spec: Any) -> str: """ import execnet - return json.dumps( - { - "id": f"{spec.id}-worker", - "execmodel": spec.execmodel, - "wait": spec.wait or "thread", - "coordinator_version": execnet.__version__, - } - ) + config: dict[str, Any] = { + "id": f"{spec.id}-worker", + "execmodel": spec.execmodel, + "wait": spec.wait or "thread", + "coordinator_version": execnet.__version__, + } + # Startup setup applied by the worker before serving (never through + # remote_exec: an exec slot must not be claimed by bookkeeping). + if spec.chdir: + config["chdir"] = spec.chdir + if spec.nice: + config["nice"] = int(spec.nice) + if spec.env: + config["env"] = spec.env + return json.dumps(config) def _worker_tokens(config: str) -> list[str]: diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index fc364772..66a892e8 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -46,6 +46,7 @@ from .gateway_base import TimeoutError from .gateway_base import Unserializer from .gateway_base import dumps_internal +from .gateway_base import gateway_info from .gateway_base import loads_internal from .gateway_base import trace @@ -678,6 +679,11 @@ def _dispatch(self, message: Message) -> None: } self._send_nowait(Message.CHANNEL_DATA, channelid, dumps_internal(status)) self._send_nowait(Message.CHANNEL_CLOSE, channelid) + elif code == Message.GATEWAY_INFO: + self._send_nowait( + Message.CHANNEL_DATA, channelid, dumps_internal(gateway_info()) + ) + self._send_nowait(Message.CHANNEL_CLOSE, channelid) elif code == Message.RECONFIGURE: data = loads_internal(message.data) assert isinstance(data, tuple) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index e29af4d2..9a1f1338 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -37,6 +37,7 @@ from .gateway_base import Message from .gateway_base import RemoteError from .gateway_base import dumps_internal +from .gateway_base import gateway_info from .gateway_base import loads_internal from .gateway_base import trace from .portal import LoopPortal @@ -177,6 +178,13 @@ def _dispatch(self, message: Message) -> None: try: if code == Message.STATUS: self._answer_status(message) + elif code == Message.GATEWAY_INFO: + gateway._send( + Message.CHANNEL_DATA, + message.channelid, + dumps_internal(gateway_info()), + ) + gateway._send(Message.CHANNEL_CLOSE, message.channelid) elif code == Message.CHANNEL_EXEC: channel = gateway._channelfactory.new(message.channelid) gateway._local_schedulexec(channel=channel, sourcetask=message.data) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index c5152cc2..56e64ef3 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -377,6 +377,25 @@ def _rough_version(version: str) -> tuple[int, ...]: return tuple(parts) +def _apply_worker_setup(config: dict[str, Any]) -> None: + """Apply chdir/nice/env from the worker config, before serving starts. + + This replaces the classic post-start ``remote_exec`` setup: valid for + every profile (a trio worker cannot run sync sources) and never + claims an exec slot. + """ + path = config.get("chdir") + if path: + if not os.path.exists(path): + os.mkdir(path) + os.chdir(path) + nice = config.get("nice") + if nice and hasattr(os, "nice"): + os.nice(nice) + for name, value in config.get("env", {}).items(): + os.environ[name] = value + + def _check_version(coordinator_version: str) -> None: """Warn on a real (major/minor) execnet version mismatch across the wire. @@ -416,6 +435,7 @@ def _main() -> None: config = json.loads(ns.config) _check_version(config["coordinator_version"]) + _apply_worker_setup(config) if ns.socket_fd is not None: serve_socket_trio( config["id"], diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index bec4c778..672a0830 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -26,7 +26,6 @@ "RemoteStatus", "_find_non_builtin_globals", "_source_of_function", - "rinfo_source", ] @@ -94,13 +93,19 @@ def reconfigure( self._send(Message.RECONFIGURE, data=data) def _rinfo(self, update: bool = False) -> RInfo: - """Return some sys/env information from remote.""" + """Return some sys/env information from remote. + + A native protocol request (like ``remote_status``): it never + touches the exec machinery, so it cannot claim an exec slot on + main-thread-shaped workers. + """ if update or not hasattr(self, "_cache_rinfo"): - ch = self.remote_exec(rinfo_source) - try: - self._cache_rinfo = RInfo(ch.receive()) - finally: - ch.waitclose() + channel = self.newchannel() + self._send(Message.GATEWAY_INFO, channel.id) + self._cache_rinfo = RInfo(channel.receive()) + # the other side didn't actually instantiate a channel + # so we just delete the internal id/channel mapping + self._channelfactory._local_close(channel.id) return self._cache_rinfo def hasreceiver(self) -> bool: @@ -166,18 +171,3 @@ def __getattr__(self, name: str) -> Any: ... RemoteStatus = RInfo - - -def rinfo_source(channel) -> None: - import os - import sys - - channel.send( - dict( - executable=sys.executable, - version_info=sys.version_info[:5], - platform=sys.platform, - cwd=os.getcwd(), - pid=os.getpid(), - ) - ) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 6a286302..c0ac5252 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -187,6 +187,7 @@ class only carries the framing and the code constants. CHANNEL_LAST_MESSAGE = 7 GATEWAY_START_SOCKET = 8 GATEWAY_START_SUB = 9 + GATEWAY_INFO = 10 # message code -> name _types: dict[int, str] = { @@ -200,6 +201,7 @@ class only carries the framing and the code constants. CHANNEL_LAST_MESSAGE: "CHANNEL_LAST_MESSAGE", GATEWAY_START_SOCKET: "GATEWAY_START_SOCKET", GATEWAY_START_SUB: "GATEWAY_START_SUB", + GATEWAY_INFO: "GATEWAY_INFO", } def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: @@ -243,6 +245,23 @@ def __repr__(self) -> str: return f"" +def gateway_info() -> dict[str, object]: + """Payload for ``Message.GATEWAY_INFO``: sys/env facts about this side. + + Answered natively by the dispatch loop -- an info request never + touches the exec machinery, so it cannot claim an exec slot (with + main-thread profiles, an info call stealing the primary slot used to + push the real workload onto a worker thread). + """ + return { + "executable": sys.executable, + "version_info": tuple(sys.version_info[:5]), + "platform": sys.platform, + "cwd": os.getcwd(), + "pid": os.getpid(), + } + + class FrameDecoder: """Incremental decoder for the 9-byte-header Message framing. diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 4bbfa7cf..ac88b181 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -183,25 +183,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: gw = _trio_host.makegateway_trio(self, spec) gw.spec = spec self._register(gw) - if spec.chdir or spec.nice or spec.env: - channel = gw.remote_exec( - """ - import os - path, nice, env = channel.receive() - if path: - if not os.path.exists(path): - os.mkdir(path) - os.chdir(path) - if nice and hasattr(os, 'nice'): - os.nice(nice) - if env: - for name, value in env.items(): - os.environ[name] = value - """ - ) - nice = (spec.nice and int(spec.nice)) or 0 - channel.send((spec.chdir, nice, spec.env)) - channel.waitclose() + # chdir/nice/env travel in the worker config and are applied at + # worker startup -- no remote_exec, so no exec slot is claimed. return gw def allocate_id(self, spec: XSpec) -> None: diff --git a/testing/test_basics.py b/testing/test_basics.py index c58e4728..470bec6d 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -153,22 +153,6 @@ def test_io_message(checker: Checker) -> None: assert "all passed" in out.stdout -def test_rinfo_source(checker: Checker) -> None: - out = checker.run_check( - f""" -class Channel: - def send(self, data): - assert eval(repr(data), {{}}) == data -channel = Channel() -{inspect.getsource(gateway.rinfo_source)} -print ('all passed') -""" - ) - - print(out.stdout) - assert "all passed" in out.stdout - - def test_geterrortext(checker: Checker) -> None: out = checker.run_check( standalone_gateway_base_source() diff --git a/testing/test_gateway.py b/testing/test_gateway.py index da519d4e..ae51cf0d 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -285,6 +285,19 @@ def test__rinfo(self, gw: Gateway) -> None: gw._cache_rinfo = rinfo gw.remote_exec("import os ; os.chdir(%r)" % old).waitclose() + def test__rinfo_while_exec_busy(self, gw: Gateway) -> None: + # info is a native protocol request: it must work (and not claim + # an exec slot) while an exec occupies the worker -- under + # main_thread_only an info-by-remote_exec used to either steal + # the main thread or trip the concurrency deadlock guard. + channel = gw.remote_exec("channel.send(channel.receive())") + try: + rinfo = gw._rinfo(update=True) + assert rinfo.pid != os.getpid() + finally: + channel.send("done") + assert channel.receive(TESTTIMEOUT) == "done" + class TestPopenGateway: gwtype = "popen" From cd2c8717fa481dd60a41e62da1439783558fa95b Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 16:55:17 +0200 Subject: [PATCH 44/59] feat: pure-async worker profile (execmodel=trio) C3 of the profile plan: the whole worker is one trio run owning the process main thread -- no host thread, no sync bridge, no sync channels. AsyncGateway gains a pluggable CHANNEL_EXEC handler (and live numexecuting for STATUS); TaskExec runs each source as a task on the worker nursery with an AsyncChannel. Sources must be async: strings execute with top-level await (PyCF_ALLOW_TOP_LEVEL_AWAIT), functions must be async def -- sync sources are rejected with a RemoteError before they can starve the loop. Gateway termination cancels running exec tasks. execmodel values are validated coordinator-side in makegateway against the profile registry (EXECMODEL_PROFILES); the worker CLI branches to the async entry (stdio and inherited-socket transports). Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 11 ++- src/execnet/_trio_worker.py | 139 +++++++++++++++++++++++++++++++-- src/execnet/gateway_base.py | 10 ++- src/execnet/multi.py | 6 ++ testing/test_execmodel_trio.py | 124 +++++++++++++++++++++++++++++ 5 files changed, 283 insertions(+), 7 deletions(-) create mode 100644 testing/test_execmodel_trio.py diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 66a892e8..81ea76d0 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -461,6 +461,12 @@ class AsyncGateway: _error: BaseException | None = None #: where the peer lives, when the transport knows (ssh host, socket addr) remoteaddress: str | None = None + #: CHANNEL_EXEC handler ``(gateway, channelid, data) -> None`` -- set by + #: an exec strategy (the pure-async worker's TaskExec); without one, + #: exec requests are rejected. + _exec_handler: Callable[[AsyncGateway, int, bytes], None] | None = None + #: the exec strategy (for STATUS numexecuting), when serving as a worker + _task_exec: Any = None def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None: self._stream = stream @@ -671,10 +677,13 @@ def _dispatch(self, message: Message) -> None: self._channel_for(channelid)._close_from_remote(None, sendonly=True) elif code == Message.GATEWAY_TERMINATE: raise GatewayReceivedTerminate(self) + elif code == Message.CHANNEL_EXEC and self._exec_handler is not None: + self._exec_handler(self, channelid, message.data) elif code == Message.STATUS: + task_exec = self._task_exec status = { "numchannels": len(self._channels), - "numexecuting": 0, + "numexecuting": task_exec.active_count() if task_exec else 0, "execmodel": "trio", } self._send_nowait(Message.CHANNEL_DATA, channelid, dumps_internal(status)) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 56e64ef3..49e53a69 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -15,6 +15,7 @@ from .gateway_base import MAIN_THREAD_ONLY_DEADLOCK_TEXT from .gateway_base import WorkerGateway from .gateway_base import get_execmodel +from .gateway_base import geterrortext from .gateway_base import loads_internal from .gateway_base import trace from .portal import Mailbox @@ -106,16 +107,93 @@ def trigger_shutdown(self) -> None: self._primary.put(None) -# execmodel profile -> exec strategy. Placement strategies to come: -# "trio" (tasks on a main-thread loop), classic hybrid for "thread" -# (primary main + pool overflow), "gevent" (greenlets on a main-thread -# hub), and eventually subinterpreters. +# execmodel profile -> exec strategy for the sync-facade worker; the +# "trio" profile serves a plain AsyncGateway instead (TaskExec below). +# Placement strategies to come: classic hybrid for "thread" (primary +# main + pool overflow), "gevent" (greenlets on a main-thread hub), and +# eventually subinterpreters. WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { "thread": PoolExec, "main_thread_only": MainExec, } +class TaskExec: + """Exec strategy for the pure-async profile (``execmodel=trio``). + + Sources run as tasks on the worker's own loop, in the one and only + thread of the process, and receive an ``AsyncChannel``. Sources must + be async: a plain function or a source string without top-level + ``await`` is rejected before it can starve the loop. Gateway + termination cancels running exec tasks (``trio.Cancelled`` inside the + source). + """ + + def __init__(self, gateway: Any, nursery: trio.Nursery) -> None: + self.gateway = gateway + self.nursery = nursery + self._running = 0 + + def active_count(self) -> int: + return self._running + + def handle_exec(self, gateway: Any, channelid: int, data: bytes) -> None: + """CHANNEL_EXEC hook; runs inline on the dispatch task.""" + item = loads_internal(data) + assert isinstance(item, tuple) + channel = gateway.open_channel(channelid) + self.nursery.start_soon(self._run_exec, channel, item) + + async def _run_exec(self, channel: Any, item: ExecItem) -> None: + import ast + import inspect + + source, file_name, call_name, kwargs = item + self._running += 1 + try: + trace(f"async execution starts[{channel.id}]: {source[:50]!r}") + loc: dict[str, Any] = {"channel": channel, "__name__": "__channelexec__"} + co = compile( + source + "\n", + file_name or "", + "exec", + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + ) + toplevel_await = bool(co.co_flags & inspect.CO_COROUTINE) + if not toplevel_await and not call_name: + await channel.aclose( + "sync source under execmodel=trio: the source must use" + " top-level await (or pass an async function)" + ) + return + if toplevel_await: + await eval(co, loc) + else: + exec(co, loc) # define the function (no top-level awaits) + if call_name: + function = loc[call_name] + result = function(channel, **kwargs) + if not hasattr(result, "__await__"): + await channel.aclose( + f"sync function {call_name!r} under execmodel=trio:" + " remote_exec functions must be async" + ) + return + await result + except trio.Cancelled: + raise + except EOFError: + trace("ignoring EOFError from async exec") + except BaseException as exc: + trace(f"async exec got exception: {exc!r}") + await channel.aclose(geterrortext(exc)) + return + finally: + self._running -= 1 + trace("async execution finished") + await channel.aclose() + + class TrioWorkerExec: """FIFO admission pump feeding an exec placement strategy. @@ -326,6 +404,51 @@ async def _make_fd_io(read_fd: int, write_fd: int) -> Any: return _trio_gateway.staple_fd_stream(read_fd, write_fd) +async def _serve_async_worker(stream: Any, id: str) -> None: + """Serve a plain AsyncGateway with task-based exec (execmodel=trio). + + The whole worker is this one trio run on the process main thread: the + dispatch loop and every exec'd source share it. Termination cancels + running exec tasks. + """ + from ._trio_gateway import AsyncGateway + + gateway = AsyncGateway(stream, id=id, _startcount=2) + async with trio.open_nursery() as nursery: + task_exec = TaskExec(gateway, nursery) + gateway._exec_handler = task_exec.handle_exec + gateway._task_exec = task_exec + await nursery.start(gateway._serve) + await gateway.wait_closed() + nursery.cancel_scope.cancel() + + +def serve_popen_async(id: str) -> None: + """Serve the pure-async worker over the stdio pipes (execmodel=trio).""" + from ._trio_gateway import staple_fd_stream + + read_fd, write_fd = _prepare_protocol_fds() + os.write(write_fd, b"1") + + async def main() -> None: + await _serve_async_worker(staple_fd_stream(read_fd, write_fd), id) + + trio.run(main) + os._exit(0) + + +def serve_socket_async(id: str, socket_fd: int) -> None: + """Serve the pure-async worker over an inherited socket fd.""" + from . import _trio_host + + async def main() -> None: + stream = await _trio_host.adopt_socket(socket_fd) + await _serve_async_worker(stream, id) + + trio.run(main) + os._exit(0) + + def serve_popen_trio(id: str, execmodel: str = "thread", wait: str = "thread") -> None: """Serve a WorkerGateway over the stdio pipes (popen / ssh worker).""" from . import _trio_host @@ -436,7 +559,13 @@ def _main() -> None: config = json.loads(ns.config) _check_version(config["coordinator_version"]) _apply_worker_setup(config) - if ns.socket_fd is not None: + if config["execmodel"] == "trio": + # pure-async profile: one thread, the loop owns the main thread + if ns.socket_fd is not None: + serve_socket_async(config["id"], ns.socket_fd) + else: + serve_popen_async(config["id"]) + elif ns.socket_fd is not None: serve_socket_trio( config["id"], config["execmodel"], diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index c0ac5252..77fd81df 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -123,10 +123,18 @@ def Event(self) -> threading.Event: return threading.Event() +#: worker profiles: where exec'd code runs relative to the protocol loop +EXECMODEL_PROFILES = ( + "thread", # exec on pool threads, loop on a side thread + "main_thread_only", # exec serialized on the main thread (GUI/pytest) + "trio", # pure async: loop owns the main thread, async sources as tasks +) + + def get_execmodel(backend: str | ExecModel) -> ExecModel: if isinstance(backend, ExecModel): return backend - if backend in ("thread", "main_thread_only"): + if backend in EXECMODEL_PROFILES: return ExecModel(backend) raise ValueError(f"unknown execmodel {backend!r}") diff --git a/src/execnet/multi.py b/src/execnet/multi.py index ac88b181..87b17786 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -24,6 +24,7 @@ from typing import overload from ._boundary import wakener_names +from .gateway_base import EXECMODEL_PROFILES from .gateway_base import Channel from .gateway_base import ExecModel from .gateway_base import get_execmodel @@ -172,6 +173,11 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: self.allocate_id(spec) if spec.execmodel is None: spec.execmodel = self.remote_execmodel.backend + elif spec.execmodel not in EXECMODEL_PROFILES: + raise ValueError( + f"unknown execmodel {spec.execmodel!r}" + f" (known profiles: {list(EXECMODEL_PROFILES)})" + ) if spec.wait is not None and spec.wait not in wakener_names(): raise ValueError( f"unknown wait backend {spec.wait!r} (known: {wakener_names()})" diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py new file mode 100644 index 00000000..d23a72cf --- /dev/null +++ b/testing/test_execmodel_trio.py @@ -0,0 +1,124 @@ +"""The pure-async worker profile (execmodel=trio). + +One single thread in the worker: the trio loop owns the main thread and +exec'd sources run as tasks on it, talking through AsyncChannels. Sync +sources are rejected before they can starve the loop. +""" + +from __future__ import annotations + +from collections.abc import Callable + +import pytest +import trio as trio_lib + +import execnet +import execnet.trio +from execnet.gateway import Gateway + +TESTTIMEOUT = 10.0 + + +@pytest.fixture +def trio_gw(makegateway: Callable[[str], Gateway]) -> Gateway: + return makegateway("popen//execmodel=trio") + + +class TestSyncCoordinator: + def test_top_level_await_roundtrip(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec( + "await channel.send(await channel.receive() + 1)" + ) + channel.send(41) + assert channel.receive(TESTTIMEOUT) == 42 + + def test_async_function_source(self, trio_gw: Gateway) -> None: + async def source(channel, delta) -> None: + value = await channel.receive() + await channel.send(value + delta) + + channel = trio_gw.remote_exec(source, delta=5) + channel.send(37) + assert channel.receive(TESTTIMEOUT) == 42 + + def test_iteration_and_eof(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec( + """ + for i in range(3): + await channel.send(i * 2) + """ + ) + assert list(channel) == [0, 2, 4] + + def test_single_thread_on_main(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec( + """ + import threading + await channel.send( + ( + threading.active_count(), + threading.current_thread() is threading.main_thread(), + ) + ) + """ + ) + active, on_main = channel.receive(TESTTIMEOUT) + assert on_main + assert active == 1 + + def test_concurrent_execs_cooperate(self, trio_gw: Gateway) -> None: + # two execs run as tasks on one loop: the first parks in receive + # while the second completes -- no threads involved. + blocked = trio_gw.remote_exec( + "await channel.send(await channel.receive())" + ) + side = trio_gw.remote_exec("await channel.send('side')") + assert side.receive(TESTTIMEOUT) == "side" + blocked.send("go") + assert blocked.receive(TESTTIMEOUT) == "go" + + def test_sync_source_rejected(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec("x = 40 + 2") + with pytest.raises(channel.RemoteError, match="sync source"): + channel.receive(TESTTIMEOUT) + + def test_sync_function_rejected(self, trio_gw: Gateway) -> None: + def source(channel) -> None: + pass + + channel = trio_gw.remote_exec(source) + with pytest.raises(channel.RemoteError, match="must be async"): + channel.receive(TESTTIMEOUT) + + def test_remote_error_traceback(self, trio_gw: Gateway) -> None: + async def source(channel) -> None: + raise ValueError(17) + + channel = trio_gw.remote_exec(source) + with pytest.raises(channel.RemoteError, match="ValueError"): + channel.receive(TESTTIMEOUT) + + def test_status_and_rinfo(self, trio_gw: Gateway) -> None: + assert trio_gw.remote_status().execmodel == "trio" + rinfo = trio_gw._rinfo() + assert rinfo.pid + assert rinfo.version_info + + +def test_unknown_execmodel_rejected(makegateway: Callable[[str], Gateway]) -> None: + with pytest.raises(ValueError, match="unknown execmodel"): + makegateway("popen//execmodel=nope") + + +def test_trio_native_coordinator() -> None: + async def main() -> None: + async with execnet.trio.AsyncGroup() as group: + gateway = await group.makegateway("popen//execmodel=trio") + channel = await gateway.remote_exec( + "await channel.send(await channel.receive() * 2)" + ) + await channel.send(21) + with trio_lib.fail_after(TESTTIMEOUT): + assert await channel.receive() == 42 + + trio_lib.run(main) From 36e904e52dd22e0a8aea0e3f8761107d0363cc76 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 17:18:34 +0200 Subject: [PATCH 45/59] feat!: restore classic hybrid exec for execmodel=thread C4 of the profile plan: the thread profile regains its pre-port shape -- a request arriving while the worker main thread is idle claims it (the primary workload gets a true main thread again), and requests arriving while it is busy overflow to pool threads instead of queueing. The claim is decided during FIFO admission, so the first exec always lands on the main thread. HybridExec composes MainExec (primary mailbox + integrate loop) with PoolExec overflow; main-thread workers now integrate as primary instead of parking in join(). Together with C2 (native info/setup) this cannot be starved by coordinator bookkeeping: only real remote_exec work claims the slot. xdist is unaffected (it forces main_thread_only); the -n 12 suite runs are the live canary. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_worker.py | 45 +++++++++++++++++++++++++++++++--- testing/test_execmodel_trio.py | 8 ++---- testing/test_gateway.py | 32 ++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 49e53a69..c6042bb8 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -107,13 +107,50 @@ def trigger_shutdown(self) -> None: self._primary.put(None) +class HybridExec(MainExec): + """Exec strategy: primary on the main thread, overflow on pool threads. + + The classic ``thread`` execmodel shape: a request arriving while the + main thread is idle claims it (pytest and friends get a true main + thread); requests arriving while it is busy run on worker threads + instead of queueing. The claim is decided during FIFO admission so + the *first* request always gets the main thread. + """ + + def __init__(self, gateway: WorkerGateway) -> None: + super().__init__(gateway) + self._pool = PoolExec(gateway) + self._claim_lock = threading.Lock() + self._primary_busy = False + self._claimed: set[int] = set() + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + with self._claim_lock: + if not self._primary_busy: + self._primary_busy = True + self._claimed.add(channel.id) + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + with self._claim_lock: + claimed = channel.id in self._claimed + self._claimed.discard(channel.id) + if not claimed: + await self._pool.run(channel, item) + return + try: + await super().run(channel, item) + finally: + with self._claim_lock: + self._primary_busy = False + + # execmodel profile -> exec strategy for the sync-facade worker; the # "trio" profile serves a plain AsyncGateway instead (TaskExec below). -# Placement strategies to come: classic hybrid for "thread" (primary -# main + pool overflow), "gevent" (greenlets on a main-thread hub), and -# eventually subinterpreters. +# Placement strategies to come: "gevent" (greenlets on a main-thread +# hub), and eventually subinterpreters. WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { - "thread": PoolExec, + "thread": HybridExec, "main_thread_only": MainExec, } diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py index d23a72cf..84585ba9 100644 --- a/testing/test_execmodel_trio.py +++ b/testing/test_execmodel_trio.py @@ -26,9 +26,7 @@ def trio_gw(makegateway: Callable[[str], Gateway]) -> Gateway: class TestSyncCoordinator: def test_top_level_await_roundtrip(self, trio_gw: Gateway) -> None: - channel = trio_gw.remote_exec( - "await channel.send(await channel.receive() + 1)" - ) + channel = trio_gw.remote_exec("await channel.send(await channel.receive() + 1)") channel.send(41) assert channel.receive(TESTTIMEOUT) == 42 @@ -69,9 +67,7 @@ def test_single_thread_on_main(self, trio_gw: Gateway) -> None: def test_concurrent_execs_cooperate(self, trio_gw: Gateway) -> None: # two execs run as tasks on one loop: the first parks in receive # while the second completes -- no threads involved. - blocked = trio_gw.remote_exec( - "await channel.send(await channel.receive())" - ) + blocked = trio_gw.remote_exec("await channel.send(await channel.receive())") side = trio_gw.remote_exec("await channel.send('side')") assert side.receive(TESTTIMEOUT) == "side" blocked.send("go") diff --git a/testing/test_gateway.py b/testing/test_gateway.py index ae51cf0d..6b6d5cf5 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -9,6 +9,7 @@ import shutil import signal import sys +import time from collections.abc import Callable from textwrap import dedent @@ -285,6 +286,37 @@ def test__rinfo(self, gw: Gateway) -> None: gw._cache_rinfo = rinfo gw.remote_exec("import os ; os.chdir(%r)" % old).waitclose() + def test_hybrid_primary_then_overflow( + self, makegateway: Callable[[str], Gateway] + ) -> None: + # classic thread-model shape: the first exec claims the true main + # thread; while it is busy, further execs overflow to worker + # threads; once released the main thread is claimable again. + gw = makegateway("popen//execmodel=thread") + report = """ + import threading + channel.send(threading.current_thread() is threading.main_thread()) + channel.receive() + """ + first = gw.remote_exec(report) + assert first.receive(TESTTIMEOUT) is True + second = gw.remote_exec(report) + assert second.receive(TESTTIMEOUT) is False + second.send(None) + second.waitclose(TESTTIMEOUT) + first.send(None) + first.waitclose(TESTTIMEOUT) + # the primary slot frees shortly after the exec finishes + for _ in range(50): + probe = gw.remote_exec(report) + on_main = probe.receive(TESTTIMEOUT) + probe.send(None) + probe.waitclose(TESTTIMEOUT) + if on_main: + break + time.sleep(0.05) + assert on_main + def test__rinfo_while_exec_busy(self, gw: Gateway) -> None: # info is a native protocol request: it must work (and not claim # an exec slot) while an exec occupies the worker -- under From cbce1830630cd02388992e8d9773b0639693b8ae Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 19:21:54 +0200 Subject: [PATCH 46/59] feat: gevent worker profile and gevent-safe management ops C5 of the profile plan, completing the gevent/loop integration: - execmodel=gevent: GreenletExec runs the hub on the worker main thread and spawns a greenlet per remote_exec; the worker's wait backend defaults to gevent so channel operations park greenlets, letting concurrent execs cooperate on one thread. The integrate loop itself waits on a gevent wakener (a thread-blocking get would stall the hub). - Provisioning: execnet grows a [gevent] extra; every uv launcher (popen python=, ssh, via sub-spawn) derives an additional `--with gevent` from the worker config when the profile or wait backend needs it. - Coordinator side: TrioHost.call_pending resolves a OneShot from a host task; makegateway (wait!=thread), SyncIOHandle.wait/kill, and Group.terminate wait on it with the gateway's wakener so a gevent app's management ops park only the calling greenlet. wait=thread keeps the KI-deferred portal.run path unchanged. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 3 ++ src/execnet/_provision.py | 23 +++++++++-- src/execnet/_trio_host.py | 63 +++++++++++++++++++++++++++-- src/execnet/_trio_worker.py | 54 ++++++++++++++++++++++++- src/execnet/gateway_base.py | 3 +- src/execnet/multi.py | 24 ++++++++++- testing/test_gevent.py | 81 +++++++++++++++++++++++++++++++++++++ uv.lock | 6 ++- 8 files changed, 246 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aedd6b59..6e12c8da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,9 @@ classifiers = [ execnet-socketserver = "execnet.script.socketserver:main" [project.optional-dependencies] +gevent = [ + "gevent", +] testing = [ "pre-commit", "pytest>8.0", diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index ffe9ffaa..4da4f689 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -164,10 +164,12 @@ def worker_cli_arg(spec: Any) -> str: """ import execnet + # the gevent profile parks its greenlets on gevent wakeners + default_wait = "gevent" if spec.execmodel == "gevent" else "thread" config: dict[str, Any] = { "id": f"{spec.id}-worker", "execmodel": spec.execmodel, - "wait": spec.wait or "thread", + "wait": spec.wait or default_wait, "coordinator_version": execnet.__version__, } # Startup setup applied by the worker before serving (never through @@ -203,13 +205,28 @@ def _uv_tokens(python: str | None) -> list[str]: return prefix +def _extra_with_tokens(config: str) -> list[str]: + """Additional ``--with`` requirements the worker env needs. + + Derived from the worker config itself so every uv launcher (popen, + ssh, via sub-spawn) provisions the same: the gevent profile / wait + backend needs gevent importable in the worker. + """ + parsed = json.loads(config) + if parsed.get("execmodel") == "gevent" or parsed.get("wait") == "gevent": + return ["--with", "gevent"] + return [] + + def uv_worker_argv(spec: Any) -> list[str]: """``uv run`` argv to launch the Trio worker locally (wheel path is local).""" + config = worker_cli_arg(spec) return [ *_uv_tokens(spec.python), "--with", coordinator_requirement(), - *worker_module_tokens(spec), + *_extra_with_tokens(config), + *_worker_tokens(config), ] @@ -228,7 +245,7 @@ def _remote_shell_command( wheel bytes are returned as the preamble to stream before the protocol. """ worker = _worker_tokens(config) - uv = _uv_tokens(python) + uv = [*_uv_tokens(python), *_extra_with_tokens(config)] if wheel is None: assert requirement is not None return shlex.join([*uv, "--with", requirement, *worker]), b"" diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 9a1f1338..597cbdb4 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -111,7 +111,7 @@ async def _wait() -> int | None: return code try: - return self._session.host.call(_wait) + return self._session.host_call(_wait) except Exception: return process.returncode @@ -126,7 +126,7 @@ async def _kill() -> None: await process.wait() try: - self._session.host.call(_kill) + self._session.host_call(_kill) except Exception as exc: trace("ERROR killing trio process:", exc) @@ -293,6 +293,21 @@ def release_channel(self, channelid: int) -> None: with suppress(trio.RunFinishedError): self.host.portal.post(self._forget_channel, channelid) + def host_call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: + """Blocking host call that parks correctly for the wait backend. + + ``wait=thread`` keeps the KI-deferred ``portal.run`` path; other + backends (gevent) wait on a OneShot with the gateway's wakener so + only the calling greenlet parks, not the whole hub. + """ + gateway = self.sync_gateway + if gateway._wait_backend == "thread": + return self.host.call(async_fn, *args) + pending = self.host.call_pending( + async_fn, *args, wakener=gateway._new_wakener() + ) + return pending.wait() + def run_on_loop(self, sync_fn: Callable[[], T]) -> T: """Run ``sync_fn`` on the host loop, excluding dispatch interleaving. @@ -428,6 +443,39 @@ async def _main(self) -> None: def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: return self.portal.run(async_fn, *args) + def call_pending( + self, + async_fn: Callable[..., Awaitable[T]], + *args: Any, + wakener: Any = None, + ) -> OneShot[T]: + """Run ``async_fn`` as a host task, resolving a :class:`OneShot`. + + The non-blocking counterpart of :meth:`call` for consumers that + must not block their OS thread (a gevent hub: waiting on the + OneShot with a gevent wakener parks only the calling greenlet). + Unlike ``portal.run`` the wait is KeyboardInterrupt-interruptible. + """ + result: OneShot[T] = OneShot(wakener) + + async def runner() -> None: + try: + value = await async_fn(*args) + except trio.Cancelled: + if not result.is_set(): + result.set_error(RuntimeError("trio host was shut down")) + raise + except BaseException as exc: + result.set_error(exc) + else: + result.set(value) + + def spawn() -> None: + self.start_soon(runner) + + self.portal.post(spawn) + return result + def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: return self.portal.run_sync(sync_fn, *args) @@ -556,7 +604,16 @@ def makegateway_trio(group: Group, spec: Any) -> Gateway: """Create a sync-facade Gateway for ``spec`` on the group's Trio host.""" host: TrioHost = group._ensure_trio_host() async_group: FacadeAsyncGroup = group._ensure_async_group() - bridge = host.call(async_group.makegateway, spec) + if spec.wait and spec.wait != "thread": + # e.g. a gevent app: wait on a OneShot so only the calling + # greenlet parks while the gateway comes up, not the whole hub. + from ._boundary import make_wakener + + bridge = host.call_pending( + async_group.makegateway, spec, wakener=make_wakener(spec.wait) + ).wait() + else: + bridge = host.call(async_group.makegateway, spec) assert isinstance(bridge, SyncBridgeGateway) gw: Gateway = bridge.sync_gateway # type: ignore[assignment] gw._io = SyncIOHandle( diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index c6042bb8..d8c6cc0e 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -145,13 +145,63 @@ async def run(self, channel: Channel, item: ExecItem) -> None: self._primary_busy = False +class GreenletExec: + """Exec strategy: greenlets on a gevent hub owning the main thread. + + ``execmodel=gevent``: each request runs as a greenlet spawned by the + integrate loop; the worker's ``wait=gevent`` wakeners make channel + operations park the greenlet, so concurrent remote_execs cooperate on + the one main thread. Requires gevent in the worker environment + (provisioning adds the ``gevent`` requirement automatically). + """ + + needs_primary_thread = True + + def __init__(self, gateway: WorkerGateway) -> None: + from ._boundary import make_wakener + + self.gateway = gateway + # The integrate loop blocks in get() on the hub thread: a gevent + # wakener parks only its root greenlet, letting exec greenlets run. + self._primary: Mailbox[tuple[Channel, ExecItem, threading.Event] | None] = ( + Mailbox(make_wakener("gevent")) + ) + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + done = threading.Event() + self._primary.put((channel, item, done)) + await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) + + def integrate_as_primary_thread(self) -> None: + """Run the hub on the main thread, spawning a greenlet per exec.""" + import gevent + + def run_exec(channel: Channel, item: ExecItem, done: threading.Event) -> None: + try: + self.gateway.executetask((channel, item)) + finally: + done.set() + + while True: + task = self._primary.get() + if task is None: + break + gevent.spawn(run_exec, *task) + + def trigger_shutdown(self) -> None: + self._primary.put(None) + + # execmodel profile -> exec strategy for the sync-facade worker; the # "trio" profile serves a plain AsyncGateway instead (TaskExec below). -# Placement strategies to come: "gevent" (greenlets on a main-thread -# hub), and eventually subinterpreters. +# Future placement strategies slot in here (e.g. subinterpreters). WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { "thread": HybridExec, "main_thread_only": MainExec, + "gevent": GreenletExec, } diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 77fd81df..2b183c1e 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -125,9 +125,10 @@ def Event(self) -> threading.Event: #: worker profiles: where exec'd code runs relative to the protocol loop EXECMODEL_PROFILES = ( - "thread", # exec on pool threads, loop on a side thread + "thread", # hybrid: primary on the main thread, overflow on pool threads "main_thread_only", # exec serialized on the main thread (GUI/pytest) "trio", # pure async: loop owns the main thread, async sources as tasks + "gevent", # greenlets on a main-thread hub, one per remote_exec ) diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 87b17786..7c0810a7 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -248,13 +248,35 @@ def terminate(self, timeout: float | None = None) -> None: # each with a GATEWAY_TERMINATE + timeout grace, then kill; # bounded at roughly twice the timeout (issues #43 / #221). try: - self._trio_host.call(self._async_group.terminate, timeout) + self._host_terminate(timeout) except Exception as exc: trace("group terminate error:", exc) for gw in self._gateways_to_join: gw.join() self._gateways_to_join[:] = [] + def _host_terminate(self, timeout: float | None) -> None: + """Terminate the async group, parking correctly for wait backends. + + A member gateway created with a non-thread ``wait=`` implies the + caller may be a greenlet: wait on a OneShot instead of blocking + the OS thread (which would stall the hub for the whole grace). + """ + backends = {gw._wait_backend for gw in self._gateways_to_join} | { + gw._wait_backend for gw in self + } + backends.discard("thread") + if backends: + from ._boundary import make_wakener + + self._trio_host.call_pending( + self._async_group.terminate, + timeout, + wakener=make_wakener(backends.pop()), + ).wait() + else: + self._trio_host.call(self._async_group.terminate, timeout) + def remote_exec( self, source: str | types.FunctionType | Callable[..., object] | types.ModuleType, diff --git a/testing/test_gevent.py b/testing/test_gevent.py index e95edfc7..91f74ef4 100644 --- a/testing/test_gevent.py +++ b/testing/test_gevent.py @@ -84,3 +84,84 @@ def test_waitclose_and_endmarker(self, gevent_gw: execnet.Gateway) -> None: channel = gevent_gw.remote_exec("channel.send(1)") assert gevent.spawn(channel.receive, TESTTIMEOUT).get(timeout=TESTTIMEOUT) == 1 gevent.spawn(channel.waitclose, TESTTIMEOUT).get(timeout=TESTTIMEOUT) + + def test_makegateway_parks_greenlet_not_hub(self) -> None: + # management ops (makegateway/terminate) from a greenlet must not + # stall the hub: they wait on a OneShot with a gevent wakener. + group = execnet.Group() + progressed: list[int] = [] + + def other() -> None: + for i in range(5): + progressed.append(i) + gevent.sleep(0.01) + + try: + ticker = gevent.spawn(other) + maker = gevent.spawn(group.makegateway, "popen//wait=gevent") + gw = maker.get(timeout=TESTTIMEOUT) + channel = gw.remote_exec("channel.send(42)") + assert gevent.spawn(channel.receive, TESTTIMEOUT).get(TESTTIMEOUT) == 42 + ticker.get(timeout=TESTTIMEOUT) + assert progressed == [0, 1, 2, 3, 4] + finally: + gevent.spawn(group.terminate, 5.0).get(timeout=TESTTIMEOUT) + + +class TestGeventWorkerProfile: + """execmodel=gevent: exec'd code runs as greenlets on the main-thread hub.""" + + @pytest.fixture + def worker_gw(self): + group = execnet.Group() + try: + yield group.makegateway("popen//execmodel=gevent") + finally: + group.terminate(timeout=5.0) + + def test_execs_are_greenlets_on_main_thread(self, worker_gw) -> None: + report = """ + import threading + channel.send(threading.current_thread() is threading.main_thread()) + channel.receive() + """ + first = worker_gw.remote_exec(report) + second = worker_gw.remote_exec(report) + # both run concurrently on the one main thread: greenlets + assert first.receive(TESTTIMEOUT) is True + assert second.receive(TESTTIMEOUT) is True + first.send(None) + second.send(None) + first.waitclose(TESTTIMEOUT) + second.waitclose(TESTTIMEOUT) + + def test_execs_cooperate_via_gevent(self, worker_gw) -> None: + # the first exec parks in channel.receive() (gevent wakener) while + # the second completes -- with a blocked hub this would deadlock. + blocked = worker_gw.remote_exec("channel.send(channel.receive())") + side = worker_gw.remote_exec( + """ + import gevent + gevent.sleep(0.01) + channel.send('side') + """ + ) + assert side.receive(TESTTIMEOUT) == "side" + blocked.send("go") + assert blocked.receive(TESTTIMEOUT) == "go" + + def test_status_reports_gevent(self, worker_gw) -> None: + assert worker_gw.remote_status().execmodel == "gevent" + + +def test_provisioning_adds_gevent_requirement() -> None: + from execnet._provision import _extra_with_tokens + from execnet._provision import worker_cli_arg + from execnet.xspec import XSpec + + spec = XSpec("popen//id=g1//execmodel=gevent") + config = worker_cli_arg(spec) + assert '"wait": "gevent"' in config + assert _extra_with_tokens(config) == ["--with", "gevent"] + plain = worker_cli_arg(XSpec("popen//id=g2//execmodel=thread")) + assert _extra_with_tokens(plain) == [] diff --git a/uv.lock b/uv.lock index b885a4b7..3b8bb46e 100644 --- a/uv.lock +++ b/uv.lock @@ -387,6 +387,9 @@ dependencies = [ ] [package.optional-dependencies] +gevent = [ + { name = "gevent" }, +] testing = [ { name = "asyncssh" }, { name = "hatch" }, @@ -411,6 +414,7 @@ testing = [ [package.metadata] requires-dist = [ { name = "asyncssh", marker = "extra == 'testing'" }, + { name = "gevent", marker = "extra == 'gevent'" }, { name = "hatch", marker = "extra == 'testing'" }, { name = "pre-commit", marker = "extra == 'testing'" }, { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, @@ -419,7 +423,7 @@ requires-dist = [ { name = "trio", specifier = ">=0.32" }, { name = "uv", marker = "extra == 'testing'" }, ] -provides-extras = ["testing"] +provides-extras = ["gevent", "testing"] [package.metadata.requires-dev] dev = [{ name = "pytest-xdist", specifier = ">=3.8.0" }] From 87c58430e31f7c849db7cef2fbb35d788a6324be Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 19:35:25 +0200 Subject: [PATCH 47/59] docs: record the landed Phase C worker profiles in the handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-c-worker-axes.md | 74 ++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index 40c222b6..01382e39 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -71,44 +71,48 @@ File map (src/execnet/): `threading.Event`/`queue.Queue` so KeyboardInterrupt can interrupt them; `portal.run` (KI-deferred) is only for management ops. -## Phase C — DECIDED PLAN (Ronny, 2026-07-26, after the boundary rethink) +## Phase C — LANDED: use-case worker profiles on `execmodel=` (2026-07-26) -Predates below gap analysis; where they conflict, this section wins. -Context: the boundary rethink (`handoff-boundary-protocol-rethink.md`) -landed first — `wait=` axis exists, ExecModel is a preset, the sync -Channel is a facade over the async core. +Where this conflicts with anything below or elsewhere, this section +wins. The axes framing (`loop=`/`exec=` as spec keys, a reserved +`backend=` axis) was **dropped** in the final rethink with Ronny: +different use-cases get named profiles, `execmodel=` is the public mode +key (xdist already passes it), and `wait=` is the only other public +knob. Implemented in commits `b2f43c3..cbce183`: -**Worker matrix** (`loop=` = main-thread role, `exec=` = placement). -The main thread is the scarce resource (signals, interrupt_main, GUI, -tests calling asyncio.run/trio.run themselves): - -| loop= | exec= | main thread | exec'd code | channel | consumer | +| `execmodel=` | loop thread | exec'd code runs | channel | worker wait | extra deps | |---|---|---|---|---|---| -| main | thread | host loop | worker threads | sync | default preset for execmodel=thread; no parked thread | -| main | task | host loop | tasks on the loop | AsyncChannel | async-native sources (C4) | -| thread | main | exec'd code (serialized) | true main thread | sync | pytest/xdist (forces main_thread_only today), GUI, signals | -| thread | thread | parked | worker threads | sync | valid, non-default (today's shape) | -| thread | task | parked | tasks on side loop | AsyncChannel | valid, niche | -| main | main | — | — | — | invalid | - -Interaction rule: exec'd code may start its own event loop only when it -does not share a thread with ours (exec=thread / exec=main); exec=task -sources are ``async def`` and join our loop — a sync source under -exec=task is a hard error. ``wait=`` composes orthogonally. - -**Backend decisions**: the core STAYS trio-only and trio stays the -default and a hard dependency (pure-python wheels; deployment cost -accepted). The anyio/asyncio-core port ("C0") is REJECTED for now; -asyncio apps use the ``execnet.aio`` bridge. ``backend=`` still lands -as a per-gateway spec axis, but reserved: only ``trio`` validates -(exactly how ``wait=`` landed with only ``thread``), keeping an -asyncio-core door open without paying for it. Consequence: exec=task -sources are trio-typed. - -Sequencing: C1 axes (`backend=`/`loop=`/`exec=` + validity matrix + -presets) → C2 `loop=main` worker restructure → C3 `exec=main` re-home → -C4 `exec=task`. Presets: `execmodel=thread` → `loop=main, exec=thread`; -`execmodel=main_thread_only` → `loop=thread, exec=main`. +| `thread` (default) | side thread | **classic hybrid restored**: primary on the main thread, overflow on pool threads (claim decided during FIFO admission) | sync | thread | — | +| `main_thread_only` | side thread | main thread, serialized (deadlock guard) | sync | thread | — | +| `trio` (new) | **main thread** | async sources as tasks — one single thread total; top-level await or async def; sync sources rejected; termination cancels tasks | AsyncChannel | (loop) | — | +| `gevent` (revived) | side thread | greenlets on a main-thread hub, one per remote_exec | sync | gevent (derived) | `execnet[gevent]`, auto-added by uv provisioning | + +Architecture: `TrioWorkerExec` is a pure FIFO admission pump delegating +to strategy objects (`WORKER_EXEC_STRATEGIES` in `_trio_worker.py`: +PoolExec building block, MainExec, HybridExec, GreenletExec; TaskExec +serves a plain AsyncGateway via its pluggable `_exec_handler` — no sync +bridge at all in the trio profile). Subinterpreters: future strategy +slot, not built. `EXECMODEL_PROFILES` (gateway_base) validates +coordinator-side in makegateway. + +**Native info/setup** (the pytest fix): `Message.GATEWAY_INFO` (code 10) +answers `_rinfo()` from the dispatch loop; chdir/nice/env ship in the +worker config JSON and apply at startup. Coordinator bookkeeping can no +longer claim an exec slot (previously an info call could occupy the main +thread so pytest landed on a worker thread) — `rinfo_source` and the +post-start remote_exec setup block are gone. + +Coordinator-gevent integration: `TrioHost.call_pending` (OneShot from a +host task) backs makegateway / `SyncIOHandle.wait/kill` / +`Group.terminate` whenever the wait backend is not `thread`, so a gevent +app's management ops park only the calling greenlet. `wait=thread` +keeps the KI-deferred `portal.run` path. + +Still decided/standing: core stays trio-only (anyio/asyncio-core port +rejected; asyncio apps use `execnet.aio`); eventlet stays dead; exec'd +code may start its own loop in every profile except `trio`. + +## Phase C — original gap analysis (pre-decision record, superseded) ## Phase C — original gap analysis (pre-decision record) From 3031810da1bcf6d907d067565c6e9125a5438383 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 10:00:04 +0200 Subject: [PATCH 48/59] feat!: run channel callbacks in an off-loop consumer task setcallback no longer delivers inline on the loop/reader thread and no longer pins the channel in a ChannelFactory._callback_channels registry. Instead Channel.setcallback -> BaseGateway._start_channel_consumer -> SyncBridgeGateway.attach_consumer starts a per-channel consumer *task* on the host loop that drains the channel into an inbox and runs each callback in a trio.to_thread pool thread (bounded by TrioHost.callback_limiter). The task holds a strong reference to the channel, so a callback channel's lifecycle is bound to consumption (and to GC once the stream closes) rather than to a strong-ref registry. Delivery keeps flowing through the existing _sync_payload -> Channel._deliver_payload path (diverted to the inbox once a consumer is attached), so nothing rebinds the raw channel and there is no race with the bind posted at newchannel(). waitclose() now waits on a new Channel._consumer_done flag for callback channels, so it still returns only after every callback -- including the endmarker -- has run. A slow callback no longer blocks the reader, and callbacks for different channels run concurrently (per-channel order kept). Deleted: Channel._callback/_endmarker/_fire_endmarker, ChannelFactory. _callback_channels/_register_callback_channel, and the callback branch of _deliver_payload (now uniformly mailbox.put); __del__'s CHANNEL_LAST_MESSAGE case folds away (a callback channel is never GC'd while open). Resolves the standing "callbacks on the loop thread" open decision (moved). Co-Authored-By: Claude Opus 4.8 (1M context) --- handoff-boundary-protocol-rethink.md | 6 +- handoff-phase-c-worker-axes.md | 54 +++++++- src/execnet/_trio_host.py | 181 +++++++++++++++++++++++++-- src/execnet/gateway_base.py | 181 +++++++++++++++------------ testing/test_channel.py | 44 +++++++ 5 files changed, 369 insertions(+), 97 deletions(-) diff --git a/handoff-boundary-protocol-rethink.md b/handoff-boundary-protocol-rethink.md index 59cd1141..70146c7d 100644 --- a/handoff-boundary-protocol-rethink.md +++ b/handoff-boundary-protocol-rethink.md @@ -139,8 +139,10 @@ namespaces + the axes/presets. one portal FIFO. - After close: `OSError("cannot send (already closed?)")`. - exec admission order = message arrival order (TrioWorkerExec._pump). -- Channel callbacks run on the loop thread (revisit-later stands; the - kit makes moving them cheap). +- Channel callbacks run off the loop thread: a per-channel consumer task + drains into a threadpool call (moved 2026-07-26; the kit made it cheap, + as predicted). See `handoff-phase-c-worker-axes.md` "Callback consumer + tasks". - `Group.terminate(timeout)` never hangs (~2×timeout). - Sync blocking waits stay KeyboardInterrupt-interruptible on the main thread (thread Wakener keeps the Event.wait + drain pattern). diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index 01382e39..102cb78f 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -63,8 +63,12 @@ File map (src/execnet/): - exec admission order = message arrival order (`TrioWorkerExec._pump`; `test_main_thread_only_concurrent_remote_exec_deadlock` guards this — trio shuffles its run batch, so never rely on task-spawn order). -- Channel callbacks run on the receiver (loop) thread — keep for now - (open decision: revisit in D). +- Channel callbacks now run in a threadpool thread driven by a per-channel + consumer *task* on the loop (RESOLVED 2026-07-26, was the "revisit in D" + open decision — moved off the loop). A slow callback no longer blocks the + reader; per-channel order is still strict, and `waitclose()` still returns + only after every callback (incl. the endmarker) has run. See "Callback + consumer tasks" below. - `Group.terminate(timeout)` never hangs (~2×timeout bound, issues #43/#221). - Sync blocking waits (send-ack, receive, waitclose, join) stay on @@ -167,8 +171,8 @@ Compat mapping: `execmodel=thread` → `loop=main` + `exec=thread`; this branch plus real `-n` smoke runs (crash/endmarker tests, `main_thread_only` GUI case). Our own suite running `-n 12` green is necessary but not sufficient. -4. **Close the open decision**: channel callbacks on the loop thread — - keep or move. +4. **Open decision CLOSED (2026-07-26)**: channel callbacks moved off the + loop thread — see "Callback consumer tasks" below. 5. **xfail markers audit (done 2026-07-26, keep as-is)**: the 11 consistent XPASSes were investigated — trio's single-loop dispatch + FIFO admission makes them pass reliably when idle, but under @@ -185,6 +189,48 @@ compat mapping unblocks the ExecModel retirement, then `exec=task`; run the xdist verification (D.3) mid-C as an early canary rather than only at the end. +## Callback consumer tasks (`setcallback`, LANDED 2026-07-26) + +`setcallback` no longer delivers inline on the loop thread and no longer +pins the channel in a `ChannelFactory._callback_channels` registry. Instead +`Channel.setcallback` → `BaseGateway._start_channel_consumer` → +`SyncBridgeGateway.attach_consumer`, which on the loop: + +- moves any already-buffered mailbox items into a fresh per-channel inbox + (a `trio` memory channel), diverts future raw payloads there + (`raw.set_consumer(inbox.send_nowait, on_close)`), and nulls `_mailbox`; +- starts a consumer *task* on the host root nursery (`_run_consumer`) that is + handed a **strong** reference to the sync `Channel` — so the channel's + lifecycle is now bound to the task (and to GC once the stream closes), + replacing the strong-ref registry. + +`_run_consumer` does `async for data in inbox:` and runs each callback via +`trio.to_thread.run_sync(..., limiter=host.callback_limiter)` — off the loop, +one thread of a bounded pool (`DEFAULT_CALLBACK_THREADS=40`), sequential per +channel (order preserved), concurrent across channels. A raising callback +(or a `loads_internal` failure) → `CHANNEL_CLOSE_ERROR` + local close +(`_consumer_failed`). On EOF/close the endmarker fires (shielded, bounded by +`CONSUMER_ENDMARKER_GRACE`) and the task sets `channel._consumer_done`. + +Key invariant preserved: `waitclose()` waits on `_consumer_done` (not +`_receiveclosed`) for callback channels, so it still returns only after every +callback — including the endmarker — has run (`test_waiting_for_callbacks`, +`test_channel_endmarker_callback`). Deleted along the way: `_callback`, +`_endmarker`, `_fire_endmarker`, `_callback_channels`, +`_register_callback_channel`, and the callback branch of `_deliver_payload` +(now uniformly `mailbox.put`). `__del__`'s `CHANNEL_LAST_MESSAGE` case folds +away (a callback channel is never GC'd while open). + +Stress coverage: `testing/test_channel_stress.py` (Hypothesis) with a +`--stress=N` pytest option (profiles registered in +`testing/conftest.py::pytest_configure`; default quick profile). Hypothesis +also surfaced and we fixed a latent serializer bug: `_save_integral` only +bounds-checked the upper int4 limit, so a negative int below `-2**31` +overflowed `struct.pack('!i', …)` instead of taking the long path +(`FOUR_BYTE_INT_MIN` added; regression `test_serializer.test_int_boundaries`). + +`hypothesis` was added to the `testing` extra in `pyproject.toml`. + ## Invariants (do not regress) - No source shipping, ever. Workers import installed execnet+trio; diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 597cbdb4..e119ca68 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -10,6 +10,8 @@ import functools import itertools import json +import math +import queue as _queue import subprocess import sys import threading @@ -23,6 +25,7 @@ import trio +from ._boundary import Flag from ._trio_gateway import RECEIVE_CHUNK from ._trio_gateway import AsyncGateway from ._trio_gateway import AsyncGroup @@ -31,6 +34,8 @@ from ._trio_gateway import open_popen_process from ._trio_gateway import read_handshake_ack from ._trio_gateway import ssh_transport_args +from .gateway_base import ENDMARKER +from .gateway_base import NO_ENDMARKER_WANTED from .gateway_base import ExecModel from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -43,6 +48,17 @@ from .portal import LoopPortal from .portal import OneShot +#: default cap on concurrent threadpool threads running receiver callbacks +DEFAULT_CALLBACK_THREADS = 40 +#: bound on how long the endmarker callback may run during host shutdown +CONSUMER_ENDMARKER_GRACE = 10.0 + + +def _run_callback(callback: Callable[[Any], Any], data: bytes, channel: Any) -> None: + """Deserialize one payload and invoke the receiver callback (in a thread).""" + callback(loads_internal(data, channel)) + + if TYPE_CHECKING: from .gateway import Gateway from .gateway_base import BaseGateway @@ -255,16 +271,21 @@ def bind_sync_channel(self, channel: Any) -> None: """ ref = weakref.ref(channel) channelid = channel.id + with suppress(trio.RunFinishedError): + self.host.portal.post(self._install_sync_consumer, ref, channelid) - def bind() -> None: - raw = self._channel_for(channelid) - raw.set_consumer( - functools.partial(self._sync_payload, ref), - functools.partial(self._sync_close, ref, channelid), - ) + def _install_sync_consumer(self, ref: weakref.ref[Any], channelid: int) -> None: + """Route ``channelid``'s raw payloads/close to the sync channel (loop). - with suppress(trio.RunFinishedError): - self.host.portal.post(bind) + Idempotent: re-binding to the same hooks just re-flushes the (empty) + raw buffer, so ``attach_consumer`` can call this itself instead of + depending on the separately-posted bind having run first. + """ + raw = self._channel_for(channelid) + raw.set_consumer( + functools.partial(self._sync_payload, ref), + functools.partial(self._sync_close, ref, channelid), + ) def _sync_payload(self, ref: weakref.ref[Any], data: bytes) -> None: channel = ref() @@ -293,6 +314,141 @@ def release_channel(self, channelid: int) -> None: with suppress(trio.RunFinishedError): self.host.portal.post(self._forget_channel, channelid) + # -- receiver callbacks: a consumer task per channel -- + + def attach_consumer( + self, + channel: Any, + callback: Callable[[Any], Any], + endmarker: object, + ) -> None: + """Switch ``channel`` to callback mode: a loop task drains it. + + The switch runs on the loop (so it cannot interleave with delivery): + it moves any already-buffered items into the task's inbox, points the + channel's delivery/close at that inbox, and starts the consumer task. + It deliberately does *not* touch the raw channel's consumer (the sync + payload/close hooks bound by :meth:`bind_sync_channel` stay in place); + delivery keeps flowing through ``Channel._deliver_payload`` / + ``_close_from_remote``, which divert to the inbox once ``_has_consumer`` + is set. Rebinding the raw channel here would race the still-queued + ``bind()`` post (``run_sync`` and ``run_sync_soon`` are not mutually + ordered) and could be clobbered back to the mailbox. + + The task is handed a strong reference to ``channel`` and thus keeps it + alive for as long as it consumes -- the channel's lifecycle is bound to + the task (and to GC once the stream closes), not to a registry. + """ + + def switch() -> None: + mailbox = channel._mailbox + if mailbox is None: + raise OSError(f"{channel!r} has callback already registered") + inbox_send, inbox_recv = trio.open_memory_channel[bytes](math.inf) + + def feed(data: bytes) -> None: + with suppress(trio.BrokenResourceError, trio.ClosedResourceError): + inbox_send.send_nowait(data) + + def close_inbox() -> None: + with suppress(trio.ClosedResourceError): + inbox_send.close() + + # Drain items buffered before the switch into the task's inbox, + # preserving order. An ENDMARKER means the channel already closed. + saw_end = False + while True: + try: + item = mailbox.get_nowait() + except _queue.Empty: + break + if item is ENDMARKER: + saw_end = True + break + inbox_send.send_nowait(item) + channel._mailbox = None + done = Flag(channel.gateway._new_wakener()) + channel._consumer_done = done + channel._consumer_feed = feed + channel._consumer_close_inbox = close_inbox + + def stop() -> None: + # thread-safe: end the task's inbox from any thread (local close) + with suppress(trio.RunFinishedError, trio.ClosedResourceError): + self.host.portal.post(inbox_send.close) + + channel._consumer_stop = stop + channel._has_consumer = True + + # Guarantee the raw channel routes to us right now (idempotent with + # the bind posted at newchannel()): otherwise, if that bind has not + # run yet, inbound payloads would buffer unread in the raw channel + # and the consumer task would wait forever. + self._install_sync_consumer(weakref.ref(channel), channel.id) + + if saw_end: + close_inbox() + self.host.start_soon( + self._run_consumer, channel, inbox_recv, callback, endmarker, done + ) + + self.run_on_loop(switch) + + async def _run_consumer( + self, + channel: Any, + inbox: trio.MemoryReceiveChannel[bytes], + callback: Callable[[Any], Any], + endmarker: object, + done: Flag, + ) -> None: + """Drain ``inbox`` into ``callback`` (each call off the loop thread). + + Runs on the host loop; ``channel`` is held for the task's lifetime so + the channel stays alive while consuming. Items are delivered in order + and each callback runs in a threadpool thread. On completion (EOF, + local close, or a raising callback) the endmarker fires and ``done`` + is set -- which is what ``waitclose()`` waits on. + """ + limiter = self.host.callback_limiter + try: + async for data in inbox: + try: + await trio.to_thread.run_sync( + functools.partial(_run_callback, callback, data, channel), + limiter=limiter, + ) + except Exception as exc: + # trio.Cancelled is a BaseException and propagates past + # here (host shutdown); only a real callback/deserialize + # failure closes the channel with the error. + self._consumer_failed(channel, exc) + break + finally: + # Fire the endmarker and signal done even while the host is being + # torn down, but never let a stuck callback hang shutdown forever. + with trio.CancelScope(shield=True): + if endmarker is not NO_ENDMARKER_WANTED: + with ( + trio.move_on_after(CONSUMER_ENDMARKER_GRACE), + suppress(BaseException), + ): + await trio.to_thread.run_sync( + functools.partial(callback, endmarker), limiter=limiter + ) + done.set() + + def _consumer_failed(self, channel: Any, exc: BaseException) -> None: + """A callback (or its deserialization) raised: close with the error.""" + gateway = self.sync_gateway + gateway._trace("exception during callback: %s" % exc) + errortext = gateway._geterrortext(exc) + with suppress(OSError): + gateway._send( + Message.CHANNEL_CLOSE_ERROR, channel.id, dumps_internal(errortext) + ) + channel._close_from_remote(RemoteError(errortext), sendonly=False) + def host_call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: """Blocking host call that parks correctly for the wait backend. @@ -406,6 +562,7 @@ def __init__(self, name: str = "execnet-trio-host") -> None: self._ready = threading.Event() self._shutdown: trio.Event | None = None self._started = False + self._callback_limiter: trio.CapacityLimiter | None = None def start(self) -> None: if self._started: @@ -422,6 +579,13 @@ def portal(self) -> LoopPortal: raise RuntimeError("TrioHost is not running") return self._portal + @property + def callback_limiter(self) -> trio.CapacityLimiter: + """Bound on concurrent threadpool threads running receiver callbacks.""" + if self._callback_limiter is None: + raise RuntimeError("TrioHost is not running") + return self._callback_limiter + def is_host_thread(self) -> bool: return self._portal is not None and self._portal.is_loop_thread() @@ -431,6 +595,7 @@ def _run(self) -> None: async def _main(self) -> None: self._portal = LoopPortal() self._shutdown = trio.Event() + self._callback_limiter = trio.CapacityLimiter(DEFAULT_CALLBACK_THREADS) try: async with trio.open_nursery() as nursery: self._nursery = nursery diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 2b183c1e..8eaa770e 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -13,7 +13,6 @@ import builtins import os -import queue as _queue import struct import sys import threading @@ -371,6 +370,23 @@ class Channel: TimeoutError = TimeoutError _INTERNALWAKEUP = 1000 _executing = False + #: set once a receiver callback is attached. A consumer *task* on the + #: loop drains this channel and runs the callback in a threadpool thread; + #: the task holds the channel alive, so a callback channel's lifecycle is + #: bound to consumption (and to GC once the stream closes) rather than to + #: a strong registry. + _has_consumer = False + #: loop-side hooks installed while a consumer is attached: divert one + #: payload into / close the consumer task's inbox (set by the Trio session, + #: so they encapsulate the trio memory channel; gateway_base stays trio-free). + _consumer_feed: Callable[[bytes], None] | None = None + _consumer_close_inbox: Callable[[], None] | None = None + #: thread-safe "stop the consumer" hook -- ends the task's inbox. + _consumer_stop: Callable[[], None] | None = None + #: set by the consumer task once it has drained every item and fired the + #: endmarker; ``waitclose()`` waits on this (instead of ``_receiveclosed``) + #: so it still guarantees "all callbacks have run" before returning. + _consumer_done: Flag | None = None def __init__(self, gateway: BaseGateway, id: int) -> None: """:private:""" @@ -380,10 +396,8 @@ def __init__(self, gateway: BaseGateway, id: int) -> None: # XXX: defaults copied from Unserializer self._strconfig = getattr(gateway, "_strconfig", (True, False)) self.id = id - # serialized payloads (or ENDMARKER); None once a callback is set + # serialized payloads (or ENDMARKER); None once a consumer is attached self._mailbox: Mailbox[Any] | None = Mailbox(gateway._new_wakener()) - self._callback: Callable[[Any], Any] | None = None - self._endmarker: object = NO_ENDMARKER_WANTED self._closed = False self._receiveclosed = Flag(gateway._new_wakener()) self._remoteerrors: list[RemoteError] = [] @@ -398,37 +412,19 @@ def setcallback( ) -> None: """Set a callback function for receiving items. - All already-queued items will immediately trigger the callback. - Afterwards the callback will execute in the receiver (loop) thread - for each received data item and calls to ``receive()`` will - raise an error. - If an endmarker is specified the callback will eventually - be called with the endmarker when the channel closes. + A consumer task on the gateway's loop drains this channel and runs + ``callback`` for each received item in a threadpool thread (so a slow + callback never blocks the loop); items for one channel are delivered + strictly in order. Already-queued items are delivered first. After + this call ``receive()`` raises an error. + + The task keeps the channel alive for as long as it is consuming, so a + callback channel need not be referenced elsewhere. If an endmarker is + specified the callback is eventually called with it when the channel + closes, and ``waitclose()`` does not return until every callback + (including the endmarker) has run. """ - - def switch() -> None: - # Runs on the loop thread (inline without a session), so the - # switch-over cannot interleave with payload delivery. - mailbox = self._mailbox - if mailbox is None: - raise OSError(f"{self!r} has callback already registered") - self._mailbox = None - while 1: - try: - olditem = mailbox.get_nowait() - except _queue.Empty: - if not (self._closed or self._receiveclosed.is_set()): - self._callback = callback - self._endmarker = endmarker - self.gateway._channelfactory._register_callback_channel(self) - break - if olditem is ENDMARKER: - if endmarker is not NO_ENDMARKER_WANTED: - callback(endmarker) - break - callback(loads_internal(olditem, self)) - - self.gateway._run_on_loop(switch) + self.gateway._start_channel_consumer(self, callback, endmarker) def __repr__(self) -> str: flag = (self.isclosed() and "closed") or "open" @@ -454,17 +450,16 @@ def __del__(self) -> None: # in which case the process will go away and we probably # don't need to try to send a closing or last message # (and often it won't work anymore to send things out) + # A callback channel is held by its consumer task until the stream + # closes, so by the time __del__ runs it is never in the "opened" + # state -- this branch only ever fires for a plain receive channel. if Message is not None: - if self._mailbox is None: # has_callback - msgcode = Message.CHANNEL_LAST_MESSAGE - else: - msgcode = Message.CHANNEL_CLOSE with suppress(OSError, ValueError): # ignore problems with sending # Never wait during GC: post the close best-effort. send = getattr( self.gateway, "_send_nonblocking", self.gateway._send ) - send(msgcode, self.id) + send(Message.CHANNEL_CLOSE, self.id) with suppress(Exception): self.gateway._release_channel(self.id) @@ -482,46 +477,43 @@ def _getremoteerror(self): # loop-side delivery (called by the session's raw-channel consumer) # def _deliver_payload(self, data: bytes) -> None: - """Route one inbound serialized payload (loop thread).""" + """Route one inbound serialized payload (loop thread). + + A callback channel diverts payloads into its consumer task's inbox; a + plain channel queues them for ``receive()``. + """ if self._closed: return # late data for a locally closed channel: drop - callback = self._callback - if callback is None: - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(data) - # no mailbox and no callback: closed for receiving -- drop - else: - try: - callback(loads_internal(data, self)) - except Exception as exc: - self.gateway._trace("exception during callback: %s" % exc) - errortext = self.gateway._geterrortext(exc) - self.gateway._send( - Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(errortext) - ) - self._close_from_remote(RemoteError(errortext)) + feed = self._consumer_feed + if feed is not None: + feed(data) + return + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(data) + # no consumer and no mailbox: closed for receiving -- drop def _close_from_remote(self, remoteerror=None, *, sendonly: bool = False) -> None: - """Close initiated by the peer or session shutdown (loop thread).""" + """Close initiated by the peer or session shutdown (loop thread). + + For a callback channel the consumer task ends separately (its inbox is + closed) and fires the endmarker; here we only record the state. + """ if remoteerror: self._remoteerrors.append(remoteerror) - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(ENDMARKER) - self._fire_endmarker() + if self._has_consumer: + close_inbox = self._consumer_close_inbox + if close_inbox is not None: + close_inbox() # ends the consumer task (it fires the endmarker) + else: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) self.gateway._channelfactory._no_longer_opened(self.id) if not sendonly: # otherwise #--> "sendonly" self._closed = True # --> "closed" self._receiveclosed.set() - def _fire_endmarker(self) -> None: - callback = self._callback - if callback is not None: - self._callback = None - if self._endmarker is not NO_ENDMARKER_WANTED: - callback(self._endmarker) - # # public API for channel objects # @@ -588,10 +580,16 @@ def close(self, error=None) -> None: self._remoteerrors.append(error) self._closed = True # --> "closed" self._receiveclosed.set() - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(ENDMARKER) - self._fire_endmarker() + if self._has_consumer: + # End the consumer task's inbox; it drains any buffered items, + # fires the endmarker, and sets _consumer_done. + stop = self._consumer_stop + if stop is not None: + stop() + else: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) self.gateway._channelfactory._no_longer_opened(self.id) self.gateway._release_channel(self.id) @@ -611,9 +609,16 @@ def waitclose(self, timeout: float | None = None) -> None: self.TimeoutError is raised after the specified number of seconds (default is None, i.e. wait indefinitely). """ - # wait for non-"opened" state - self._receiveclosed.wait(timeout=timeout) - if not self._receiveclosed.is_set(): + # For a callback channel wait on the consumer task finishing (so every + # callback, including the endmarker, has run); otherwise wait for the + # non-"opened" state directly. + signal = ( + self._consumer_done + if self._consumer_done is not None + else (self._receiveclosed) + ) + signal.wait(timeout=timeout) + if not signal.is_set(): raise self.TimeoutError("Timeout after %r seconds" % timeout) error = self._getremoteerror() if error: @@ -693,16 +698,14 @@ class ChannelFactory: Message routing lives in the Trio session (the sync channel binds a consumer on the session's raw channel); the factory only tracks live channels -- weakly, so dropping the last user reference triggers - ``Channel.__del__``'s close message -- and keeps channels with a - registered callback strongly alive until they close. + ``Channel.__del__``'s close message. A callback channel is kept alive by + its consumer task rather than by any registry here. """ def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: self._channels: weakref.WeakValueDictionary[int, Channel] = ( weakref.WeakValueDictionary() ) - # channels kept strongly alive while their callback is registered - self._callback_channels: dict[int, Channel] = {} self._writelock = threading.Lock() self.gateway = gateway self.count = startcount @@ -739,12 +742,8 @@ def channels(self) -> list[Channel]: # # internal methods, called from the loop thread (or local close paths) # - def _register_callback_channel(self, channel: Channel) -> None: - self._callback_channels[channel.id] = channel - def _no_longer_opened(self, id: int) -> None: self._channels.pop(id, None) - self._callback_channels.pop(id, None) def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: """Close ``id`` as if the peer had closed it (no message is sent).""" @@ -885,6 +884,22 @@ def _run_on_loop(self, sync_fn: Callable[[], Any]) -> Any: return sync_fn() return session.run_on_loop(sync_fn) + def _start_channel_consumer( + self, + channel: Channel, + callback: Callable[[Any], Any], + endmarker: object, + ) -> None: + """Attach a receiver callback: hand the channel to a consumer task. + + The task drains the channel on the loop and runs ``callback`` in a + threadpool thread, holding the channel alive while it consumes. + """ + session = self._trio_session + if session is None: + raise OSError(f"cannot set callback on {channel!r}: no active session") + session.attach_consumer(channel, callback, endmarker) + def _terminate_execution(self) -> None: pass diff --git a/testing/test_channel.py b/testing/test_channel.py index f06b8632..f1b0b454 100644 --- a/testing/test_channel.py +++ b/testing/test_channel.py @@ -305,6 +305,50 @@ def f(item): channel.send(1) channel.waitclose() + @needs_early_gc + def test_callback_channel_collected_after_close(self, gw: Gateway) -> None: + # A callback channel is kept alive only by its consumer task: while it + # is consuming it survives with no user reference, and once the stream + # closes and the callback has run the object becomes collectable. + import gc + import weakref + + received: list[int] = [] + channel = gw.remote_exec("channel.send(1); channel.send(2)") + channel.setcallback(received.append) + ref = weakref.ref(channel) + del channel # only the consumer task holds it now + + deadline = time.time() + TESTTIMEOUT + while ref() is not None and time.time() < deadline: + gc.collect() + time.sleep(0.05) + assert received == [1, 2] + assert ref() is None # consumer finished -> no strong refs -> reclaimed + + def test_callbacks_run_off_the_loop_thread(self, gw: Gateway) -> None: + # A slow callback on one channel must not block delivery to another: + # callbacks run in threadpool threads, not inline on the loop. + import threading + + release = threading.Event() + fast_ran = threading.Event() + + def slow(item: object) -> None: + release.wait(TESTTIMEOUT) + + chan_slow = gw.remote_exec("channel.send(1); channel.receive()") + chan_fast = gw.remote_exec("channel.send(1)") + chan_slow.setcallback(slow) + chan_fast.setcallback(lambda item: fast_ran.set()) + + # the fast callback fires even while the slow one is still blocking + assert fast_ran.wait(TESTTIMEOUT) + release.set() + chan_slow.send(0) # let the remote finish and close + chan_slow.waitclose(TESTTIMEOUT) + chan_fast.waitclose(TESTTIMEOUT) + class TestChannelFile: def test_channel_file_write(self, gw: Gateway) -> None: From 34c9953dadc2c1eba3888e23de3e2faf39086772 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 10:00:55 +0200 Subject: [PATCH 49/59] fix: serialize ints below the signed int4 minimum via the long opcode _save_integral only bounds-checked the upper int4 limit, so a negative int below -2**31 took the short (INT/LONG) path and overflowed struct.pack('!i', ...) with a struct.error instead of using the arbitrary-precision long opcode. Gate the short path on both ends (FOUR_BYTE_INT_MIN..MAX). Found by the new channel-callback Hypothesis stress tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/execnet/gateway_base.py | 5 ++++- testing/test_serializer.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 8eaa770e..c59f4437 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1051,6 +1051,7 @@ def bchr(n: int) -> bytes: DUMPFORMAT_VERSION = bchr(2) FOUR_BYTE_INT_MAX = 2147483647 +FOUR_BYTE_INT_MIN = -2147483648 FLOAT_FORMAT = "!d" FLOAT_FORMAT_SIZE = struct.calcsize(FLOAT_FORMAT) @@ -1407,7 +1408,9 @@ def _write_byte_sequence(self, bytes_: bytes) -> None: self._write(bytes_) def _save_integral(self, i: int, short_op: bytes, long_op: bytes) -> None: - if i <= FOUR_BYTE_INT_MAX: + # The short op packs a signed 4-byte int; anything outside that range + # (in either direction) goes through the arbitrary-precision long op. + if FOUR_BYTE_INT_MIN <= i <= FOUR_BYTE_INT_MAX: self._write(short_op) self._write_int4(i) else: diff --git a/testing/test_serializer.py b/testing/test_serializer.py index cb2d422c..87adf04c 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -126,6 +126,26 @@ def test_long(load, dump) -> None: assert v == really_big +@pytest.mark.parametrize( + "value", + [ + "2147483647", # int4 max: short path + "-2147483648", # int4 min: short path + "2147483648", # just over max: long path + "-2147483649", # just under min: long path (used to crash in struct.pack) + "9223372036854775807324234", + "-9223372036854775807324234", + ], +) +def test_int_boundaries(value, dump, load) -> None: + # regression: negative ints below the signed int4 minimum must take the + # arbitrary-precision long path instead of overflowing the 4-byte pack. + p = dump(value) + tp, v = load(p) + assert tp == "int" + assert v == value + + def test_bytes(dump, load) -> None: p = dump("b'hi'") tp, v = load(p) From 0d36f7fd8a05524e3dbba03e4f669d7921e7b8dc Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 10:01:35 +0200 Subject: [PATCH 50/59] test: hypothesis stress tests for channel callbacks with a --stress knob testing/test_channel_stress.py hammers the setcallback consumer-task path with randomised traffic: ordering/completeness, many-channel isolation, receive-then-switch, endmarker-last, and GC-reclaim-after-close. A --stress=N pytest option scales the number of Hypothesis examples (profiles registered in conftest.pytest_configure; a quick profile runs by default). hypothesis added to the testing extra. Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 1 + testing/conftest.py | 37 ++++++++++ testing/test_channel_stress.py | 123 +++++++++++++++++++++++++++++++++ uv.lock | 75 ++++++++++++++++++++ 4 files changed, 236 insertions(+) create mode 100644 testing/test_channel_stress.py diff --git a/pyproject.toml b/pyproject.toml index 6e12c8da..f29b24d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ testing = [ "pre-commit", "pytest>8.0", "pytest-timeout", + "hypothesis", "tox", "hatch", "uv", diff --git a/testing/conftest.py b/testing/conftest.py index 7c3ed7a6..becf10f2 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -69,6 +69,43 @@ def pytest_addoption(parser: pytest.Parser) -> None: "page on invalid addresses" ), ) + group.addoption( + "--stress", + action="store", + dest="stress", + default=None, + metavar="N", + help=( + "how hard the Hypothesis stress tests try: number of examples " + "per test (e.g. --stress=500). Without it a quick profile runs." + ), + ) + + +def pytest_configure(config: pytest.Config) -> None: + # Register Hypothesis profiles scaled by --stress. The stress tests reuse + # a function-scoped gateway across examples on purpose (spawning one per + # example would dominate the runtime), and each round-trip can be slow, so + # the health checks for those are suppressed. + try: + from hypothesis import HealthCheck + from hypothesis import settings + except ImportError: + return + common = dict( + deadline=None, + suppress_health_check=[ + HealthCheck.function_scoped_fixture, + HealthCheck.too_slow, + ], + ) + settings.register_profile("execnet-quick", max_examples=15, **common) + stress = config.getoption("stress") + if stress is not None: + settings.register_profile("execnet-stress", max_examples=int(stress), **common) + settings.load_profile("execnet-stress") + else: + settings.load_profile("execnet-quick") @pytest.fixture diff --git a/testing/test_channel_stress.py b/testing/test_channel_stress.py new file mode 100644 index 00000000..8090367d --- /dev/null +++ b/testing/test_channel_stress.py @@ -0,0 +1,123 @@ +"""Hypothesis stress tests for the channel callback (consumer-task) machinery. + +These hammer ``setcallback`` -- the loop-task-plus-threadpool consumer -- with +randomised traffic to check the invariants that must hold no matter the timing: + +* every item reaches the callback, exactly once and in send order; +* many callback channels run concurrently without cross-talk; +* switching to a callback after some ``receive()`` calls loses nothing; +* the endmarker is always delivered last; +* a callback channel with no user reference is kept alive by its consumer. + +Use ``--stress=N`` to raise the number of examples per test (default: a quick +profile registered in ``conftest.pytest_configure``). +""" + +from __future__ import annotations + +import gc +import weakref + +import pytest + +hypothesis = pytest.importorskip("hypothesis") +from hypothesis import given # noqa: E402 +from hypothesis import strategies as st # noqa: E402 + +from execnet.gateway import Gateway # noqa: E402 + +# High --stress levels replay one test many times; lift the per-test timeout +# well above the default so that only a real hang (bounded by TESTTIMEOUT on +# every blocking call below) fails, not sheer example count. +pytestmark = pytest.mark.timeout(600) + +TESTTIMEOUT = 10.0 + +# Varied payloads: unbounded ints exercise both the short and long serializer +# paths (and their boundaries), on top of the callback machinery itself. +payload_strategy = st.one_of( + st.integers(), + st.text(max_size=20), + st.booleans(), + st.none(), +) +items_strategy = st.lists(payload_strategy, max_size=40) + + +def _echo(channel, items): + """Remote: send every item, in order, then let the channel close.""" + for item in items: + channel.send(item) + + +class TestCallbackStress: + # popen only: fast to spawn and the callback path is transport-independent + gwtype = "popen" + + @given(data=items_strategy) + def test_callback_receives_all_in_order( + self, gw: Gateway, data: list[int] + ) -> None: + collected: list[int] = [] + channel = gw.remote_exec(_echo, items=data) + channel.setcallback(collected.append) + channel.waitclose(TESTTIMEOUT) + assert collected == data + + @given(batches=st.lists(items_strategy, min_size=1, max_size=6)) + def test_many_channels_stay_ordered_and_isolated( + self, gw: Gateway, batches: list[list[int]] + ) -> None: + results: list[list[int]] = [[] for _ in batches] + channels = [] + for index, items in enumerate(batches): + channel = gw.remote_exec(_echo, items=items) + channel.setcallback(results[index].append) + channels.append(channel) + for channel in channels: + channel.waitclose(TESTTIMEOUT) + assert results == batches + + @given(data=st.data()) + def test_receive_then_switch_loses_nothing( + self, gw: Gateway, data: st.DataObject + ) -> None: + items = data.draw(items_strategy) + split = data.draw(st.integers(min_value=0, max_value=len(items))) + channel = gw.remote_exec(_echo, items=items) + first = [channel.receive(TESTTIMEOUT) for _ in range(split)] + rest: list[int] = [] + channel.setcallback(rest.append) + channel.waitclose(TESTTIMEOUT) + assert first + rest == items + + @given(data=items_strategy) + def test_endmarker_is_always_last(self, gw: Gateway, data: list[int]) -> None: + endmarker = object() + collected: list[object] = [] + channel = gw.remote_exec(_echo, items=data) + channel.setcallback(collected.append, endmarker=endmarker) + channel.waitclose(TESTTIMEOUT) + assert collected == [*data, endmarker] + + @pytest.mark.skipif( + "not hasattr(sys, 'getrefcount')", reason="needs refcount GC semantics" + ) + @given(data=st.lists(payload_strategy, min_size=1, max_size=40)) + def test_callback_channel_kept_alive_then_collected( + self, gw: Gateway, data: list[object] + ) -> None: + collected: list[int] = [] + channel = gw.remote_exec(_echo, items=data) + channel.setcallback(collected.append) + ref = weakref.ref(channel) + del channel # only the consumer task holds it now + + import time + + deadline = time.time() + TESTTIMEOUT + while ref() is not None and time.time() < deadline: + gc.collect() + time.sleep(0.02) + assert collected == data + assert ref() is None # consumer finished -> reclaimed diff --git a/uv.lock b/uv.lock index 3b8bb46e..311eb0d8 100644 --- a/uv.lock +++ b/uv.lock @@ -393,6 +393,7 @@ gevent = [ testing = [ { name = "asyncssh" }, { name = "hatch" }, + { name = "hypothesis" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-timeout" }, @@ -416,6 +417,7 @@ requires-dist = [ { name = "asyncssh", marker = "extra == 'testing'" }, { name = "gevent", marker = "extra == 'gevent'" }, { name = "hatch", marker = "extra == 'testing'" }, + { name = "hypothesis", marker = "extra == 'testing'" }, { name = "pre-commit", marker = "extra == 'testing'" }, { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, { name = "pytest-timeout", marker = "extra == 'testing'" }, @@ -680,6 +682,79 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/aa/8caf6a0a3e62863cbb9dab27135660acba46903b703e224f14f447e57934/hyperlink-21.0.0-py2.py3-none-any.whl", hash = "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4", size = 74638, upload-time = "2021-01-08T05:51:22.906Z" }, ] +[[package]] +name = "hypothesis" +version = "6.161.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/17/94/d208ced653376e7e0a2f0429ee5be864dd0b59393b98a8b41a35ceb4d035/hypothesis-6.161.5.tar.gz", hash = "sha256:ba73a3c3b68e63a0bee5ea1a8a13efce60bcc7ee5fc7e71df2954db39c225b95", size = 486653, upload-time = "2026-07-25T14:39:34.866Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/58/e48aa878d119474631fc097d511b5e70807dfe56be4b244cce0275f1805e/hypothesis-6.161.5-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:4c89e0c35d7cd70af2a0a5f0b5ad69d1898c369adf15115c6d4271d47bf1b280", size = 766970, upload-time = "2026-07-25T14:38:42.613Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ff/c83c1a4d5d3c4b6a4dc1fcb5069245171b7a30ad5dd31f44282e8881e089/hypothesis-6.161.5-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:1bb987e519a6d00675bab124c925000637e2e59384196b0ded5d108dc1851449", size = 762574, upload-time = "2026-07-25T14:38:15.427Z" }, + { url = "https://files.pythonhosted.org/packages/d2/04/7a5ba16cd14340009d1d8aa46083bf3a3a55c5b0d1ccf553e6f6748ee0e8/hypothesis-6.161.5-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5990e9e2e145ff5369c95ad684bd6eee7ebd2d4de37d37eea4c6ca5e339e2a52", size = 1091754, upload-time = "2026-07-25T14:38:38.562Z" }, + { url = "https://files.pythonhosted.org/packages/70/69/031913f7408ebb41e31a9b20e5bca75c5a64ab151d57ca75e944a09ba6e0/hypothesis-6.161.5-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2a5c6400c58c9c3d616ad7177ada12ae577e5103b3b0df6e79920729250bf100", size = 1120372, upload-time = "2026-07-25T14:38:18.962Z" }, + { url = "https://files.pythonhosted.org/packages/b9/95/360884e86ab99099340fca781531286c26d40ce32c853097e76537cc0726/hypothesis-6.161.5-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71aa718e08bdfbacd1a5d8f86ffd55f1d86e64a57cec6a107144d883b522823e", size = 1141230, upload-time = "2026-07-25T14:39:03.757Z" }, + { url = "https://files.pythonhosted.org/packages/bc/5b/6f3c9fbe9191432f33c9aaf951cf5a5416533848999f2e2c6a20ec00098b/hypothesis-6.161.5-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:66912038467e1af791f76336bc079d669f7c8bbf7aeb85bdd52e83259d885122", size = 1096611, upload-time = "2026-07-25T14:38:34.688Z" }, + { url = "https://files.pythonhosted.org/packages/32/c7/d126b9f66295fed5d5745f98ad92f485c98dee303dd67ea7a53fe4a903aa/hypothesis-6.161.5-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:99d8b2380d0bba602df963d6e42d56bb50cf1734376c1b9b14dcab3cec21814e", size = 1133382, upload-time = "2026-07-25T14:38:20.27Z" }, + { url = "https://files.pythonhosted.org/packages/56/de/277a17091687f298079c197cadd806d64908c5a08c9c91bb27b834192503/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:19e57e621c6abd98123a91bd4b022bde7760ba709f1ca121f822d9aa291bd001", size = 1265592, upload-time = "2026-07-25T14:38:11.734Z" }, + { url = "https://files.pythonhosted.org/packages/d7/e5/3f15b1a43cb70b223427ce3a9a6afbe430bf1754b3346ac546a3df38b96b/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:3b74cced62995ed2e0096fa63d0b7babd1fa08ce1944cff41134275516848708", size = 1393424, upload-time = "2026-07-25T14:38:46.738Z" }, + { url = "https://files.pythonhosted.org/packages/1a/1d/a3cf1441550dbf5a362dcfa19c831721f3bb6a749eb53ecbdfc983959027/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:fd37a0257647288be8c27d56a78a31da2b65e3b6254511f03164f5e1c786187a", size = 1266197, upload-time = "2026-07-25T14:39:33.052Z" }, + { url = "https://files.pythonhosted.org/packages/57/e5/2616432d6144057b6c8f4409c95d95610bdbdf07fecb6618e98761861c24/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cf9dfba054164e481f05774c70b65ea88ba735b986cdc44f5210ef40871a32e9", size = 1308330, upload-time = "2026-07-25T14:39:27.86Z" }, + { url = "https://files.pythonhosted.org/packages/f7/cd/9e9c7a88858f84447e44e45c8b9fd1058120b4d5fa9bab9c3ceb9b9cf96b/hypothesis-6.161.5-cp310-abi3-win32.whl", hash = "sha256:8763938787e6c98e461f8c11139981597d083e6915a9956db1c26cc609bc7b19", size = 652819, upload-time = "2026-07-25T14:38:35.915Z" }, + { url = "https://files.pythonhosted.org/packages/03/23/aae37b5bb2014e0e01e372498f7f5977b9fd61a0179143854b7fb0538b1c/hypothesis-6.161.5-cp310-abi3-win_amd64.whl", hash = "sha256:79b5d069095a726dca5b6b7e4c5d268acc809a6595c46d7eee860708f960cb8f", size = 658970, upload-time = "2026-07-25T14:39:05.239Z" }, + { url = "https://files.pythonhosted.org/packages/70/77/fbda7005f891f534af0a74cb5acd7a67cb1a7fbee1ae170d176c7dcd5e0a/hypothesis-6.161.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c73c86afe87b1bf81af586402d41bff69d93c8c34517a81d3a88a1e28e8662e2", size = 767688, upload-time = "2026-07-25T14:39:13.418Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8d/f884e10029018504504df6100faf3795918e1dd14d4b6f4102c3fcb21acf/hypothesis-6.161.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7b289d33f0154850baed802d088564bba83dae58bae6fb9157e498c43d758afd", size = 763399, upload-time = "2026-07-25T14:38:10.49Z" }, + { url = "https://files.pythonhosted.org/packages/40/23/75d9c1f7ad8a52a14c5981c9ba97b1d4abfd263cd48ce7ffb036ce1f5a16/hypothesis-6.161.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60afc424627a4119d89f02425049a97e500e022aef6a06d76e987e266e3d720", size = 1092278, upload-time = "2026-07-25T14:38:51.187Z" }, + { url = "https://files.pythonhosted.org/packages/be/52/efc7c45352636fc71d24590893e52f0e8b4ca42984aea870b47831720fef/hypothesis-6.161.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:defa4f24aa62b3a6d07271a4d54f31c7bb07ce1f3fa86cc0705890a071a58301", size = 1141822, upload-time = "2026-07-25T14:38:48.298Z" }, + { url = "https://files.pythonhosted.org/packages/6b/48/697b194412fed03cbf9757771b9c3b51ca062e1d78fe2b5ddf833a25b7f2/hypothesis-6.161.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4f5e8be93cf0da10b8a2f1044c12f2aa506e80c5071e84dffeec4311fddeb8d7", size = 1266206, upload-time = "2026-07-25T14:39:31.414Z" }, + { url = "https://files.pythonhosted.org/packages/2f/b8/56c7996272a8310737c44378fe181befbe3ab41f72ee3d7180e32269cb7c/hypothesis-6.161.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:46aa94fa4107f97760b63c83142cd2fe208f38a6708736494cabfcf219b137cb", size = 1308570, upload-time = "2026-07-25T14:39:09.772Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ba/ed5baa948085a08ed38362e0e88ddad7a4a524b0a2a1581c8138ce7b307a/hypothesis-6.161.5-cp310-cp310-win_amd64.whl", hash = "sha256:44e25c35123a2b77ebd2df356e1a78bf2c9abbf875222ff252b3012801bfb143", size = 658822, upload-time = "2026-07-25T14:38:45.402Z" }, + { url = "https://files.pythonhosted.org/packages/9d/82/60f7213dd5262863646bbf31ca28e1dba68730080c14f39744d110733932/hypothesis-6.161.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f6d55821d13890875d5a50a1433448c8a86f7f83e85103a527dad232d18c470e", size = 767446, upload-time = "2026-07-25T14:38:39.904Z" }, + { url = "https://files.pythonhosted.org/packages/20/b3/145fe198d155ac394cac067ae852074adbca2c015a7244fdb3df2f655c19/hypothesis-6.161.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ecb5878b81beb1dfda4e573375a816b1ffa7931d7899a1a2d7216afa5e1efb4f", size = 763215, upload-time = "2026-07-25T14:38:30.624Z" }, + { url = "https://files.pythonhosted.org/packages/fc/c0/0ea73426ba5b2504d4e3f4e6c7589f5808ad6eef4c922f15766c0537bbc2/hypothesis-6.161.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bff9062c1af02e391c0e48aa75f989393651c6e07f1b50eddeadaa8ca440f944", size = 1092115, upload-time = "2026-07-25T14:39:18.121Z" }, + { url = "https://files.pythonhosted.org/packages/dc/65/2a52daac1d39d0a1a88f96a602ff997e4665c5a9acc4d029e3168e536cd1/hypothesis-6.161.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:570ab5afc34dd7a78e4d1ab1a0ea4e026bbae6caee79dc04245d0347f1d548a7", size = 1141577, upload-time = "2026-07-25T14:38:55.841Z" }, + { url = "https://files.pythonhosted.org/packages/91/cd/77693dbb6766345d980fdfa41f7e7d3d0a676469d156c8e9455c1ebda67d/hypothesis-6.161.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6a7541433984c5b5fb4ad433a715808579cba16ebaca6a214a99d2f8c35ed49d", size = 1265879, upload-time = "2026-07-25T14:38:21.759Z" }, + { url = "https://files.pythonhosted.org/packages/ed/af/ea72a466210a43b35f3e9d86350aca94529a00d2aff698b4cdd490238955/hypothesis-6.161.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f3118cce6e66366e2c64389a5dd6e9bc49c697c366d62709e6fda0b3a8a7d997", size = 1308593, upload-time = "2026-07-25T14:38:23.221Z" }, + { url = "https://files.pythonhosted.org/packages/50/8c/d0ae6738baed9061ca29b18de506f5bc8dc84649b216b55213474ec7da9b/hypothesis-6.161.5-cp311-cp311-win_amd64.whl", hash = "sha256:d8cbd7c938b191d5f9bf846a79e81484135a239cd45bf194993949645495ee6f", size = 658671, upload-time = "2026-07-25T14:38:54.453Z" }, + { url = "https://files.pythonhosted.org/packages/bf/37/71a53545f2732574bdd7a45a6e073043bce3447fec4aa722211ebf742f2e/hypothesis-6.161.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:37016b6b4842993b06ee453dda4dedac5ef328cefa6daa36372d261abf12db43", size = 768565, upload-time = "2026-07-25T14:38:14.362Z" }, + { url = "https://files.pythonhosted.org/packages/e6/56/7f32ba1443d5819b324444e7d5ebcf207ba4a0f9219a1693a7994293fd3f/hypothesis-6.161.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d60a204b86936b29d8914f93a99c59a2ecd4e311be793bf32a3f94774d41e016", size = 760188, upload-time = "2026-07-25T14:38:07.894Z" }, + { url = "https://files.pythonhosted.org/packages/07/0d/699436929d2980103c9ddba153872ebed55cfcc13f0f78c1741ac6128205/hypothesis-6.161.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cee9f1a563830a9137b9d1505c9c4fac760bc43dfbfa5873bebefe36c8dca4ec", size = 1090570, upload-time = "2026-07-25T14:39:00.492Z" }, + { url = "https://files.pythonhosted.org/packages/8a/a5/2c28b58d16443fd8ad63e4f287440291a42135073748e5e511084f32a6f3/hypothesis-6.161.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7fcf44f9876b7b4fdc40b607342219c634cf6a1acc708dc0ff88e8e48ae8ea2", size = 1140601, upload-time = "2026-07-25T14:39:21.403Z" }, + { url = "https://files.pythonhosted.org/packages/4e/fe/f0f87c38dc741687bace80553478f8c9796ba5b6e68793a80990d79ec5ac/hypothesis-6.161.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:59088f6458d2a6ef04724a2a639418c97dd8b8c280bfd222fcc5566854560caf", size = 1263341, upload-time = "2026-07-25T14:38:41.257Z" }, + { url = "https://files.pythonhosted.org/packages/44/db/02e30bcdb3434f4eee8fdbebf5cf151fecd37d2a76f2fc19f2745c93ca38/hypothesis-6.161.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ea8e7e6bed5407ab902026d8cac74d10cac894a496d82e261a0beaca499114dc", size = 1307604, upload-time = "2026-07-25T14:39:19.867Z" }, + { url = "https://files.pythonhosted.org/packages/1d/70/1f6349a244f13c5a383a62ca674d3175018c7fc30fbaad10faa859d5c2cf/hypothesis-6.161.5-cp312-cp312-win_amd64.whl", hash = "sha256:a4ad11f4eafd561a672cb9fa977d0f58ab4ae9cf4b160dd3c12c351089f3e2be", size = 656103, upload-time = "2026-07-25T14:38:57.639Z" }, + { url = "https://files.pythonhosted.org/packages/52/07/86f98ebd945f3f8a821c1e7bf9b530902675145d93ed44fa56a309cc24fd/hypothesis-6.161.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3b4c308ff19741ab9f795cac61cce61227f55a1b9767ad525a34de8b01391d1d", size = 768455, upload-time = "2026-07-25T14:38:27.821Z" }, + { url = "https://files.pythonhosted.org/packages/70/11/93c00ba5c77b886017ea8eaa42de8ca937e319032c4251b0540fc1838ae1/hypothesis-6.161.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ef16fac46cd5675504a4d3824f443f6842b80fae8b6d15ecffa9c90e0fcf4522", size = 760099, upload-time = "2026-07-25T14:38:04.918Z" }, + { url = "https://files.pythonhosted.org/packages/86/13/c8787cfae81c816976c1b3aaa43294c3abd6fd055a0147122edd7357a349/hypothesis-6.161.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc4c6aab8dbabb98b9c5ca8da8fa9294d2e5173432a12abcb447e83e666fa430", size = 1090486, upload-time = "2026-07-25T14:39:26.243Z" }, + { url = "https://files.pythonhosted.org/packages/f8/be/48f4f56bfb24787aa27e5ae851d1aee5214ab6e95311cf8c88a56707941f/hypothesis-6.161.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f485e3c7ead1f76a8c060b16c6b50e5615264819e95b794c6fc6882a9cbfdc4", size = 1140433, upload-time = "2026-07-25T14:38:59.043Z" }, + { url = "https://files.pythonhosted.org/packages/2e/50/c119bfec24d267e4af5bdefda8fd490f7b3a7ee36a52d5ac0b88e07c10b3/hypothesis-6.161.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:75bb899b2db5c45dbfa6ef704c26259bfd27fb3418179c3bd5788b7f2ecce72d", size = 1263296, upload-time = "2026-07-25T14:38:06.719Z" }, + { url = "https://files.pythonhosted.org/packages/81/89/eeb97a9684e3eaae801dd538058202b0b65c2aaecea0211e17d79f731170/hypothesis-6.161.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:894e5d9e2cd97798c3dc2731699098c7f800a7905572db1677e0d27d3b41f541", size = 1307324, upload-time = "2026-07-25T14:38:24.881Z" }, + { url = "https://files.pythonhosted.org/packages/15/9d/e0c64a64721ba169dff5a6f12236d5102f8c76f661598d7837591ec7b375/hypothesis-6.161.5-cp313-cp313-win_amd64.whl", hash = "sha256:62452bb19d73496a74919d82afc6306bf9bd42b8cf1dfce21da3802c596a59b1", size = 656067, upload-time = "2026-07-25T14:38:37.157Z" }, + { url = "https://files.pythonhosted.org/packages/31/27/0c6884785ab6afb41305296b2a5fac2c5d1d26dba40e85c70909b8692fce/hypothesis-6.161.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:35b41648b547a233dd89bed37a3ce8c8298a454fd7133b27b0a37ced135eddfe", size = 768668, upload-time = "2026-07-25T14:39:15.033Z" }, + { url = "https://files.pythonhosted.org/packages/7b/12/17c63bba85201d5fa9db89841e8548c4fa6b240b2437c80d606a65889eb6/hypothesis-6.161.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:429d2179b01787a54f8ccd150c02077a6feb973f14ef31664227148c6a1fadff", size = 760240, upload-time = "2026-07-25T14:38:43.93Z" }, + { url = "https://files.pythonhosted.org/packages/a2/df/1084fd695a1516120a5ea26c026d9f0a071fdf9efcba774c94c09332b372/hypothesis-6.161.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b23fc5441ef297d56797dd372889dec38ddf3662468772b347db2fd8fd43eced", size = 1090987, upload-time = "2026-07-25T14:39:06.646Z" }, + { url = "https://files.pythonhosted.org/packages/a0/a3/bef99daeaf6d1d2db09790cd4ec9ce0cdcac52faaf1fd80b4eaba5d5ea5e/hypothesis-6.161.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f42dcc7fee0d4d56d011218b8c4d5afa6cdee5dbc690652d2a1a598d21969a3", size = 1140613, upload-time = "2026-07-25T14:38:26.284Z" }, + { url = "https://files.pythonhosted.org/packages/9f/09/ce1f6e29d897c7dcb7de1665a9e30d1e00500933caec697a4338db9e7bd0/hypothesis-6.161.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2825fac0d0428cb5e7826347aa4e60106d28f32948cd58c837097f3bd96dbf6c", size = 1263802, upload-time = "2026-07-25T14:38:33.277Z" }, + { url = "https://files.pythonhosted.org/packages/81/e5/7b33aae590bc457fffba70c4f19cc150e3ca356f8e15322f9224c5372a64/hypothesis-6.161.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e9f859ecfd3e00ebb8a36fbb36a8951513fd8e09103a56c645b102e0a51dd1bc", size = 1307642, upload-time = "2026-07-25T14:39:29.729Z" }, + { url = "https://files.pythonhosted.org/packages/d4/8b/4156a9e70bf8c69f54954539ca805e7311048516b665c5f87bee2c1955b7/hypothesis-6.161.5-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:b2647c2d5da341467c1ffaad0dd86d8612e27da2419e76b3b6ee79a33bba7d86", size = 600146, upload-time = "2026-07-25T14:38:17.815Z" }, + { url = "https://files.pythonhosted.org/packages/f9/db/48a518f3f2facd7b280592f81dbb0ba09109656be4ad3dbf0d95a02b4c02/hypothesis-6.161.5-cp314-cp314-win_amd64.whl", hash = "sha256:7c40dd1a3e99497d48a3cfd4c5d71bdb0cd70402a9e09eceb160f045edc92b3a", size = 655981, upload-time = "2026-07-25T14:39:16.52Z" }, + { url = "https://files.pythonhosted.org/packages/25/92/1e9f9f44ef75fc052cc0e3700cfc7d52fd4af1ed9e8b2945d19956bb83cb/hypothesis-6.161.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:6c1b7e1d508a3cc5c10ea7a7a4a7d3b0b56fe6291e8936bb9d052af530380b15", size = 767251, upload-time = "2026-07-25T14:38:52.608Z" }, + { url = "https://files.pythonhosted.org/packages/48/31/0c3f824bbb1fccc0338930c8ef855880065d9d58dddf9547a1b9ae4e664d/hypothesis-6.161.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b0926ba51452c24b92fbcbb30d28cc43fd6cd9e636ea2134fbc1cc2c9da109de", size = 758710, upload-time = "2026-07-25T14:38:09.156Z" }, + { url = "https://files.pythonhosted.org/packages/ab/6b/5811dbb4c1dc0a772519fb9af9c4089128bceeb5deb4c36a887e1e731920/hypothesis-6.161.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba8d0346308b558bfbb49f965ed1d9d5bb65758f4530cb30ce1f9bd911a130b4", size = 1089583, upload-time = "2026-07-25T14:38:49.83Z" }, + { url = "https://files.pythonhosted.org/packages/5f/29/f9cbfd9d0aaea5370a7ceb966f6c8d47e9101ee9a6623606e9f564a59c6e/hypothesis-6.161.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c64079ef4633ab7b540f532107188d75999ee5b342116636312e09ada15783e", size = 1139525, upload-time = "2026-07-25T14:38:29.262Z" }, + { url = "https://files.pythonhosted.org/packages/99/27/f56a03be4ee21f25828e8fba8a502d6840826aea4ff784ff54cbd1f51175/hypothesis-6.161.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ed3eede1c23689edb4773b4c2303ee91518e82056a0a4893fc8415b18e5c3e8b", size = 1261963, upload-time = "2026-07-25T14:38:31.931Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ea/72e11fe6832a68674271655be5cb6f035502dd328ea629b3ee510fb130a5/hypothesis-6.161.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:82b1088ebcad5ab6d146caf9a1552019472a8a9302727f1be5939e30460a101b", size = 1306382, upload-time = "2026-07-25T14:38:13.011Z" }, + { url = "https://files.pythonhosted.org/packages/df/8b/bb039d11a805db0977904c245fa3dcf6d266808482d9dfc6dad3e5f1b256/hypothesis-6.161.5-cp314-cp314t-win_amd64.whl", hash = "sha256:bbe8705ff394a573624cd53f2192dad6255c86346704adf9873cb3acb9b940e5", size = 656131, upload-time = "2026-07-25T14:38:16.712Z" }, + { url = "https://files.pythonhosted.org/packages/21/71/0b28e9ac10f692d9b85a0df74615d308ec1e77140c61773ab107070b82d3/hypothesis-6.161.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:64167d94bcd4a15c1b0aa44c3176f019614898ff10907a2065c142ed34acf828", size = 768375, upload-time = "2026-07-25T14:39:23Z" }, + { url = "https://files.pythonhosted.org/packages/6d/5c/f518bdc85ee5a9b9153aa01b3ed6a77304b8f573a461da1a6694a6992001/hypothesis-6.161.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:22bc641a290428cfd01c62b2b7fe7686007b104dfb5e8ed5420fd3d3a281ebb9", size = 764276, upload-time = "2026-07-25T14:39:02.167Z" }, + { url = "https://files.pythonhosted.org/packages/af/3b/6c93b03adb5804b01854d2801a1f74705ea73d685cef7861484ad8804856/hypothesis-6.161.5-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35527a795992396b82293fec2294ba5b9c0768350c8cb89085e3f5f76ed7d08e", size = 1093089, upload-time = "2026-07-25T14:39:11.442Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3d/6ebfbd76534a71c7ffdd48e0d42373a14152e45714feabf0c25bd7d6b10a/hypothesis-6.161.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ae81035628d2b80435320c43cb09f5b421bac4941bd8ea9b2778d278ae65996", size = 1142876, upload-time = "2026-07-25T14:39:24.674Z" }, + { url = "https://files.pythonhosted.org/packages/fd/07/b57cfd34f55c14a2e823284a796d4228c356e338e1195affc0546bca567b/hypothesis-6.161.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:20e3ab3b0dc300a84b58f41ddd7f5190757521282aad58701c4c76f28a3d981a", size = 659768, upload-time = "2026-07-25T14:39:08.166Z" }, +] + [[package]] name = "identify" version = "2.6.12" From f911ce7589d7e748725c8a5ff3639a4cb0c5cea9 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 10:01:35 +0200 Subject: [PATCH 51/59] refactor: name the main_thread_only deadlock-admission timeout Extract MAIN_THREAD_ONLY_ADMIT_TIMEOUT (unchanged, 1s) so the deadlock window is documented in one place. It stays small: a genuine second concurrent remote_exec never returns, so this only bounds how fast that is reported. main_thread_only is an xdist-only transitional profile; the whole guard goes away with it. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/execnet/_trio_worker.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index d8c6cc0e..4f3a65f2 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -59,13 +59,23 @@ def trigger_shutdown(self) -> None: pass +#: How long admission waits for the previous main-thread exec to finish +#: before declaring a deadlock. Kept small: a genuine second concurrent +#: remote_exec never returns, so this only affects how fast that is +#: reported. (Under heavy CPU contention a merely slow predecessor can be +#: misread as a deadlock, but main_thread_only is an xdist-only transitional +#: profile slated for removal, so the whole guard goes away with it.) +MAIN_THREAD_ONLY_ADMIT_TIMEOUT = 1.0 + + class MainExec: """Exec strategy: serialize each request onto the process main thread. The ``main_thread_only`` profile (GUI/signal-safe: pytest under xdist runs this way). Admission waits for the previous request to finish and - closes the channel with the deadlock text when it cannot within a - second (a second concurrent remote_exec would deadlock the requester). + closes the channel with the deadlock text when it cannot within + ``MAIN_THREAD_ONLY_ADMIT_TIMEOUT`` (a second concurrent remote_exec would + deadlock the requester). """ needs_primary_thread = True @@ -79,7 +89,9 @@ def __init__(self, gateway: WorkerGateway) -> None: async def admit(self, channel: Channel, item: ExecItem) -> bool: complete = self.gateway._executetask_complete assert complete is not None - wait_slot = functools.partial(complete.wait, timeout=1) + wait_slot = functools.partial( + complete.wait, timeout=MAIN_THREAD_ONLY_ADMIT_TIMEOUT + ) if not await trio.to_thread.run_sync(wait_slot, abandon_on_cancel=True): channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) return False From 8eee047f5e4b87f0dbce30a9c63d555bb4189804 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 14:40:29 +0200 Subject: [PATCH 52/59] feat!: hide the standalone serializer from the public API dumps/loads/dump/load are no longer re-exported on any public namespace (execnet, execnet.sync, execnet.trio, execnet.aio); they remain internal to execnet.gateway_base. A channel carries only simple builtin data (plus channel references) -- encoding rich objects is the caller's job, and execnet deliberately does not provide or manage a codec. The serializer error types (DataFormatError / DumpError / LoadError) stay exposed on every surface for `except` and documentation, and now spell out that hitting one is a caller error to resolve (encode the value to simple data first), pointing at pydantic model_dump/model_validate and pytest's report to/from-serializable hooks as the mechanisms to use. Docs: the stale "Cross-interpreter serialization" section (which documented the removed execnet.dumps, with an autofunction + doctest that would now break the build) becomes "Sending objects over a channel" guidance; the dumps/loads anchors are kept so existing refs resolve. BREAKING CHANGE: execnet.dumps, execnet.loads, execnet.dump and execnet.load are removed from the public API (use your own encoding, e.g. pydantic or the pytest report hooks). Shipping with the trio backend as a major release. Co-Authored-By: Claude Opus 4.8 (1M context) --- doc/basics.rst | 56 ++++++++++++++++++++++++++++--------- doc/index.rst | 8 +++--- src/execnet/__init__.py | 8 ------ src/execnet/aio.py | 14 +++------- src/execnet/gateway_base.py | 33 ++++++++++++++++++++-- src/execnet/sync.py | 8 ------ src/execnet/trio.py | 14 +++------- testing/test_basics.py | 24 ++++++++++------ testing/test_namespaces.py | 5 ++-- testing/test_serializer.py | 2 +- 10 files changed, 105 insertions(+), 67 deletions(-) diff --git a/doc/basics.rst b/doc/basics.rst index a6b256b7..94f43ad8 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -216,23 +216,53 @@ configure a tracing mechanism: .. _`dumps/loads`: .. _`dumps/loads API`: +.. _`serialization`: -Cross-interpreter serialization of Python objects +Sending objects over a channel ======================================================= -.. versionadded:: 1.1 +A channel carries only **simple builtin data**: ``None``, ``bool``, +``int``, ``float``, ``complex``, ``bytes``, ``str`` and arbitrarily nested +``list`` / ``tuple`` / ``set`` / ``frozenset`` / ``dict`` of those -- plus +**channel references**, which arrive as channels on the peer. That is the +entire contract. -Execnet exposes a function pair which you can safely use to -store and load values from different Python interpreters -(e.g. Python2 and Python3, PyPy and Jython). Here is -a basic example:: +execnet does **not** pickle and does **not** encode rich objects for you: +arbitrary instances, functions, ``datetime``, dataclasses, pydantic models, +numpy arrays, enums, etc. have no wire representation. This is deliberate; +encoded / rich-object channels are out of scope for execnet. - >>> import execnet - >>> dump = execnet.dumps([1,2,3]) - >>> execnet.loads(dump) - [1,2,3] +Sending an unsupported value raises ``DumpError`` (a subclass of +``DataFormatError``); a corrupt or protocol-mismatched payload on receive +raises ``LoadError``. These signal a **caller error to resolve** -- reduce +the value to simple data before sending -- not a transport failure. The +standalone serializer itself is an internal implementation detail +(``execnet.gateway_base``) and is not part of the public API. -For more examples see :ref:`dumps/loads examples`. +Encode rich objects yourself +------------------------------------------------------- -.. autofunction:: execnet.dumps(spec) -.. autofunction:: execnet.loads(spec) +Turning a rich object into simple data (and back) is the caller's job. Use +an established encoding mechanism rather than expecting the channel to do it: + +- **pydantic**: ``model.model_dump(mode="json")`` reduces a model to simple + data (``datetime`` -> ISO string, ``UUID`` / ``Enum`` / ``Decimal`` -> + primitives); ``Model.model_validate(...)`` rebuilds it on the other side. + ``TypeAdapter`` covers non-model types. + + :: + + channel.send(model.model_dump(mode="json")) + # peer: + model = MyModel.model_validate(channel.receive()) + +- **pytest** does exactly this above execnet: pytest-xdist ships + ``TestReport`` objects with the ``pytest_report_to_serializable`` / + ``pytest_report_from_serializable`` hooks (rich report <-> simple dict) + around ``channel.send`` / ``channel.receive``. + +- **stdlib**: ``dataclasses.asdict(obj)``, ``dt.isoformat()`` / + ``datetime.fromisoformat``, or ``json`` with a ``default=`` hook. + +Channels are the one non-builtin you *can* send: nested channel references +pass through intact, so callbacks and sub-streams need no encoding. diff --git a/doc/index.rst b/doc/index.rst index 19a20eb8..ce161b47 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -31,10 +31,10 @@ Features * Automatic bootstrapping: no manual remote installation. * Safe and simple serialization of Python builtin - types for sending/receiving structured data messages. - (New in 1.1) execnet offers a new :ref:`dumps/loads ` - API which allows cross-interpreter compatible serialization - of Python builtin types. + types for sending/receiving structured data messages; + see :ref:`sending objects over a channel `. + Encoding rich objects is the caller's job (execnet stays + builtin-types-only). * Flexible communication: synchronous send/receive as well as callback/queue mechanisms supported diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 66c61821..b81aaedd 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -34,10 +34,6 @@ from .sync import TimeoutError from .sync import XSpec from .sync import default_group -from .sync import dump -from .sync import dumps -from .sync import load -from .sync import loads from .sync import makegateway from .sync import set_execmodel @@ -56,10 +52,6 @@ "XSpec", "__version__", "default_group", - "dump", - "dumps", - "load", - "loads", "makegateway", "set_execmodel", ] diff --git a/src/execnet/aio.py b/src/execnet/aio.py index bbfe06e3..7151ab5d 100644 --- a/src/execnet/aio.py +++ b/src/execnet/aio.py @@ -23,8 +23,10 @@ async def main(): side only -- the host-side task runs to completion (a cancelled ``receive`` may still consume the next item). -The serialization helpers and error types are shared with -:mod:`execnet.sync` and :mod:`execnet.trio`. +The error types are shared with :mod:`execnet.sync` and +:mod:`execnet.trio`. Items you send must already be simple builtin data +(plus channels); the standalone serializer is intentionally not part of +the public API -- see ``DumpError``. """ from __future__ import annotations @@ -53,10 +55,6 @@ async def main(): from .gateway_base import LoadError from .gateway_base import RemoteError from .gateway_base import TimeoutError -from .gateway_base import dump -from .gateway_base import dumps -from .gateway_base import load -from .gateway_base import loads from .xspec import XSpec if TYPE_CHECKING: @@ -73,10 +71,6 @@ async def main(): "RemoteError", "TimeoutError", "XSpec", - "dump", - "dumps", - "load", - "loads", "open_popen_gateway", ] diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index c59f4437..75e5895a 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1033,15 +1033,42 @@ def executetask( class DataFormatError(Exception): - pass + """A value could not cross the channel in execnet's simple wire format. + + execnet only moves *simple* builtin data over a channel -- ``None``, + ``bool``, ``int``, ``float``, ``complex``, ``bytes``, ``str``, and + (arbitrarily nested) ``list``/``tuple``/``set``/``frozenset``/``dict`` of + those -- plus channel references, which pass through as channels. It does + **not** pickle: arbitrary instances, functions, ``datetime``, dataclasses, + pydantic models, numpy arrays, etc. have no wire representation. + + A ``DataFormatError`` therefore signals a caller error to resolve, not a + transport failure: reduce the value to simple data before sending (and + reconstruct it after receiving) with an encoding mechanism of your own -- + e.g. pydantic ``model_dump`` / ``model_validate`` or pytest's + ``pytest_report_to_serializable`` / ``pytest_report_from_serializable`` + hooks. See the docs, "Sending objects over a channel". + """ class DumpError(DataFormatError): - """Error while serializing an object.""" + """A value being **sent** is not simple wire data; convert it first. + + Raised by ``channel.send`` (and the internal serializer) when an object is + not one of execnet's simple wire types. Fix it at the call site by turning + the rich object into simple data -- e.g. ``dt.isoformat()``, + ``dataclasses.asdict(obj)``, ``model.model_dump(mode="json")`` -- rather + than expecting the channel to pickle it. Channels are the one non-builtin + that *is* sendable, so nested channel references are fine. + """ class LoadError(DataFormatError): - """Error while unserializing an object.""" + """Received bytes could not be turned back into an object. + + Raised while **receiving** (deserializing) -- a corrupted or + protocol-incompatible payload, or data produced by a mismatched peer. + """ def bchr(n: int) -> bytes: diff --git a/src/execnet/sync.py b/src/execnet/sync.py index 0d3c835f..e5e026c7 100644 --- a/src/execnet/sync.py +++ b/src/execnet/sync.py @@ -14,10 +14,6 @@ from .gateway_base import LoadError from .gateway_base import RemoteError from .gateway_base import TimeoutError -from .gateway_base import dump -from .gateway_base import dumps -from .gateway_base import load -from .gateway_base import loads from .multi import Group from .multi import MultiChannel from .multi import default_group @@ -40,10 +36,6 @@ "TimeoutError", "XSpec", "default_group", - "dump", - "dumps", - "load", - "loads", "makegateway", "set_execmodel", ] diff --git a/src/execnet/trio.py b/src/execnet/trio.py index df0ab493..4b712c0a 100644 --- a/src/execnet/trio.py +++ b/src/execnet/trio.py @@ -14,8 +14,10 @@ async def main(): trio.run(main) -The serialization helpers and error types are shared with the blocking -API in :mod:`execnet.sync`. +The error types are shared with the blocking API in :mod:`execnet.sync`. +Items you send must already be simple builtin data (plus channels); the +standalone serializer is intentionally not part of the public API -- see +``DumpError``. """ from ._trio_gateway import AsyncChannel @@ -32,10 +34,6 @@ async def main(): from .gateway_base import LoadError from .gateway_base import RemoteError from .gateway_base import TimeoutError -from .gateway_base import dump -from .gateway_base import dumps -from .gateway_base import load -from .gateway_base import loads from .xspec import XSpec __all__ = [ @@ -52,10 +50,6 @@ async def main(): "RemoteError", "TimeoutError", "XSpec", - "dump", - "dumps", - "load", - "loads", "open_popen_gateway", "serve_gateway", ] diff --git a/testing/test_basics.py b/testing/test_basics.py index 470bec6d..d5606610 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -28,37 +28,45 @@ ) +# The standalone serializer is an internal detail (execnet.gateway_base), +# not part of the public API -- see the docs, "Sending objects over a channel". @pytest.mark.parametrize("val", ["123", 42, [1, 2, 3], ["23", 25]]) class TestSerializeAPI: def test_serializer_api(self, val: object) -> None: - dumped = execnet.dumps(val) - val2 = execnet.loads(dumped) + dumped = gateway_base.dumps(val) + val2 = gateway_base.loads(dumped) assert val == val2 def test_mmap(self, tmp_path: Path, val: object) -> None: mmap = pytest.importorskip("mmap").mmap p = tmp_path / "data.bin" - p.write_bytes(execnet.dumps(val)) + p.write_bytes(gateway_base.dumps(val)) with p.open("r+b") as f: m = mmap(f.fileno(), 0) - val2 = execnet.load(m) + val2 = gateway_base.load(m) assert val == val2 def test_bytesio(self, val: object) -> None: f = BytesIO() - execnet.dump(f, val) + gateway_base.dump(f, val) read = BytesIO(f.getvalue()) - val2 = execnet.load(read) + val2 = gateway_base.load(read) assert val == val2 +def test_serializer_not_public() -> None: + # dumps/loads/dump/load are internal to gateway_base only. + for name in ("dumps", "loads", "dump", "load"): + assert not hasattr(execnet, name), name + + def test_serializer_api_version_error(monkeypatch: pytest.MonkeyPatch) -> None: bchr = gateway_base.bchr monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(1)) - dumped = execnet.dumps(42) + dumped = gateway_base.dumps(42) monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(2)) - pytest.raises(execnet.DataFormatError, lambda: execnet.loads(dumped)) + pytest.raises(execnet.DataFormatError, lambda: gateway_base.loads(dumped)) def test_errors_on_execnet() -> None: diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index a8bfef0c..0317bc2b 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -32,9 +32,10 @@ def test_trio_namespace_exposes_async_core() -> None: assert execnet.trio.AsyncGateway is _trio_gateway.AsyncGateway assert execnet.trio.AsyncChannel is _trio_gateway.AsyncChannel assert execnet.trio.open_popen_gateway is _trio_gateway.open_popen_gateway - # serialization + errors are shared with the sync surface + # error types are shared with the sync surface; the standalone serializer + # is intentionally not exposed on any public namespace assert execnet.trio.RemoteError is execnet.RemoteError - assert execnet.trio.dumps is execnet.dumps + assert not hasattr(execnet.trio, "dumps") def test_portal_namespace() -> None: diff --git a/testing/test_serializer.py b/testing/test_serializer.py index 87adf04c..49332c97 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -189,4 +189,4 @@ def test_tuple_nested_with_empty_in_between(dump, load) -> None: def test_py2_string_loads() -> None: """Regression test for #267.""" - assert execnet.loads(b"\x02M\x00\x00\x00\x01aQ") == b"a" + assert execnet.gateway_base.loads(b"\x02M\x00\x00\x00\x01aQ") == b"a" From 6a3371673d492d7a17cb835cd6c29105e6295067 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 14:40:29 +0200 Subject: [PATCH 53/59] test: type-check the callback stress tests under mypy Add hypothesis to the mypy pre-commit hook's dependencies so @given is typed, and pass Hypothesis settings.register_profile keywords explicitly (a homogeneous **dict tripped the arg-type check). Also a ruff-format tidy. Co-Authored-By: Claude Opus 4.8 (1M context) --- .pre-commit-config.yaml | 1 + testing/conftest.py | 17 +++++++++-------- testing/test_channel_stress.py | 4 +--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f7a9f86c..91db1f29 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,4 +32,5 @@ repos: additional_dependencies: - pytest - trio + - hypothesis - types-pywin32 diff --git a/testing/conftest.py b/testing/conftest.py index becf10f2..04dbe631 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -92,17 +92,18 @@ def pytest_configure(config: pytest.Config) -> None: from hypothesis import settings except ImportError: return - common = dict( - deadline=None, - suppress_health_check=[ - HealthCheck.function_scoped_fixture, - HealthCheck.too_slow, - ], + suppress = [HealthCheck.function_scoped_fixture, HealthCheck.too_slow] + settings.register_profile( + "execnet-quick", max_examples=15, deadline=None, suppress_health_check=suppress ) - settings.register_profile("execnet-quick", max_examples=15, **common) stress = config.getoption("stress") if stress is not None: - settings.register_profile("execnet-stress", max_examples=int(stress), **common) + settings.register_profile( + "execnet-stress", + max_examples=int(stress), + deadline=None, + suppress_health_check=suppress, + ) settings.load_profile("execnet-stress") else: settings.load_profile("execnet-quick") diff --git a/testing/test_channel_stress.py b/testing/test_channel_stress.py index 8090367d..12f69d47 100644 --- a/testing/test_channel_stress.py +++ b/testing/test_channel_stress.py @@ -55,9 +55,7 @@ class TestCallbackStress: gwtype = "popen" @given(data=items_strategy) - def test_callback_receives_all_in_order( - self, gw: Gateway, data: list[int] - ) -> None: + def test_callback_receives_all_in_order(self, gw: Gateway, data: list[int]) -> None: collected: list[int] = [] channel = gw.remote_exec(_echo, items=data) channel.setcallback(collected.append) From 12fdbbec13564e6576f4c8ab4f284b0b90aea983 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 14:57:26 +0200 Subject: [PATCH 54/59] refactor!: drop the injected remote shell script execnet.script.shell connected to a socket server, sent repr() of its own source as the first line, and expected the server to exec it with clientsock bound in globals. That handshake belonged to the pre-Trio socket server; the Trio one speaks the execnet frame protocol from the first byte and never execs an incoming line, so the module has been inert since that rewrite. Nothing imports it, no test covers it, and a remote prompt is better served as a remote_exec example than as a socket-protocol side channel. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 3 ++ src/execnet/script/shell.py | 91 ------------------------------------- 2 files changed, 3 insertions(+), 91 deletions(-) delete mode 100644 src/execnet/script/shell.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index eac3c8d6..3e99f1d5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,9 @@ * Trio host-thread Message IO for local ``popen`` + import bootstrap (coordinator and worker). Adds a hard ``trio`` dependency. Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types and greenlet execmodels keep the legacy thread path. +* Removed ``execnet.script.shell``, an interactive remote prompt that injected its own + source into the pre-Trio socket server. The Trio socket server never execs an incoming + source line, so the module could no longer work against it. 2.1.2 (2025-11-11) diff --git a/src/execnet/script/shell.py b/src/execnet/script/shell.py deleted file mode 100644 index de6f8ded..00000000 --- a/src/execnet/script/shell.py +++ /dev/null @@ -1,91 +0,0 @@ -#! /usr/bin/env python -""" -a remote python shell - -for injection into startserver.py -""" - -import os -import select -import socket -import sys -from threading import Thread -from traceback import print_exc -from typing import NoReturn - - -def clientside() -> NoReturn: - print("client side starting") - host, portstr = sys.argv[1].split(":") - port = int(portstr) - with open(os.path.abspath(sys.argv[0])) as fd: - myself = fd.read() - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) - sock.sendall((repr(myself) + "\n").encode()) - print("send boot string") - inputlist = [sock, sys.stdin] - try: - while 1: - r, _w, _e = select.select(inputlist, [], []) - if sys.stdin in r: - line = input() - sock.sendall((line + "\n").encode()) - if sock in r: - line = sock.recv(4096).decode() - sys.stdout.write(line) - sys.stdout.flush() - except BaseException: - import traceback - - traceback.print_exc() - - sys.exit(1) - - -class promptagent(Thread): - def __init__(self, clientsock) -> None: - print("server side starting") - super().__init__() # type: ignore[call-overload] - self.clientsock = clientsock - - def run(self) -> None: - print("Entering thread prompt loop") - clientfile = self.clientsock.makefile("w") - - filein = self.clientsock.makefile("r") - loc = self.clientsock.getsockname() - - while 1: - try: - clientfile.write("{} {} >>> ".format(*loc)) - clientfile.flush() - line = filein.readline() - if not line: - raise EOFError("nothing") - if line.strip(): - oldout, olderr = sys.stdout, sys.stderr - sys.stdout, sys.stderr = clientfile, clientfile - try: - try: - exec(compile(line + "\n", "", "single")) - except BaseException: - print_exc() - finally: - sys.stdout = oldout - sys.stderr = olderr - clientfile.flush() - except EOFError: - sys.stderr.write("connection close, prompt thread returns") - break - - self.clientsock.close() - - -sock = globals().get("clientsock") -if sock is not None: - prompter = promptagent(sock) - prompter.start() - print("promptagent - thread started") -else: - clientside() From 6718808a37ff099cbe5f57739f3d1aec6deac821 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 14:58:56 +0200 Subject: [PATCH 55/59] refactor!: drop the quitserver script execnet.script.quitserver stopped a socket server by connecting and sending '"raise KeyboardInterrupt"' for the server to exec. Like the shell script it depended on the pre-Trio exec-a-source-line handshake; against the Trio socket server it is just a malformed first frame. Stopping the server is now the job of whatever supervises the execnet-socketserver process. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 2 ++ src/execnet/script/quitserver.py | 17 ----------------- 2 files changed, 2 insertions(+), 17 deletions(-) delete mode 100644 src/execnet/script/quitserver.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3e99f1d5..1ac8a0ef 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,8 @@ * Removed ``execnet.script.shell``, an interactive remote prompt that injected its own source into the pre-Trio socket server. The Trio socket server never execs an incoming source line, so the module could no longer work against it. +* Removed ``execnet.script.quitserver``, which shut a socket server down by sending it + ``"raise KeyboardInterrupt"`` to exec. It relied on the same removed handshake. 2.1.2 (2025-11-11) diff --git a/src/execnet/script/quitserver.py b/src/execnet/script/quitserver.py deleted file mode 100644 index 4c94c383..00000000 --- a/src/execnet/script/quitserver.py +++ /dev/null @@ -1,17 +0,0 @@ -""" - -send a "quit" signal to a remote server - -""" - -from __future__ import annotations - -import socket -import sys - -host, port = sys.argv[1].split(":") -hostport = (host, int(port)) - -sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -sock.connect(hostport) -sock.sendall(b'"raise KeyboardInterrupt"\n') From 7124fc0cfeb51a65bc0d5454296be1486cd46a4c Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 15:00:37 +0200 Subject: [PATCH 56/59] refactor!: drop the socketserver restart loop script loop_socketserver re-Popen'd "python /socketserver.py" in a while loop, from the days when a single connection ended the server process. The Trio socket server accepts connections in a loop via trio.serve_listeners and only exits after one connection when asked with --once, so the wrapper has nothing left to restart. Its sibling-file lookup also only ever worked from a source checkout, not from an installed execnet. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 4 ++++ src/execnet/script/loop_socketserver.py | 14 -------------- 2 files changed, 4 insertions(+), 14 deletions(-) delete mode 100644 src/execnet/script/loop_socketserver.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1ac8a0ef..7bde0c0f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,10 @@ source line, so the module could no longer work against it. * Removed ``execnet.script.quitserver``, which shut a socket server down by sending it ``"raise KeyboardInterrupt"`` to exec. It relied on the same removed handshake. +* Removed ``execnet.script.loop_socketserver``, a restart loop around a sibling + ``socketserver.py`` file. The socket server serves connections in a loop itself + (``--once`` opts out), and the sibling-file path never resolved for an installed + execnet anyway. 2.1.2 (2025-11-11) diff --git a/src/execnet/script/loop_socketserver.py b/src/execnet/script/loop_socketserver.py deleted file mode 100644 index a4688a80..00000000 --- a/src/execnet/script/loop_socketserver.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -import subprocess -import sys - -if __name__ == "__main__": - directory = os.path.dirname(os.path.abspath(sys.argv[0])) - script = os.path.join(directory, "socketserver.py") - while 1: - cmdlist = ["python", script] - cmdlist.extend(sys.argv[1:]) - text = "starting subcommand: " + " ".join(cmdlist) - print(text) - process = subprocess.Popen(cmdlist) - process.wait() From 47e5b78ead6be7c9d96c9649747c18254f0cb2c4 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 15:05:36 +0200 Subject: [PATCH 57/59] refactor!: drop the pywin32 windows service wrapper socketserverservice.py registered the socket server as a Windows service via pywin32. pywin32 was never a dependency, nothing imported the module, no test ever ran it, and no entry point installed it -- it was a recipe living in the package. Now that the server is a real console command, any service host can supervise it. Document that instead: an NSSM (or WinSW) wrapper on Windows, a systemd unit on Linux, plus why sc.exe alone will not start a plain console program. While rewriting that section, replace the stale instruction to download socketserver.py from raw.githubusercontent and run it standalone. The Trio server imports execnet and trio, so it is no longer a version-independent single file; "uvx --from execnet execnet-socketserver" is the install-free equivalent. Also document --once, port 0, and that the port is unauthenticated. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 3 + doc/example/test_info.rst | 65 ++++++++++++++-- src/execnet/script/socketserverservice.py | 90 ----------------------- 3 files changed, 61 insertions(+), 97 deletions(-) delete mode 100644 src/execnet/script/socketserverservice.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7bde0c0f..0b66b38a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,9 @@ ``socketserver.py`` file. The socket server serves connections in a loop itself (``--once`` opts out), and the sibling-file path never resolved for an installed execnet anyway. +* Removed ``execnet.script.socketserverservice``, the pywin32 Windows service wrapper. + Wrap the ``execnet-socketserver`` console command with a service host such as NSSM + instead; the socket gateway example documents how. 2.1.2 (2025-11-11) diff --git a/doc/example/test_info.rst b/doc/example/test_info.rst index eefe91b3..7a5ead4f 100644 --- a/doc/example/test_info.rst +++ b/doc/example/test_info.rst @@ -172,15 +172,23 @@ sends back the results. Instantiate gateways through sockets ----------------------------------------------------- -.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/main/src/execnet/script/socketserver.py +In cases where you do not have SSH-access to a machine you need a +bootstrapping-point that listens on a socket. execnet ships one as the +``execnet-socketserver`` console command; run it on the target machine:: -In cases where you do not have SSH-access to a machine -you need to download a small version-independent standalone -`socketserver.py`_ script to provide a remote bootstrapping-point. -You do not need to install the execnet package remotely. -Simply run the script like this:: + execnet-socketserver :8888 # bind to all IPs, port 8888 - python socketserver.py :8888 # bind to all IPs, port 8888 +If execnet is not installed there, uv_ can fetch and run it in one step +without leaving anything behind:: + + uvx --from execnet execnet-socketserver :8888 + +.. _uv: https://docs.astral.sh/uv/ + +The server accepts connections in a loop and serves each one in its own +worker subprocess; pass ``--once`` to serve a single connection and exit. +Passing port ``0`` binds an ephemeral port -- the bound address is printed +on the first line of output either way. You can then instruct execnet on your local machine to bootstrap itself into the remote socket endpoint:: @@ -191,4 +199,47 @@ itself into the remote socket endpoint:: That's it, you can now use the gateway object just like a popen- or SSH-based one. +.. warning:: + + The socket server performs no authentication and its traffic is not + encrypted -- anyone who can reach the port can execute code on that + machine. Bind it to a trusted network only, or tunnel it over SSH. + +Keeping the socket server running ++++++++++++++++++++++++++++++++++ + +``execnet-socketserver`` is an ordinary long-running process, so restarts and +boot-time startup are the job of your platform's service manager. + +On Linux, a systemd unit does it:: + + # /etc/systemd/system/execnet-socketserver.service + [Unit] + Description=execnet socket server + + [Service] + ExecStart=/usr/local/bin/execnet-socketserver :8888 + Restart=always + + [Install] + WantedBy=multi-user.target + +On Windows, wrap the console command with a service host such as NSSM_ or +WinSW_. Use ``where execnet-socketserver`` to find the installed +``execnet-socketserver.exe`` (it lives in the ``Scripts`` directory of the +environment it was installed into), then:: + + nssm install ExecNetSocketServer C:\path\to\Scripts\execnet-socketserver.exe :8888 + nssm set ExecNetSocketServer Start SERVICE_AUTO_START + net start ExecNetSocketServer + +NSSM restarts the process if it exits. Note that ``sc.exe create`` on its own +is not enough: it expects a binary that implements the Windows service control +protocol, which a plain console program does not. If you would rather not +install a service host, a Task Scheduler task with an *At startup* trigger +works too. + +.. _NSSM: https://nssm.cc/ +.. _WinSW: https://github.com/winsw/winsw + .. include:: test_ssh_fileserver.rst diff --git a/src/execnet/script/socketserverservice.py b/src/execnet/script/socketserverservice.py deleted file mode 100644 index 1c991307..00000000 --- a/src/execnet/script/socketserverservice.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -A windows service wrapper for the py.execnet socketserver. - -To use, run: - python socketserverservice.py register - net start ExecNetSocketServer -""" - -import sys -import threading - -import servicemanager -import win32event -import win32evtlogutil -import win32service -import win32serviceutil - -from . import socketserver - -appname = "ExecNetSocketServer" - - -class SocketServerService(win32serviceutil.ServiceFramework): - _svc_name_ = appname - _svc_display_name_ = "%s" % appname - _svc_deps_ = ["EventLog"] - - def __init__(self, args) -> None: - # The exe-file has messages for the Event Log Viewer. - # Register the exe-file as event source. - # - # Probably it would be better if this is done at installation time, - # so that it also could be removed if the service is uninstalled. - # Unfortunately it cannot be done in the 'if __name__ == "__main__"' - # block below, because the 'frozen' exe-file does not run this code. - # - win32evtlogutil.AddSourceToRegistry( - self._svc_display_name_, servicemanager.__file__, "Application" - ) - super().__init__(args) - self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) - self.WAIT_TIME = 1000 # in milliseconds - - def SvcStop(self) -> None: - self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) - win32event.SetEvent(self.hWaitStop) - - def SvcDoRun(self) -> None: - # Redirect stdout and stderr to prevent "IOError: [Errno 9] - # Bad file descriptor". Windows services don't have functional - # output streams. - sys.stdout = sys.stderr = open("nul", "w") - - # Write a 'started' event to the event log... - win32evtlogutil.ReportEvent( - self._svc_display_name_, - servicemanager.PYS_SERVICE_STARTED, - 0, # category - servicemanager.EVENTLOG_INFORMATION_TYPE, - (self._svc_name_, ""), - ) - print("Begin: %s" % self._svc_display_name_) - - hostport = ":8888" - print("Starting py.execnet SocketServer on %s" % hostport) - thread = threading.Thread( - target=socketserver.main, args=([hostport],), daemon=True - ) - thread.start() - - # wait to be stopped or self.WAIT_TIME to pass - while True: - result = win32event.WaitForSingleObject(self.hWaitStop, self.WAIT_TIME) - if result == win32event.WAIT_OBJECT_0: - break - - # write a 'stopped' event to the event log. - win32evtlogutil.ReportEvent( - self._svc_display_name_, - servicemanager.PYS_SERVICE_STOPPED, - 0, # category - servicemanager.EVENTLOG_INFORMATION_TYPE, - (self._svc_name_, ""), - ) - print("End: %s" % appname) - - -if __name__ == "__main__": - # Note that this code will not be run in the 'frozen' exe-file!!! - win32serviceutil.HandleCommandLine(SocketServerService) From fa0d7a8b7b94f563372a0c51e393b808f8033c74 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 15:13:39 +0200 Subject: [PATCH 58/59] refactor: move the socket server to execnet._socketserver The Trio rewrite turned socketserver.py from a copy-me standalone script into ordinary package code: it imports trio and execnet._trio_host, and reaches users as the execnet-socketserver console command. Sitting in an execnet.script package still advertised the old download-and-run story. Move it next to the other private implementation modules (_trio_host, _trio_worker, _provision) and drop the execnet.script package, which is now empty. The console command and its behaviour are unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 3 +++ pyproject.toml | 2 +- src/execnet/{script/socketserver.py => _socketserver.py} | 6 ++++-- src/execnet/script/__init__.py | 1 - 4 files changed, 8 insertions(+), 4 deletions(-) rename src/execnet/{script/socketserver.py => _socketserver.py} (91%) delete mode 100644 src/execnet/script/__init__.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0b66b38a..3a1e9422 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,9 @@ * Removed ``execnet.script.socketserverservice``, the pywin32 Windows service wrapper. Wrap the ``execnet-socketserver`` console command with a service host such as NSSM instead; the socket gateway example documents how. +* Moved the socket server to ``execnet._socketserver`` and removed the now empty + ``execnet.script`` package. The ``execnet-socketserver`` console command is unchanged + and remains the supported way to run it. 2.1.2 (2025-11-11) diff --git a/pyproject.toml b/pyproject.toml index f29b24d0..68029e90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ classifiers = [ ] [project.scripts] -execnet-socketserver = "execnet.script.socketserver:main" +execnet-socketserver = "execnet._socketserver:main" [project.optional-dependencies] gevent = [ diff --git a/src/execnet/script/socketserver.py b/src/execnet/_socketserver.py similarity index 91% rename from src/execnet/script/socketserver.py rename to src/execnet/_socketserver.py index eb2a86ae..c27609f6 100644 --- a/src/execnet/script/socketserver.py +++ b/src/execnet/_socketserver.py @@ -1,9 +1,11 @@ -#! /usr/bin/env python """Trio socket server for execnet gateways. Listens on a TCP port and hands each accepted connection (by fd) to a fresh ``python -m execnet._trio_worker`` subprocess that serves the gateway over it. -No code is executed inline. Run directly, e.g. provisioned anywhere with +No code is executed inline. + +This module implements the ``execnet-socketserver`` console command, which is +the supported entry point -- run it on the target host, or install-free with ``uvx --from execnet execnet-socketserver``. """ diff --git a/src/execnet/script/__init__.py b/src/execnet/script/__init__.py deleted file mode 100644 index 792d6005..00000000 --- a/src/execnet/script/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# From 871dfd730b3bd80407bdf271844359fed4f04a83 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 16:14:08 +0200 Subject: [PATCH 59/59] refactor!: drop the py2/py3 string coercion machinery reconfigure() configured how str/bytes cross a Python2 <-> Python3 gateway. execnet cannot have such a peer anymore, so remove it: * Gateway.reconfigure, Channel.reconfigure and AsyncChannel.reconfigure, plus the _strconfig attribute they fed. * the py2str_as_py3str / py3str_as_py2str arguments of gateway_base.loads and .load, and the strconfig argument of loads_internal. py2str_as_py3str was already unreachable on Python3 -- it only gates the PY2STRING opcode, which no Python3 serializer emits. * the retired PY2STRING and UNICODE opcodes; PY3STRING becomes STRING. Opcode bytes are unchanged for every surviving type, and M/S stay unassigned. Values dumped by execnet on Python2 no longer load. * the no-op reconfigure() call in rsync, which cost a round-trip per channel to set what was already the default. * doc/example/hybridpython.rst and py3topy2.py, along with the remaining Python2 references in basics.rst and index.rst. The RECONFIGURE message code stays reserved, but is no longer sent or handled. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 9 ++ doc/basics.rst | 8 +- doc/example/hybridpython.rst | 153 ---------------------- doc/example/py3topy2.py | 19 --- doc/examples.rst | 1 - doc/index.rst | 4 - src/execnet/_trio_gateway.py | 32 +---- src/execnet/_trio_host.py | 7 - src/execnet/gateway.py | 12 -- src/execnet/gateway_base.py | 89 ++----------- src/execnet/rsync.py | 1 - testing/test_compatibility_regressions.py | 6 +- testing/test_serializer.py | 7 +- testing/test_trio_gateway.py | 13 -- 14 files changed, 35 insertions(+), 326 deletions(-) delete mode 100644 doc/example/hybridpython.rst delete mode 100644 doc/example/py3topy2.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3a1e9422..9f2d3f38 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,15 @@ * Moved the socket server to ``execnet._socketserver`` and removed the now empty ``execnet.script`` package. The ``execnet-socketserver`` console command is unchanged and remains the supported way to run it. +* Removed ``Gateway.reconfigure`` and ``Channel.reconfigure`` along with the + ``py2str_as_py3str`` / ``py3str_as_py2str`` arguments of ``gateway_base.loads`` and + ``gateway_base.load``. They configured string coercion between Python2 and Python3 + peers, which execnet can no longer have: ``py2str_as_py3str`` was already unreachable + because only a Python2 serializer emits the opcode it gates. The ``RECONFIGURE`` + message code stays reserved but is no longer sent or handled. +* The serializer dropped the retired ``PY2STRING`` and ``UNICODE`` opcodes and renamed + ``PY3STRING`` to ``STRING``. Values dumped by execnet running on Python2 no longer + load. Opcode bytes are unchanged for every type that survives. 2.1.2 (2025-11-11) diff --git a/doc/basics.rst b/doc/basics.rst index 94f43ad8..a22ad7de 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -43,8 +43,8 @@ Examples for valid gateway specifications ``default`` via SSH through Vagrant's ``vagrant ssh`` command. It supports the same additional parameters as regular SSH connections. -* ``popen//python=python2.7//nice=20`` specification of - a python subprocess using the ``python2.7`` executable which must be +* ``popen//python=python3.13//nice=20`` specification of + a python subprocess using the ``python3.13`` executable which must be discoverable through the system ``PATH``; running with the lowest CPU priority ("nice" level). By default current dir will be the current dir of the instantiator. @@ -86,10 +86,6 @@ a channel object whose symmetric counterpart channel is available to the remotely executing source. -.. method:: Gateway.reconfigure([py2str_as_py3str=True, py3str_as_py2str=False]) - - Reconfigures the string-coercion behaviour of the gateway - .. _`Channel`: .. _`channel-api`: diff --git a/doc/example/hybridpython.rst b/doc/example/hybridpython.rst deleted file mode 100644 index f4ad465a..00000000 --- a/doc/example/hybridpython.rst +++ /dev/null @@ -1,153 +0,0 @@ -Connecting different Python interpreters -========================================== - -.. _`dumps/loads examples`: - -Dumping and loading values across interpreter versions ----------------------------------------------------------- - -.. versionadded:: 1.1 - -Execnet offers a new safe and fast :ref:`dumps/loads API` which you -can use to dump builtin python data structures and load them -later with the same or a different python interpreter (including -between Python2 and Python3). The standard library offers -the pickle and marshal modules but they do not work safely -between different interpreter versions. Using xml/json -requires a mapping of Python objects and is not easy to -get right. Moreover, execnet allows to control handling -of bytecode/strings/unicode types. Here is an example:: - - # using python2 - import execnet - with open("data.py23", "wb") as f: - f.write(execnet.dumps(["hello", "world"])) - - # using Python3 - import execnet - with open("data.py23", "rb") as f: - val = execnet.loads(f.read(), py2str_as_py3str=True) - assert val == ["hello", "world"] - -See the :ref:`dumps/loads API` for more details on string -conversion options. Please note, that you can not dump -user-level instances, only builtin python types. - -Connect to Python2/Numpy from Python3 ----------------------------------------- - -Here we run a Python3 interpreter to connect to a Python2.7 interpreter -that has numpy installed. We send items to be added to an array and -receive back the remote "repr" of the array:: - - import execnet - gw = execnet.makegateway("popen//python=python2.7") - channel = gw.remote_exec(""" - import numpy - array = numpy.array([1,2,3]) - while 1: - x = channel.receive() - if x is None: - break - array = numpy.append(array, x) - channel.send(repr(array)) - """) - for x in range(10): - channel.send(x) - channel.send(None) - print (channel.receive()) - -will print on the CPython3.1 side:: - - array([1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) - -A more refined real-life example of python3/python2 interaction -is the anyvc_ project which uses version-control bindings in -a Python2 subprocess in order to offer Python3-based library -functionality. - -.. _anyvc: http://bitbucket.org/RonnyPfannschmidt/anyvc/overview/ - - -Reconfiguring the string coercion between python2 and python3 -------------------------------------------------------------- - -Sometimes the default configuration of string coercion (2str to 3str, 3str to 2unicode) -is inconvient, thus it can be reconfigured via `gw.reconfigure` and `channel.reconfigure`. Here is an example session on a Python2 interpreter:: - - - >>> import execnet - >>> execnet.makegateway("popen//python=python3.2") - - >>> gw=execnet.makegateway("popen//python=python3.2") - >>> gw.remote_exec("channel.send('hello')").receive() - u'hello' - >>> gw.reconfigure(py3str_as_py2str=True) - >>> gw.remote_exec("channel.send('hello')").receive() - 'hello' - >>> ch = gw.remote_exec('channel.send(type(channel.receive()).__name__)') - >>> ch.send('a') - >>> ch.receive() - 'str' - >>> ch = gw.remote_exec('channel.send(type(channel.receive()).__name__)') - >>> ch.reconfigure(py2str_as_py3str=False) - >>> ch.send('a') - >>> ch.receive() - u'bytes' - - -Work with Java objects from CPython ----------------------------------------- - -Use your CPython interpreter to connect to a `Jython 2.5.1`_ interpreter -and work with Java types:: - - import execnet - gw = execnet.makegateway("popen//python=jython") - channel = gw.remote_exec(""" - from java.util import Vector - v = Vector() - v.add('aaa') - v.add('bbb') - for val in v: - channel.send(val) - """) - - for item in channel: - print (item) - -will print on the CPython side:: - - aaa - bbb - -.. _`Jython 2.5.1`: http://www.jython.org - -Work with C# objects from CPython ----------------------------------------- - -(Experimental) use your CPython interpreter to connect to a IronPython_ interpreter -which can work with C# classes. Here is an example for instantiating -a CLR Array instance and sending back its representation:: - - import execnet - gw = execnet.makegateway("popen//python=ipy") - - channel = gw.remote_exec(""" - import clr - clr.AddReference("System") - from System import Array - array = Array[float]([1,2]) - channel.send(str(array)) - """) - print (channel.receive()) - -using Mono 2.0 and IronPython-1.1 this will print on the CPython side:: - - System.Double[](1.0, 2.0) - -.. note:: - Using IronPython needs more testing, likely newer versions - will work better. please feedback if you have information. - -.. _IronPython: http://ironpython.net diff --git a/doc/example/py3topy2.py b/doc/example/py3topy2.py deleted file mode 100644 index 3dcf0437..00000000 --- a/doc/example/py3topy2.py +++ /dev/null @@ -1,19 +0,0 @@ -import execnet - -gw = execnet.makegateway("popen//python=python2") -channel = gw.remote_exec( - """ - import numpy - array = numpy.array([1,2,3]) - while 1: - x = channel.receive() - if x is None: - break - array = numpy.append(array, x) - channel.send(repr(array)) -""" -) -for x in range(10): - channel.send(x) -channel.send(None) -print(channel.receive()) diff --git a/doc/examples.rst b/doc/examples.rst index 767fa4fc..3f09197a 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -14,7 +14,6 @@ Note: all examples with `>>>` prompts are automatically tested. example/test_group example/test_proxy example/test_multi - example/hybridpython example/test_debug .. toctree:: diff --git a/doc/index.rst b/doc/index.rst index ce161b47..47a918cc 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -59,9 +59,6 @@ Known uses project to launch computation processes through ssh. He also compares `disco and execnet`_ in a subsequent post. -* Ronny Pfannschmidt uses it for his `anyvc`_ VCS-abstraction project - to bridge the Python2/Python3 version gap. - * Sysadmins and developers are using it for ad-hoc custom scripting .. _`quora`: http://quora.com @@ -71,7 +68,6 @@ Known uses .. _`distributed testing`: https://pypi.python.org/pypi/pytest-xdist .. _`Distributed NTLK with execnet`: http://streamhacker.com/2009/11/29/distributed-nltk-execnet/ .. _`disco and execnet`: http://streamhacker.com/2009/12/14/execnet-disco-distributed-nltk/ -.. _`anyvc`: http://bitbucket.org/RonnyPfannschmidt/anyvc/ Project status -------------------------- diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 81ea76d0..3972db4f 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -8,7 +8,7 @@ Two-level channel model: * :class:`RawChannel` (this module) -- id-routed raw byte payload streams - over the gateway: no serialization, no strconfig, no callbacks. + over the gateway: no serialization and no callbacks. ``CHANNEL_DATA`` payloads route to the channel verbatim; the layer on top decides what the bytes mean. * ``AsyncChannel`` -- the serialized object API layered on a RawChannel. @@ -44,7 +44,6 @@ from .gateway_base import Message from .gateway_base import RemoteError from .gateway_base import TimeoutError -from .gateway_base import Unserializer from .gateway_base import dumps_internal from .gateway_base import gateway_info from .gateway_base import loads_internal @@ -149,8 +148,6 @@ class RawChannel: the peer drains, hits EOF, and may keep sending to us. """ - _strconfig: tuple[bool, bool] | None = None - def __init__(self, gateway: AsyncGateway, id: int) -> None: self.gateway = gateway self.id = id @@ -405,16 +402,6 @@ async def wait_closed(self) -> None: if error is not None: raise error - async def reconfigure( - self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False - ) -> None: - """Set the string coercion for both ends of this channel.""" - strconfig = (py2str_as_py3str, py3str_as_py2str) - self._raw._strconfig = strconfig - await self.gateway._send( - Message.RECONFIGURE, self.id, dumps_internal(strconfig) - ) - def __aiter__(self) -> AsyncChannel: return self @@ -424,12 +411,8 @@ async def __anext__(self) -> Any: except EOFError: raise StopAsyncIteration from None - # Unserializer duck-type: loads_internal(data, self) reads _strconfig - # and _channelfactory off the object to resolve CHANNEL opcodes. - - @property - def _strconfig(self) -> tuple[bool, bool]: - return self._raw._strconfig or self.gateway._strconfig + # Unserializer duck-type: loads_internal(data, self) reads + # _channelfactory off the object to resolve CHANNEL opcodes. @property def _channelfactory(self) -> _AsyncChannelFactory: @@ -475,7 +458,6 @@ def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None self._async_channels: dict[int, AsyncChannel] = {} self._channelfactory = _AsyncChannelFactory(self) self._count = _startcount - self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) self._outbound_send, self._outbound = trio.open_memory_channel[ tuple[bytes, Callable[[BaseException | None], None] | None] ](math.inf) @@ -693,14 +675,6 @@ def _dispatch(self, message: Message) -> None: Message.CHANNEL_DATA, channelid, dumps_internal(gateway_info()) ) self._send_nowait(Message.CHANNEL_CLOSE, channelid) - elif code == Message.RECONFIGURE: - data = loads_internal(message.data) - assert isinstance(data, tuple) - if channelid == 0: - self._strconfig = data - else: - # picked up by the serialized channel layer - self._channel_for(channelid)._strconfig = data else: # CHANNEL_EXEC / GATEWAY_START_*: not served by the async core self._trace("rejecting unsupported message", message) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index e119ca68..1d9149dd 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -204,13 +204,6 @@ def _dispatch(self, message: Message) -> None: elif code == Message.CHANNEL_EXEC: channel = gateway._channelfactory.new(message.channelid) gateway._local_schedulexec(channel=channel, sourcetask=message.data) - elif code == Message.RECONFIGURE: - data = loads_internal(message.data, gateway) - assert isinstance(data, tuple) - if message.channelid == 0: - gateway._strconfig = data - else: - gateway._channelfactory.new(message.channelid)._strconfig = data elif code == Message.GATEWAY_START_SOCKET: handle_start_socket(gateway, message.channelid, message.data) elif code == Message.GATEWAY_START_SUB: diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 672a0830..ac41705c 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -80,18 +80,6 @@ def exit(self) -> None: self._trace("io-error: could not send termination sequence") self._trace(" exception: %r" % exc) - def reconfigure( - self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False - ) -> None: - """Set the string coercion for this gateway. - - The default is to try to convert py2 str as py3 str, but not to try and - convert py3 str to py2 str. - """ - self._strconfig = (py2str_as_py3str, py3str_as_py2str) - data = gateway_base.dumps_internal(self._strconfig) - self._send(Message.RECONFIGURE, data=data) - def _rinfo(self, update: bool = False) -> RInfo: """Return some sys/env information from remote. diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 75e5895a..ce19eb0f 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -186,6 +186,8 @@ class only carries the framing and the code constants. """ STATUS = 0 + #: retired: the py2/py3 string coercion switch. Nothing sends or + #: handles it anymore, the code stays reserved for reuse. RECONFIGURE = 1 GATEWAY_TERMINATE = 2 CHANNEL_EXEC = 3 @@ -393,8 +395,6 @@ def __init__(self, gateway: BaseGateway, id: int) -> None: assert isinstance(id, int) assert not isinstance(gateway, type) self.gateway = gateway - # XXX: defaults copied from Unserializer - self._strconfig = getattr(gateway, "_strconfig", (True, False)) self.id = id # serialized payloads (or ENDMARKER); None once a consumer is attached self._mailbox: Mailbox[Any] | None = Mailbox(gateway._new_wakener()) @@ -672,18 +672,6 @@ def next(self) -> Any: __next__ = next - def reconfigure( - self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False - ) -> None: - """Set the string coercion for this channel. - - The default is to try to convert py2 str as py3 str, - but not to try and convert py3 str to py2 str - """ - self._strconfig = (py2str_as_py3str, py3str_as_py2str) - data = dumps_internal(self._strconfig) - self.gateway._send(Message.RECONFIGURE, self.id, data=data) - ENDMARKER = object() INTERRUPT_TEXT = "keyboard-interrupted" @@ -838,7 +826,6 @@ def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel self._io = io self.id = id - self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) self._channelfactory = ChannelFactory(self, _startcount) # globals may be NONE at process-termination self.__trace = trace @@ -1106,35 +1093,26 @@ class opcode: NEWDICT = b"J" NEWLIST = b"K" NONE = b"L" - PY2STRING = b"M" - PY3STRING = b"N" + STRING = b"N" SET = b"O" SETITEM = b"P" STOP = b"Q" TRUE = b"R" - UNICODE = b"S" COMPLEX = b"T" class Unserializer: num2func: dict[bytes, Callable[[Unserializer], None]] = {} - py2str_as_py3str = True # True - py3str_as_py2str = False # false means py2 will get unicode def __init__( self, stream: ReadIO, channel_or_gateway: Channel | BaseGateway | None = None, - strconfig: tuple[bool, bool] | None = None, ) -> None: if isinstance(channel_or_gateway, Channel): gw: BaseGateway | None = channel_or_gateway.gateway else: gw = channel_or_gateway - if channel_or_gateway is not None: - strconfig = channel_or_gateway._strconfig - if strconfig: - self.py2str_as_py3str, self.py3str_as_py2str = strconfig self.stream = stream if gw is None: self.channelfactory = None @@ -1219,25 +1197,10 @@ def _read_byte_string(self) -> bytes: as_bytes = self.stream.read(length) return as_bytes - def load_py3string(self) -> None: - as_bytes = self._read_byte_string() - if self.py3str_as_py2str: - # XXX Should we try to decode into latin-1? - self.stack.append(as_bytes) - else: - self.stack.append(as_bytes.decode("utf-8")) - - num2func[opcode.PY3STRING] = load_py3string - - def load_py2string(self) -> None: - as_bytes = self._read_byte_string() - if self.py2str_as_py3str: - s: bytes | str = as_bytes.decode("latin-1") - else: - s = as_bytes - self.stack.append(s) + def load_string(self) -> None: + self.stack.append(self._read_byte_string().decode("utf-8")) - num2func[opcode.PY2STRING] = load_py2string + num2func[opcode.STRING] = load_string def load_bytes(self) -> None: s = self._read_byte_string() @@ -1245,11 +1208,6 @@ def load_bytes(self) -> None: num2func[opcode.BYTES] = load_bytes - def load_unicode(self) -> None: - self.stack.append(self._read_byte_string().decode("utf-8")) - - num2func[opcode.UNICODE] = load_unicode - def load_newlist(self) -> None: length = self._read_int4() self.stack.append([None] * length) @@ -1323,46 +1281,27 @@ def dump(byteio, obj: object) -> None: _Serializer(write=byteio.write).save(obj, versioned=True) -def loads( - bytestring: bytes, py2str_as_py3str: bool = False, py3str_as_py2str: bool = False -) -> Any: +def loads(bytestring: bytes) -> Any: """Deserialize the given bytestring to an object. - py2str_as_py3str: If true then string (str) objects previously - dumped on Python2 will be loaded as Python3 - strings which really are text objects. - py3str_as_py2str: If true then string (str) objects previously - dumped on Python3 will be loaded as Python2 - strings instead of unicode objects. - If the bytestring was dumped with an incompatible protocol version or if the bytestring is corrupted, the ``execnet.DataFormatError`` will be raised. """ - io = BytesIO(bytestring) - return load( - io, py2str_as_py3str=py2str_as_py3str, py3str_as_py2str=py3str_as_py2str - ) + return load(BytesIO(bytestring)) -def load( - io: ReadIO, py2str_as_py3str: bool = False, py3str_as_py2str: bool = False -) -> Any: +def load(io: ReadIO) -> Any: """Derserialize an object form the specified stream. - Behaviour and parameters are otherwise the same as with ``loads`` + Behaviour is otherwise the same as with ``loads`` """ - strconfig = (py2str_as_py3str, py3str_as_py2str) - return Unserializer(io, strconfig=strconfig).load(versioned=True) + return Unserializer(io).load(versioned=True) -def loads_internal( - bytestring: bytes, - channelfactory=None, - strconfig: tuple[bool, bool] | None = None, -) -> Any: +def loads_internal(bytestring: bytes, channelfactory=None) -> Any: io = BytesIO(bytestring) - return Unserializer(io, channelfactory, strconfig).load() + return Unserializer(io, channelfactory).load() def dumps_internal(obj: object) -> bytes: @@ -1420,7 +1359,7 @@ def save_bytes(self, bytes_: bytes) -> None: self._write_byte_sequence(bytes_) def save_str(self, s: str) -> None: - self._write(opcode.PY3STRING) + self._write(opcode.STRING) self._write_unicode_string(s) def _write_unicode_string(self, s: str) -> None: diff --git a/src/execnet/rsync.py b/src/execnet/rsync.py index f92bc37a..876e38fc 100644 --- a/src/execnet/rsync.py +++ b/src/execnet/rsync.py @@ -173,7 +173,6 @@ def itemcallback(req) -> None: self._receivequeue.put((channel, req)) channel = gateway.remote_exec(execnet.rsync_remote) - channel.reconfigure(py2str_as_py3str=False, py3str_as_py2str=False) channel.setcallback(itemcallback, endmarker=None) channel.send((str(destdir), options)) self._channels[channel] = finishedcallback diff --git a/testing/test_compatibility_regressions.py b/testing/test_compatibility_regressions.py index 5343e7d5..837c0c6d 100644 --- a/testing/test_compatibility_regressions.py +++ b/testing/test_compatibility_regressions.py @@ -18,13 +18,13 @@ def test_opcodes() -> None: "NEWDICT": b"J", "NEWLIST": b"K", "NONE": b"L", - "PY2STRING": b"M", - "PY3STRING": b"N", + # b"M" (py2 str) and b"S" (py2 unicode) are retired along with + # Python2 support -- the bytes stay unused rather than reassigned. + "STRING": b"N", "SET": b"O", "SETITEM": b"P", "STOP": b"Q", "TRUE": b"R", - "UNICODE": b"S", # added in 1.4 # causes a regression since it was ordered in # between CHANNEL and FALSE as "C" moving the other items diff --git a/testing/test_serializer.py b/testing/test_serializer.py index 49332c97..ec8e86ae 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -187,6 +187,7 @@ def test_tuple_nested_with_empty_in_between(dump, load) -> None: assert s == "(1, (), 3)" -def test_py2_string_loads() -> None: - """Regression test for #267.""" - assert execnet.gateway_base.loads(b"\x02M\x00\x00\x00\x01aQ") == b"a" +def test_py2_string_opcode_is_retired() -> None: + """The py2 ``str`` opcode ``M`` is gone; only Python2 ever emitted it.""" + with pytest.raises(execnet.DataFormatError, match="unknown opcode"): + execnet.gateway_base.loads(b"\x02M\x00\x00\x00\x01aQ") diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index a6f27981..ef52e232 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -403,16 +403,3 @@ async def main() -> None: await group.makegateway("id=notype") trio.run(main) - - -def test_channel_reconfigure_string_coercion() -> None: - async def main() -> None: - async with gateway_pair() as (left, right): - sender = left.open_channel() - receiver = right.open_channel(sender.id) - await sender.reconfigure(py3str_as_py2str=True) - await receiver.send("text") - # our side now loads py3 strings as bytes - assert await sender.receive() == b"text" - - trio.run(main)