feat(watchdog): supervise the netns sidecars, on their own health

Dockerfile.test

@@ -25,7 +25,7 @@ COPY tests ./tests
 # docker-compose.yml.j2 is here because the compose-generation test renders the
 # REAL template — a stub would assert nothing about what actually ships.
 COPY requirements-api.txt requirements-mcpunifier.txt docker-compose.yml.example docker-compose.yml.j2 run.sh ./
-COPY scripts/config_helper.py scripts/start.bat scripts/check_health.py scripts/healthcheck.sh scripts/verify_binaries.py scripts/wickworks-healthcheck.py scripts/recreate-vm.sh ./scripts/
+COPY scripts/config_helper.py scripts/start.bat scripts/check_health.py scripts/healthcheck.sh scripts/verify_binaries.py scripts/wickworks-healthcheck.py scripts/vm-watchdog.py scripts/recreate-vm.sh ./scripts/
 COPY assets/binaries.lock.json ./assets/
 
 ENV PYTHONPATH=/app

Dockerfile.watchdog

@@ -0,0 +1,30 @@
+# VM crash watchdog.
+#
+# Recovery is a coordinated compose recreate (VM + the sidecars sharing its
+# netns), not a Docker-API restart — see scripts/vm-watchdog.py for why a
+# restart strands the sidecar. That means this image needs the docker CLI and
+# the compose plugin, plus bash and PyYAML for scripts/recreate-vm.sh, which is
+# the helper it shells out to.
+#
+# Pinned by digest, not tag: this container mounts the Docker socket, which is
+# root-equivalent on the host, and a moving tag would hand that to whatever the
+# registry serves tomorrow. The digest is the multi-arch OCI index for
+# python:3.12-alpine, so it still resolves per-platform.
+# To update: docker buildx imagetools inspect python:3.12-alpine
+FROM python:3.12-alpine@sha256:d09d15e60962ca365d1cd544a48773bac9d33f2fb1b00f2aa0deec78ade7dc31
+
+# docker-cli-compose provides `docker compose`; recreate-vm.sh is bash and
+# parses the compose file with PyYAML. The Python dependency is pinned by
+# version and hash (see requirements-watchdog.txt for why); --require-hashes
+# makes pip fail closed on anything not in that file.
+COPY requirements-watchdog.txt /tmp/requirements-watchdog.txt
+RUN apk add --no-cache bash docker-cli docker-cli-compose \
+    && pip install --no-cache-dir --require-hashes -r /tmp/requirements-watchdog.txt \
+    && rm /tmp/requirements-watchdog.txt
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
+
+# The watchdog script and recreate-vm.sh stay bind-mounted by compose rather
+# than baked in, so a fix to either is a container restart and not a rebuild.
+CMD ["python", "-u", "/vm-watchdog.py"]

docker-compose.yml.example

@@ -91,21 +91,48 @@ services:
       - ./scripts/rotate-logs.sh:/rotate.sh:ro
     command: ["sh", "/rotate.sh"]
 
-  # nginx is the single entry point for all terminal APIs. Routes
-  # /<broker>/<account>/... to mt5:<terminal_port> (per-terminal Python
-  # API process inside the Windows VM, reachable via mt5 container's
-  # iptables DNAT). Auto-generated from config/config.yaml by run.sh.
-  # Bound to 127.0.0.1:8888 so it's loopback-only by default — LAN
-  # exposure is opt-in (change the host bind), tailnet exposure is via
-  # the optional tailscale sidecar below.
-  # Unified MCP endpoint. One MCP session that reaches every terminal, with
-  # broker/account as tool parameters, instead of one endpoint per terminal.
-  # The per-terminal /<broker>/<account>/mcp endpoints keep working unchanged;
-  # nginx routes /mcp/ here.
-  #
-  # Reads the same config/config.yaml that generates the nginx routing, so it
-  # cannot route somewhere nginx does not. It never waits for terminals: a
-  # terminal that is down fails only the calls naming it.
+  # VM crash watchdog. dockurr/windows keeps the container up while the
+  # Windows guest may have crashed internally, so restart: unless-stopped
+  # never fires and every terminal API in that VM stays dead. This sidecar
+  # polls Docker health through the socket and restarts a VM only after its
+  # health has stayed unhealthy for a sustained FailingStreak, with
+  # exponential backoff and bounded retries; state lives on a named volume.
+  # Runs inside the compose project (docker compose up -d), no host cron.
+  # See scripts/vm-watchdog.py.
+  vm-watchdog:
+    # Needs the docker CLI + compose plugin to run scripts/recreate-vm.sh, so
+    # it is built rather than pulled. The base image is digest-pinned inside
+    # the Dockerfile (this container mounts the Docker socket).
+    build:
+      context: .
+      dockerfile: Dockerfile.watchdog
+    restart: unless-stopped
+    command: ["python", "-u", "/vm-watchdog.py"]
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./scripts/vm-watchdog.py:/vm-watchdog.py:ro
+      - vm-watchdog-state:/state
+      # The project itself, at the SAME absolute path the host uses. Compose
+      # resolves the relative bind mounts in this file client-side, so a
+      # different path in here would rewrite every mount to somewhere that
+      # does not exist on the host. run.sh exports MT5_PROJECT_DIR; the
+      # watchdog refuses to act (and says so at startup) if it is unset.
+      - ${MT5_PROJECT_DIR:?MT5_PROJECT_DIR must be the absolute host path of this project (run.sh exports it; otherwise set it in .env)}:${MT5_PROJECT_DIR}:ro
+    environment:
+      WATCHDOG_STATE_DIR: /state
+      WATCHDOG_PROJECT_DIR: ${MT5_PROJECT_DIR}
+      WATCHDOG_RECREATE_SCRIPT: ${MT5_PROJECT_DIR}/scripts/recreate-vm.sh
+      # The helper the watchdog runs calls `docker compose`, which interpolates
+      # ${MT5_PROJECT_DIR:?} in THIS file again. The watchdog sets it in the
+      # helper's environment itself; it is handed through here as well so a
+      # human running `docker compose exec vm-watchdog …/recreate-vm.sh` gets
+      # the same environment the watchdog uses.
+      MT5_PROJECT_DIR: ${MT5_PROJECT_DIR}
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
   mcpunifier:
     build:
       context: .
@@ -198,3 +225,8 @@ services:
   #     - NET_RAW
   #   depends_on:
   #     - nginx
+
+volumes:
+  # Persistent per-container watchdog state (last restart, attempts,
+  # healthy-since) so backoff survives the watchdog's own restarts.
+  vm-watchdog-state:

docker-compose.yml.j2

@@ -120,6 +120,48 @@ services:
       - ./scripts/rotate-logs.sh:/rotate.sh:ro
     command: ["sh", "/rotate.sh"]
 
+  # VM crash watchdog. dockurr/windows keeps the container up while the
+  # Windows guest may have crashed internally, so restart: unless-stopped
+  # never fires and every terminal API in that VM stays dead. This sidecar
+  # polls Docker health through the socket and restarts a VM only after its
+  # health has stayed unhealthy for a sustained FailingStreak, with
+  # exponential backoff and bounded retries; state lives on a named volume.
+  # Runs inside the compose project (docker compose up -d), no host cron.
+  # See scripts/vm-watchdog.py.
+  vm-watchdog:
+    # Needs the docker CLI + compose plugin to run scripts/recreate-vm.sh, so
+    # it is built rather than pulled. The base image is digest-pinned inside
+    # the Dockerfile (this container mounts the Docker socket).
+    build:
+      context: .
+      dockerfile: Dockerfile.watchdog
+    restart: unless-stopped
+    command: ["python", "-u", "/vm-watchdog.py"]
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./scripts/vm-watchdog.py:/vm-watchdog.py:ro
+      - vm-watchdog-state:/state
+      # The project itself, at the SAME absolute path the host uses. Compose
+      # resolves the relative bind mounts in this file client-side, so a
+      # different path in here would rewrite every mount to somewhere that
+      # does not exist on the host. run.sh exports MT5_PROJECT_DIR; the
+      # watchdog refuses to act (and says so at startup) if it is unset.
+      - ${MT5_PROJECT_DIR:?MT5_PROJECT_DIR must be the absolute host path of this project (run.sh exports it; otherwise set it in .env)}:${MT5_PROJECT_DIR}:ro
+    environment:
+      WATCHDOG_STATE_DIR: /state
+      WATCHDOG_PROJECT_DIR: ${MT5_PROJECT_DIR}
+      WATCHDOG_RECREATE_SCRIPT: ${MT5_PROJECT_DIR}/scripts/recreate-vm.sh
+      # The helper the watchdog runs calls `docker compose`, which interpolates
+      # ${MT5_PROJECT_DIR:?} in THIS file again. The watchdog sets it in the
+      # helper's environment itself; it is handed through here as well so a
+      # human running `docker compose exec vm-watchdog …/recreate-vm.sh` gets
+      # the same environment the watchdog uses.
+      MT5_PROJECT_DIR: ${MT5_PROJECT_DIR}
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
 {% if enable_mcpunifier|default(true) %}
   # Unified MCP endpoint. One MCP session that reaches every terminal, with
   # broker/account as tool parameters, instead of one endpoint per terminal.
@@ -229,3 +271,8 @@ services:
   #     - NET_RAW
   #   depends_on:
   #     - nginx
+
+volumes:
+  # Persistent per-container watchdog state (last restart, attempts,
+  # healthy-since) so backoff survives the watchdog's own restarts.
+  vm-watchdog-state:

docs/operations.md

@@ -10,6 +10,7 @@ The commands, ports, tunnels, locks, queues, and logs you need after the thing b
 - [Cloudflare Tunnel](#cloudflare-tunnel-optional)
 - [Project structure](#project-structure)
 - [Concurrency and backpressure](#concurrency-and-backpressure)
+- [Auto-recovery](#auto-recovery)
 - [Logs](#logs)
 
 ## Make Targets
@@ -251,6 +252,172 @@ or, with the sidecar list discovered from the generated compose file:
 
 This is covered by the real Compose lifecycle regression in
 `tests/integration/test_wickworks_lifecycle.py`.
+## Auto-recovery
+
+The Windows VM(s) run inside `dockurr/windows` containers with a Docker healthcheck
+(`scripts/healthcheck.sh`) that probes every terminal port this VM owns. A crash
+inside the guest — an unexpected shutdown (Event 6008), a wedged terminal, an
+OOM — leaves the **container** up while the **API** is dead, so
+`restart: unless-stopped` never fires and nothing recovers it on its own.
+
+### VM crash watchdog
+
+`vm-watchdog` is a Compose-managed sidecar for exactly that case. It is part of
+the project (`docker compose up -d` brings it up with everything else) — no
+host cron, no systemd unit, no machine-specific checkout path. It polls Docker
+health through the mounted unix socket and uses Docker's own
+`State.Health.FailingStreak` as the source of truth.
+
+Behavior:
+
+- Scopes itself to this Compose project (`com.docker.compose.project`,
+  discovered from its own container labels) and to the `dockurr/windows` VM
+  image, so the VM sweep only ever recovers the VM containers — never nginx,
+  the log rotator, or other containers. The netns sidecars get a second,
+  narrower sweep of their own (below).
+- **Recovers by recreating the VM together with its netns sidecars**, by
+  running `scripts/recreate-vm.sh <service>` — the same helper documented above
+  and covered by `tests/integration/test_wickworks_lifecycle.py`. It does not
+  use `docker restart`: that keeps the container ID but gives the VM a fresh
+  netns on start, which strands the wickworks sidecar exactly as recreating the
+  VM alone does.
+- Acts **only** after its Docker health has stayed `unhealthy`
+  for `WATCHDOG_MIN_FAILING_STREAK` consecutive healthcheck failures (default
+  `10`, i.e. ~5 minutes at the default 30s interval). A container that is
+  healthy or still starting is never touched, so running backtests on a working
+  VM are never interrupted — the healthcheck stays green the whole time a
+  terminal is serving.
+- Keeps a tiny state record per VM on a named volume, keyed by compose
+  project + service (`/state/<project>.<service>.json` — stable across the
+  recreate that recovery performs, unlike a container id): last restart,
+  attempt count, and when the VM
+  was last observed healthy.
+- Enforces exponential backoff between recovery attempts
+  (`WATCHDOG_BACKOFF_ATTEMPTS`, default `300,900,3600` — 5m → 15m → 1h), so a
+  VM that crashes again immediately after recovery is not restarted into a
+  loop.
+- Stops after `WATCHDOG_MAX_ATTEMPTS` consecutive failed recoveries (default
+  `3`) and logs loudly, instead of threshing forever.
+- Resets the attempt budget only after the VM has stayed **continuously**
+  healthy for `WATCHDOG_RESET_SECONDS` (default `1800`), so a VM that recovered
+  then crashed later gets a fresh budget. Any non-healthy observation — a
+  `starting` container after a restart, or an `unhealthy` poll below the streak
+  threshold — restarts that clock; it does not carry over from an earlier
+  healthy run.
+- Never selects itself, whatever `WATCHDOG_IMAGE_FILTER` is set to — it resolves
+  its own full container id at startup and excludes it by equality (falling
+  back to Docker's short-id hostname only if that inspect fails) — and matches
+  the image **repository exactly** (`dockurr/windows`, `dockurr/windows:5.14`,
+  `dockurr/windows@sha256:…` — not `dockurr/windows-something`).
+- `WATCHDOG_DRY_RUN=1` (or `--dry-run`) prints what it would do without
+  touching any container.
+
+#### The stranded sidecar, which no VM ever reports
+
+A sidecar joins its VM with `network_mode: service:<vm>`. Docker resolves that
+**once**, at the sidecar's own start, into an immutable
+`NetworkMode=container:<owner-id>`, and it builds a **fresh namespace every
+time the owner starts**. So restarting the owner strands the sidecar: the
+container id is unchanged, so nothing about the binding looks wrong, but the
+namespace it points at is gone. The VM comes back perfectly healthy and the VM
+sweep has nothing to act on.
+
+That is not hypothetical here. On 2026-09-07 the `mt5` container exited cleanly
+and `restart: unless-stopped` brought it back; its `wickworks` sidecar sat in
+the dead namespace for **two days** — `FailingStreak` 13,700, only `lo` left,
+every `/rates/ta` call 502-ing — with the VM green throughout.
+
+The failing streak is the point. The sidecar's own healthcheck saw the fault
+the whole time. There was simply no supervisor for it. So a second sweep
+follows the VM sweep and applies the **same rule** — Docker health, past the
+same `WATCHDOG_MIN_FAILING_STREAK`, the same backoff and attempt cap — to the
+netns sidecars:
+
+- It recreates **that sidecar alone** (`recreate-vm.sh <sidecar>` →
+  `up -d --force-recreate --no-deps`). Nothing declares
+  `network_mode: service:<sidecar>`, so the helper's own discovery returns
+  nothing for it and exactly one container is touched. That is the documented
+  repair, it is what an operator does by hand, and it leaves the VM and its
+  terminals alone.
+- It acts **only while the owner is a running, healthy VM of this project**. An
+  unhealthy owner belongs to the VM sweep, which recreates owner and sidecars
+  together. An owner that is not running is left alone too, and that one is
+  load-bearing rather than a default: `/containers/json` lists running
+  containers only, so a **stopped** owner is indistinguishable from a destroyed
+  one. Recreating a sidecar under a stopped owner cannot work — the helper
+  stops it first, then `up --no-deps` has no namespace to join — so the sidecar
+  would end up stopped, invisible to both sweeps, and never retried. A
+  `docker compose stop mt5` for maintenance must not cost you the sidecar.
+- `container:<name>` and `container:<short-id>` are legal to write by hand and
+  are not normalised anywhere, so the owner reference is matched by full id, by
+  a 12-character-or-longer prefix, and by container name.
+
+**This makes an orphan-aware sidecar healthcheck a requirement, not a nicety.**
+A check that only probes loopback stays green inside a dead namespace, and
+Docker health is this daemon's only source of truth, so nothing here will ever
+fire for it. `scripts/wickworks-healthcheck.py` is the worked example: it
+probes the owner's gateway services, which disappear the moment the namespace
+does.
+
+Set `WATCHDOG_WATCH_SIDECARS=0` to restore the VM-only scope.
+
+Environment overrides: `WATCHDOG_INTERVAL_SECONDS`, `WATCHDOG_MIN_FAILING_STREAK`,
+`WATCHDOG_IMAGE_FILTER`, `WATCHDOG_WATCH_SIDECARS`, `WATCHDOG_BACKOFF_ATTEMPTS`,
+`WATCHDOG_MAX_ATTEMPTS`, `WATCHDOG_RESET_SECONDS`, `WATCHDOG_COMPOSE_PROJECT`,
+`WATCHDOG_STATE_DIR`, `WATCHDOG_DOCKER_SOCKET`, `WATCHDOG_DRY_RUN`,
+`WATCHDOG_PROJECT_DIR`, `WATCHDOG_RECREATE_SCRIPT`, `WATCHDOG_RECREATE_TIMEOUT`.
+
+`WATCHDOG_PROJECT_DIR` is the **host** path of this project, and the compose
+service mounts the project through at that same absolute path. Compose resolves
+the relative bind mounts in `docker-compose.yml` client-side, so a
+container-local path would rewrite every mount to something that does not exist
+on the host. If it is missing the watchdog reports it at startup and refuses to
+act, rather than falling back to a restart that looks like recovery and is not.
+
+That path reaches compose as `MT5_PROJECT_DIR`, and compose interpolates it on
+**every** command against `docker-compose.yml`, not only the first `up`:
+
+- `run.sh` exports it for its own run **and writes it to `.env`**, so `make
+  down`, `make logs` and a manual `docker compose …` keep working after `run.sh`
+  has exited. Starting the stack some other way? Put
+  `MT5_PROJECT_DIR=<absolute host path of this directory>` in `.env` yourself.
+- The watchdog sets it explicitly in the environment of the `recreate-vm.sh` it
+  runs (from its own `WATCHDOG_PROJECT_DIR`), because the container is not
+  handed the host's shell variables. `tests/test_vm_watchdog.py` runs the real
+  helper under exactly that environment, and
+  `tests/integration/test_vm_watchdog_lifecycle.py` drives a real recovery
+  through the built sidecar on a disposable Compose project.
+
+### Busy is not dead — and hung is not busy
+
+`healthcheck.sh` reports a port **healthy** when the TCP handshake completes but
+no HTTP answer arrives inside the probe window: something is listening, the
+guest is just saturated (a compile, a Strategy Tester run). Restarting a VM for
+being busy would turn a slow batch into an outage.
+
+That tolerance is **bounded**. A port that accepts TCP but stays silent for
+`HEALTHCHECK_SLOW_GRACE` consecutive checks (default `10`, ≈5 minutes at the 30s
+interval) is reported as `hung` and the check fails — from there the watchdog's
+own streak gate (`WATCHDOG_MIN_FAILING_STREAK`, another ≈5 minutes) applies, so
+a wedged API is recovered in roughly ten minutes rather than never. The
+per-port counters live in `HEALTHCHECK_STATE_DIR` (default `/tmp/healthcheck-slow`
+inside the VM container); an HTTP answer or a refused connection resets a port's
+count, and a recreate starts every count from zero. A refused connection
+(nothing listening) is `DOWN` immediately, as before. If the counters cannot be
+written (a full disk), the bound is off for that check and the verdict says so:
+`ok (slow but listening: …) [slow-state unwritable: hung detection off]`.
+
+**Blast radius.** One hung terminal API is enough to mark the whole VM `DOWN`,
+and the watchdog's recovery is the whole VM — every other terminal on it, and
+whatever they were running, goes with it. That is the same rule the check has
+always applied to a dead port; it is just now applied to a hung one after the
+grace. When you catch a single wedged terminal before the watchdog does,
+`POST /terminal/restart` on that terminal (see `docs/rest-api.md`) is the
+cheaper first response.
+
+This complements the in-VM `MT5AutoReboot` scheduled task, which reboots on a
+fixed timer and can interrupt long-running backtests; operators who disable that
+task still get crash recovery from the watchdog.
 
 ## Logs