Fix nested Docker exec root isolation

.github/workflows/server-test.yaml

@@ -62,6 +62,38 @@ jobs:
         run: make test-runtime
         working-directory: server
 
+  test-hypeman-nested-docker:
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    needs: [build-headless]
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: "server/go.mod"
+          cache: true
+          cache-dependency-path: server/go.sum
+
+      - name: Compute short SHA for image
+        id: vars
+        shell: bash
+        run: echo "short_sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
+
+      - name: Test nested Docker exec on Hypeman
+        working-directory: server
+        run: go test -v -count=1 -timeout 15m ./e2e/ -run '^TestNestedDockerExecUsesContainerRoot$'
+        env:
+          KI_E2E_BACKEND: hypeman
+          HYPEMAN_BASE_URL: ${{ vars.HYPEMAN_BASE_URL }}
+          HYPEMAN_API_KEY: ${{ secrets.HYPEMAN_API_KEY }}
+          E2E_CHROMIUM_HEADLESS_IMAGE: onkernel/chromium-headless:${{ steps.vars.outputs.short_sha }}
+
   test-server-e2e:
     runs-on: ubuntu-latest
     needs: [build-headful, build-headless]

images/chromium-headful/Dockerfile

@@ -39,6 +39,19 @@ RUN --mount=type=cache,target=/root/.cache/go-build,id=$CACHEIDPREFIX-go-build \
     GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \
     go build -ldflags="-s -w" -o /out/kernel-images-supervisord-shim ./cmd/supervisord-shim
 
+FROM docker.io/golang:1.25.0-bookworm AS runc-hypeman-builder
+ARG TARGETARCH
+ARG TARGETOS
+ARG CACHEIDPREFIX=${TARGETOS:-linux}-${TARGETARCH:-amd64}-runc-hypeman
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=$CACHEIDPREFIX-apt-cache \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked,id=$CACHEIDPREFIX-apt-lib \
+    apt-get update && \
+    apt-get --no-install-recommends -y install ca-certificates curl libseccomp-dev patch pkg-config
+COPY shared/docker/build-runc-hypeman.sh shared/docker/runc-hypeman.patch /build/
+RUN --mount=type=cache,target=/root/.cache/go-build,id=$CACHEIDPREFIX-go-build \
+    --mount=type=cache,target=/go/pkg/mod,id=$CACHEIDPREFIX-go-pkg-mod \
+    /build/build-runc-hypeman.sh /out/runc-hypeman
+
 # webrtc client
 FROM node:22-bullseye-slim AS client
 WORKDIR /src
@@ -353,6 +366,7 @@ COPY --from=docker \
     /usr/local/bin/dockerd \
     /usr/local/bin/runc \
     /usr/local/bin/
+COPY --from=runc-hypeman-builder /out/runc-hypeman /usr/local/bin/runc-hypeman
 COPY shared/docker/daemon.json /etc/docker/daemon.json
 COPY shared/docker/install-docker-dependencies.sh /tmp/install-docker-dependencies.sh
 RUN /tmp/install-docker-dependencies.sh && rm /tmp/install-docker-dependencies.sh && \

images/chromium-headless/image/Dockerfile

@@ -40,6 +40,19 @@ RUN --mount=type=cache,target=/root/.cache/go-build,id=$CACHEIDPREFIX-go-build \
     GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \
     go build -ldflags="-s -w" -o /out/kernel-images-supervisord-shim ./cmd/supervisord-shim
 
+FROM docker.io/golang:1.25.0-bookworm AS runc-hypeman-builder
+ARG TARGETARCH
+ARG TARGETOS
+ARG CACHEIDPREFIX=${TARGETOS:-linux}-${TARGETARCH:-amd64}-runc-hypeman
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=$CACHEIDPREFIX-apt-cache \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked,id=$CACHEIDPREFIX-apt-lib \
+    apt-get update && \
+    apt-get --no-install-recommends -y install ca-certificates curl libseccomp-dev patch pkg-config
+COPY shared/docker/build-runc-hypeman.sh shared/docker/runc-hypeman.patch /build/
+RUN --mount=type=cache,target=/root/.cache/go-build,id=$CACHEIDPREFIX-go-build \
+    --mount=type=cache,target=/go/pkg/mod,id=$CACHEIDPREFIX-go-pkg-mod \
+    /build/build-runc-hypeman.sh /out/runc-hypeman
+
 FROM docker.io/ubuntu:22.04 AS ffmpeg-downloader
 
 # Allow cross-compilation when building with BuildKit platforms
@@ -244,6 +257,7 @@ COPY --from=docker \
     /usr/local/bin/dockerd \
     /usr/local/bin/runc \
     /usr/local/bin/
+COPY --from=runc-hypeman-builder /out/runc-hypeman /usr/local/bin/runc-hypeman
 COPY shared/docker/daemon.json /etc/docker/daemon.json
 COPY shared/docker/install-docker-dependencies.sh /tmp/install-docker-dependencies.sh
 RUN /tmp/install-docker-dependencies.sh && rm /tmp/install-docker-dependencies.sh && \

server/e2e/e2e_nested_docker_test.go

@@ -0,0 +1,151 @@
+package e2e
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/coder/websocket"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNestedDockerExecUsesContainerRoot(t *testing.T) {
+	if backendKindFromEnv() != BackendHypeman {
+		t.Skip("nested Docker rootfs regression requires the Hypeman backend")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+
+	c := NewTestContainer(t, headlessImage)
+	require.NoError(t, c.Start(ctx, ContainerConfig{}), "failed to start Hypeman instance")
+	defer func() {
+		cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cleanupCancel()
+		require.NoError(t, c.Stop(cleanupCtx), "failed to stop Hypeman instance")
+	}()
+	const script = `set -euxo pipefail
+rm -f /var/run/docker.pid /var/run/docker.sock
+rm -rf /var/lib/docker-exec-e2e /run/docker-exec-e2e
+
+dockerd \
+  --host=unix:///var/run/docker.sock \
+  --data-root=/var/lib/docker-exec-e2e \
+  --exec-root=/run/docker-exec-e2e \
+  >/tmp/docker-exec-e2e.log 2>&1 &
+dockerd_pid=$!
+cleanup() {
+  docker rm -f nested-exec-e2e >/dev/null 2>&1 || true
+  kill "$dockerd_pid" >/dev/null 2>&1 || true
+  wait "$dockerd_pid" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+for _ in $(seq 1 60); do
+  if docker info >/dev/null 2>&1; then
+    break
+  fi
+  sleep 1
+done
+if ! docker info >/dev/null 2>&1; then
+  cat /tmp/docker-exec-e2e.log >&2
+  exit 1
+fi
+
+docker pull alpine:3.20 >/dev/null
+docker run -d --name nested-exec-e2e alpine:3.20 \
+  sh -c 'echo pid1-created >/runtime-created; exec sleep 300' >/dev/null
+container_pid=$(docker inspect --format '{{.State.Pid}}' nested-exec-e2e)
+for _ in $(seq 1 50); do
+  if test -f "/proc/$container_pid/root/runtime-created"; then
+    break
+  fi
+  sleep 0.1
+done
+
+pid1_image=$(cat "/proc/$container_pid/root/etc/alpine-release")
+exec_image=$(docker exec nested-exec-e2e cat /etc/alpine-release)
+printf 'pid1 image: %s\ndocker exec image: %s\n' "$pid1_image" "$exec_image"
+test "$exec_image" = "$pid1_image"
+test "$(docker exec nested-exec-e2e cat /runtime-created)" = pid1-created
+
+docker exec nested-exec-e2e sh -c 'echo exec-created >/exec-created'
+test "$(docker exec nested-exec-e2e cat /exec-created)" = exec-created
+`
+
+	exitCode, output, err := execViaHypemanControlPlane(ctx, c, []string{"bash", "-lc", script})
+	require.NoError(t, err, "failed to execute nested Docker regression")
+	require.Equalf(t, 0, exitCode, "nested docker exec used the wrong rootfs:\n%s", output)
+}
+
+func execViaHypemanControlPlane(ctx context.Context, c *TestContainer, command []string) (int, string, error) {
+	backend, ok := c.backend.(*hypemanBackend)
+	if !ok {
+		return -1, "", fmt.Errorf("backend is %T, not *hypemanBackend", c.backend)
+	}
+
+	execURL, err := url.Parse(backend.cfg.BaseURL)
+	if err != nil {
+		return -1, "", fmt.Errorf("parse Hypeman URL: %w", err)
+	}
+	switch execURL.Scheme {
+	case "https":
+		execURL.Scheme = "wss"
+	case "http":
+		execURL.Scheme = "ws"
+	default:
+		return -1, "", fmt.Errorf("unsupported Hypeman URL scheme %q", execURL.Scheme)
+	}
+	execURL.Path = strings.TrimSuffix(execURL.Path, "/") + "/instances/" + url.PathEscape(backend.instanceID) + "/exec"
+
+	header := http.Header{}
+	header.Set("Authorization", "Bearer "+backend.cfg.Token)
+	conn, _, err := websocket.Dial(ctx, execURL.String(), &websocket.DialOptions{HTTPHeader: header})
+	if err != nil {
+		return -1, "", fmt.Errorf("connect to Hypeman exec: %w", err)
+	}
+	defer conn.CloseNow()
+
+	request, err := json.Marshal(map[string]any{
+		"command":        command,
+		"timeout":        300,
+		"wait_for_agent": 60,
+	})
+	if err != nil {
+		return -1, "", fmt.Errorf("marshal Hypeman exec request: %w", err)
+	}
+	if err := conn.Write(ctx, websocket.MessageText, request); err != nil {
+		return -1, "", fmt.Errorf("write Hypeman exec request: %w", err)
+	}
+
+	var output strings.Builder
+	for {
+		messageType, message, err := conn.Read(ctx)
+		if err != nil {
+			return -1, output.String(), fmt.Errorf("read Hypeman exec response: %w", err)
+		}
+		if messageType == websocket.MessageBinary {
+			output.Write(message)
+			continue
+		}
+
+		var result struct {
+			ExitCode *int   `json:"exitCode"`
+			Error    string `json:"error"`
+		}
+		if err := json.Unmarshal(message, &result); err != nil {
+			return -1, output.String(), fmt.Errorf("decode Hypeman exec response %q: %w", message, err)
+		}
+		if result.Error != "" {
+			return -1, output.String(), fmt.Errorf("Hypeman exec: %s", result.Error)
+		}
+		if result.ExitCode != nil {
+			return *result.ExitCode, output.String(), nil
+		}
+	}
+}

shared/docker/build-runc-hypeman.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+RUNC_VERSION=1.3.4
+RUNC_ARCHIVE_SHA256=a9f9646c4c8990239f6462b408b22d9aa40ba0473a9fc642b9d6576126495eee
+PATCH_PATH=${RUNC_HYPEMAN_PATCH:-"$(dirname "$0")/runc-hypeman.patch"}
+OUTPUT_PATH=${1:-/out/runc-hypeman}
+
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+
+curl -fsSL --retry 3 --connect-timeout 15 \
+  -o "$tmp/runc.tar.gz" \
+  "https://github.com/opencontainers/runc/archive/refs/tags/v${RUNC_VERSION}.tar.gz"
+echo "$RUNC_ARCHIVE_SHA256  $tmp/runc.tar.gz" | sha256sum -c -
+mkdir "$tmp/source"
+tar -xzf "$tmp/runc.tar.gz" --strip-components=1 -C "$tmp/source"
+patch -d "$tmp/source" -p1 < "$PATCH_PATH"
+make -C "$tmp/source" static
+strip --strip-unneeded "$tmp/source/runc"
+install -D -m 0755 "$tmp/source/runc" "$OUTPUT_PATH"
+"$OUTPUT_PATH" --version

shared/docker/daemon.json

@@ -1,3 +1,9 @@
 {
-  "storage-driver": "fuse-overlayfs"
+  "storage-driver": "fuse-overlayfs",
+  "default-runtime": "hypeman-runc",
+  "runtimes": {
+    "hypeman-runc": {
+      "path": "/usr/local/bin/runc-hypeman"
+    }
+  }
 }

shared/docker/install-hypeman-docker-runtime.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+KERNEL_IMAGES_REF=${1:-}
+if [[ ! "$KERNEL_IMAGES_REF" =~ ^[0-9a-f]{40}$ ]]; then
+  echo "usage: $0 <kernel-images-commit-sha>" >&2
+  exit 2
+fi
+if [[ $(uname -m) != x86_64 ]]; then
+  echo "the dynamic Hypeman Docker runtime installer requires x86_64" >&2
+  exit 1
+fi
+if [[ $(id -u) -ne 0 ]]; then
+  echo "the dynamic Hypeman Docker runtime installer must run as root" >&2
+  exit 1
+fi
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get -o DPkg::Lock::Timeout=300 update
+apt-get -o DPkg::Lock::Timeout=300 --no-install-recommends -y install \
+  build-essential \
+  ca-certificates \
+  curl \
+  fuse-overlayfs \
+  iptables \
+  libseccomp-dev \
+  patch \
+  pkg-config
+update-alternatives --set iptables /usr/sbin/iptables-legacy
+update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+
+if ! command -v runc-hypeman >/dev/null 2>&1; then
+  tmp=$(mktemp -d)
+  trap 'rm -rf "$tmp"' EXIT
+
+  readonly raw_base="https://raw.githubusercontent.com/kernel/kernel-images/${KERNEL_IMAGES_REF}/shared/docker"
+  curl -fsSL --retry 3 --connect-timeout 15 \
+    -o "$tmp/build-runc-hypeman.sh" "$raw_base/build-runc-hypeman.sh"
+  curl -fsSL --retry 3 --connect-timeout 15 \
+    -o "$tmp/runc-hypeman.patch" "$raw_base/runc-hypeman.patch"
+  chmod 0755 "$tmp/build-runc-hypeman.sh"
+
+  readonly go_archive_sha256=2852af0cb20a13139b3448992e69b868e50ed0f8a1e5940ee1de9e19a123b613
+  curl -fsSL --retry 3 --connect-timeout 15 \
+    -o "$tmp/go.tar.gz" https://go.dev/dl/go1.25.0.linux-amd64.tar.gz
+  echo "$go_archive_sha256  $tmp/go.tar.gz" | sha256sum -c -
+  mkdir "$tmp/go"
+  tar -xzf "$tmp/go.tar.gz" --strip-components=1 -C "$tmp/go"
+
+  PATH="$tmp/go/bin:$PATH" \
+    RUNC_HYPEMAN_PATCH="$tmp/runc-hypeman.patch" \
+    "$tmp/build-runc-hypeman.sh" /usr/local/bin/runc-hypeman
+fi
+
+mkdir -p /etc/docker
+cat >/etc/docker/daemon.json <<'JSON'
+{
+  "storage-driver": "fuse-overlayfs",
+  "default-runtime": "hypeman-runc",
+  "runtimes": {
+    "hypeman-runc": {
+      "path": "/usr/local/bin/runc-hypeman"
+    }
+  }
+}
+JSON
+
+runc-hypeman --version
+if command -v dockerd >/dev/null 2>&1; then
+  dockerd --validate --config-file=/etc/docker/daemon.json
+fi

shared/docker/runc-hypeman.patch

@@ -0,0 +1,45 @@
+diff --git a/libcontainer/nsenter/nsexec.c b/libcontainer/nsenter/nsexec.c
+index 728e68b..b04a3b2 100644
+--- a/libcontainer/nsenter/nsexec.c
++++ b/libcontainer/nsenter/nsexec.c
+@@ -609,0 +610 @@ void join_namespaces(char *nsspec)
++	int rootfd = -1;
+@@ -617,0 +619,27 @@ void join_namespaces(char *nsspec)
++	/*
++	 * If runc itself runs inside a chroot, joining a container's mount
++	 * namespace does not update the calling process's root directory. Keep an
++	 * fd for the container init's root so the exec process can explicitly
++	 * enter it after setns.
++	 */
++	for (size_t i = 0; i < ns_len; i++) {
++		struct namespace_t *ns = &ns_list[i];
++		const char *suffix = "/ns/mnt";
++		size_t path_len = strlen(ns->path);
++		size_t suffix_len = strlen(suffix);
++
++		if (nstype(ns->type) != CLONE_NEWNS || path_len <= suffix_len ||
++		    strcmp(ns->path + path_len - suffix_len, suffix) != 0)
++			continue;
++
++		char rootpath[PATH_MAX];
++		int n = snprintf(rootpath, sizeof(rootpath), "%.*s/root",
++				 (int)(path_len - suffix_len), ns->path);
++		if (n < 0 || (size_t)n >= sizeof(rootpath))
++			bail("container root path is too long");
++		rootfd = open(rootpath, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
++		if (rootfd < 0)
++			bail("failed to open container root %s", rootpath);
++		break;
++	}
++
+@@ -637,0 +666,10 @@ void join_namespaces(char *nsspec)
++
++	if (rootfd >= 0) {
++		if (fchdir(rootfd) < 0)
++			bail("failed to change directory to container root");
++		if (chroot(".") < 0)
++			bail("failed to chroot to container root");
++		if (chdir("/") < 0)
++			bail("failed to change directory after chroot");
++		close(rootfd);
++	}