diff --git a/.github/workflows/server-test.yaml b/.github/workflows/server-test.yaml index 07288872..360dcb23 100644 --- a/.github/workflows/server-test.yaml +++ b/.github/workflows/server-test.yaml @@ -62,6 +62,38 @@ jobs: run: make test-runtime working-directory: server + test-hypeman-nested-docker: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + needs: [build-headless] + permissions: + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: "server/go.mod" + cache: true + cache-dependency-path: server/go.sum + + - name: Compute short SHA for image + id: vars + shell: bash + run: echo "short_sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + + - name: Test nested Docker exec on Hypeman + working-directory: server + run: go test -v -count=1 -timeout 15m ./e2e/ -run '^TestNestedDockerExecUsesContainerRoot$' + env: + KI_E2E_BACKEND: hypeman + HYPEMAN_BASE_URL: ${{ vars.HYPEMAN_BASE_URL }} + HYPEMAN_API_KEY: ${{ secrets.HYPEMAN_API_KEY }} + E2E_CHROMIUM_HEADLESS_IMAGE: onkernel/chromium-headless:${{ steps.vars.outputs.short_sha }} + test-server-e2e: runs-on: ubuntu-latest needs: [build-headful, build-headless] diff --git a/images/chromium-headful/Dockerfile b/images/chromium-headful/Dockerfile index db66f7a5..d2b9fbc8 100644 --- a/images/chromium-headful/Dockerfile +++ b/images/chromium-headful/Dockerfile @@ -39,6 +39,19 @@ RUN --mount=type=cache,target=/root/.cache/go-build,id=$CACHEIDPREFIX-go-build \ GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ go build -ldflags="-s -w" -o /out/kernel-images-supervisord-shim ./cmd/supervisord-shim +FROM docker.io/golang:1.25.0-bookworm AS runc-hypeman-builder +ARG TARGETARCH +ARG TARGETOS +ARG CACHEIDPREFIX=${TARGETOS:-linux}-${TARGETARCH:-amd64}-runc-hypeman +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=$CACHEIDPREFIX-apt-cache \ + --mount=type=cache,target=/var/lib/apt,sharing=locked,id=$CACHEIDPREFIX-apt-lib \ + apt-get update && \ + apt-get --no-install-recommends -y install ca-certificates curl libseccomp-dev patch pkg-config +COPY shared/docker/build-runc-hypeman.sh shared/docker/runc-hypeman.patch /build/ +RUN --mount=type=cache,target=/root/.cache/go-build,id=$CACHEIDPREFIX-go-build \ + --mount=type=cache,target=/go/pkg/mod,id=$CACHEIDPREFIX-go-pkg-mod \ + /build/build-runc-hypeman.sh /out/runc-hypeman + # webrtc client FROM node:22-bullseye-slim AS client WORKDIR /src @@ -353,6 +366,7 @@ COPY --from=docker \ /usr/local/bin/dockerd \ /usr/local/bin/runc \ /usr/local/bin/ +COPY --from=runc-hypeman-builder /out/runc-hypeman /usr/local/bin/runc-hypeman COPY shared/docker/daemon.json /etc/docker/daemon.json COPY shared/docker/install-docker-dependencies.sh /tmp/install-docker-dependencies.sh RUN /tmp/install-docker-dependencies.sh && rm /tmp/install-docker-dependencies.sh && \ diff --git a/images/chromium-headless/image/Dockerfile b/images/chromium-headless/image/Dockerfile index 4fbcadbd..1eae5bce 100644 --- a/images/chromium-headless/image/Dockerfile +++ b/images/chromium-headless/image/Dockerfile @@ -40,6 +40,19 @@ RUN --mount=type=cache,target=/root/.cache/go-build,id=$CACHEIDPREFIX-go-build \ GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ go build -ldflags="-s -w" -o /out/kernel-images-supervisord-shim ./cmd/supervisord-shim +FROM docker.io/golang:1.25.0-bookworm AS runc-hypeman-builder +ARG TARGETARCH +ARG TARGETOS +ARG CACHEIDPREFIX=${TARGETOS:-linux}-${TARGETARCH:-amd64}-runc-hypeman +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=$CACHEIDPREFIX-apt-cache \ + --mount=type=cache,target=/var/lib/apt,sharing=locked,id=$CACHEIDPREFIX-apt-lib \ + apt-get update && \ + apt-get --no-install-recommends -y install ca-certificates curl libseccomp-dev patch pkg-config +COPY shared/docker/build-runc-hypeman.sh shared/docker/runc-hypeman.patch /build/ +RUN --mount=type=cache,target=/root/.cache/go-build,id=$CACHEIDPREFIX-go-build \ + --mount=type=cache,target=/go/pkg/mod,id=$CACHEIDPREFIX-go-pkg-mod \ + /build/build-runc-hypeman.sh /out/runc-hypeman + FROM docker.io/ubuntu:22.04 AS ffmpeg-downloader # Allow cross-compilation when building with BuildKit platforms @@ -244,6 +257,7 @@ COPY --from=docker \ /usr/local/bin/dockerd \ /usr/local/bin/runc \ /usr/local/bin/ +COPY --from=runc-hypeman-builder /out/runc-hypeman /usr/local/bin/runc-hypeman COPY shared/docker/daemon.json /etc/docker/daemon.json COPY shared/docker/install-docker-dependencies.sh /tmp/install-docker-dependencies.sh RUN /tmp/install-docker-dependencies.sh && rm /tmp/install-docker-dependencies.sh && \ diff --git a/server/e2e/e2e_nested_docker_test.go b/server/e2e/e2e_nested_docker_test.go new file mode 100644 index 00000000..d87dbbbd --- /dev/null +++ b/server/e2e/e2e_nested_docker_test.go @@ -0,0 +1,151 @@ +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/coder/websocket" + "github.com/stretchr/testify/require" +) + +func TestNestedDockerExecUsesContainerRoot(t *testing.T) { + if backendKindFromEnv() != BackendHypeman { + t.Skip("nested Docker rootfs regression requires the Hypeman backend") + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + c := NewTestContainer(t, headlessImage) + require.NoError(t, c.Start(ctx, ContainerConfig{}), "failed to start Hypeman instance") + defer func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cleanupCancel() + require.NoError(t, c.Stop(cleanupCtx), "failed to stop Hypeman instance") + }() + const script = `set -euxo pipefail +rm -f /var/run/docker.pid /var/run/docker.sock +rm -rf /var/lib/docker-exec-e2e /run/docker-exec-e2e + +dockerd \ + --host=unix:///var/run/docker.sock \ + --data-root=/var/lib/docker-exec-e2e \ + --exec-root=/run/docker-exec-e2e \ + >/tmp/docker-exec-e2e.log 2>&1 & +dockerd_pid=$! +cleanup() { + docker rm -f nested-exec-e2e >/dev/null 2>&1 || true + kill "$dockerd_pid" >/dev/null 2>&1 || true + wait "$dockerd_pid" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +for _ in $(seq 1 60); do + if docker info >/dev/null 2>&1; then + break + fi + sleep 1 +done +if ! docker info >/dev/null 2>&1; then + cat /tmp/docker-exec-e2e.log >&2 + exit 1 +fi + +docker pull alpine:3.20 >/dev/null +docker run -d --name nested-exec-e2e alpine:3.20 \ + sh -c 'echo pid1-created >/runtime-created; exec sleep 300' >/dev/null +container_pid=$(docker inspect --format '{{.State.Pid}}' nested-exec-e2e) +for _ in $(seq 1 50); do + if test -f "/proc/$container_pid/root/runtime-created"; then + break + fi + sleep 0.1 +done + +pid1_image=$(cat "/proc/$container_pid/root/etc/alpine-release") +exec_image=$(docker exec nested-exec-e2e cat /etc/alpine-release) +printf 'pid1 image: %s\ndocker exec image: %s\n' "$pid1_image" "$exec_image" +test "$exec_image" = "$pid1_image" +test "$(docker exec nested-exec-e2e cat /runtime-created)" = pid1-created + +docker exec nested-exec-e2e sh -c 'echo exec-created >/exec-created' +test "$(docker exec nested-exec-e2e cat /exec-created)" = exec-created +` + + exitCode, output, err := execViaHypemanControlPlane(ctx, c, []string{"bash", "-lc", script}) + require.NoError(t, err, "failed to execute nested Docker regression") + require.Equalf(t, 0, exitCode, "nested docker exec used the wrong rootfs:\n%s", output) +} + +func execViaHypemanControlPlane(ctx context.Context, c *TestContainer, command []string) (int, string, error) { + backend, ok := c.backend.(*hypemanBackend) + if !ok { + return -1, "", fmt.Errorf("backend is %T, not *hypemanBackend", c.backend) + } + + execURL, err := url.Parse(backend.cfg.BaseURL) + if err != nil { + return -1, "", fmt.Errorf("parse Hypeman URL: %w", err) + } + switch execURL.Scheme { + case "https": + execURL.Scheme = "wss" + case "http": + execURL.Scheme = "ws" + default: + return -1, "", fmt.Errorf("unsupported Hypeman URL scheme %q", execURL.Scheme) + } + execURL.Path = strings.TrimSuffix(execURL.Path, "/") + "/instances/" + url.PathEscape(backend.instanceID) + "/exec" + + header := http.Header{} + header.Set("Authorization", "Bearer "+backend.cfg.Token) + conn, _, err := websocket.Dial(ctx, execURL.String(), &websocket.DialOptions{HTTPHeader: header}) + if err != nil { + return -1, "", fmt.Errorf("connect to Hypeman exec: %w", err) + } + defer conn.CloseNow() + + request, err := json.Marshal(map[string]any{ + "command": command, + "timeout": 300, + "wait_for_agent": 60, + }) + if err != nil { + return -1, "", fmt.Errorf("marshal Hypeman exec request: %w", err) + } + if err := conn.Write(ctx, websocket.MessageText, request); err != nil { + return -1, "", fmt.Errorf("write Hypeman exec request: %w", err) + } + + var output strings.Builder + for { + messageType, message, err := conn.Read(ctx) + if err != nil { + return -1, output.String(), fmt.Errorf("read Hypeman exec response: %w", err) + } + if messageType == websocket.MessageBinary { + output.Write(message) + continue + } + + var result struct { + ExitCode *int `json:"exitCode"` + Error string `json:"error"` + } + if err := json.Unmarshal(message, &result); err != nil { + return -1, output.String(), fmt.Errorf("decode Hypeman exec response %q: %w", message, err) + } + if result.Error != "" { + return -1, output.String(), fmt.Errorf("Hypeman exec: %s", result.Error) + } + if result.ExitCode != nil { + return *result.ExitCode, output.String(), nil + } + } +} diff --git a/shared/docker/build-runc-hypeman.sh b/shared/docker/build-runc-hypeman.sh new file mode 100755 index 00000000..eff5ec2e --- /dev/null +++ b/shared/docker/build-runc-hypeman.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +RUNC_VERSION=1.3.4 +RUNC_ARCHIVE_SHA256=a9f9646c4c8990239f6462b408b22d9aa40ba0473a9fc642b9d6576126495eee +PATCH_PATH=${RUNC_HYPEMAN_PATCH:-"$(dirname "$0")/runc-hypeman.patch"} +OUTPUT_PATH=${1:-/out/runc-hypeman} + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +curl -fsSL --retry 3 --connect-timeout 15 \ + -o "$tmp/runc.tar.gz" \ + "https://github.com/opencontainers/runc/archive/refs/tags/v${RUNC_VERSION}.tar.gz" +echo "$RUNC_ARCHIVE_SHA256 $tmp/runc.tar.gz" | sha256sum -c - +mkdir "$tmp/source" +tar -xzf "$tmp/runc.tar.gz" --strip-components=1 -C "$tmp/source" +patch -d "$tmp/source" -p1 < "$PATCH_PATH" +make -C "$tmp/source" static +strip --strip-unneeded "$tmp/source/runc" +install -D -m 0755 "$tmp/source/runc" "$OUTPUT_PATH" +"$OUTPUT_PATH" --version diff --git a/shared/docker/daemon.json b/shared/docker/daemon.json index 514d027b..45b00e11 100644 --- a/shared/docker/daemon.json +++ b/shared/docker/daemon.json @@ -1,3 +1,9 @@ { - "storage-driver": "fuse-overlayfs" + "storage-driver": "fuse-overlayfs", + "default-runtime": "hypeman-runc", + "runtimes": { + "hypeman-runc": { + "path": "/usr/local/bin/runc-hypeman" + } + } } diff --git a/shared/docker/install-hypeman-docker-runtime.sh b/shared/docker/install-hypeman-docker-runtime.sh new file mode 100755 index 00000000..ae32b5d6 --- /dev/null +++ b/shared/docker/install-hypeman-docker-runtime.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +KERNEL_IMAGES_REF=${1:-} +if [[ ! "$KERNEL_IMAGES_REF" =~ ^[0-9a-f]{40}$ ]]; then + echo "usage: $0 " >&2 + exit 2 +fi +if [[ $(uname -m) != x86_64 ]]; then + echo "the dynamic Hypeman Docker runtime installer requires x86_64" >&2 + exit 1 +fi +if [[ $(id -u) -ne 0 ]]; then + echo "the dynamic Hypeman Docker runtime installer must run as root" >&2 + exit 1 +fi + +export DEBIAN_FRONTEND=noninteractive +apt-get -o DPkg::Lock::Timeout=300 update +apt-get -o DPkg::Lock::Timeout=300 --no-install-recommends -y install \ + build-essential \ + ca-certificates \ + curl \ + fuse-overlayfs \ + iptables \ + libseccomp-dev \ + patch \ + pkg-config +update-alternatives --set iptables /usr/sbin/iptables-legacy +update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy + +if ! command -v runc-hypeman >/dev/null 2>&1; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + + readonly raw_base="https://raw.githubusercontent.com/kernel/kernel-images/${KERNEL_IMAGES_REF}/shared/docker" + curl -fsSL --retry 3 --connect-timeout 15 \ + -o "$tmp/build-runc-hypeman.sh" "$raw_base/build-runc-hypeman.sh" + curl -fsSL --retry 3 --connect-timeout 15 \ + -o "$tmp/runc-hypeman.patch" "$raw_base/runc-hypeman.patch" + chmod 0755 "$tmp/build-runc-hypeman.sh" + + readonly go_archive_sha256=2852af0cb20a13139b3448992e69b868e50ed0f8a1e5940ee1de9e19a123b613 + curl -fsSL --retry 3 --connect-timeout 15 \ + -o "$tmp/go.tar.gz" https://go.dev/dl/go1.25.0.linux-amd64.tar.gz + echo "$go_archive_sha256 $tmp/go.tar.gz" | sha256sum -c - + mkdir "$tmp/go" + tar -xzf "$tmp/go.tar.gz" --strip-components=1 -C "$tmp/go" + + PATH="$tmp/go/bin:$PATH" \ + RUNC_HYPEMAN_PATCH="$tmp/runc-hypeman.patch" \ + "$tmp/build-runc-hypeman.sh" /usr/local/bin/runc-hypeman +fi + +mkdir -p /etc/docker +cat >/etc/docker/daemon.json <<'JSON' +{ + "storage-driver": "fuse-overlayfs", + "default-runtime": "hypeman-runc", + "runtimes": { + "hypeman-runc": { + "path": "/usr/local/bin/runc-hypeman" + } + } +} +JSON + +runc-hypeman --version +if command -v dockerd >/dev/null 2>&1; then + dockerd --validate --config-file=/etc/docker/daemon.json +fi diff --git a/shared/docker/runc-hypeman.patch b/shared/docker/runc-hypeman.patch new file mode 100644 index 00000000..a0eb1c3b --- /dev/null +++ b/shared/docker/runc-hypeman.patch @@ -0,0 +1,45 @@ +diff --git a/libcontainer/nsenter/nsexec.c b/libcontainer/nsenter/nsexec.c +index 728e68b..b04a3b2 100644 +--- a/libcontainer/nsenter/nsexec.c ++++ b/libcontainer/nsenter/nsexec.c +@@ -609,0 +610 @@ void join_namespaces(char *nsspec) ++ int rootfd = -1; +@@ -617,0 +619,27 @@ void join_namespaces(char *nsspec) ++ /* ++ * If runc itself runs inside a chroot, joining a container's mount ++ * namespace does not update the calling process's root directory. Keep an ++ * fd for the container init's root so the exec process can explicitly ++ * enter it after setns. ++ */ ++ for (size_t i = 0; i < ns_len; i++) { ++ struct namespace_t *ns = &ns_list[i]; ++ const char *suffix = "/ns/mnt"; ++ size_t path_len = strlen(ns->path); ++ size_t suffix_len = strlen(suffix); ++ ++ if (nstype(ns->type) != CLONE_NEWNS || path_len <= suffix_len || ++ strcmp(ns->path + path_len - suffix_len, suffix) != 0) ++ continue; ++ ++ char rootpath[PATH_MAX]; ++ int n = snprintf(rootpath, sizeof(rootpath), "%.*s/root", ++ (int)(path_len - suffix_len), ns->path); ++ if (n < 0 || (size_t)n >= sizeof(rootpath)) ++ bail("container root path is too long"); ++ rootfd = open(rootpath, O_RDONLY | O_DIRECTORY | O_CLOEXEC); ++ if (rootfd < 0) ++ bail("failed to open container root %s", rootpath); ++ break; ++ } ++ +@@ -637,0 +666,10 @@ void join_namespaces(char *nsspec) ++ ++ if (rootfd >= 0) { ++ if (fchdir(rootfd) < 0) ++ bail("failed to change directory to container root"); ++ if (chroot(".") < 0) ++ bail("failed to chroot to container root"); ++ if (chdir("/") < 0) ++ bail("failed to change directory after chroot"); ++ close(rootfd); ++ }