From a71114550ec86d746cf02ec1646bb89b186fea05 Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Thu, 10 Sep 2026 11:17:40 -0700 Subject: [PATCH 1/3] fix: handle busy cgroup mounts on Bottlerocket Bottlerocket can retain references to the inherited cgroup2 mount, causing the normal unmount to fail with EBnUSY and preventing dockerd from starting. Fall back to a lazy detach before installing the correctly rooted mount. If both unmount attempts fail, retain the inherited mount and continue with potentially degraded cgroup attribution rather than failing the workspace. Refs: https://linear.app/codercom/issue/PLAT-637 --- cli/docker_test.go | 40 +++++++++++++++++++++++++++++++++------- cli/wrap_dockerd.sh | 34 ++++++++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/cli/docker_test.go b/cli/docker_test.go index 9074ce6..ec5d912 100644 --- a/cli/docker_test.go +++ b/cli/docker_test.go @@ -197,16 +197,38 @@ if [ -f /sys/fs/cgroup/cgroup.controllers ]; then # When mounts are stacked at /sys/fs/cgroup, the visible mount is the # one whose ID is not another same-location mount's parent (see # proc_pid_mountinfo(5)). - cgroup_mount_root=$(awk ' - $5 == "/sys/fs/cgroup" { root[$1] = $4; isparent[$2] = 1 } - END { for (id in root) if (!(id in isparent)) print root[id] } - ' /proc/self/mountinfo) + get_cgroup_mount_root() { + awk ' + $5 == "/sys/fs/cgroup" { root[$1] = $4; isparent[$2] = 1 } + END { for (id in root) if (!(id in isparent)) print root[id] } + ' /proc/self/mountinfo + } + cgroup_mount_root=$(get_cgroup_mount_root) if [ "$cgroup_mount_root" != "/" ]; then # Remount /sys/fs/cgroup so the new cgroup namespace's view becomes the # fs root; inner container cgroups end up under the envbox container's # cgroup on the host. - umount /sys/fs/cgroup || { echo "envbox: failed to umount /sys/fs/cgroup" >&2; exit 1; } - mount -t cgroup2 cgroup /sys/fs/cgroup || { echo "envbox: failed to mount cgroup2 on /sys/fs/cgroup" >&2; exit 1; } + # A regular unmount can fail with EBUSY on runtimes that retain references + # to the inherited mount. A lazy detach keeps retained references valid + # while freeing the mount point for a correctly rooted replacement. + cgroup_unmounted=false + if ! umount /sys/fs/cgroup; then + echo "envbox: normal umount of /sys/fs/cgroup failed; trying lazy detach" >&2 + if umount -l /sys/fs/cgroup; then + cgroup_unmounted=true + else + echo "envbox: failed to detach /sys/fs/cgroup; continuing with inherited mount (inner container cgroup attribution may be incorrect)" >&2 + fi + else + cgroup_unmounted=true + fi + if [ "$cgroup_unmounted" = true ]; then + mount -t cgroup2 cgroup /sys/fs/cgroup || { echo "envbox: failed to mount cgroup2 on /sys/fs/cgroup" >&2; exit 1; } + cgroup_mount_root=$(get_cgroup_mount_root) + if [ "$cgroup_mount_root" != "/" ]; then + echo "envbox: cgroup2 mount root is '$cgroup_mount_root' after remount; inner container cgroup attribution may be incorrect" >&2 + fi + fi fi # move the processes from the root group to the /init group, @@ -836,11 +858,15 @@ func TestWrapDockerdCmd(t *testing.T) { script := args[3] require.Contains(t, script, fmt.Sprintf("envbox_max_attempts=%d", cli.DockerdSubtreeControlMaxAttempts)) require.Contains(t, script, "[ -f /sys/fs/cgroup/cgroup.controllers ]") + require.Contains(t, script, `get_cgroup_mount_root() {`) require.Contains(t, script, `$5 == "/sys/fs/cgroup" { root[$1] = $4; isparent[$2] = 1 }`) require.Contains(t, script, `END { for (id in root) if (!(id in isparent)) print root[id] }`) require.Contains(t, script, `if [ "$cgroup_mount_root" != "/" ]; then`) - require.Contains(t, script, "umount /sys/fs/cgroup") + require.Contains(t, script, "if ! umount /sys/fs/cgroup") + require.Contains(t, script, "umount -l /sys/fs/cgroup") + require.Contains(t, script, "continuing with inherited mount") require.Contains(t, script, "mount -t cgroup2 cgroup /sys/fs/cgroup") + require.Contains(t, script, "inner container cgroup attribution may be incorrect") require.Contains(t, script, "mkdir -p /sys/fs/cgroup/init") require.Contains(t, script, "/sys/fs/cgroup/cgroup.subtree_control") require.Contains(t, script, `ge "$envbox_max_attempts" ]`) diff --git a/cli/wrap_dockerd.sh b/cli/wrap_dockerd.sh index ac0af9a..2194d58 100644 --- a/cli/wrap_dockerd.sh +++ b/cli/wrap_dockerd.sh @@ -11,16 +11,38 @@ if [ -f /sys/fs/cgroup/cgroup.controllers ]; then # When mounts are stacked at /sys/fs/cgroup, the visible mount is the # one whose ID is not another same-location mount's parent (see # proc_pid_mountinfo(5)). - cgroup_mount_root=$(awk ' - $5 == "/sys/fs/cgroup" { root[$1] = $4; isparent[$2] = 1 } - END { for (id in root) if (!(id in isparent)) print root[id] } - ' /proc/self/mountinfo) + get_cgroup_mount_root() { + awk ' + $5 == "/sys/fs/cgroup" { root[$1] = $4; isparent[$2] = 1 } + END { for (id in root) if (!(id in isparent)) print root[id] } + ' /proc/self/mountinfo + } + cgroup_mount_root=$(get_cgroup_mount_root) if [ "$cgroup_mount_root" != "/" ]; then # Remount /sys/fs/cgroup so the new cgroup namespace's view becomes the # fs root; inner container cgroups end up under the envbox container's # cgroup on the host. - umount /sys/fs/cgroup || { echo "envbox: failed to umount /sys/fs/cgroup" >&2; exit 1; } - mount -t cgroup2 cgroup /sys/fs/cgroup || { echo "envbox: failed to mount cgroup2 on /sys/fs/cgroup" >&2; exit 1; } + # A regular unmount can fail with EBUSY on runtimes that retain references + # to the inherited mount. A lazy detach keeps retained references valid + # while freeing the mount point for a correctly rooted replacement. + cgroup_unmounted=false + if ! umount /sys/fs/cgroup; then + echo "envbox: normal umount of /sys/fs/cgroup failed; trying lazy detach" >&2 + if umount -l /sys/fs/cgroup; then + cgroup_unmounted=true + else + echo "envbox: failed to detach /sys/fs/cgroup; continuing with inherited mount (inner container cgroup attribution may be incorrect)" >&2 + fi + else + cgroup_unmounted=true + fi + if [ "$cgroup_unmounted" = true ]; then + mount -t cgroup2 cgroup /sys/fs/cgroup || { echo "envbox: failed to mount cgroup2 on /sys/fs/cgroup" >&2; exit 1; } + cgroup_mount_root=$(get_cgroup_mount_root) + if [ "$cgroup_mount_root" != "/" ]; then + echo "envbox: cgroup2 mount root is '$cgroup_mount_root' after remount; inner container cgroup attribution may be incorrect" >&2 + fi + fi fi # move the processes from the root group to the /init group, From d5faa186deb526d009c4e9ce921a642f671b4747 Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Mon, 14 Sep 2026 15:58:43 -0700 Subject: [PATCH 2/3] fix: guard cgroup nesting setup on a correctly rooted mount Only configure /init delegation after confirming that the cgroup mount is rooted at / in the new cgroup namespace. If neither unmount method can detach the inherited mount, or the replacement mount remains incorrectly rooted, start dockerd without nesting setup and warn that inner container cgroup attribution may be incorrect. --- cli/docker_test.go | 37 ++++++++++++++++++++++--------------- cli/wrap_dockerd.sh | 28 ++++++++++++++++------------ 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/cli/docker_test.go b/cli/docker_test.go index ec5d912..7d070ac 100644 --- a/cli/docker_test.go +++ b/cli/docker_test.go @@ -203,34 +203,38 @@ if [ -f /sys/fs/cgroup/cgroup.controllers ]; then END { for (id in root) if (!(id in isparent)) print root[id] } ' /proc/self/mountinfo } - cgroup_mount_root=$(get_cgroup_mount_root) - if [ "$cgroup_mount_root" != "/" ]; then + configure_cgroup_nesting=true + if [ "$(get_cgroup_mount_root)" != "/" ]; then # Remount /sys/fs/cgroup so the new cgroup namespace's view becomes the # fs root; inner container cgroups end up under the envbox container's - # cgroup on the host. + # cgroup on the host. Starting dockerd with potentially incorrect cgroup + # attribution is preferable to failing the workspace if neither unmount + # method can detach the inherited mount. # A regular unmount can fail with EBUSY on runtimes that retain references # to the inherited mount. A lazy detach keeps retained references valid # while freeing the mount point for a correctly rooted replacement. - cgroup_unmounted=false if ! umount /sys/fs/cgroup; then echo "envbox: normal umount of /sys/fs/cgroup failed; trying lazy detach" >&2 - if umount -l /sys/fs/cgroup; then - cgroup_unmounted=true - else - echo "envbox: failed to detach /sys/fs/cgroup; continuing with inherited mount (inner container cgroup attribution may be incorrect)" >&2 + if ! umount -l /sys/fs/cgroup; then + configure_cgroup_nesting=false + echo "envbox: failed to detach /sys/fs/cgroup; skipping cgroup nesting setup (inner container cgroup attribution may be incorrect)" >&2 fi - else - cgroup_unmounted=true fi - if [ "$cgroup_unmounted" = true ]; then + if [ "$configure_cgroup_nesting" = true ]; then mount -t cgroup2 cgroup /sys/fs/cgroup || { echo "envbox: failed to mount cgroup2 on /sys/fs/cgroup" >&2; exit 1; } cgroup_mount_root=$(get_cgroup_mount_root) if [ "$cgroup_mount_root" != "/" ]; then - echo "envbox: cgroup2 mount root is '$cgroup_mount_root' after remount; inner container cgroup attribution may be incorrect" >&2 + configure_cgroup_nesting=false + echo "envbox: cgroup2 mount root is '$cgroup_mount_root' after remount; skipping cgroup nesting setup (inner container cgroup attribution may be incorrect)" >&2 fi fi fi + if [ "$configure_cgroup_nesting" = false ]; then + # exec replaces this shell, so the nesting setup below is not run. + exec "$0" "$@" + fi + # move the processes from the root group to the /init group, # otherwise writing subtree_control fails with EBUSY. # An error during moving non-existent process (i.e., "cat") is ignored. @@ -852,7 +856,8 @@ func TestWrapDockerdCmd(t *testing.T) { // reference it as a variable) // - guard the v2-only block on cgroup.controllers existing // - only remount when /sys/fs/cgroup is still rooted at a nested path - // - delegate via /init + subtree_control + // - delegate via /init + subtree_control only after establishing the + // correct cgroup mount root // - bound the retry loop against envbox_max_attempts // - exec into dockerd script := args[3] @@ -861,12 +866,14 @@ func TestWrapDockerdCmd(t *testing.T) { require.Contains(t, script, `get_cgroup_mount_root() {`) require.Contains(t, script, `$5 == "/sys/fs/cgroup" { root[$1] = $4; isparent[$2] = 1 }`) require.Contains(t, script, `END { for (id in root) if (!(id in isparent)) print root[id] }`) - require.Contains(t, script, `if [ "$cgroup_mount_root" != "/" ]; then`) + require.Contains(t, script, `if [ "$(get_cgroup_mount_root)" != "/" ]; then`) require.Contains(t, script, "if ! umount /sys/fs/cgroup") require.Contains(t, script, "umount -l /sys/fs/cgroup") - require.Contains(t, script, "continuing with inherited mount") + require.Contains(t, script, "skipping cgroup nesting setup") require.Contains(t, script, "mount -t cgroup2 cgroup /sys/fs/cgroup") require.Contains(t, script, "inner container cgroup attribution may be incorrect") + require.Contains(t, script, `if [ "$configure_cgroup_nesting" = true ]; then`) + require.Contains(t, script, `if [ "$configure_cgroup_nesting" = false ]; then`) require.Contains(t, script, "mkdir -p /sys/fs/cgroup/init") require.Contains(t, script, "/sys/fs/cgroup/cgroup.subtree_control") require.Contains(t, script, `ge "$envbox_max_attempts" ]`) diff --git a/cli/wrap_dockerd.sh b/cli/wrap_dockerd.sh index 2194d58..d37494a 100644 --- a/cli/wrap_dockerd.sh +++ b/cli/wrap_dockerd.sh @@ -17,34 +17,38 @@ if [ -f /sys/fs/cgroup/cgroup.controllers ]; then END { for (id in root) if (!(id in isparent)) print root[id] } ' /proc/self/mountinfo } - cgroup_mount_root=$(get_cgroup_mount_root) - if [ "$cgroup_mount_root" != "/" ]; then + configure_cgroup_nesting=true + if [ "$(get_cgroup_mount_root)" != "/" ]; then # Remount /sys/fs/cgroup so the new cgroup namespace's view becomes the # fs root; inner container cgroups end up under the envbox container's - # cgroup on the host. + # cgroup on the host. Starting dockerd with potentially incorrect cgroup + # attribution is preferable to failing the workspace if neither unmount + # method can detach the inherited mount. # A regular unmount can fail with EBUSY on runtimes that retain references # to the inherited mount. A lazy detach keeps retained references valid # while freeing the mount point for a correctly rooted replacement. - cgroup_unmounted=false if ! umount /sys/fs/cgroup; then echo "envbox: normal umount of /sys/fs/cgroup failed; trying lazy detach" >&2 - if umount -l /sys/fs/cgroup; then - cgroup_unmounted=true - else - echo "envbox: failed to detach /sys/fs/cgroup; continuing with inherited mount (inner container cgroup attribution may be incorrect)" >&2 + if ! umount -l /sys/fs/cgroup; then + configure_cgroup_nesting=false + echo "envbox: failed to detach /sys/fs/cgroup; skipping cgroup nesting setup (inner container cgroup attribution may be incorrect)" >&2 fi - else - cgroup_unmounted=true fi - if [ "$cgroup_unmounted" = true ]; then + if [ "$configure_cgroup_nesting" = true ]; then mount -t cgroup2 cgroup /sys/fs/cgroup || { echo "envbox: failed to mount cgroup2 on /sys/fs/cgroup" >&2; exit 1; } cgroup_mount_root=$(get_cgroup_mount_root) if [ "$cgroup_mount_root" != "/" ]; then - echo "envbox: cgroup2 mount root is '$cgroup_mount_root' after remount; inner container cgroup attribution may be incorrect" >&2 + configure_cgroup_nesting=false + echo "envbox: cgroup2 mount root is '$cgroup_mount_root' after remount; skipping cgroup nesting setup (inner container cgroup attribution may be incorrect)" >&2 fi fi fi + if [ "$configure_cgroup_nesting" = false ]; then + # exec replaces this shell, so the nesting setup below is not run. + exec "$0" "$@" + fi + # move the processes from the root group to the /init group, # otherwise writing subtree_control fails with EBUSY. # An error during moving non-existent process (i.e., "cat") is ignored. From 665ecddc31467a6bfefdb776f1f5e996cd19c1e6 Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Mon, 14 Sep 2026 18:09:17 -0700 Subject: [PATCH 3/3] fix: fall back to the original cgroup namespace Starting dockerd in the new cgroup namespace with the inherited mount leaves the namespace and mount roots misaligned, causing the runtime to look for workspace cgroups at paths that do not exist. If neither unmount method can detach the inherited mount, re-enter the parent cgroup namespace before starting dockerd. This preserves workspace functionality while accepting potentially incorrect inner cgroup attribution. Fail explicitly if a replacement mount remains incorrectly rooted, since the original mount has already been detached. --- cli/docker_test.go | 46 ++++++++++++++++++++++----------------------- cli/wrap_dockerd.sh | 41 ++++++++++++++++++++-------------------- 2 files changed, 42 insertions(+), 45 deletions(-) diff --git a/cli/docker_test.go b/cli/docker_test.go index 7d070ac..dd8e504 100644 --- a/cli/docker_test.go +++ b/cli/docker_test.go @@ -203,38 +203,37 @@ if [ -f /sys/fs/cgroup/cgroup.controllers ]; then END { for (id in root) if (!(id in isparent)) print root[id] } ' /proc/self/mountinfo } - configure_cgroup_nesting=true if [ "$(get_cgroup_mount_root)" != "/" ]; then # Remount /sys/fs/cgroup so the new cgroup namespace's view becomes the # fs root; inner container cgroups end up under the envbox container's - # cgroup on the host. Starting dockerd with potentially incorrect cgroup - # attribution is preferable to failing the workspace if neither unmount - # method can detach the inherited mount. - # A regular unmount can fail with EBUSY on runtimes that retain references - # to the inherited mount. A lazy detach keeps retained references valid - # while freeing the mount point for a correctly rooted replacement. + # cgroup on the host. A regular unmount can fail with EBUSY on runtimes + # that retain references to the inherited mount. A lazy detach keeps + # retained references valid while freeing the mount point for a correctly + # rooted replacement. + # If neither unmount method can detach the inherited mount, re-enter the + # parent cgroup namespace before starting dockerd. This keeps the namespace + # and mount aligned, preserving workspace functionality at the cost of + # potentially incorrect inner container cgroup attribution. if ! umount /sys/fs/cgroup; then echo "envbox: normal umount of /sys/fs/cgroup failed; trying lazy detach" >&2 if ! umount -l /sys/fs/cgroup; then - configure_cgroup_nesting=false - echo "envbox: failed to detach /sys/fs/cgroup; skipping cgroup nesting setup (inner container cgroup attribution may be incorrect)" >&2 + echo "envbox: failed to detach /sys/fs/cgroup; falling back to the original cgroup namespace (inner container cgroup attribution may be incorrect)" >&2 + # exec replaces this shell, so the remount and nesting setup below + # are not run. + exec nsenter --target "$PPID" --cgroup -- "$0" "$@" fi fi - if [ "$configure_cgroup_nesting" = true ]; then - mount -t cgroup2 cgroup /sys/fs/cgroup || { echo "envbox: failed to mount cgroup2 on /sys/fs/cgroup" >&2; exit 1; } - cgroup_mount_root=$(get_cgroup_mount_root) - if [ "$cgroup_mount_root" != "/" ]; then - configure_cgroup_nesting=false - echo "envbox: cgroup2 mount root is '$cgroup_mount_root' after remount; skipping cgroup nesting setup (inner container cgroup attribution may be incorrect)" >&2 - fi + mount -t cgroup2 cgroup /sys/fs/cgroup || { echo "envbox: failed to mount cgroup2 on /sys/fs/cgroup" >&2; exit 1; } + cgroup_mount_root=$(get_cgroup_mount_root) + if [ "$cgroup_mount_root" != "/" ]; then + # The inherited mount is already detached, so re-entering the parent + # cgroup namespace would not restore the original namespace/mount + # alignment. + echo "envbox: cgroup2 mount root is '$cgroup_mount_root' after remount" >&2 + exit 1 fi fi - if [ "$configure_cgroup_nesting" = false ]; then - # exec replaces this shell, so the nesting setup below is not run. - exec "$0" "$@" - fi - # move the processes from the root group to the /init group, # otherwise writing subtree_control fails with EBUSY. # An error during moving non-existent process (i.e., "cat") is ignored. @@ -869,11 +868,10 @@ func TestWrapDockerdCmd(t *testing.T) { require.Contains(t, script, `if [ "$(get_cgroup_mount_root)" != "/" ]; then`) require.Contains(t, script, "if ! umount /sys/fs/cgroup") require.Contains(t, script, "umount -l /sys/fs/cgroup") - require.Contains(t, script, "skipping cgroup nesting setup") + require.Contains(t, script, "falling back to the original cgroup namespace") + require.Contains(t, script, `exec nsenter --target "$PPID" --cgroup -- "$0" "$@"`) require.Contains(t, script, "mount -t cgroup2 cgroup /sys/fs/cgroup") require.Contains(t, script, "inner container cgroup attribution may be incorrect") - require.Contains(t, script, `if [ "$configure_cgroup_nesting" = true ]; then`) - require.Contains(t, script, `if [ "$configure_cgroup_nesting" = false ]; then`) require.Contains(t, script, "mkdir -p /sys/fs/cgroup/init") require.Contains(t, script, "/sys/fs/cgroup/cgroup.subtree_control") require.Contains(t, script, `ge "$envbox_max_attempts" ]`) diff --git a/cli/wrap_dockerd.sh b/cli/wrap_dockerd.sh index d37494a..616cc73 100644 --- a/cli/wrap_dockerd.sh +++ b/cli/wrap_dockerd.sh @@ -17,38 +17,37 @@ if [ -f /sys/fs/cgroup/cgroup.controllers ]; then END { for (id in root) if (!(id in isparent)) print root[id] } ' /proc/self/mountinfo } - configure_cgroup_nesting=true if [ "$(get_cgroup_mount_root)" != "/" ]; then # Remount /sys/fs/cgroup so the new cgroup namespace's view becomes the # fs root; inner container cgroups end up under the envbox container's - # cgroup on the host. Starting dockerd with potentially incorrect cgroup - # attribution is preferable to failing the workspace if neither unmount - # method can detach the inherited mount. - # A regular unmount can fail with EBUSY on runtimes that retain references - # to the inherited mount. A lazy detach keeps retained references valid - # while freeing the mount point for a correctly rooted replacement. + # cgroup on the host. A regular unmount can fail with EBUSY on runtimes + # that retain references to the inherited mount. A lazy detach keeps + # retained references valid while freeing the mount point for a correctly + # rooted replacement. + # If neither unmount method can detach the inherited mount, re-enter the + # parent cgroup namespace before starting dockerd. This keeps the namespace + # and mount aligned, preserving workspace functionality at the cost of + # potentially incorrect inner container cgroup attribution. if ! umount /sys/fs/cgroup; then echo "envbox: normal umount of /sys/fs/cgroup failed; trying lazy detach" >&2 if ! umount -l /sys/fs/cgroup; then - configure_cgroup_nesting=false - echo "envbox: failed to detach /sys/fs/cgroup; skipping cgroup nesting setup (inner container cgroup attribution may be incorrect)" >&2 + echo "envbox: failed to detach /sys/fs/cgroup; falling back to the original cgroup namespace (inner container cgroup attribution may be incorrect)" >&2 + # exec replaces this shell, so the remount and nesting setup below + # are not run. + exec nsenter --target "$PPID" --cgroup -- "$0" "$@" fi fi - if [ "$configure_cgroup_nesting" = true ]; then - mount -t cgroup2 cgroup /sys/fs/cgroup || { echo "envbox: failed to mount cgroup2 on /sys/fs/cgroup" >&2; exit 1; } - cgroup_mount_root=$(get_cgroup_mount_root) - if [ "$cgroup_mount_root" != "/" ]; then - configure_cgroup_nesting=false - echo "envbox: cgroup2 mount root is '$cgroup_mount_root' after remount; skipping cgroup nesting setup (inner container cgroup attribution may be incorrect)" >&2 - fi + mount -t cgroup2 cgroup /sys/fs/cgroup || { echo "envbox: failed to mount cgroup2 on /sys/fs/cgroup" >&2; exit 1; } + cgroup_mount_root=$(get_cgroup_mount_root) + if [ "$cgroup_mount_root" != "/" ]; then + # The inherited mount is already detached, so re-entering the parent + # cgroup namespace would not restore the original namespace/mount + # alignment. + echo "envbox: cgroup2 mount root is '$cgroup_mount_root' after remount" >&2 + exit 1 fi fi - if [ "$configure_cgroup_nesting" = false ]; then - # exec replaces this shell, so the nesting setup below is not run. - exec "$0" "$@" - fi - # move the processes from the root group to the /init group, # otherwise writing subtree_control fails with EBUSY. # An error during moving non-existent process (i.e., "cat") is ignored.