diff --git a/packages/envd/internal/api/init.go b/packages/envd/internal/api/init.go index 0c61f0b14f..c6264a0b1e 100644 --- a/packages/envd/internal/api/init.go +++ b/packages/envd/internal/api/init.go @@ -618,6 +618,12 @@ var nfsOptions = strings.Join([]string{ "nfsvers=3", // nfs proxy is nfs version 3 "noacl", // no reason for acl in the sandbox + // Resolve locks node-locally. The nfs proxy has no lock manager and its + // portmap never registers NLM, so without this a hard-mounted lock request + // (e.g. flock) is sent to the proxy, gets port 0, and retries forever in + // uninterruptible sleep. See github.com/e2b-dev/infra issue #3619. + "nolock", + // disable caching so that pause/resume works correctly "noac", "lookupcache=none", diff --git a/packages/envd/internal/services/cgroups/hierarchy.go b/packages/envd/internal/services/cgroups/hierarchy.go index ba3c56628c..51fe554ca3 100644 --- a/packages/envd/internal/services/cgroups/hierarchy.go +++ b/packages/envd/internal/services/cgroups/hierarchy.go @@ -16,24 +16,23 @@ const ProcSelfCgroup = "/proc/self/cgroup" // path in the resume, not by caution -- a list that grew with the customer population // would defeat the point of walking the hierarchy in the first place. // -// - init.scope holds systemd (PID 1). The resume thaw is deferred inside the /init -// handler, so it runs after setupNFS; volume mounts are nfsvers=3 without nolock, so -// mount.nfs needs rpc.statd, which it asks systemd to start. Freeze PID 1 and every -// volume-mounting sandbox hangs to the NFS mount timeout and fails. Keeping systemd -// live is also what lets envd's own Restart=always fire if envd dies mid-resume. +// - init.scope holds systemd (PID 1). Keeping it live is what lets envd's own +// Restart=always fire if envd dies mid-resume, and PID 1 must never be frozen +// regardless. (Volume mounts now carry nolock, so mount.nfs no longer needs +// rpc.statd during setupNFS -- see the rpcbind note below.) // - systemd-journald.service drains the socket envd logs to (its unit has // Wants=systemd-journald.socket). Freeze it and once the socket buffer fills, envd's // own log writes block -- turning a slow resume into a wedged one. // - socats is envd's port forwarding. It is already excluded from the freeze today // (ProcessTypeSocat is absent from WorkloadProcessTypes); this preserves that. // - rpcbind.service holds the local portmapper, and rpcbind.socket is its activation -// pair. nfsvers=3 mounts carry no `nolock`, so mount.nfs starts rpc.statd, which -// registers with the LOCAL portmapper -- and the resume thaw is deferred inside the -// /init handler, so it runs after setupNFS. Measured on a dev guest: with rpcbind -// frozen, `rpcinfo -p 127.0.0.1` goes from answering in 0.145s to timing out, and a -// v3 mount attempt from a clean 3.2s error to a 25s hang. The socket unit holds no -// processes today, so freezing it stops nothing -- it is here because activation could -// later place a process into a cgroup we froze, and rpcbind is one service in practice. +// pair. Volume mounts now carry `nolock` (issue #3619), so mount.nfs no longer starts +// rpc.statd and the resume mount path no longer depends on the local portmapper. +// These entries are kept defensively: before nolock, a frozen rpcbind turned a v3 +// mount from a clean 3.2s error into a 25s hang (measured on a dev guest), so keeping +// OUR own portmapper live across a pause is cheap insurance against any other resume +// path that might come to rely on it. The socket unit holds no processes today, so +// freezing it stops nothing; it rides along with its service for the same reason. // // Deliberately NOT here, so the list stays justified rather than merely cautious: // run-rpc_pipefs.mount (a mount unit, 0 processes, so freezing its cgroup stops nothing) diff --git a/packages/orchestrator/pkg/nfsproxy/e2e_test.sh b/packages/orchestrator/pkg/nfsproxy/e2e_test.sh index a8bea3fb21..4bbfd898b1 100644 --- a/packages/orchestrator/pkg/nfsproxy/e2e_test.sh +++ b/packages/orchestrator/pkg/nfsproxy/e2e_test.sh @@ -109,4 +109,16 @@ test ! -f "${tmpdir}/dir1/subdir/b.txt" rm -rf "${tmpdir}/dir1" test ! -d "${tmpdir}/dir1" +# 13) advisory locking (flock) must resolve locally and NOT hang. +# The proxy has no lock manager, so the volume must be mounted with nolock +# (see packages/envd/internal/api/init.go). Without it the kernel sends the +# lock to the proxy as an NLM request, gets port 0, and a hard mount retries +# forever -- flock wedges in uninterruptible sleep. Guard against that +# regression: a 10s timeout turns a hang into a loud failure instead of a +# stuck suite. See github.com/e2b-dev/infra issue #3619. +if ! timeout 10 flock "${tmpdir}/test.txt" -c true; then + echo "flock did not resolve within 10s under ${test_path} (missing nolock?)" >&2 + exit 1 +fi + echo "POSIX filesystem operations OK under ${test_path}"