From f43a685128f4157501d803da73293ab247ac7db2 Mon Sep 17 00:00:00 2001 From: hiroTamada <88675973+hiroTamada@users.noreply.github.com> Date: Sun, 13 Sep 2026 16:12:42 +0000 Subject: [PATCH] Fence vault fills at the Chromium process owner --- server/cmd/api/main.go | 9 ++ server/cmd/chromium-launcher/main.go | 150 +++++++++----------------- server/lib/fillfence/README.md | 24 +++++ server/lib/fillfence/fence.go | 150 ++++++++++++++++++++++++++ server/lib/fillfence/fence_test.go | 120 +++++++++++++++++++++ server/lib/fillfence/process.go | 138 ++++++++++++++++++++++++ server/lib/fillfence/process_linux.go | 130 ++++++++++++++++++++++ 7 files changed, 623 insertions(+), 98 deletions(-) create mode 100644 server/lib/fillfence/README.md create mode 100644 server/lib/fillfence/fence.go create mode 100644 server/lib/fillfence/fence_test.go create mode 100644 server/lib/fillfence/process.go create mode 100644 server/lib/fillfence/process_linux.go diff --git a/server/cmd/api/main.go b/server/cmd/api/main.go index b3384c6f..c4397e8d 100644 --- a/server/cmd/api/main.go +++ b/server/cmd/api/main.go @@ -6,6 +6,7 @@ import ( "fmt" "log/slog" "net/http" + "net/http/httputil" "net/url" "os" "os/exec" @@ -30,6 +31,7 @@ import ( "github.com/kernel/kernel-images/server/lib/chromedriverproxy" "github.com/kernel/kernel-images/server/lib/devtoolsproxy" "github.com/kernel/kernel-images/server/lib/events" + "github.com/kernel/kernel-images/server/lib/fillfence" "github.com/kernel/kernel-images/server/lib/forkidentity" "github.com/kernel/kernel-images/server/lib/logger" "github.com/kernel/kernel-images/server/lib/metrics" @@ -332,7 +334,14 @@ func main() { // Checked once per forwarded client frame, so it reads the session's // lock-free view rather than taking the telemetry lock. controlEnabled := func() bool { return telemetrySession.CategoryEnabled(events.Control) } + fillProxy := httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "http", Host: fillfence.Address}) rDevtools.Get("/*", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("kernelVaultFill") == "1" { + // The launcher owns admission and command completion. This hop must + // neither reconnect an upgraded stream nor log its secret payloads. + fillProxy.ServeHTTP(w, r) + return + } devtoolsproxy.WebSocketProxyHandler(upstreamMgr, slogger, config.LogCDPMessages, stz, telemetrySession.Publish, controlEnabled, telemetrySession.ExcludedCdpMethods, wsRegistry).ServeHTTP(w, r) }) diff --git a/server/cmd/chromium-launcher/main.go b/server/cmd/chromium-launcher/main.go index d08d040b..c106af45 100644 --- a/server/cmd/chromium-launcher/main.go +++ b/server/cmd/chromium-launcher/main.go @@ -1,17 +1,21 @@ package main import ( + "context" "flag" "fmt" - "net" "os" "os/exec" - "path/filepath" + "os/signal" + "os/user" + "runtime" + "strconv" "strings" "syscall" "time" "github.com/kernel/kernel-images/server/lib/chromiumflags" + "github.com/kernel/kernel-images/server/lib/fillfence" "github.com/kernel/kernel-images/server/lib/x11" ) @@ -30,32 +34,22 @@ const ( ) func main() { + // Pdeathsig is tied to the creating thread. Keep the browser's parent thread + // alive for the owner's entire lifetime. + runtime.LockOSThread() + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() headless := flag.Bool("headless", false, "Run Chromium with headless flags") chromiumPath := flag.String("chromium", "chromium", "Chromium binary path (default: chromium)") runtimeFlagsPath := flag.String("runtime-flags", "/chromium/flags", "Path to runtime flags overlay file") flag.Parse() - // Clean up stale lock file from previous SIGKILL termination - // Chromium creates this lock and doesn't clean it up when killed - _ = os.Remove("/home/kernel/user-data/SingletonLock") - _ = os.Remove("/home/kernel/user-data/SingletonSocket") - _ = os.Remove("/home/kernel/user-data/SingletonCookie") - - // Kill any existing chromium processes to ensure clean restart. - // This is necessary because supervisord's stopwaitsecs=0 doesn't wait for - // the old process to fully die before starting the new one, which can cause - // the new process to fall back to IPv6 while the old one holds IPv4. - killExistingChromium() - // Inputs internalPort := strings.TrimSpace(os.Getenv("INTERNAL_PORT")) if internalPort == "" { internalPort = "9223" } - // Wait for devtools port to be available (handles SIGKILL socket cleanup delay) - waitForPort(internalPort, 5*time.Second) - // Wait for the X server. The wrapper starts chromium in parallel with // xorg/xvfb, so the display socket may not be ready yet — without this // gate chromium would fail on connect and supervisord would restart us. @@ -105,8 +99,7 @@ func main() { runAsRoot := strings.EqualFold(strings.TrimSpace(os.Getenv("RUN_AS_ROOT")), "true") // Prepare environment. PULSE_SERVER/PULSE_SINK route chromium's audio into the - // recorder's sink; the root path below relies on this inherited env, while the - // non-root path re-asserts them in its runuser env allowlist. + // recorder's sink, for both root and kernel-user launches. env := os.Environ() env = append(env, "DISPLAY=:1", @@ -115,49 +108,26 @@ func main() { "PULSE_SINK="+pulseSink, ) - if runAsRoot { - // Replace current process with Chromium - if p, err := execLookPath(*chromiumPath); err == nil { - if err := syscall.Exec(p, append([]string{filepath.Base(p)}, chromiumArgs...), env); err != nil { - fmt.Fprintf(os.Stderr, "exec chromium failed: %v\n", err) - os.Exit(1) - } - } else { - fmt.Fprintf(os.Stderr, "chromium binary not found: %v\n", err) - os.Exit(1) - } - return - } - - // Not running as root: call runuser to exec as kernel user, providing env vars inside - runuserPath, err := execLookPath("runuser") + path, err := execLookPath(*chromiumPath) if err != nil { - fmt.Fprintf(os.Stderr, "runuser not found: %v\n", err) + fmt.Fprintf(os.Stderr, "chromium binary not found: %v\n", err) os.Exit(1) } - - // Build: runuser -u kernel -- env DISPLAY=... DBUS_... XDG_... HOME=... chromium - // PULSE_SERVER tells libpulse which daemon socket to connect to; without it - // chromium-as-kernel-user can't reach the recorder's PulseAudio instance and - // has no audio output at all. PULSE_SINK then selects which sink within that - // daemon playback lands on: Chromium's AudioManagerPulse honors it to redirect - // playback into KernelOutput (see media/audio/pulse/audio_manager_pulse.cc - // GetDefaultOutputDeviceID), which is the sink the recorder captures. - inner := []string{ - "env", - "DISPLAY=:1", - "DBUS_SESSION_BUS_ADDRESS=unix:path=/run/dbus/system_bus_socket", - "PULSE_SERVER=" + pulseServer, - "PULSE_SINK=" + pulseSink, - "XDG_CONFIG_HOME=/home/kernel/.config", - "XDG_CACHE_HOME=/home/kernel/.cache", - "HOME=/home/kernel", - *chromiumPath, - } - inner = append(inner, chromiumArgs...) - argv := append([]string{filepath.Base(runuserPath), "-u", "kernel", "--"}, inner...) - if err := syscall.Exec(runuserPath, argv, env); err != nil { - fmt.Fprintf(os.Stderr, "exec runuser failed: %v\n", err) + cmd := exec.Command(path, chromiumArgs...) + cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} + if !runAsRoot { + credential, err := kernelCredential() + if err != nil { + fmt.Fprintf(os.Stderr, "kernel user unavailable: %v\n", err) + os.Exit(1) + } + cmd.SysProcAttr.Credential = credential + env = append(env, "USER=kernel", "LOGNAME=kernel", "HOME=/home/kernel", "XDG_CONFIG_HOME=/home/kernel/.config", "XDG_CACHE_HOME=/home/kernel/.cache") + } + cmd.Env, cmd.Stdout, cmd.Stderr = env, os.Stdout, os.Stderr + browser := fillfence.Browser{Command: cmd, Executable: cmd.Path, ProfileDir: "/home/kernel/user-data", DevToolsPort: internalPort, Address: fillfence.Address, Identity: fillfence.Identity} + if err := browser.Run(ctx); err != nil { + fmt.Fprintf(os.Stderr, "browser owner stopped: %v\n", err) os.Exit(1) } } @@ -171,7 +141,6 @@ func withDefaultPrivateNetworkBypass(flags []string) []string { return append(flags, defaultPrivateNetworkBypassFlag) } -// execLookPath helps satisfy syscall.Exec's requirement to pass an absolute path. func execLookPath(file string) (string, error) { if strings.ContainsRune(file, os.PathSeparator) { return file, nil @@ -179,45 +148,30 @@ func execLookPath(file string) (string, error) { return exec.LookPath(file) } -// waitForPort waits until the devtools port can be bound on IPv4. After SIGKILL -// the old listener may linger briefly; this loop waits for that to clear. -// Go's net.Listen sets SO_REUSEADDR, matching chromium's DevTools bind — so -// TIME_WAIT sockets from prior CDP client connections do not block the probe. -// Only IPv4 is checked because IPv6 is disabled at the kernel level in the VM. -func waitForPort(port string, timeout time.Duration) { - deadline := time.Now().Add(timeout) - addr := "127.0.0.1:" + port - - for time.Now().Before(deadline) { - ln, err := net.Listen("tcp", addr) - if err == nil { - ln.Close() - return - } - time.Sleep(50 * time.Millisecond) +func kernelCredential() (*syscall.Credential, error) { + u, err := user.Lookup("kernel") + if err != nil { + return nil, err } - // Timeout reached, proceed anyway and let chromium report the error -} - -// killExistingChromium kills any existing chromium browser processes and waits for them to die. -// This ensures a clean restart where the new process can bind to IPv4. -// Note: We use -x for exact match to avoid killing chromium-launcher itself. -func killExistingChromium() { - // Kill chromium processes by exact name match. - // Using -x prevents matching "chromium-launcher" which would kill this process. - _ = exec.Command("pkill", "-9", "-x", "chromium").Run() - - // Wait up to 2 seconds for processes to fully terminate - deadline := time.Now().Add(2 * time.Second) - for time.Now().Before(deadline) { - // Check if any chromium browser processes are still running (exact match) - output, err := exec.Command("pgrep", "-x", "chromium").Output() - if err != nil || len(strings.TrimSpace(string(output))) == 0 { - // No processes found, we're done - return + uid, err := strconv.ParseUint(u.Uid, 10, 32) + if err != nil { + return nil, err + } + gid, err := strconv.ParseUint(u.Gid, 10, 32) + if err != nil { + return nil, err + } + groups, err := u.GroupIds() + if err != nil { + return nil, err + } + credential := &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)} + for _, group := range groups { + id, err := strconv.ParseUint(group, 10, 32) + if err != nil { + return nil, err } - time.Sleep(100 * time.Millisecond) + credential.Groups = append(credential.Groups, uint32(id)) } - // Timeout - processes may still exist but we continue anyway - fmt.Fprintf(os.Stderr, "warning: chromium processes may still be running after kill attempt\n") + return credential, nil } diff --git a/server/lib/fillfence/README.md b/server/lib/fillfence/README.md new file mode 100644 index 00000000..fbb3d4c1 --- /dev/null +++ b/server/lib/fillfence/README.md @@ -0,0 +1,24 @@ +# Vault fill owner protocol (work in progress) + +This path is implemented but has not yet completed end-to-end validation. Do not deploy it or enable the API fill gate based on unit tests. + +The Chromium launcher remains the browser's parent instead of execing it. It reserves the loopback owner listener before checking for surviving Chromium processes. The next browser is not started and the owner HTTP handler is not served until the previous browser/renderer/zygote/crashpad processes are gone. Inspection errors, unsupported pidfds, permission failures, an occupied DevTools port, and unsuccessful termination fail closed. A successful supervisorctl response is not used as proof of fencing. + +The image CDP proxy forwards `kernelVaultFill=1` to this owner without reconnecting upgraded streams or logging command payloads. The API must complete a nonce-bound `kernel.vault-fill.v1` handshake, including the actual instance name, before sending CDP commands. Older images or proxies cannot complete this handshake and receive no secret-bearing command on the new API path. + +One operation occupies the owner. Commands are sequential, ID-checked and use one fixed upstream socket. A finish message is accepted only between acknowledged commands; it closes that upstream before releasing admission and makes the old client terminal. Disconnect, cancellation, protocol failure or loss of a command response leaves the generation occupied. Neither Redis key deletion nor a timeout clears it. There is no reconnect, replay or force-unlock endpoint. Lost finish acknowledgment does not make a browser write unknown: the client already acknowledged every command by sending finish. + +API/proxy restart does not restart the launcher or clear its state. Launcher restart must fence surviving Chrome processes before admitting a replacement. Parent-death signaling closes the pre-exec orphan window; the startup process census also covers Chrome surviving owner death. This relies on Linux pidfd/proc semantics, not durable Redis or filesystem lease records. The image's immutable Chromium executable and its shipped process helpers are the supported process cohort; out-of-band browser binaries or subprocess launch wrappers are not a supported fenced configuration. + +Standby/restore retains the owner's state together with the browser. An active or quarantined generation does not become idle on disconnect or resume. The owner captures its instance identity at birth and checks it on admission and every command. A template waiting for fork identity cannot fill, and an owner cannot adopt a different identity in place. Fork handoff requires a launcher restart after identity application before fill is supported; that restart must confirm the old browser generation exited. A restored active operation must never be used as a clean template. These platform lifecycle cases still require validation beyond protocol unit tests. + +The protocol covers the API's synchronous guarded CDP scripts and their completion. It does not serialize unrelated customer CDP clients or application JavaScript timers. It is not a security boundary against privileged process/file mutation inside the browser. + +## Integration order + +1. Complete real-image lifecycle and API adversarial validation. +2. Merge/release the image companion, then integrate that image through the normal release path. Drain incompatible pool/template inventory; validate the target platform's process primitives and fork reset behavior. +3. Merge the API transport change. Its protocol negotiation is mandatory; no fallback to ordinary CDP is permitted. +4. Consider the default-off fill gate separately. Neither companion PR enables it. + +Pending evidence: actual image owner restart with surviving Chrome, rejected/uncertain stop, lost-response quarantine, terminal old clients, independent browsers, two full public HTTP runs and the formerly failing lost-key regression. Protocol unit tests and API authorization tests against stock Chromium are not substitutes for these checks. diff --git a/server/lib/fillfence/fence.go b/server/lib/fillfence/fence.go new file mode 100644 index 00000000..6811d9f1 --- /dev/null +++ b/server/lib/fillfence/fence.go @@ -0,0 +1,150 @@ +// Package fillfence serializes vault fills at the process that owns Chromium. +// A Fence belongs to exactly one Chrome generation. It must never be reused for +// a replacement browser or constructed against an inherited, unfenced browser. +package fillfence + +import ( + "context" + "encoding/json" + "net/http" + "sync" + "time" + + "github.com/coder/websocket" +) + +const Protocol = "kernel.vault-fill.v1" +const Address = "127.0.0.1:9226" + +type message struct { + ID int64 `json:"id"` + Method string `json:"method,omitempty"` + SessionID string `json:"sessionId,omitempty"` + Params json.RawMessage `json:"params,omitempty"` +} + +// Fence is fail-closed: only the explicit end message following acknowledged +// commands releases admission. Disconnect, timeout and protocol failure leave +// this generation quarantined, even if an outstanding command later completes. +// There is no timer, reconnect, or administrative unlock that clears quarantine. +type Fence struct { + upstream string + identity func() string + bornAs string + mu sync.Mutex + occupied bool +} + +func New(upstream string, identity func() string) *Fence { + return &Fence{upstream: upstream, identity: identity, bornAs: identity()} +} + +func (f *Fence) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + if f.occupied { + f.mu.Unlock() + http.Error(w, "fill unavailable", http.StatusConflict) + return + } + f.occupied = true + f.mu.Unlock() + claimed := false + defer func() { + if !claimed { + f.mu.Lock() + f.occupied = false + f.mu.Unlock() + } + }() + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute) + defer cancel() + client, err := websocket.Accept(w, r, &websocket.AcceptOptions{OriginPatterns: []string{"*"}}) + if err != nil { + return + } + defer client.CloseNow() + client.SetReadLimit(8 << 20) + handshake, stop := context.WithTimeout(ctx, 5*time.Second) + _, raw, err := client.Read(handshake) + stop() + var begin message + var params struct{ Protocol, Nonce, Instance string } + if err != nil || json.Unmarshal(raw, &begin) != nil || begin.ID <= 0 || begin.Method != "Kernel.vaultFill.begin" || begin.SessionID != "" || json.Unmarshal(begin.Params, ¶ms) != nil || params.Protocol != Protocol || len(params.Nonce) != 32 { + return + } + if f.bornAs == "" || f.identity() != f.bornAs || params.Instance != f.bornAs { + return + } + claimed = true + + // Never redial this connection, including on browser or proxy restarts. + upstream, _, err := websocket.Dial(ctx, f.upstream, nil) + if err != nil { + return + } + defer upstream.CloseNow() + upstream.SetReadLimit(8 << 20) + if !reply(ctx, client, begin.ID, map[string]string{"protocol": Protocol, "nonce": params.Nonce}) { + return + } + lastID := begin.ID + for { + _, raw, err := client.Read(ctx) + var command message + if err != nil || json.Unmarshal(raw, &command) != nil || command.ID <= lastID || f.identity() != f.bornAs { + return + } + lastID = command.ID + if command.Method == "Kernel.vaultFill.end" && command.SessionID == "" { + // No command is outstanding here. Close the only command channel before + // releasing; queued data on the old client can never reach Chrome again. + upstream.CloseNow() + f.mu.Lock() + f.occupied = false + f.mu.Unlock() + reply(ctx, client, command.ID, struct{}{}) + return + } + if !allowed(command.Method) { + return + } + if upstream.Write(ctx, websocket.MessageText, raw) != nil { + return + } + for { + _, data, err := upstream.Read(ctx) + var response message + if err != nil || json.Unmarshal(data, &response) != nil { + return + } + if response.ID != 0 && (response.ID != command.ID || response.SessionID != command.SessionID) { + return + } + if client.Write(ctx, websocket.MessageText, data) != nil { + return + } + if response.ID == command.ID { + break + } + } + } +} + +func reply(ctx context.Context, conn *websocket.Conn, id int64, result any) bool { + data, err := json.Marshal(struct { + ID int64 `json:"id"` + Result any `json:"result"` + }{id, result}) + return err == nil && conn.Write(ctx, websocket.MessageText, data) == nil +} + +// This path cannot create a second command channel, reset Chrome, or enqueue +// detached protocol work. Runtime calls are synchronous (the API supplies the +// guarded scripts); only their response confirms completion of that command. +func allowed(method string) bool { + switch method { + case "Target.getTargets", "Target.attachToTarget", "Page.enable", "Page.getFrameTree", "Page.createIsolatedWorld", "DOM.describeNode", "DOM.resolveNode", "Runtime.callFunctionOn", "Runtime.releaseObject", "Runtime.evaluate", "Runtime.getProperties": + return true + } + return false +} diff --git a/server/lib/fillfence/fence_test.go b/server/lib/fillfence/fence_test.go new file mode 100644 index 00000000..16c32c90 --- /dev/null +++ b/server/lib/fillfence/fence_test.go @@ -0,0 +1,120 @@ +package fillfence + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" + "github.com/stretchr/testify/require" +) + +func TestFenceCompletionAndQuarantine(t *testing.T) { + for _, loss := range []bool{false, true} { + t.Run(map[bool]string{false: "release", true: "lost-response"}[loss], func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + entered, resume, completed := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var writes atomic.Int32 + chrome := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + defer c.CloseNow() + _, data, err := c.Read(ctx) + if err != nil { + return + } + var m message + require.NoError(t, json.Unmarshal(data, &m)) + writes.Add(1) + close(entered) + <-resume + reply(ctx, c, m.ID, struct{}{}) + close(completed) + _, _, _ = c.Read(ctx) + })) + defer chrome.Close() + owner := httptest.NewServer(New("ws"+strings.TrimPrefix(chrome.URL, "http"), func() string { return "browser" })) + defer owner.Close() + a := claim(t, ctx, owner.URL, "browser", true) + defer a.CloseNow() + require.NoError(t, a.Write(ctx, websocket.MessageText, []byte(`{"id":2,"method":"Runtime.evaluate","params":{}}`))) + <-entered + b := claim(t, ctx, owner.URL, "browser", false) + if b != nil { + b.CloseNow() + } + if loss { + a.CloseNow() + } + close(resume) + <-completed + if loss { + b = claim(t, ctx, owner.URL, "browser", false) + if b != nil { + b.CloseNow() + } + } else { + _, _, err := a.Read(ctx) + require.NoError(t, err) + require.NoError(t, a.Write(ctx, websocket.MessageText, []byte(`{"id":3,"method":"Kernel.vaultFill.end"}`))) + _, _, err = a.Read(ctx) + require.NoError(t, err) + // A released channel is terminal, not a reusable lease. + _ = a.Write(ctx, websocket.MessageText, []byte(`{"id":4,"method":"Runtime.evaluate","params":{}}`)) + _, _, err = a.Read(ctx) + require.Error(t, err) + b = claim(t, ctx, owner.URL, "browser", true) + b.CloseNow() + } + require.EqualValues(t, 1, writes.Load()) + }) + } +} + +func claim(t *testing.T, ctx context.Context, base, instance string, ready bool) *websocket.Conn { + t.Helper() + c, response, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(base, "http"), nil) + if err != nil { + require.False(t, ready) + require.NotNil(t, response) + require.Equal(t, http.StatusConflict, response.StatusCode) + return nil + } + raw, err := json.Marshal(map[string]any{"id": 1, "method": "Kernel.vaultFill.begin", "params": map[string]string{"protocol": Protocol, "nonce": strings.Repeat("a", 32), "instance": instance}}) + require.NoError(t, err) + require.NoError(t, c.Write(ctx, websocket.MessageText, raw)) + _, data, err := c.Read(ctx) + if ready { + require.NoError(t, err) + require.Contains(t, string(data), Protocol) + } else { + require.Error(t, err) + } + return c +} + +func TestIdentityCannotBeAdoptedInPlace(t *testing.T) { + for _, born := range []string{"", "template"} { + t.Run("born-"+born, func(t *testing.T) { + var identity atomic.Value + identity.Store(born) + owner := httptest.NewServer(New("ws://127.0.0.1:1", func() string { return identity.Load().(string) })) + defer owner.Close() + identity.Store("fork") + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + c := claim(t, ctx, owner.URL, "fork", false) + if c != nil { + c.CloseNow() + } + }) + } +} diff --git a/server/lib/fillfence/process.go b/server/lib/fillfence/process.go new file mode 100644 index 00000000..ff23f28f --- /dev/null +++ b/server/lib/fillfence/process.go @@ -0,0 +1,138 @@ +package fillfence + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/kernel/kernel-images/server/lib/forkidentity" +) + +// Browser contains the process configuration supplied by chromium-launcher. +// Address is loopback-only in the image; tests use their own isolated listener. +type Browser struct { + Command *exec.Cmd + Executable string + ProfileDir string + DevToolsPort string + Address string + Identity func() string +} + +func (b Browser) Run(ctx context.Context) (result error) { + // The listener is also the kernel-enforced single-owner exclusion. A second + // launcher cannot start a browser while this owner still holds the port. + listener, err := net.Listen("tcp4", b.Address) + if err != nil { + return err + } + defer listener.Close() + stop := func() error { + deadline, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return StopPrevious(deadline, b.Executable) + } + if err := stop(); err != nil { + return err + } + port, err := net.Listen("tcp4", "127.0.0.1:"+b.DevToolsPort) + if err != nil { + return fmt.Errorf("previous DevTools listener still present: %w", err) + } + port.Close() + for _, name := range []string{"SingletonLock", "SingletonSocket", "SingletonCookie"} { + if err := os.Remove(filepath.Join(b.ProfileDir, name)); err != nil && !os.IsNotExist(err) { + return err + } + } + if err := b.Command.Start(); err != nil { + return err + } + exited := make(chan error, 1) + go func() { exited <- b.Command.Wait() }() + // Keep the listener reserved until teardown has attempted to fence Chrome. + // If this fails, the next launcher must still pass its own strict exit check. + var server *http.Server + defer func() { + result = errors.Join(result, stop()) + if server != nil { + server.Close() + } + }() + upstream, err := b.waitUpstream(ctx) + if err != nil { + return err + } + server = &http.Server{Handler: New(upstream, b.Identity), ReadHeaderTimeout: 5 * time.Second, BaseContext: func(net.Listener) context.Context { return ctx }} + served := make(chan error, 1) + go func() { served <- server.Serve(listener) }() + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-exited: + return err + case err := <-served: + return err + } +} + +func (b Browser) waitUpstream(ctx context.Context) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + client := &http.Client{Timeout: time.Second} + for { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://127.0.0.1:"+b.DevToolsPort+"/json/version", nil) + if err != nil { + return "", err + } + response, err := client.Do(req) + if err == nil { + var result struct { + URL string `json:"webSocketDebuggerUrl"` + } + err = json.NewDecoder(io.LimitReader(response.Body, 64<<10)).Decode(&result) + response.Body.Close() + if err == nil && result.URL != "" { + return result.URL, nil + } + } + select { + case <-ctx.Done(): + return "", fmt.Errorf("new browser not ready: %w", ctx.Err()) + case <-time.After(50 * time.Millisecond): + } + } +} + +// Identity never adopts a fork in place. New captures its birth identity; a +// template waiting for handoff is disabled for its entire owner lifetime. After +// handoff it needs a launcher restart (and confirmed old-Chrome exit) to fill. +func Identity() string { + wait, err := forkidentity.WaitEnabled() + if err != nil { + return "" + } + if _, err := os.Stat(forkidentity.ReadyFile); err == nil { + applied, err := forkidentity.ReadAppliedMarker() + if err != nil || applied == "" { + return "" + } + payload, err := forkidentity.ReadPayload() + if err != nil || payload.InstanceName() != applied { + return "" + } + return applied + } + if wait { + return "" + } + return forkidentity.FirstNonEmpty(os.Getenv("INSTANCE_NAME"), os.Getenv("INST_NAME")) +} diff --git a/server/lib/fillfence/process_linux.go b/server/lib/fillfence/process_linux.go new file mode 100644 index 00000000..bb0aebba --- /dev/null +++ b/server/lib/fillfence/process_linux.go @@ -0,0 +1,130 @@ +package fillfence + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +// StopPrevious runs only inside the single-browser image's process namespace, +// before a new browser starts. Both browser and renderer/zygote executables are +// covered, including orphans left when the launcher was killed. A disconnected +// DevTools socket or a successful supervisorctl response is not proof of exit. +// Permission errors, unsupported pidfds and unkillable processes fail closed. +func StopPrevious(ctx context.Context, executable string) error { + self, err := os.Readlink("/proc/self") + if err != nil || self != strconv.Itoa(os.Getpid()) { + return errors.New("proc process namespace mismatch") + } + executable, err = filepath.Abs(executable) + if err != nil { + return err + } + executable, err = filepath.EvalSymlinks(executable) + if err != nil { + return err + } + return stopProcesses(ctx, executable, unix.PidfdSendSignal) +} + +func stopProcesses(ctx context.Context, executable string, signal func(int, unix.Signal, *unix.Siginfo, int) error) error { + for { + entries, err := os.ReadDir("/proc") + if err != nil { + return err + } + found := false + for _, entry := range entries { + pid, err := strconv.Atoi(entry.Name()) + if err != nil { + continue + } + path, err := os.Readlink(filepath.Join("/proc", entry.Name(), "exe")) + if errors.Is(err, os.ErrNotExist) { + // /proc//exe can also disappear when only the main thread exits. + // A readable pidfd, unlike a zombie-looking leader, proves group exit. + exited, checkErr := processExited(pid) + if checkErr != nil { + return checkErr + } + if !exited { + kernel, err := kernelThread(pid) + if err != nil || !kernel { + return errors.New("cannot inspect live process executable") + } + } + continue + } + if err != nil { + return fmt.Errorf("inspect process: %w", err) + } + path = strings.TrimSuffix(path, " (deleted)") + if path != executable && path != filepath.Join(filepath.Dir(executable), "chrome_crashpad_handler") && path != filepath.Join(filepath.Dir(executable), "chrome_sandbox") { + continue + } + found = true + fd, err := unix.PidfdOpen(pid, 0) + if errors.Is(err, unix.ESRCH) { + continue + } + if err != nil { + return fmt.Errorf("open process handle: %w", err) + } + // Check the executable again after acquiring a stable process handle. A PID + // reused for an unrelated program must not be signalled. + current, readErr := os.Readlink(filepath.Join("/proc", entry.Name(), "exe")) + if readErr == nil && strings.TrimSuffix(current, " (deleted)") == path { + err = signal(fd, unix.SIGKILL, nil, 0) + } + unix.Close(fd) + if err != nil && !errors.Is(err, unix.ESRCH) { + return fmt.Errorf("kill previous browser: %w", err) + } + if readErr != nil && !errors.Is(readErr, os.ErrNotExist) { + return readErr + } + } + if !found { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("previous browser exit not confirmed: %w", ctx.Err()) + case <-time.After(20 * time.Millisecond): + } + } +} + +func kernelThread(pid int) (bool, error) { + stat, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat")) + if err != nil { + return false, err + } + fields := strings.Fields(string(stat)[strings.LastIndexByte(string(stat), ')')+1:]) + if len(fields) < 7 { + return false, errors.New("invalid process stat") + } + flags, err := strconv.ParseUint(fields[6], 10, 64) + return flags&0x00200000 != 0, err // Linux PF_KTHREAD; cannot execute userspace. +} + +func processExited(pid int) (bool, error) { + fd, err := unix.PidfdOpen(pid, 0) + if errors.Is(err, unix.ESRCH) { + return true, nil + } + if err != nil { + return false, err + } + defer unix.Close(fd) + events := []unix.PollFd{{Fd: int32(fd), Events: unix.POLLIN}} + _, err = unix.Poll(events, 0) + return events[0].Revents&unix.POLLIN != 0, err +}