Fence vault fills at the Chromium process owner

server/cmd/api/main.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"log/slog"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
 	"os"
 	"os/exec"
@@ -30,6 +31,7 @@ import (
 	"github.com/kernel/kernel-images/server/lib/chromedriverproxy"
 	"github.com/kernel/kernel-images/server/lib/devtoolsproxy"
 	"github.com/kernel/kernel-images/server/lib/events"
+	"github.com/kernel/kernel-images/server/lib/fillfence"
 	"github.com/kernel/kernel-images/server/lib/forkidentity"
 	"github.com/kernel/kernel-images/server/lib/logger"
 	"github.com/kernel/kernel-images/server/lib/metrics"
@@ -332,7 +334,14 @@ func main() {
 	// Checked once per forwarded client frame, so it reads the session's
 	// lock-free view rather than taking the telemetry lock.
 	controlEnabled := func() bool { return telemetrySession.CategoryEnabled(events.Control) }
+	fillProxy := httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "http", Host: fillfence.Address})
 	rDevtools.Get("/*", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("kernelVaultFill") == "1" {
+			// The launcher owns admission and command completion. This hop must
+			// neither reconnect an upgraded stream nor log its secret payloads.
+			fillProxy.ServeHTTP(w, r)
+			return
+		}
 		devtoolsproxy.WebSocketProxyHandler(upstreamMgr, slogger, config.LogCDPMessages, stz, telemetrySession.Publish, controlEnabled, telemetrySession.ExcludedCdpMethods, wsRegistry).ServeHTTP(w, r)
 	})
 

server/cmd/chromium-launcher/main.go

@@ -1,17 +1,21 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
-	"net"
 	"os"
 	"os/exec"
-	"path/filepath"
+	"os/signal"
+	"os/user"
+	"runtime"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
 
 	"github.com/kernel/kernel-images/server/lib/chromiumflags"
+	"github.com/kernel/kernel-images/server/lib/fillfence"
 	"github.com/kernel/kernel-images/server/lib/x11"
 )
 
@@ -30,32 +34,22 @@ const (
 )
 
 func main() {
+	// Pdeathsig is tied to the creating thread. Keep the browser's parent thread
+	// alive for the owner's entire lifetime.
+	runtime.LockOSThread()
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer cancel()
 	headless := flag.Bool("headless", false, "Run Chromium with headless flags")
 	chromiumPath := flag.String("chromium", "chromium", "Chromium binary path (default: chromium)")
 	runtimeFlagsPath := flag.String("runtime-flags", "/chromium/flags", "Path to runtime flags overlay file")
 	flag.Parse()
 
-	// Clean up stale lock file from previous SIGKILL termination
-	// Chromium creates this lock and doesn't clean it up when killed
-	_ = os.Remove("/home/kernel/user-data/SingletonLock")
-	_ = os.Remove("/home/kernel/user-data/SingletonSocket")
-	_ = os.Remove("/home/kernel/user-data/SingletonCookie")
-
-	// Kill any existing chromium processes to ensure clean restart.
-	// This is necessary because supervisord's stopwaitsecs=0 doesn't wait for
-	// the old process to fully die before starting the new one, which can cause
-	// the new process to fall back to IPv6 while the old one holds IPv4.
-	killExistingChromium()
-
 	// Inputs
 	internalPort := strings.TrimSpace(os.Getenv("INTERNAL_PORT"))
 	if internalPort == "" {
 		internalPort = "9223"
 	}
 
-	// Wait for devtools port to be available (handles SIGKILL socket cleanup delay)
-	waitForPort(internalPort, 5*time.Second)
-
 	// Wait for the X server. The wrapper starts chromium in parallel with
 	// xorg/xvfb, so the display socket may not be ready yet — without this
 	// gate chromium would fail on connect and supervisord would restart us.
@@ -105,8 +99,7 @@ func main() {
 	runAsRoot := strings.EqualFold(strings.TrimSpace(os.Getenv("RUN_AS_ROOT")), "true")
 
 	// Prepare environment. PULSE_SERVER/PULSE_SINK route chromium's audio into the
-	// recorder's sink; the root path below relies on this inherited env, while the
-	// non-root path re-asserts them in its runuser env allowlist.
+	// recorder's sink, for both root and kernel-user launches.
 	env := os.Environ()
 	env = append(env,
 		"DISPLAY=:1",
@@ -115,49 +108,26 @@ func main() {
 		"PULSE_SINK="+pulseSink,
 	)
 
-	if runAsRoot {
-		// Replace current process with Chromium
-		if p, err := execLookPath(*chromiumPath); err == nil {
-			if err := syscall.Exec(p, append([]string{filepath.Base(p)}, chromiumArgs...), env); err != nil {
-				fmt.Fprintf(os.Stderr, "exec chromium failed: %v\n", err)
-				os.Exit(1)
-			}
-		} else {
-			fmt.Fprintf(os.Stderr, "chromium binary not found: %v\n", err)
-			os.Exit(1)
-		}
-		return
-	}
-
-	// Not running as root: call runuser to exec as kernel user, providing env vars inside
-	runuserPath, err := execLookPath("runuser")
+	path, err := execLookPath(*chromiumPath)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "runuser not found: %v\n", err)
+		fmt.Fprintf(os.Stderr, "chromium binary not found: %v\n", err)
 		os.Exit(1)
 	}
-
-	// Build: runuser -u kernel -- env DISPLAY=... DBUS_... XDG_... HOME=... chromium <args>
-	// PULSE_SERVER tells libpulse which daemon socket to connect to; without it
-	// chromium-as-kernel-user can't reach the recorder's PulseAudio instance and
-	// has no audio output at all. PULSE_SINK then selects which sink within that
-	// daemon playback lands on: Chromium's AudioManagerPulse honors it to redirect
-	// playback into KernelOutput (see media/audio/pulse/audio_manager_pulse.cc
-	// GetDefaultOutputDeviceID), which is the sink the recorder captures.
-	inner := []string{
-		"env",
-		"DISPLAY=:1",
-		"DBUS_SESSION_BUS_ADDRESS=unix:path=/run/dbus/system_bus_socket",
-		"PULSE_SERVER=" + pulseServer,
-		"PULSE_SINK=" + pulseSink,
-		"XDG_CONFIG_HOME=/home/kernel/.config",
-		"XDG_CACHE_HOME=/home/kernel/.cache",
-		"HOME=/home/kernel",
-		*chromiumPath,
-	}
-	inner = append(inner, chromiumArgs...)
-	argv := append([]string{filepath.Base(runuserPath), "-u", "kernel", "--"}, inner...)
-	if err := syscall.Exec(runuserPath, argv, env); err != nil {
-		fmt.Fprintf(os.Stderr, "exec runuser failed: %v\n", err)
+	cmd := exec.Command(path, chromiumArgs...)
+	cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL}
+	if !runAsRoot {
+		credential, err := kernelCredential()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "kernel user unavailable: %v\n", err)
+			os.Exit(1)
+		}
+		cmd.SysProcAttr.Credential = credential
+		env = append(env, "USER=kernel", "LOGNAME=kernel", "HOME=/home/kernel", "XDG_CONFIG_HOME=/home/kernel/.config", "XDG_CACHE_HOME=/home/kernel/.cache")
+	}
+	cmd.Env, cmd.Stdout, cmd.Stderr = env, os.Stdout, os.Stderr
+	browser := fillfence.Browser{Command: cmd, Executable: cmd.Path, ProfileDir: "/home/kernel/user-data", DevToolsPort: internalPort, Address: fillfence.Address, Identity: fillfence.Identity}
+	if err := browser.Run(ctx); err != nil {
+		fmt.Fprintf(os.Stderr, "browser owner stopped: %v\n", err)
 		os.Exit(1)
 	}
 }
@@ -171,53 +141,37 @@ func withDefaultPrivateNetworkBypass(flags []string) []string {
 	return append(flags, defaultPrivateNetworkBypassFlag)
 }
 
-// execLookPath helps satisfy syscall.Exec's requirement to pass an absolute path.
 func execLookPath(file string) (string, error) {
 	if strings.ContainsRune(file, os.PathSeparator) {
 		return file, nil
 	}
 	return exec.LookPath(file)
 }
 
-// waitForPort waits until the devtools port can be bound on IPv4. After SIGKILL
-// the old listener may linger briefly; this loop waits for that to clear.
-// Go's net.Listen sets SO_REUSEADDR, matching chromium's DevTools bind — so
-// TIME_WAIT sockets from prior CDP client connections do not block the probe.
-// Only IPv4 is checked because IPv6 is disabled at the kernel level in the VM.
-func waitForPort(port string, timeout time.Duration) {
-	deadline := time.Now().Add(timeout)
-	addr := "127.0.0.1:" + port
-
-	for time.Now().Before(deadline) {
-		ln, err := net.Listen("tcp", addr)
-		if err == nil {
-			ln.Close()
-			return
-		}
-		time.Sleep(50 * time.Millisecond)
+func kernelCredential() (*syscall.Credential, error) {
+	u, err := user.Lookup("kernel")
+	if err != nil {
+		return nil, err
 	}
-	// Timeout reached, proceed anyway and let chromium report the error
-}
-
-// killExistingChromium kills any existing chromium browser processes and waits for them to die.
-// This ensures a clean restart where the new process can bind to IPv4.
-// Note: We use -x for exact match to avoid killing chromium-launcher itself.
-func killExistingChromium() {
-	// Kill chromium processes by exact name match.
-	// Using -x prevents matching "chromium-launcher" which would kill this process.
-	_ = exec.Command("pkill", "-9", "-x", "chromium").Run()
-
-	// Wait up to 2 seconds for processes to fully terminate
-	deadline := time.Now().Add(2 * time.Second)
-	for time.Now().Before(deadline) {
-		// Check if any chromium browser processes are still running (exact match)
-		output, err := exec.Command("pgrep", "-x", "chromium").Output()
-		if err != nil || len(strings.TrimSpace(string(output))) == 0 {
-			// No processes found, we're done
-			return
+	uid, err := strconv.ParseUint(u.Uid, 10, 32)
+	if err != nil {
+		return nil, err
+	}
+	gid, err := strconv.ParseUint(u.Gid, 10, 32)
+	if err != nil {
+		return nil, err
+	}
+	groups, err := u.GroupIds()
+	if err != nil {
+		return nil, err
+	}
+	credential := &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}
+	for _, group := range groups {
+		id, err := strconv.ParseUint(group, 10, 32)
+		if err != nil {
+			return nil, err
 		}
-		time.Sleep(100 * time.Millisecond)
+		credential.Groups = append(credential.Groups, uint32(id))
 	}
-	// Timeout - processes may still exist but we continue anyway
-	fmt.Fprintf(os.Stderr, "warning: chromium processes may still be running after kill attempt\n")
+	return credential, nil
 }

server/lib/fillfence/README.md

@@ -0,0 +1,24 @@
+# Vault fill owner protocol (work in progress)
+
+This path is implemented but has not yet completed end-to-end validation. Do not deploy it or enable the API fill gate based on unit tests.
+
+The Chromium launcher remains the browser's parent instead of execing it. It reserves the loopback owner listener before checking for surviving Chromium processes. The next browser is not started and the owner HTTP handler is not served until the previous browser/renderer/zygote/crashpad processes are gone. Inspection errors, unsupported pidfds, permission failures, an occupied DevTools port, and unsuccessful termination fail closed. A successful supervisorctl response is not used as proof of fencing.
+
+The image CDP proxy forwards `kernelVaultFill=1` to this owner without reconnecting upgraded streams or logging command payloads. The API must complete a nonce-bound `kernel.vault-fill.v1` handshake, including the actual instance name, before sending CDP commands. Older images or proxies cannot complete this handshake and receive no secret-bearing command on the new API path.
+
+One operation occupies the owner. Commands are sequential, ID-checked and use one fixed upstream socket. A finish message is accepted only between acknowledged commands; it closes that upstream before releasing admission and makes the old client terminal. Disconnect, cancellation, protocol failure or loss of a command response leaves the generation occupied. Neither Redis key deletion nor a timeout clears it. There is no reconnect, replay or force-unlock endpoint. Lost finish acknowledgment does not make a browser write unknown: the client already acknowledged every command by sending finish.
+
+API/proxy restart does not restart the launcher or clear its state. Launcher restart must fence surviving Chrome processes before admitting a replacement. Parent-death signaling closes the pre-exec orphan window; the startup process census also covers Chrome surviving owner death. This relies on Linux pidfd/proc semantics, not durable Redis or filesystem lease records. The image's immutable Chromium executable and its shipped process helpers are the supported process cohort; out-of-band browser binaries or subprocess launch wrappers are not a supported fenced configuration.
+
+Standby/restore retains the owner's state together with the browser. An active or quarantined generation does not become idle on disconnect or resume. The owner captures its instance identity at birth and checks it on admission and every command. A template waiting for fork identity cannot fill, and an owner cannot adopt a different identity in place. Fork handoff requires a launcher restart after identity application before fill is supported; that restart must confirm the old browser generation exited. A restored active operation must never be used as a clean template. These platform lifecycle cases still require validation beyond protocol unit tests.
+
+The protocol covers the API's synchronous guarded CDP scripts and their completion. It does not serialize unrelated customer CDP clients or application JavaScript timers. It is not a security boundary against privileged process/file mutation inside the browser.
+
+## Integration order
+
+1. Complete real-image lifecycle and API adversarial validation.
+2. Merge/release the image companion, then integrate that image through the normal release path. Drain incompatible pool/template inventory; validate the target platform's process primitives and fork reset behavior.
+3. Merge the API transport change. Its protocol negotiation is mandatory; no fallback to ordinary CDP is permitted.
+4. Consider the default-off fill gate separately. Neither companion PR enables it.
+
+Pending evidence: actual image owner restart with surviving Chrome, rejected/uncertain stop, lost-response quarantine, terminal old clients, independent browsers, two full public HTTP runs and the formerly failing lost-key regression. Protocol unit tests and API authorization tests against stock Chromium are not substitutes for these checks.