diff --git a/server/cmd/api/api/api.go b/server/cmd/api/api/api.go
index 222ac9a50..676fc5cc7 100644
--- a/server/cmd/api/api/api.go
+++ b/server/cmd/api/api/api.go
@@ -27,6 +27,8 @@ type cdpMonitorController interface {
Start(ctx context.Context) error
Stop()
IsRunning() bool
+ SetTelemetry(bool) error
+ NetworkSnapshot() cdpmonitor.NetworkSnapshot
}
var _ cdpMonitorController = (*cdpmonitor.Monitor)(nil)
@@ -152,6 +154,7 @@ func New(
screenshotEnabled := func() bool { return telemetrySession.CategoryEnabled(events.Screenshot) }
mon := cdpmonitor.New(upstreamMgr, telemetrySession.Publish, displayNum, slog.Default(), screenshotEnabled)
+ _ = mon.SetTelemetry(false)
ctx, cancel := context.WithCancel(context.Background())
return &ApiService{
@@ -430,10 +433,22 @@ func (s *ApiService) ListRecorders(ctx context.Context, _ oapi.ListRecordersRequ
return oapi.ListRecorders200JSONResponse(infos), nil
}
+// StartNetworkMonitor starts process-lifetime capture, independently of customer telemetry.
+func (s *ApiService) StartNetworkMonitor() error {
+ s.monitorMu.Lock()
+ defer s.monitorMu.Unlock()
+ return s.cdpMonitor.Start(s.lifecycleCtx)
+}
+
+func (s *ApiService) NetworkMetrics() (resets, completed uint64, up bool) {
+ snapshot := s.cdpMonitor.NetworkSnapshot()
+ return snapshot.Resets, snapshot.Completed, snapshot.Up
+}
+
func (s *ApiService) Shutdown(ctx context.Context) error {
+ s.lifecycleCancel()
_ = s.webmcp.Close()
s.monitorMu.Lock()
- s.lifecycleCancel()
s.cdpMonitor.Stop()
s.telemetrySession.Stop()
s.monitorMu.Unlock()
diff --git a/server/cmd/api/api/network_metrics_test.go b/server/cmd/api/api/network_metrics_test.go
new file mode 100644
index 000000000..0be2476e9
--- /dev/null
+++ b/server/cmd/api/api/network_metrics_test.go
@@ -0,0 +1,51 @@
+package api
+
+import (
+ "context"
+ "sync"
+ "testing"
+
+ "github.com/kernel/kernel-images/server/lib/oapi"
+ "github.com/stretchr/testify/require"
+)
+
+func TestNetworkMonitorOutlivesCustomerTelemetry(t *testing.T) {
+ svc, err := newSvc(t, newMockRecordManager())
+ require.NoError(t, err)
+ require.NoError(t, svc.StartNetworkMonitor())
+ defer svc.Shutdown(context.Background())
+ require.True(t, svc.cdpMonitor.IsRunning())
+ resets, completed, up := svc.NetworkMetrics()
+ require.Zero(t, resets)
+ require.Zero(t, completed)
+ require.False(t, up)
+ _, err = svc.PutTelemetry(context.Background(), oapi.PutTelemetryRequestObject{})
+ require.NoError(t, err)
+ require.True(t, svc.cdpMonitor.IsRunning())
+ _, err = svc.PutTelemetry(context.Background(), oapi.PutTelemetryRequestObject{Body: &oapi.BrowserTelemetryConfig{Browser: allCategoriesDisabled()}})
+ require.NoError(t, err)
+ require.False(t, svc.telemetrySession.Active())
+ require.True(t, svc.cdpMonitor.IsRunning(), "disabling telemetry must not stop network monitoring")
+ require.NoError(t, svc.Shutdown(context.Background()))
+ require.False(t, svc.cdpMonitor.IsRunning())
+ require.Error(t, svc.StartNetworkMonitor(), "a stopped API lifecycle must not restart capture")
+}
+
+func TestNetworkMonitorTelemetryShutdownRace(t *testing.T) {
+ svc, err := newSvc(t, newMockRecordManager())
+ require.NoError(t, err)
+ require.NoError(t, svc.StartNetworkMonitor())
+ var wg sync.WaitGroup
+ for range 2 {
+ wg.Go(func() {
+ for range 20 {
+ _, _ = svc.PutTelemetry(context.Background(), oapi.PutTelemetryRequestObject{})
+ _, _ = svc.PutTelemetry(context.Background(), oapi.PutTelemetryRequestObject{Body: &oapi.BrowserTelemetryConfig{Browser: allCategoriesDisabled()}})
+ }
+ })
+ }
+ require.NoError(t, svc.Shutdown(context.Background()))
+ wg.Wait()
+ require.False(t, svc.cdpMonitor.IsRunning())
+ require.False(t, svc.telemetrySession.Active())
+}
diff --git a/server/cmd/api/api/telemetry.go b/server/cmd/api/api/telemetry.go
index 278264538..c752fd19f 100644
--- a/server/cmd/api/api/telemetry.go
+++ b/server/cmd/api/api/telemetry.go
@@ -116,13 +116,9 @@ func (s *ApiService) PatchTelemetry(ctx context.Context, req oapi.PatchTelemetry
return oapi.PatchTelemetry200JSONResponse(s.buildTelemetryResponse()), nil
}
-// reconcileTelemetryState reconciles the CDP collector and the api_call
-// middleware to the desired category set. The collector runs iff a CDP category
-// is captured; the middleware emits iff control or platform is, since it is the
-// sole producer of both api_call and platform_api_call. Callers commit the
-// session config first so the filter is live before the collector emits; this
-// returns an error only when the collector fails to start, leaving the caller to
-// roll back.
+// reconcileTelemetryState reconciles optional CDP capture and api_call middleware.
+// Network counters stay active independently. Callers commit the session config
+// first so publication is gated before capture changes, and roll back on failure.
func (s *ApiService) reconcileTelemetryState(cats []oapi.TelemetryEventCategory) error {
if containsCategory(cats, events.Control) || containsCategory(cats, events.Platform) {
EnableTelemetryMiddleware()
@@ -130,19 +126,14 @@ func (s *ApiService) reconcileTelemetryState(cats []oapi.TelemetryEventCategory)
DisableTelemetryMiddleware()
}
- switch {
- case events.HasCDPCategory(cats) && !s.cdpMonitor.IsRunning():
- return s.cdpMonitor.Start(s.lifecycleCtx)
- case !events.HasCDPCategory(cats) && s.cdpMonitor.IsRunning():
- s.cdpMonitor.Stop()
+ if err := s.lifecycleCtx.Err(); err != nil {
+ return err
}
- return nil
+ return s.cdpMonitor.SetTelemetry(events.HasCDPCategory(cats))
}
-// rollbackTelemetry restores telemetry to its prior state after a failed apply.
-// A fresh session is torn down; an updated session is reverted to prev. Reverting
-// never requires a fallible collector start (the failed start left it stopped),
-// so the reconcile here cannot fail.
+// rollbackTelemetry restores the previous desired capture state after a failed
+// apply. A fresh session is torn down; an updated session is reverted to prev.
func (s *ApiService) rollbackTelemetry(wasActive bool, prev telemetry.TelemetryConfig) {
if !wasActive {
s.telemetrySession.Stop()
@@ -187,11 +178,11 @@ func (s *ApiService) reconcileExport(ctx context.Context) {
}
}
-// stopTelemetryState tears down the collector and middleware after a session is
+// stopTelemetryState tears down optional capture and middleware after a session is
// cleared. Export is reconciled separately, after monitorMu is released.
func (s *ApiService) stopTelemetryState() {
- if s.cdpMonitor.IsRunning() {
- s.cdpMonitor.Stop()
+ if err := s.cdpMonitor.SetTelemetry(false); err != nil {
+ logger.FromContext(s.lifecycleCtx).Warn("failed to clean up telemetry capture", "err", err)
}
DisableTelemetryMiddleware()
}
diff --git a/server/cmd/api/api/telemetry_async_test.go b/server/cmd/api/api/telemetry_async_test.go
new file mode 100644
index 000000000..620810f00
--- /dev/null
+++ b/server/cmd/api/api/telemetry_async_test.go
@@ -0,0 +1,214 @@
+package api
+
+import (
+ "context"
+ "io"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/coder/websocket"
+ "github.com/coder/websocket/wsjson"
+ "github.com/kernel/kernel-images/server/lib/cdpmonitor"
+ "github.com/kernel/kernel-images/server/lib/oapi"
+ "github.com/stretchr/testify/require"
+)
+
+type telemetryUpstream string
+
+func (u telemetryUpstream) Current() string { return string(u) }
+func (u telemetryUpstream) Subscribe() (<-chan string, func()) { return nil, func() {} }
+
+func TestTelemetryCleanupDoesNotBlockAPI(t *testing.T) {
+ for _, method := range []string{"PUT", "PATCH"} {
+ for _, ending := range []string{"resume", "timeout", "shutdown"} {
+ t.Run(method+"/"+ending, func(t *testing.T) { testTelemetryCleanupDoesNotBlockAPI(t, method, ending) })
+ }
+ }
+}
+
+func testTelemetryCleanupDoesNotBlockAPI(t *testing.T, method, ending string) {
+ svc, err := newSvc(t, newMockRecordManager())
+ require.NoError(t, err)
+ registered, blocked, release, disabledDomains := make(chan struct{}), make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var registerOnce, blockOnce, releaseOnce, disableOnce sync.Once
+ var runtimeEnables, connections atomic.Int32
+ unblock := func() { releaseOnce.Do(func() { close(release) }) }
+ var socketMu, writes sync.Mutex
+ var socket *websocket.Conn
+ send := func(ctx context.Context, conn *websocket.Conn, value any) error {
+ writes.Lock()
+ defer writes.Unlock()
+ return wsjson.Write(ctx, conn, value)
+ }
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := websocket.Accept(w, r, nil)
+ if err != nil {
+ return
+ }
+ defer conn.CloseNow()
+ connections.Add(1)
+ socketMu.Lock()
+ socket = conn
+ socketMu.Unlock()
+ for {
+ var command struct {
+ ID int `json:"id"`
+ Method string `json:"method"`
+ }
+ if wsjson.Read(r.Context(), conn, &command) != nil {
+ return
+ }
+ result := map[string]any{}
+ switch command.Method {
+ case "Target.getTargets":
+ result["targetInfos"] = []any{map[string]any{"targetId": "page", "type": "page"}}
+ case "Target.attachToTarget":
+ result["sessionId"] = "session"
+ case "Runtime.enable":
+ runtimeEnables.Add(1)
+ case "Runtime.disable":
+ disableOnce.Do(func() { close(disabledDomains) })
+ case "Page.addScriptToEvaluateOnNewDocument":
+ result["identifier"] = "script"
+ case "Page.removeScriptToEvaluateOnNewDocument":
+ delay := false
+ blockOnce.Do(func() { delay = true; close(blocked) })
+ if delay {
+ go func(id int) {
+ select {
+ case <-release:
+ case <-time.After(11 * time.Second):
+ case <-r.Context().Done():
+ return
+ }
+ _ = send(r.Context(), conn, map[string]any{"id": id, "result": map[string]any{}})
+ }(command.ID)
+ continue
+ }
+ }
+ if send(r.Context(), conn, map[string]any{"id": command.ID, "result": result}) != nil {
+ return
+ }
+ if command.Method == "Page.addScriptToEvaluateOnNewDocument" {
+ registerOnce.Do(func() { close(registered) })
+ }
+ }
+ }))
+ defer server.Close()
+ mon := cdpmonitor.New(telemetryUpstream("ws"+strings.TrimPrefix(server.URL, "http")), svc.telemetrySession.Publish, 0, slog.New(slog.NewTextHandler(io.Discard, nil)), nil)
+ svc.cdpMonitor = mon
+ require.NoError(t, mon.SetTelemetry(false))
+ require.NoError(t, svc.StartNetworkMonitor())
+ defer svc.Shutdown(context.Background())
+ defer unblock()
+ require.Eventually(t, func() bool { return mon.NetworkSnapshot().Up }, time.Second, time.Millisecond)
+ on := true
+ enabled := oapi.PutTelemetryRequestObject{Body: &oapi.BrowserTelemetryConfig{Browser: &oapi.BrowserTelemetryCategoriesConfig{Console: &oapi.BrowserTelemetryCategoryConfig{Enabled: &on}}}}
+ _, err = svc.PutTelemetry(context.Background(), enabled)
+ require.NoError(t, err)
+ select {
+ case <-registered:
+ case <-time.After(time.Second):
+ t.Fatal("optional registration not reached")
+ }
+ disabled := &oapi.BrowserTelemetryConfig{Browser: allCategoriesDisabled()}
+ disable := func(ctx context.Context) any {
+ if method == "PATCH" {
+ response, _ := svc.PatchTelemetry(ctx, oapi.PatchTelemetryRequestObject{Body: disabled})
+ return response
+ }
+ response, _ := svc.PutTelemetry(ctx, oapi.PutTelemetryRequestObject{Body: disabled})
+ return response
+ }
+ assertDisabled := func(response any) {
+ t.Helper()
+ if method == "PATCH" {
+ require.IsType(t, oapi.PatchTelemetry200JSONResponse{}, response)
+ } else {
+ require.IsType(t, oapi.PutTelemetry200JSONResponse{}, response)
+ }
+ }
+ requestCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ done := make(chan any, 1)
+ started := time.Now()
+ go func() { done <- disable(requestCtx) }()
+ select {
+ case <-blocked:
+ case <-time.After(2 * time.Second):
+ t.Fatal("cleanup command not reached")
+ }
+ select {
+ case response := <-done:
+ require.Less(t, time.Since(started), time.Second)
+ t.Logf("%s disable response while cleanup is blocked: %s", method, time.Since(started))
+ assertDisabled(response)
+ case <-requestCtx.Done():
+ t.Fatal("telemetry response exceeded a one-second request budget")
+ }
+ cancel() // Cleanup must outlive this request.
+ require.False(t, svc.telemetrySession.Active())
+ // GET and a newer enable both finish while the previous cleanup is blocked.
+ requests := make(chan struct{})
+ var getResponse oapi.GetTelemetryResponseObject
+ go func() {
+ getResponse, _ = svc.GetTelemetry(context.Background(), oapi.GetTelemetryRequestObject{})
+ _, _ = svc.PutTelemetry(context.Background(), enabled)
+ close(requests)
+ }()
+ select {
+ case <-requests:
+ case <-time.After(time.Second):
+ t.Fatal("cleanup held the API-wide lock")
+ }
+ require.IsType(t, oapi.GetTelemetry404JSONResponse{}, getResponse)
+ require.True(t, svc.telemetrySession.Active())
+ seq := svc.eventStream.Seq()
+ socketMu.Lock()
+ conn := socket
+ socketMu.Unlock()
+ before := mon.NetworkSnapshot()
+ require.NoError(t, send(context.Background(), conn, map[string]any{"method": "Runtime.consoleAPICalled", "sessionId": "session", "params": map[string]any{"type": "log", "args": []any{map[string]any{"type": "string", "value": "old-capture"}}}}))
+ require.NoError(t, send(context.Background(), conn, map[string]any{"method": "Network.loadingFailed", "sessionId": "session", "params": map[string]any{"requestId": "during-cleanup", "errorText": "net::ERR_CONNECTION_RESET"}}))
+ require.Eventually(t, func() bool { return mon.NetworkSnapshot().Resets == before.Resets+1 }, time.Second, time.Millisecond)
+ require.Equal(t, seq, svc.eventStream.Seq(), "old capture published into the newer telemetry session")
+ finalDisable := make(chan any, 1)
+ go func() { finalDisable <- disable(context.Background()) }()
+ select {
+ case response := <-finalDisable:
+ assertDisabled(response)
+ case <-time.After(time.Second):
+ t.Fatal("newer disable blocked on cleanup")
+ }
+ if ending == "resume" {
+ unblock()
+ select {
+ case <-disabledDomains:
+ case <-time.After(time.Second):
+ t.Fatal("cleanup did not resume")
+ }
+ require.Never(t, func() bool { return runtimeEnables.Load() != 1 }, 250*time.Millisecond, time.Millisecond, "obsolete enable revision ran after the final disable")
+ require.False(t, svc.telemetrySession.Active())
+ }
+ if ending == "timeout" {
+ require.Eventually(t, func() bool { return connections.Load() >= 2 && mon.NetworkSnapshot().Up }, 6*time.Second, 10*time.Millisecond)
+ require.EqualValues(t, 1, runtimeEnables.Load(), "cleanup recovery enabled stale telemetry")
+ require.False(t, svc.telemetrySession.Active())
+ }
+ require.Equal(t, before.Resets+1, mon.NetworkSnapshot().Resets)
+ require.Equal(t, before.Completed+1, mon.NetworkSnapshot().Completed)
+ shutdown := make(chan struct{})
+ go func() { _ = svc.Shutdown(context.Background()); close(shutdown) }()
+ select {
+ case <-shutdown:
+ case <-time.After(2 * time.Second):
+ t.Fatal("shutdown deadlocked behind cleanup")
+ }
+ require.False(t, mon.IsRunning())
+}
diff --git a/server/cmd/api/api/telemetry_test.go b/server/cmd/api/api/telemetry_test.go
index 8db4243cf..cb14eebc2 100644
--- a/server/cmd/api/api/telemetry_test.go
+++ b/server/cmd/api/api/telemetry_test.go
@@ -8,6 +8,7 @@ import (
"testing"
"time"
+ "github.com/kernel/kernel-images/server/lib/cdpmonitor"
"github.com/kernel/kernel-images/server/lib/events"
oapi "github.com/kernel/kernel-images/server/lib/oapi"
"github.com/kernel/kernel-images/server/lib/recorder"
@@ -415,13 +416,28 @@ func newTestService(t *testing.T, mgr recorder.RecordManager) *ApiService {
type stubCdpMonitor struct{}
+func (s *stubCdpMonitor) SetTelemetry(bool) error { return nil }
+func (s *stubCdpMonitor) NetworkSnapshot() cdpmonitor.NetworkSnapshot {
+ return cdpmonitor.NetworkSnapshot{}
+}
+
func (s *stubCdpMonitor) Start(_ context.Context) error { return nil }
func (s *stubCdpMonitor) Stop() {}
func (s *stubCdpMonitor) IsRunning() bool { return false }
-// failingCdpMonitor always fails to start, to exercise the reconcile-before-commit path.
+// failingCdpMonitor rejects optional capture to exercise configuration rollback.
type failingCdpMonitor struct{ running bool }
+func (f *failingCdpMonitor) SetTelemetry(enabled bool) error {
+ if enabled {
+ return errors.New("collector configuration failed")
+ }
+ return nil
+}
+func (f *failingCdpMonitor) NetworkSnapshot() cdpmonitor.NetworkSnapshot {
+ return cdpmonitor.NetworkSnapshot{}
+}
+
func (f *failingCdpMonitor) Start(_ context.Context) error {
return errors.New("collector start failed")
}
diff --git a/server/cmd/api/main.go b/server/cmd/api/main.go
index b3384c6f5..4fa731764 100644
--- a/server/cmd/api/main.go
+++ b/server/cmd/api/main.go
@@ -248,6 +248,11 @@ func main() {
os.Exit(1)
}
+ if err := apiService.StartNetworkMonitor(); err != nil {
+ slogger.Error("failed to start network monitor", "err", err)
+ os.Exit(1)
+ }
+
// api_call event emission. Off until the telemetry handlers flip it on.
r.Use(api.TelemetryHTTPMiddleware(telemetrySession.Publish))
r.Use(api.WebMCPRequestSizeMiddleware)
@@ -374,6 +379,7 @@ func main() {
rMetrics := chi.NewRouter()
rMetrics.Use(chiMiddleware.Recoverer)
metricsCollectors := []metrics.Collector{
+ metrics.NewNetworkCollector(apiService.NetworkMetrics),
metrics.NewChromeCollector(upstreamMgr),
metrics.NewGPUCollector(),
metrics.NewSystemCollector(),
diff --git a/server/lib/browsersurface/frames.go b/server/lib/browsersurface/frames.go
index 00641d63e..e8b4a7706 100644
--- a/server/lib/browsersurface/frames.go
+++ b/server/lib/browsersurface/frames.go
@@ -68,9 +68,18 @@ func (t *Tracker) attachDedicatedWorkers(sessionID string) {
"autoAttach": true, "flatten": true, "waitForDebuggerOnStart": false,
"filter": []map[string]any{{"type": "worker"}},
}, sessionID)
- if err != nil && ctx.Err() == nil && t.SessionExists(sessionID) {
- t.logger.Warn("failed to attach dedicated workers", "session_id", sessionID, "err", err)
+ if err != nil {
+ if t.SessionExists(sessionID) && !t.protocol.IsClosed() {
+ t.logger.Warn("failed to attach dedicated workers", "session_id", sessionID, "err", err)
+ t.publish(Event{Kind: EventDiscoveryFailed, SessionID: sessionID})
+ }
+ return
}
+ t.stateMu.Lock()
+ if sess := t.sessions[sessionID]; sess != nil {
+ sess.workersAttached = true
+ }
+ t.stateMu.Unlock()
}
func (t *Tracker) initializeSession(sessionID string) {
diff --git a/server/lib/browsersurface/targets.go b/server/lib/browsersurface/targets.go
index 1c44e87c9..c8698c161 100644
--- a/server/lib/browsersurface/targets.go
+++ b/server/lib/browsersurface/targets.go
@@ -70,10 +70,14 @@ func (t *Tracker) attachPage(tabID int, target targetInfo) {
t.logger.Warn("failed to attach browser tab", "tab_id", tabID, "err", err)
}
t.stateMu.Lock()
- if t.tabsByTarget[target.TargetID] == tabID {
+ stillPresent := t.tabsByTarget[target.TargetID] == tabID
+ if stillPresent {
t.trackingTarget[target.TargetID] = false
}
t.stateMu.Unlock()
+ if stillPresent {
+ t.publish(Event{Kind: EventDiscoveryFailed})
+ }
t.signalChanged()
}
@@ -129,8 +133,12 @@ func (t *Tracker) trackNonPageTarget(target targetInfo) {
t.logger.Warn("failed to attach browser target", "target_id", target.TargetID, "type", target.Type, "err", err)
}
t.stateMu.Lock()
+ stillPresent := t.trackingNonPageTarget[target.TargetID]
delete(t.trackingNonPageTarget, target.TargetID)
t.stateMu.Unlock()
+ if stillPresent {
+ t.publish(Event{Kind: EventDiscoveryFailed})
+ }
t.signalChanged()
}()
}
diff --git a/server/lib/browsersurface/tracker.go b/server/lib/browsersurface/tracker.go
index 041416138..897c06722 100644
--- a/server/lib/browsersurface/tracker.go
+++ b/server/lib/browsersurface/tracker.go
@@ -194,6 +194,36 @@ func (t *Tracker) IsClosed() bool {
}
}
+// CaptureSessions returns the attached session IDs and whether known targets
+// and worker discovery are initialized. Consumers also check their own domains.
+func (t *Tracker) CaptureSessions() ([]string, bool) {
+ if t.IsClosed() {
+ return nil, false
+ }
+ t.stateMu.RLock()
+ defer t.stateMu.RUnlock()
+ attached := make(map[string]bool, len(t.sessions))
+ ids := make([]string, 0, len(t.sessions))
+ for id, sess := range t.sessions {
+ ids = append(ids, id)
+ attached[sess.target.TargetID] = true
+ if t.tracksTarget("worker") && sess.target.Type != "service_worker" && !sess.workersAttached {
+ return nil, false
+ }
+ }
+ for target := range t.tabsByTarget {
+ if !attached[target] {
+ return nil, false
+ }
+ }
+ for target := range t.trackingNonPageTarget {
+ if !attached[target] {
+ return nil, false
+ }
+ }
+ return ids, true
+}
+
func (t *Tracker) HasTabs() bool {
t.stateMu.RLock()
defer t.stateMu.RUnlock()
diff --git a/server/lib/browsersurface/types.go b/server/lib/browsersurface/types.go
index 1f667cc58..d45d3d062 100644
--- a/server/lib/browsersurface/types.go
+++ b/server/lib/browsersurface/types.go
@@ -26,6 +26,8 @@ const (
// EventSessionAttached precedes page/frame initialization. Consumers can
// enable domains without waiting for a tab or frame location to resolve.
EventSessionAttached
+ // EventDiscoveryFailed reports attachment or worker-discovery setup failure.
+ EventDiscoveryFailed
)
type SessionTarget struct {
@@ -104,12 +106,13 @@ type frameTree struct {
}
type session struct {
- id string
- parentID string
- target targetInfo
- tabID int
- initializing bool
- initialized bool
+ id string
+ parentID string
+ target targetInfo
+ tabID int
+ initializing bool
+ initialized bool
+ workersAttached bool
}
type window struct {
diff --git a/server/lib/cdpmonitor/README.md b/server/lib/cdpmonitor/README.md
index c8bcee28c..d25498880 100644
--- a/server/lib/cdpmonitor/README.md
+++ b/server/lib/cdpmonitor/README.md
@@ -1,12 +1,90 @@
# CDP Monitor
-The monitor is the browser-facing layer of the kernel browser logging pipeline. It owns a connection to Chrome's DevTools endpoint, uses `browsersurface` to track page, iframe, and worker sessions, and converts raw CDP notifications into typed `events.Event` values for downstream consumers.
+The API starts the monitor independently of customer telemetry. It owns one persistent connection to Chrome's DevTools endpoint and uses `browsersurface` to track page, iframe, worker, and background-page sessions. Metrics-only mode enables Network and target discovery; customer telemetry optionally adds typed events, computed timers, body retrieval, screenshots, and interaction capture.
## Overview
`cdpmonitor` manages a Chrome DevTools Protocol (CDP) WebSocket connection to a running Chrome browser. It subscribes to CDP events across all attached tabs, translates them into structured `events.Event` values, and publishes them via a caller-supplied `PublishFunc`. It also derives synthetic events from sequences of CDP events and takes screenshots on significant page activity.
-Chrome can restart independently of the monitor. When that happens, `UpstreamProvider` pushes a new DevTools URL and the monitor reconnects automatically, emitting lifecycle events so consumers can track continuity.
+Chrome can restart independently of the monitor. The monitor retries startup failures, upstream notifications, socket loss (including at the same URL), and required discovery/domain initialization failures. Customer events still publish exclusively through `TelemetrySession.Publish`; platform counters do not export event payloads.
+
+## Always-on network metrics
+
+The separate in-memory collector serves these **label-free** metrics on the existing
+`GET /metrics` endpoint, including zeros before capture starts:
+
+| Metric | Meaning |
+| --- | --- |
+| `kernel_chromium_connection_resets_total` | Observed terminal `Network.loadingFailed` outcomes with exactly `net::ERR_CONNECTION_RESET`. |
+| `kernel_chromium_network_requests_completed_total` | Observed terminal `Network.loadingFinished` or `Network.loadingFailed` outcomes. Includes cancellations, refusals, HTTP 500 responses, and unknown-start outcomes. |
+| `kernel_chromium_network_monitor_up` | Discovery initialized, socket open, and Network plus dedicated-worker discovery initialized for every known attached target. Reattached targets also finish any retained interaction cleanup before becoming ready. Zero during setup, failures, reconnect, and shutdown. Not browser responsiveness or proof of complete request coverage. |
+
+These count **CDP request-chain observations, not socket resets**. Internal browser
+retries are not separately counted. Redirects reuse a request ID and contribute
+one final outcome, not one per hop. HTTP status does not classify transport errors.
+Missing `requestWillBeSent` does not exclude a valid terminal event from either
+counter; metrics-only mode does not retain request-start records or bodies.
+
+Deduplication is first-terminal-wins within the most recent **8,192 distinct
+(session ID, request ID)** terminal keys in one connection generation. A fixed FIFO
+and map bound the history. Retained IDs are limited to 256 bytes each; empty,
+oversized, malformed, sessionless, and untracked-session events are ignored.
+Identities are cleared only after old connection work drains, so a reused session
+and request ID on a fresh connection counts again. Totals survive reconnections,
+Chrome restarts, and telemetry toggles; a new API process starts at zero.
+
+There is no cross-session deduplication: equal IDs on different targets may be
+unrelated, while workers/service workers can expose multiple observations of one
+logical fetch. A duplicate arriving after FIFO eviction can count again. Requests
+before attachment/domain readiness, during disconnection, or after target detach
+can be missed. These metrics are **not lossless** and are not unique HTTP-request
+or socket counts. The reset fraction uses the same observed-terminal denominator.
+
+The existing short-lived `ChromeCollector` and its UMA behavior are unchanged.
+Network metrics remain available when Chrome/UMA collection fails; scrapes do not
+reset or increment the counters. No URLs, domains, identities, or error-string
+labels are emitted. No additional configuration or kill switch is introduced.
+
+### Local verification and remaining checks
+
+```sh
+# From server/; Chromium must be installed for the opt-in suite.
+go test -race ./lib/cdpmonitor ./lib/metrics ./cmd/api/api
+KERNEL_CDPMONITOR_CHROME_E2E=1 go test -race ./lib/cdpmonitor -count=1 -v
+go test ./lib/cdpmonitor -run '^$' -bench '^BenchmarkMetricsOnlyTerminalDispatch$' -benchmem
+```
+
+The metrics fixture uses local HTTP and TCP RST (`SetLinger(0)`), POST requests to
+avoid transparent GET retries, cache-disabled responses, and settled targets.
+It independently checks Chromium's error text and exact +10/+10 scrape deltas;
+then exercises non-reset outcomes, same-process frames, OOPIFs, dedicated/shared/
+service workers, telemetry off/on/off cleanup, socket replacement, actual Chrome
+restart, and a fresh monitor's zero counters. API lifecycle/race tests cover
+startup, telemetry toggles, and shutdown. Additional regressions delay a cleanup
+command while PUT/PATCH/GET continue, fence old capture across coalesced revisions,
+and recover page/frame listeners after socket loss or failed cleanup. They check
+future navigations and the independent user CDP connection as well. Extension
+background pages are included in discovery but are not validated by a real extension
+fixture here. A delayed-Network-reply regression also verifies that pending
+attachments finish recovery before optional instrumentation starts, and that
+click events arrive without navigation through subsequent telemetry toggles.
+
+Known limitation: re-enabling telemetry can leave already-loaded same-process
+iframes without interaction listeners. This also occurs with the previous
+Stop/Start lifecycle and is not addressed by the attachment-ordering fix.
+
+The microbenchmark measures Go terminal ingestion, not total Chromium CPU/memory
+or live workload overhead. Full image/API-process restart, suspend/resume, snapshot
+fork, and staging checks are not covered by these local tests. A restored process
+retains its counters and dedup history; a fork of its memory may inherit its parent's
+baseline. Applying fork identity does not reset these process-lifetime counters.
+Scrapers must use the new instance identity and treat the first sample as a baseline,
+not a count of post-fork activity. Half-open socket detection uses a 5-second
+probe interval plus a 5-second timeout after execution resumes; health can lag a
+silent failure until that probe. A failed probe cancels the connection before
+publishing disconnection or waiting for serialized teardown, so blocked optional
+setup/body work cannot keep health up or delay recovery through its command timeout.
+Reattachment then follows the retry/setup limits below. No Chromium patch is involved.
## Real-Chromium network regression tests
@@ -49,6 +127,8 @@ targets, not requests issued before their capture domains finish initializing.
| CDP transport and command routing | `../cdpclient` |
| Target discovery and attachment lifecycle | `../browsersurface` |
| CDP domain setup per session | `domains.go` |
+| Desired telemetry revision and asynchronous reconcile | `telemetry.go` |
+| Interaction cleanup across connection replacement | `interaction_cleanup.go` |
| Event translation (CDP params to `events.Event`) | `handlers.go` |
| Synthetic event state machines | `computed.go` |
| Screenshot capture via ffmpeg | `screenshot.go` |
@@ -74,21 +154,79 @@ locations and does not subscribe to workers.
### Reconnect and shutdown
-`subscribeToUpstream` listens for new DevTools URLs. `handleUpstreamRestart` cancels
-the current connection context, closes its protocol, stops consuming its events,
-and drains capture work before clearing state and constructing a new client/tracker.
-Dial retries use capped-exponential backoff (250 ms, 500 ms, 1 s, then 2 s; at most
-10 attempts). CDP sessions and pending requests never carry into the next connection.
-
-`asyncWg` tracks the upstream listener and request sweeper. `captureWg` tracks
-tracker startup, domain initialization, body fetches, and screenshots. `Stop` cancels
-the lifecycle, waits for the lifecycle workers, and drains the current connection.
-Closing the protocol unblocks pending commands, including callers without the
-monitor's cancellation context.
+`Start` subscribes before reading the current URL. The supervisor reacts to socket
+closure, upstream notifications, and required initialization failure; a 5-second
+browser-level probe detects silent socket failure. It cancels and drains the old
+connection before clearing identities and creating the next client/tracker. Retries
+continue for the API lifecycle, with delays doubling from 250 ms to a 5-second cap.
+A healthy probe resets the delay. Dials take at most 5 seconds; initial discovery
+and Network commands have 30-second limits. Health requires domain readiness,
+not just a successful dial or probe. The existing `monitor_disconnected` payload
+retains its legacy `chrome_restarted` reason for connection replacement, including
+socket loss; use the capture-health gauge rather than that reason to diagnose
+availability. `monitor_reconnected` is emitted only after an established connection
+was lost, with duration measured from that loss across failed retries. Initial
+acquisition (including absent URLs or failed dials) emits no restart event. Each
+`Start` begins a new event lifecycle; queued pre-acquisition URL notifications are
+superseded by the current URL. Retries no longer exhaust, so
+`monitor_reconnect_failed` is not emitted.
+
+`asyncWg` tracks the supervisor, telemetry reconciler, and request sweeper. `captureWg` tracks discovery
+and domain setup; `telemetryWg` drains optional body/screenshot work. `Stop` cancels
+the lifecycle and drains all three. Closing the protocol unblocks pending commands.
+
+`SetTelemetry` commits a desired revision and fences customer publication without
+waiting for CDP work. The telemetry endpoints report this accepted desired state,
+not completion of background cleanup. A lifecycle-owned worker serializes teardown
+and setup outside the API-wide lock. An off/on pair still drains the old revision;
+a newer disable prevents an obsolete enable from running after slow cleanup.
+Reconciliation schedules optional setup only for attachment-ready sessions;
+pending sessions finish Network setup and orphan cleanup in the attachment path
+before starting optional capture.
+`TelemetrySession.Publish` continues to enforce the customer's category/session gate.
+
+The worker drains bounded setup/body work without aborting socket writes, stops
+computed timers, removes new-document registrations and live-document listeners,
+and disables optional domains. Cleanup commands share a 3-second budget; failure
+replaces only the monitor connection and retries. Setup remains bounded at 30 seconds.
+Network counters continue during the drain. Request cancellation does not cancel
+cleanup; API shutdown cancels the lifecycle and joins the worker.
+
+Closing CDP does **not** remove document listeners. Cleanup obligations therefore
+use target IDs, survive connection/session replacement, and are cleared only on
+successful cleanup or confirmed target destruction. On reconnect, a target inventory
+prunes vanished targets; reattached targets clean orphan listeners before being
+marked ready, even when telemetry is now disabled. A temporary cleanup registration
+with `runImmediately` reaches existing main worlds, including same-process frames;
+it is then removed. No Runtime/Page domain is enabled solely for this recovery.
+Registration IDs remain connection-local and are never reused on another session.
+These obligations are in memory; full API-process restart remains an unverified check.
+
+A typed CDP rejection of script registration creates no new cleanup obligation and
+never falls back to live-document injection. Prior obligations remain intact.
+For a still-live target, timeouts, transport failures, and invalid registration
+responses retain the obligation and replace the connection because registration
+may have taken effect.
+Successful registration still requires cleanup when current-document evaluation
+fails. Clean targets do not issue an unnecessary cleanup registration. A local
+Chromium proxy regression persistently rejects this Page command; it is an injected
+protocol failure, not a discovered website-specific trigger.
+
+Registrations have individual in-flight markers. Confirmed target destruction
+invalidates matching markers, so late acknowledgements or uncertain outcomes
+cannot recreate cleanup obligations or force recovery for a destroyed document.
+Ordinary detach does not invalidate them. Markers are removed on completion or
+destruction; there is no persistent destroyed-target tombstone set.
### Synchronization
-`restartMu` serializes connection replacement. `lifeMu` protects the connection
+`controlMu` serializes Start/Stop. `desiredMu` fences only in-memory publication
+against desired-revision changes; it never guards CDP work. `restartMu` serializes
+connection replacement against background reconcile, and `telemetryMu` protects
+optional setup/dispatch. Old body/screenshot/computed work is drained before the
+applied revision advances. `sessionsMu` also guards domain readiness, connection-local
+script IDs, and target-scoped interaction cleanup obligations.
+`lifeMu` protects the connection
pointer and lifecycle context/cancel function; it is released before waiting on
connection or capture work. `sessionsMu` protects target metadata and computed-state
lookup. `pendReqMu` protects requests keyed by **(CDP session ID, request ID)**;
diff --git a/server/lib/cdpmonitor/attachment_telemetry_chrome_e2e_test.go b/server/lib/cdpmonitor/attachment_telemetry_chrome_e2e_test.go
new file mode 100644
index 000000000..be9165eb0
--- /dev/null
+++ b/server/lib/cdpmonitor/attachment_telemetry_chrome_e2e_test.go
@@ -0,0 +1,180 @@
+package cdpmonitor
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/coder/websocket"
+ "github.com/stretchr/testify/require"
+)
+
+func TestTelemetryWaitsForAttachmentChrome(t *testing.T) {
+ if os.Getenv("KERNEL_CDPMONITOR_CHROME_E2E") == "" {
+ t.Skip("set KERNEL_CDPMONITOR_CHROME_E2E=1")
+ }
+ for _, toggles := range []int{1, 5} {
+ t.Run(fmt.Sprintf("toggles_%d", toggles), func(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ page := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/html")
+ fmt.Fprint(w, ``)
+ }))
+ defer page.Close()
+ ws := launchChromium(t, ctx, findChromium(t))
+ driver := dialCDP(t, ctx, ws)
+ defer driver.close()
+ target := driver.call(t, ctx, "", "Target.createTarget", map[string]any{"url": page.URL}).targetID(t)
+ session := driver.call(t, ctx, "", "Target.attachToTarget", map[string]any{"targetId": target, "flatten": true}).sessionID(t)
+ require.Eventually(t, func() bool {
+ return driver.evalBool(ctx, session, `document.readyState === 'complete' && !!document.querySelector('#go')`)
+ }, 5*time.Second, 10*time.Millisecond)
+ proxy, blocked, release := delayNetworkReplies(t, ctx, ws)
+ ec := newEventCollector()
+ mon := New(newTestUpstream(proxy), ec.publishFn(), 0, discardLogger, func() bool { return false })
+ require.NoError(t, mon.SetTelemetry(false))
+ require.NoError(t, mon.Start(ctx))
+ defer mon.Stop()
+ defer release()
+ for waiting := true; waiting; {
+ select {
+ case sid := <-blocked:
+ mon.sessionsMu.RLock()
+ waiting = mon.sessions[sid].targetID != target
+ mon.sessionsMu.RUnlock()
+ case <-ctx.Done():
+ t.Fatal("page Network.enable reply was not intercepted")
+ }
+ }
+ for i := 0; i < toggles; i++ {
+ require.NoError(t, mon.SetTelemetry(i%2 == 0))
+ require.Eventually(t, func() bool {
+ return mon.appliedTelemetry.Load() == mon.desiredTelemetry.Load() && !mon.telemetryChanging.Load()
+ }, time.Second, time.Millisecond)
+ }
+ require.False(t, mon.NetworkSnapshot().Up)
+ require.Never(t, func() bool {
+ return driver.evalBool(ctx, session, `window.__kernelEventInjected === true`)
+ }, 250*time.Millisecond, 10*time.Millisecond, "optional injection ran before attachment recovery")
+ release()
+ waitForTelemetryReconcile(t, mon, true)
+ for i := 0; i < 3; i++ {
+ require.Eventually(t, func() bool {
+ return driver.evalBool(ctx, session, `window.__kernelEventInjected === true`)
+ }, time.Second, 10*time.Millisecond)
+ checkpoint := ec.checkpoint()
+ evaluateNetworkScript(t, ctx, driver, session, `document.querySelector('#go').click(); true`)
+ ec.waitForNew(t, EventInteractionClick, checkpoint, time.Second)
+ if i < 2 {
+ require.NoError(t, mon.SetTelemetry(false))
+ waitForTelemetryReconcile(t, mon, false)
+ require.False(t, driver.evalBool(ctx, session, `window.__kernelEventInjected === true`))
+ require.NoError(t, mon.SetTelemetry(true))
+ waitForTelemetryReconcile(t, mon, true)
+ }
+ }
+ })
+ }
+}
+
+// Only Network.enable responses are held; commands, events, and other replies
+// continue to flow. The separate user CDP connection bypasses this proxy.
+func delayNetworkReplies(t *testing.T, parent context.Context, upstreamURL string) (string, <-chan string, func()) {
+ t.Helper()
+ return delayCommandReplies(t, parent, upstreamURL, func(method string) bool { return method == "Network.enable" })
+}
+
+func delayCommandReplies(t *testing.T, parent context.Context, upstreamURL string, hold func(method string) bool) (string, <-chan string, func()) {
+ t.Helper()
+ blocked := make(chan string, 16)
+ released := make(chan struct{})
+ var once sync.Once
+ release := func() { once.Do(func() { close(released) }) }
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ client, err := websocket.Accept(w, r, nil)
+ if err != nil {
+ return
+ }
+ defer client.CloseNow()
+ ctx, cancel := context.WithCancel(parent)
+ defer cancel()
+ upstream, _, err := websocket.Dial(ctx, upstreamURL, nil)
+ if err != nil {
+ return
+ }
+ defer upstream.CloseNow()
+ var mu sync.Mutex
+ pending := make(map[int]string)
+ var delayed sync.WaitGroup
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ defer cancel()
+ for {
+ kind, data, err := upstream.Read(ctx)
+ if err != nil {
+ return
+ }
+ var reply struct {
+ ID int `json:"id"`
+ }
+ if json.Unmarshal(data, &reply) != nil {
+ return
+ }
+ mu.Lock()
+ sid, hold := pending[reply.ID]
+ delete(pending, reply.ID)
+ mu.Unlock()
+ if hold {
+ delayed.Go(func() {
+ select {
+ case blocked <- sid:
+ case <-ctx.Done():
+ return
+ }
+ select {
+ case <-released:
+ _ = client.Write(ctx, kind, data)
+ case <-ctx.Done():
+ }
+ })
+ } else if client.Write(ctx, kind, data) != nil {
+ return
+ }
+ }
+ }()
+ defer func() { cancel(); <-done; delayed.Wait() }()
+ for {
+ kind, data, err := client.Read(ctx)
+ if err != nil {
+ return
+ }
+ var command struct {
+ ID int `json:"id"`
+ Method string `json:"method"`
+ SessionID string `json:"sessionId"`
+ }
+ if json.Unmarshal(data, &command) != nil {
+ return
+ }
+ if hold(command.Method) {
+ mu.Lock()
+ pending[command.ID] = command.SessionID
+ mu.Unlock()
+ }
+ if upstream.Write(ctx, kind, data) != nil {
+ return
+ }
+ }
+ }))
+ t.Cleanup(server.Close)
+ return "ws" + strings.TrimPrefix(server.URL, "http"), blocked, release
+}
diff --git a/server/lib/cdpmonitor/cdp_test.go b/server/lib/cdpmonitor/cdp_test.go
index 353b2b4bd..8e1d2451c 100644
--- a/server/lib/cdpmonitor/cdp_test.go
+++ b/server/lib/cdpmonitor/cdp_test.go
@@ -262,11 +262,11 @@ func (c *eventCollector) assertNone(t *testing.T, eventType string, d time.Durat
}
// ResponderFunc is called for each CDP command the Monitor sends.
-// Return nil to use the default empty result.
+// Return nil to use the default result.
type ResponderFunc func(msg cdpMessage) any
// listenAndRespond drains srv.msgCh, calls fn for each command, and sends the
-// response. If fn is nil or returns nil, sends {"id": msg.ID, "result": {}}.
+// response. The default is empty except for script registration's required ID.
func listenAndRespond(srv *testServer, stopCh <-chan struct{}, fn ResponderFunc) {
for {
select {
@@ -286,7 +286,11 @@ func listenAndRespond(srv *testServer, stopCh <-chan struct{}, fn ResponderFunc)
resp = fn(msg)
}
if resp == nil {
- resp = map[string]any{"id": msg.ID, "result": map[string]any{}}
+ result := map[string]any{}
+ if msg.Method == "Page.addScriptToEvaluateOnNewDocument" {
+ result["identifier"] = "test-script"
+ }
+ resp = map[string]any{"id": msg.ID, "result": result}
}
_ = wsjson.Write(context.Background(), c, resp)
case <-stopCh:
diff --git a/server/lib/cdpmonitor/domains.go b/server/lib/cdpmonitor/domains.go
index 00073695b..b2aa7e853 100644
--- a/server/lib/cdpmonitor/domains.go
+++ b/server/lib/cdpmonitor/domains.go
@@ -3,6 +3,10 @@ package cdpmonitor
import (
"context"
_ "embed"
+ "encoding/json"
+ "errors"
+
+ "github.com/kernel/kernel-images/server/lib/cdpclient"
)
// bindingName is the JS function exposed via Runtime.addBinding.
@@ -16,18 +20,13 @@ func isPageLikeTarget(targetType string) bool {
return targetType == "page" || targetType == "iframe"
}
-// enableDomains enables CDP domains, registers the event binding, and starts
-// layout-shift observation. Failures are non-fatal.
+// enableDomains enables optional telemetry domains, registers the event binding,
+// and starts layout-shift observation. Failures are non-fatal.
// Page-level domains (Page.enable, PerformanceTimeline.enable, Runtime.addBinding)
// are skipped for worker and service_worker targets that don't support them.
func (m *Monitor) enableDomains(ctx context.Context, sessionID string, targetType string) {
- for _, method := range []string{
- "Runtime.enable",
- "Network.enable",
- } {
- if _, err := m.send(ctx, method, nil, sessionID); err != nil && ctx.Err() == nil {
- m.log.Warn("cdpmonitor: failed to enable CDP domain", "method", method, "session", sessionID, "err", err)
- }
+ if _, err := m.send(ctx, "Runtime.enable", nil, sessionID); err != nil && ctx.Err() == nil {
+ m.log.Warn("cdpmonitor: failed to enable CDP domain", "method", "Runtime.enable", "session", sessionID, "err", err)
}
if !isPageLikeTarget(targetType) {
@@ -62,14 +61,60 @@ func (m *Monitor) enableDomains(ctx context.Context, sessionID string, targetTyp
//go:embed interaction.js
var injectedJS string
+type interactionInjection struct {
+ targetID string
+ destroyed bool // sessionsMu
+}
+
// injectScript installs the interaction tracker for the session on both future
// document loads and the currently-loaded document, so a page that was already
// live when the session attached is tracked without waiting for a navigation.
-// Idempotent via window.__kernelEventInjected.
+// Idempotent for the current binding; replaces listeners from an old connection.
func (m *Monitor) injectScript(ctx context.Context, sessionID string) error {
- _, err := m.send(ctx, "Page.addScriptToEvaluateOnNewDocument", map[string]any{
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ m.sessionsMu.Lock()
+ info, exists := m.sessions[sessionID]
+ injection := &interactionInjection{targetID: info.targetID}
+ if exists {
+ m.pendingInjections[injection] = struct{}{}
+ }
+ m.sessionsMu.Unlock()
+ raw, err := m.send(ctx, "Page.addScriptToEvaluateOnNewDocument", map[string]any{
"source": injectedJS,
}, sessionID)
+ var rejection *cdpclient.Error
+ rejected := errors.As(err, &rejection)
+ m.sessionsMu.Lock()
+ delete(m.pendingInjections, injection)
+ destroyed := injection.destroyed
+ // Detach alone preserves success/uncertain obligations; confirmed destruction
+ // invalidates even a late result. Rejection never erases an earlier obligation.
+ if exists && !destroyed && !rejected {
+ m.interactionTargets[info.targetID] = struct{}{}
+ }
+ m.sessionsMu.Unlock()
+ if destroyed {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ var result struct {
+ Identifier string `json:"identifier"`
+ }
+ if err := json.Unmarshal(raw, &result); err != nil {
+ return err
+ }
+ if result.Identifier == "" {
+ return errors.New("script registration returned no identifier")
+ }
+ m.sessionsMu.Lock()
+ if _, exists := m.optionalSessions[sessionID]; exists {
+ m.optionalSessions[sessionID] = result.Identifier
+ }
+ m.sessionsMu.Unlock()
// Some documents have no evaluable main-world context (e.g. chrome:// pages);
// the registration above still applies to the next load. Surface anything else.
if _, evalErr := m.send(ctx, "Runtime.evaluate", map[string]any{
@@ -77,5 +122,46 @@ func (m *Monitor) injectScript(ctx context.Context, sessionID string) error {
}, sessionID); evalErr != nil && ctx.Err() == nil {
m.log.Warn("cdpmonitor: failed to inject interaction script into current document", "session", sessionID, "err", evalErr)
}
- return err
+ return nil
+}
+
+func (m *Monitor) disableOptionalDomains(ctx context.Context, sessionID, scriptID string) error {
+ m.sessionsMu.RLock()
+ info, exists := m.sessions[sessionID]
+ _, dirty := m.interactionTargets[info.targetID]
+ m.sessionsMu.RUnlock()
+ if !exists {
+ return nil
+ }
+ var cleanupErr error
+ if scriptID != "" {
+ _, cleanupErr = m.send(ctx, "Page.removeScriptToEvaluateOnNewDocument", map[string]any{"identifier": scriptID}, sessionID)
+ }
+ if isPageLikeTarget(info.targetType) {
+ if dirty {
+ cleanupErr = errors.Join(cleanupErr, m.cleanupInteraction(ctx, sessionID))
+ }
+ for _, command := range []struct {
+ method string
+ params any
+ }{
+ {"Runtime.removeBinding", map[string]any{"name": bindingName}},
+ {"PerformanceTimeline.enable", map[string]any{"eventTypes": []string{}}},
+ {"Inspector.disable", nil},
+ {"Page.disable", nil},
+ } {
+ if _, err := m.send(ctx, command.method, command.params, sessionID); err != nil {
+ cleanupErr = errors.Join(cleanupErr, err)
+ }
+ }
+ }
+ _, err := m.send(ctx, "Runtime.disable", nil, sessionID)
+ cleanupErr = errors.Join(cleanupErr, err)
+ if cleanupErr == nil {
+ m.sessionsMu.Lock()
+ delete(m.interactionTargets, info.targetID)
+ delete(m.optionalSessions, sessionID)
+ m.sessionsMu.Unlock()
+ }
+ return cleanupErr
}
diff --git a/server/lib/cdpmonitor/handlers.go b/server/lib/cdpmonitor/handlers.go
index d347cf256..ea67ec7c8 100644
--- a/server/lib/cdpmonitor/handlers.go
+++ b/server/lib/cdpmonitor/handlers.go
@@ -99,12 +99,29 @@ func (m *Monitor) decodeParams(method string, params json.RawMessage, dst any) b
// dispatchEvent routes a CDP event to its handler.
func (m *Monitor) dispatchEvent(msg cdpMessage) {
- m.lifeMu.Lock()
- ctx := m.lifecycleCtx
- if m.conn != nil {
- ctx = m.conn.ctx
+ if msg.Method == "Network.loadingFinished" || msg.Method == "Network.loadingFailed" {
+ var p struct {
+ RequestID string `json:"requestId"`
+ ErrorText string `json:"errorText"`
+ }
+ if m.decodeParams(msg.Method, msg.Params, &p) {
+ if msg.Method == "Network.loadingFinished" {
+ p.ErrorText = ""
+ }
+ m.network.terminal(msg.SessionID, p.RequestID, p.ErrorText)
+ }
+ }
+ if !m.captureEnabled() || m.telemetryChanging.Load() || !m.telemetryMu.TryRLock() {
+ return
+ }
+ defer m.telemetryMu.RUnlock()
+ if !m.telemetryEnabled {
+ return
+ }
+ ctx := m.telemetryCtx
+ if ctx.Err() != nil {
+ return
}
- m.lifeMu.Unlock()
switch msg.Method {
case "Runtime.consoleAPICalled":
@@ -557,7 +574,7 @@ func (m *Monitor) handleLoadingFinished(ctx context.Context, p cdpNetworkLoading
cs.onLoadingFinished()
}
// Fetch response body async to avoid blocking readLoop; binary types are skipped.
- m.captureWg.Go(func() {
+ m.telemetryWg.Go(func() {
body := m.fetchResponseBody(ctx, p.RequestID, sessionID, state)
var hdrs oapi.BrowserHttpHeaders
_ = json.Unmarshal(state.resHeaders, &hdrs)
@@ -857,34 +874,52 @@ func (m *Monitor) handleAttachedToTarget(ctx context.Context, p cdpTargetAttache
return
}
m.sessions[p.SessionID] = targetInfo{
+ title: p.TargetInfo.Title,
+ openerID: p.TargetInfo.OpenerID,
targetID: p.TargetInfo.TargetID,
url: p.TargetInfo.URL,
targetType: p.TargetInfo.Type,
parentFrameID: p.TargetInfo.ParentFrameID,
}
- if p.TargetInfo.Type == targetTypePage {
- m.computedStates[p.SessionID] = newComputedState(m.publish)
- }
+ info := m.sessions[p.SessionID]
m.sessionsMu.Unlock()
-
- if p.TargetInfo.Type == targetTypePage {
- data, _ := json.Marshal(oapi.BrowserPageTabOpenedEventData{
- TargetId: p.TargetInfo.TargetID,
- TargetType: oapi.BrowserTargetType(p.TargetInfo.Type),
- Url: p.TargetInfo.URL,
- OpenerId: ptrOf(p.TargetInfo.OpenerID),
- Title: ptrOf(p.TargetInfo.Title),
- })
- m.publishEvent(EventTabOpened, events.Page, oapi.BrowserEventSource{Kind: oapi.Cdp}, "Target.attachedToTarget", data, p.SessionID)
+ if !m.telemetryChanging.Load() && m.telemetryMu.TryRLock() {
+ m.preparePageCapture(p.SessionID, info)
+ m.telemetryMu.RUnlock()
}
- targetType := p.TargetInfo.Type
- // Domain setup must not block delivery of protocol events.
+ // Network setup runs independently of optional capture configuration.
m.captureWg.Go(func() {
- m.enableDomains(ctx, p.SessionID, targetType)
- if isPageLikeTarget(targetType) {
- _ = m.injectScript(ctx, p.SessionID)
+ if _, err := m.send(ctx, "Network.enable", nil, p.SessionID); err != nil {
+ m.sessionsMu.RLock()
+ _, exists := m.sessions[p.SessionID]
+ m.sessionsMu.RUnlock()
+ if exists && ctx.Err() == nil {
+ m.log.Warn("cdpmonitor: Network.enable failed", "err", err)
+ m.lifeMu.Lock()
+ if m.conn != nil {
+ m.conn.cancel()
+ }
+ m.lifeMu.Unlock()
+ }
+ return
+ }
+ m.telemetryMu.RLock()
+ defer m.telemetryMu.RUnlock()
+ if err := m.cleanupAttachedTarget(ctx, p.SessionID, info); err != nil {
+ m.lifeMu.Lock()
+ if m.conn != nil {
+ m.conn.cancel()
+ }
+ m.lifeMu.Unlock()
+ return
}
+ m.sessionsMu.Lock()
+ if _, exists := m.sessions[p.SessionID]; exists {
+ m.networkReady[p.SessionID] = true
+ }
+ m.sessionsMu.Unlock()
+ m.enableOptionalCapture(m.telemetryCtx, p.SessionID, info)
})
}
@@ -895,6 +930,8 @@ func (m *Monitor) handleDetachedFromTarget(p cdpTargetDetachedFromTargetParams)
m.sessionsMu.Lock()
cs := m.computedStates[p.SessionID]
delete(m.sessions, p.SessionID)
+ delete(m.networkReady, p.SessionID)
+ delete(m.optionalSessions, p.SessionID)
delete(m.computedStates, p.SessionID)
m.sessionsMu.Unlock()
if cs != nil {
diff --git a/server/lib/cdpmonitor/injection_destruction_chrome_e2e_test.go b/server/lib/cdpmonitor/injection_destruction_chrome_e2e_test.go
new file mode 100644
index 000000000..db3b4390f
--- /dev/null
+++ b/server/lib/cdpmonitor/injection_destruction_chrome_e2e_test.go
@@ -0,0 +1,124 @@
+package cdpmonitor
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestLateInjectionAfterTargetDestructionChrome(t *testing.T) {
+ if os.Getenv("KERNEL_CDPMONITOR_CHROME_E2E") == "" {
+ t.Skip("set KERNEL_CDPMONITOR_CHROME_E2E=1")
+ }
+ for _, outcome := range []string{"success", "canceled"} {
+ t.Run(outcome, func(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ page := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/html")
+ fmt.Fprint(w, ``)
+ }))
+ defer page.Close()
+ ws := launchChromium(t, ctx, findChromium(t))
+ driver := dialCDP(t, ctx, ws)
+ defer driver.close()
+ target := driver.call(t, ctx, "", "Target.createTarget", map[string]any{"url": page.URL}).targetID(t)
+ session := driver.call(t, ctx, "", "Target.attachToTarget", map[string]any{"targetId": target, "flatten": true}).sessionID(t)
+ var armed atomic.Bool
+ proxy, blocked, release := delayCommandReplies(t, ctx, ws, func(method string) bool {
+ return armed.Load() && method == "Page.addScriptToEvaluateOnNewDocument"
+ })
+ m := New(newTestUpstream(proxy), newEventCollector().publishFn(), 0, discardLogger, func() bool { return false })
+ require.NoError(t, m.Start(ctx))
+ defer m.Stop()
+ defer release()
+ require.Eventually(t, func() bool {
+ return m.NetworkSnapshot().Up && driver.evalBool(ctx, session, `window.__kernelEventInjected === true`)
+ }, 5*time.Second, time.Millisecond)
+ m.captureWg.Wait()
+ completed := m.NetworkSnapshot().Completed
+ result := evaluateNetworkScript(t, ctx, driver, session, `fetch('/count', {method:'POST'}).then(async r => { await r.text(); return r.ok; })`)
+ require.JSONEq(t, "true", string(result))
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Completed == completed+1 }, time.Second, time.Millisecond)
+ m.sessionsMu.RLock()
+ monitorSession := ""
+ for id, info := range m.sessions {
+ if info.targetID == target {
+ monitorSession = id
+ }
+ }
+ _, dirty := m.interactionTargets[target]
+ m.sessionsMu.RUnlock()
+ require.NotEmpty(t, monitorSession)
+ require.True(t, dirty)
+ m.lifeMu.Lock()
+ original := m.conn
+ m.lifeMu.Unlock()
+ before := m.NetworkSnapshot()
+
+ // Retry a registration on an instrumented target. Chrome executes it;
+ // only its acknowledgement is held while the independent client closes it.
+ armed.Store(true)
+ injectCtx, cancelInjection := context.WithCancel(ctx)
+ defer cancelInjection()
+ done := make(chan error, 1)
+ m.captureWg.Go(func() {
+ m.telemetryMu.RLock()
+ err := m.injectScript(injectCtx, monitorSession)
+ m.telemetryMu.RUnlock()
+ done <- err
+ })
+ select {
+ case sid := <-blocked:
+ require.Equal(t, monitorSession, sid)
+ case <-time.After(time.Second):
+ t.Fatal("registration acknowledgement was not intercepted")
+ }
+ raw := driver.call(t, ctx, "", "Target.closeTarget", map[string]any{"targetId": target})
+ var closed struct {
+ Result struct {
+ Success bool `json:"success"`
+ } `json:"result"`
+ }
+ require.NoError(t, json.Unmarshal(raw.raw, &closed))
+ require.True(t, closed.Result.Success)
+ require.Eventually(t, func() bool {
+ m.sessionsMu.RLock()
+ defer m.sessionsMu.RUnlock()
+ _, dirty := m.interactionTargets[target]
+ return !dirty
+ }, time.Second, time.Millisecond, "monitor did not process target destruction")
+ if outcome == "success" {
+ release()
+ } else {
+ cancelInjection()
+ }
+ select {
+ case err := <-done:
+ require.NoError(t, err)
+ case <-time.After(time.Second):
+ t.Fatal("registration did not finish")
+ }
+ release()
+ require.NoError(t, m.SetTelemetry(false))
+ waitForTelemetryReconcile(t, m, false)
+ require.Never(t, func() bool {
+ m.lifeMu.Lock()
+ changed := m.conn != original
+ m.lifeMu.Unlock()
+ return changed || !m.NetworkSnapshot().Up
+ }, 300*time.Millisecond, time.Millisecond, "destroyed target forced healthy connection replacement")
+ require.Equal(t, before.Resets, m.NetworkSnapshot().Resets)
+ require.Equal(t, before.Completed, m.NetworkSnapshot().Completed)
+ driver.call(t, ctx, "", "Browser.getVersion", nil)
+ })
+ }
+}
diff --git a/server/lib/cdpmonitor/injection_destruction_test.go b/server/lib/cdpmonitor/injection_destruction_test.go
new file mode 100644
index 000000000..b0890173b
--- /dev/null
+++ b/server/lib/cdpmonitor/injection_destruction_test.go
@@ -0,0 +1,103 @@
+package cdpmonitor
+
+import (
+ "context"
+ "fmt"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/kernel/kernel-images/server/lib/browsersurface"
+ "github.com/kernel/kernel-images/server/lib/cdpclient"
+ "github.com/stretchr/testify/require"
+)
+
+func TestLateInjectionOutcomeAfterTargetRemoval(t *testing.T) {
+ for _, removal := range []string{"destroy", "detach"} {
+ for _, outcome := range []string{"success", "canceled", "timeout", "transport"} {
+ for _, prior := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/%s/prior_%t", removal, outcome, prior), func(t *testing.T) {
+ srv := newTestServer(t)
+ defer srv.close()
+ var hold atomic.Bool
+ blocked := make(chan cdpMessage, 1)
+ stop := make(chan struct{})
+ defer close(stop)
+ go listenAndRespond(srv, stop, func(msg cdpMessage) any {
+ if msg.Method == "Page.addScriptToEvaluateOnNewDocument" && hold.Load() {
+ blocked <- msg
+ return map[string]any{"method": "Test.unanswered"}
+ }
+ return nil
+ })
+ protocol, err := cdpclient.DialWithEvents(context.Background(), srv.wsURL())
+ require.NoError(t, err)
+ defer protocol.Close()
+ m := New(newTestUpstream(""), newEventCollector().publishFn(), 0, discardLogger, nil)
+ m.conn = &monitorConnection{protocol: protocol}
+ m.sessions["s"] = targetInfo{targetID: "t", targetType: "page"}
+ m.optionalSessions["s"] = ""
+ timeout := 5 * time.Second
+ if outcome == "timeout" {
+ timeout = 100 * time.Millisecond
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
+ defer cancel()
+ if prior {
+ require.NoError(t, m.injectScript(ctx, "s"))
+ require.Contains(t, m.interactionTargets, "t")
+ }
+ hold.Store(true)
+ done := make(chan error, 1)
+ go func() { done <- m.injectScript(ctx, "s") }()
+ var command cdpMessage
+ select {
+ case command = <-blocked:
+ case <-time.After(time.Second):
+ t.Fatal("registration was not sent")
+ }
+ m.sessionsMu.RLock()
+ pending := len(m.pendingInjections)
+ m.sessionsMu.RUnlock()
+ require.Equal(t, 1, pending)
+ // The tracker removes sessions before publishing target destruction.
+ m.handleDetachedFromTarget(cdpTargetDetachedFromTargetParams{SessionID: "s"})
+ if removal == "destroy" {
+ m.handleSurfaceEvent(nil, browsersurface.Event{Kind: browsersurface.EventProtocol, Message: cdpclient.Message{Method: "Target.targetDestroyed", Params: []byte(`{"targetId":"t"}`)}})
+ require.Empty(t, m.interactionTargets)
+ }
+ switch outcome {
+ case "success":
+ srv.sendToMonitor(t, map[string]any{"id": command.ID, "result": map[string]any{"identifier": "late-script"}})
+ case "canceled":
+ cancel()
+ case "transport":
+ require.NoError(t, protocol.Close())
+ }
+ select {
+ case err = <-done:
+ case <-time.After(time.Second):
+ t.Fatal("registration did not finish")
+ }
+ require.Empty(t, m.pendingInjections, "completed registrations must not leave tombstones")
+ if removal == "destroy" {
+ require.Empty(t, m.interactionTargets, "late outcome resurrected a destroyed target")
+ require.NoError(t, err, "a destroyed target cannot require uncertain-registration recovery")
+ } else {
+ require.Contains(t, m.interactionTargets, "t", "detach does not prove document destruction")
+ switch outcome {
+ case "success":
+ require.NoError(t, err)
+ case "canceled":
+ require.ErrorIs(t, err, context.Canceled)
+ case "timeout":
+ require.ErrorIs(t, err, context.DeadlineExceeded)
+ case "transport":
+ require.ErrorIs(t, err, cdpclient.ErrOutcomeUnknown)
+ }
+ }
+ })
+ }
+ }
+ }
+}
diff --git a/server/lib/cdpmonitor/injection_outcome_test.go b/server/lib/cdpmonitor/injection_outcome_test.go
new file mode 100644
index 000000000..5a2b58ab7
--- /dev/null
+++ b/server/lib/cdpmonitor/injection_outcome_test.go
@@ -0,0 +1,128 @@
+package cdpmonitor
+
+import (
+ "context"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/kernel/kernel-images/server/lib/cdpclient"
+ "github.com/stretchr/testify/require"
+)
+
+func TestInjectionCleanupObligations(t *testing.T) {
+ for _, outcome := range []string{"rejected", "unsupported", "timeout", "transport", "missing_id", "malformed_result", "evaluate_rejected", "evaluate_exception", "evaluate_timeout", "success_then_rejected"} {
+ t.Run(outcome, func(t *testing.T) {
+ srv := newTestServer(t)
+ defer srv.close()
+ var registrations, evaluations atomic.Int32
+ stop := make(chan struct{})
+ defer close(stop)
+ go listenAndRespond(srv, stop, func(msg cdpMessage) any {
+ switch msg.Method {
+ case "Page.addScriptToEvaluateOnNewDocument":
+ attempt := registrations.Add(1)
+ if outcome == "rejected" || outcome == "unsupported" || (outcome == "success_then_rejected" && attempt > 1) {
+ code := -32000
+ if outcome == "unsupported" {
+ code = -32601
+ }
+ return map[string]any{"id": msg.ID, "error": map[string]any{"code": code, "message": "fixture rejection"}}
+ }
+ if outcome == "transport" {
+ srv.connMu.Lock()
+ _ = srv.conn.CloseNow()
+ srv.connMu.Unlock()
+ return map[string]any{"method": "Test.unanswered"}
+ }
+ if outcome == "timeout" {
+ return map[string]any{"method": "Test.unanswered"}
+ }
+ if outcome == "missing_id" {
+ return map[string]any{"id": msg.ID, "result": map[string]any{}}
+ }
+ if outcome == "malformed_result" {
+ return map[string]any{"id": msg.ID, "result": "invalid"}
+ }
+ case "Runtime.evaluate":
+ evaluations.Add(1)
+ if outcome == "evaluate_rejected" {
+ return map[string]any{"id": msg.ID, "error": map[string]any{"code": -32000, "message": "fixture evaluation rejection"}}
+ }
+ if outcome == "evaluate_exception" {
+ return map[string]any{"id": msg.ID, "result": map[string]any{"exceptionDetails": map[string]any{"text": "fixture exception"}}}
+ }
+ if outcome == "evaluate_timeout" {
+ return map[string]any{"method": "Test.unanswered"}
+ }
+ }
+ return nil
+ })
+ protocol, err := cdpclient.DialWithEvents(context.Background(), srv.wsURL())
+ require.NoError(t, err)
+ defer protocol.Close()
+ m := New(newTestUpstream(""), newEventCollector().publishFn(), 0, discardLogger, nil)
+ m.conn = &monitorConnection{protocol: protocol}
+ m.sessions["s"] = targetInfo{targetID: "t", targetType: "page"}
+ m.optionalSessions["s"] = ""
+ ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ defer cancel()
+ if outcome == "success_then_rejected" {
+ require.NoError(t, m.injectScript(ctx, "s"))
+ }
+ err = m.injectScript(ctx, "s")
+ if outcome == "rejected" || outcome == "unsupported" || outcome == "success_then_rejected" {
+ var rejection *cdpclient.Error
+ require.ErrorAs(t, err, &rejection)
+ } else if outcome == "timeout" {
+ require.ErrorIs(t, err, context.DeadlineExceeded)
+ } else if outcome == "transport" {
+ require.ErrorIs(t, err, cdpclient.ErrOutcomeUnknown)
+ } else if outcome == "missing_id" || outcome == "malformed_result" {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ require.Equal(t, "test-script", m.optionalSessions["s"], "evaluation failure does not undo future-document registration")
+ }
+ if outcome == "rejected" || outcome == "unsupported" {
+ require.Empty(t, m.interactionTargets)
+ require.Zero(t, evaluations.Load(), "a rejected registration must not fall back to live-document injection")
+ require.NoError(t, m.disableOptionalDomains(context.Background(), "s", ""))
+ require.EqualValues(t, 1, registrations.Load(), "clean targets must not run script cleanup")
+ } else {
+ require.Contains(t, m.interactionTargets, "t")
+ if outcome == "success_then_rejected" {
+ require.Equal(t, "test-script", m.optionalSessions["s"])
+ require.EqualValues(t, 1, evaluations.Load())
+ require.Error(t, m.disableOptionalDomains(context.Background(), "s", "test-script"))
+ require.Contains(t, m.interactionTargets, "t", "failed cleanup must preserve the earlier obligation")
+ }
+ }
+ })
+ }
+}
+
+func TestInjectionOutcomeSurvivesDetach(t *testing.T) {
+ srv := newTestServer(t)
+ defer srv.close()
+ protocol, err := cdpclient.DialWithEvents(context.Background(), srv.wsURL())
+ require.NoError(t, err)
+ defer protocol.Close()
+ m := New(newTestUpstream(""), newEventCollector().publishFn(), 0, discardLogger, nil)
+ m.conn = &monitorConnection{protocol: protocol}
+ m.sessions["s"] = targetInfo{targetID: "t", targetType: "page"}
+ canceled, cancelBeforeSend := context.WithCancel(context.Background())
+ cancelBeforeSend()
+ require.ErrorIs(t, m.injectScript(canceled, "s"), context.Canceled)
+ require.Empty(t, m.interactionTargets, "cancellation before sending has no injection outcome")
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ done := make(chan error, 1)
+ go func() { done <- m.injectScript(ctx, "s") }()
+ command := srv.readFromMonitor(t, time.Second)
+ require.Equal(t, "Page.addScriptToEvaluateOnNewDocument", command.Method)
+ m.handleDetachedFromTarget(cdpTargetDetachedFromTargetParams{SessionID: "s"})
+ cancel()
+ require.ErrorIs(t, <-done, context.Canceled)
+ require.Contains(t, m.interactionTargets, "t")
+}
diff --git a/server/lib/cdpmonitor/injection_rejection_chrome_e2e_test.go b/server/lib/cdpmonitor/injection_rejection_chrome_e2e_test.go
new file mode 100644
index 000000000..0566f1995
--- /dev/null
+++ b/server/lib/cdpmonitor/injection_rejection_chrome_e2e_test.go
@@ -0,0 +1,101 @@
+package cdpmonitor
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestRejectedInjectionDoesNotReconnectChrome(t *testing.T) {
+ if os.Getenv("KERNEL_CDPMONITOR_CHROME_E2E") == "" {
+ t.Skip("set KERNEL_CDPMONITOR_CHROME_E2E=1")
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ page := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/html")
+ fmt.Fprint(w, ``)
+ }))
+ defer page.Close()
+ ws := launchChromium(t, ctx, findChromium(t))
+ driver := dialCDP(t, ctx, ws)
+ defer driver.close()
+ target := driver.call(t, ctx, "", "Target.createTarget", map[string]any{"url": page.URL}).targetID(t)
+ session := driver.call(t, ctx, "", "Target.attachToTarget", map[string]any{"targetId": target, "flatten": true}).sessionID(t)
+ require.Eventually(t, func() bool {
+ return driver.evalBool(ctx, session, `document.readyState === 'complete' && !!document.querySelector('#go')`)
+ }, 5*time.Second, time.Millisecond)
+ var rejections atomic.Int32
+ proxy := rejectPageCommandProxy(t, ctx, ws, func(method, source string) bool {
+ if method == "Page.addScriptToEvaluateOnNewDocument" {
+ rejections.Add(1)
+ return true
+ }
+ return false
+ })
+ defer func() { t.Logf("injected Page command rejections: %d", rejections.Load()) }()
+ m := New(newTestUpstream(proxy), newEventCollector().publishFn(), 0, discardLogger, func() bool { return false })
+ require.NoError(t, m.SetTelemetry(false))
+ require.NoError(t, m.Start(ctx))
+ defer m.Stop()
+ waitForTelemetryReconcile(t, m, false)
+ for cycle := 0; cycle < 2; cycle++ {
+ prior := rejections.Load()
+ require.NoError(t, m.SetTelemetry(true))
+ require.Eventually(t, func() bool { return rejections.Load() > prior && !m.telemetryChanging.Load() }, time.Second, time.Millisecond)
+ m.captureWg.Wait()
+ injected := driver.evalBool(ctx, session, `window.__kernelEventInjected === true`)
+ t.Logf("injected after rejected registration: %t", injected)
+ m.lifeMu.Lock()
+ original := m.conn
+ m.lifeMu.Unlock()
+ rejected := rejections.Load()
+ require.NoError(t, m.SetTelemetry(false))
+ waitForTelemetryReconcile(t, m, false)
+ require.False(t, injected, "confirmed rejection must not install live-document listeners")
+ require.Equal(t, rejected, rejections.Load(), "no cleanup registration is needed without injection")
+ assertStable := func(conn *monitorConnection) {
+ t.Helper()
+ require.Never(t, func() bool {
+ m.lifeMu.Lock()
+ changed := m.conn != conn
+ m.lifeMu.Unlock()
+ return changed || !m.NetworkSnapshot().Up
+ }, time.Second, time.Millisecond, "optional rejection caused monitor reconnects")
+ }
+ assertStable(original)
+ completed := m.NetworkSnapshot().Completed
+ result := evaluateNetworkScript(t, ctx, driver, session, `fetch('/observed', {method:'POST'}).then(async r => { await r.text(); return r.ok; })`)
+ require.JSONEq(t, "true", string(result))
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Completed == completed+1 }, time.Second, time.Millisecond)
+ before := m.NetworkSnapshot()
+ require.NoError(t, original.protocol.Close())
+ require.Eventually(t, func() bool {
+ m.lifeMu.Lock()
+ fresh := m.conn != nil && m.conn != original
+ m.lifeMu.Unlock()
+ return fresh && m.NetworkSnapshot().Up
+ }, 3*time.Second, time.Millisecond)
+ waitForTelemetryReconcile(t, m, false)
+ m.lifeMu.Lock()
+ fresh := m.conn
+ m.lifeMu.Unlock()
+ assertStable(fresh)
+ require.Equal(t, rejected, rejections.Load())
+ require.Equal(t, before.Resets, m.NetworkSnapshot().Resets)
+ require.Equal(t, before.Completed, m.NetworkSnapshot().Completed)
+ // The independent user connection remains usable, including navigation.
+ driver.call(t, ctx, session, "Page.navigate", map[string]any{"url": page.URL + fmt.Sprintf("/next-%d", cycle)})
+ require.Eventually(t, func() bool {
+ return driver.evalBool(ctx, session, fmt.Sprintf(`location.pathname === '/next-%d' && document.readyState === 'complete'`, cycle))
+ }, time.Second, time.Millisecond)
+ require.False(t, driver.evalBool(ctx, session, `window.__kernelEventInjected === true`))
+ }
+}
diff --git a/server/lib/cdpmonitor/interaction.js b/server/lib/cdpmonitor/interaction.js
index 14c34045e..4490acff6 100644
--- a/server/lib/cdpmonitor/interaction.js
+++ b/server/lib/cdpmonitor/interaction.js
@@ -1,8 +1,18 @@
(function() {
- if (window.__kernelEventInjected) return;
var send = window.__kernelEvent;
if (!send) return;
+ if (window.__kernelEventOwner === send) return;
+ if (window.__kernelEventCleanup) window.__kernelEventCleanup();
+ window.__kernelEventOwner = send;
window.__kernelEventInjected = true;
+ var controller = new AbortController();
+ window.__kernelEventCleanup = function() {
+ controller.abort();
+ if (scrollTimer) clearTimeout(scrollTimer);
+ delete window.__kernelEventInjected;
+ delete window.__kernelEventOwner;
+ delete window.__kernelEventCleanup;
+ };
function sel(el) {
return el.id ? '#' + el.id : (el.className ? '.' + String(el.className).split(' ')[0] : '');
@@ -76,7 +86,7 @@
selector: sel(t), tag: t.tagName || '',
text: text
}));
- }, true);
+ }, {capture: true, signal: controller.signal});
document.addEventListener('keydown', function(e) {
var t = e.target || {};
@@ -87,7 +97,7 @@
key: e.key,
selector: sel(t), tag: t.tagName || ''
}));
- }, true);
+ }, {capture: true, signal: controller.signal});
function scrollPos(target) {
if (target === document || target === document.documentElement) {
@@ -121,5 +131,5 @@
}
scrollTarget = null;
}, 300);
- }, true);
+ }, {capture: true, signal: controller.signal});
})();
\ No newline at end of file
diff --git a/server/lib/cdpmonitor/interaction_cleanup.go b/server/lib/cdpmonitor/interaction_cleanup.go
new file mode 100644
index 000000000..05fe0e089
--- /dev/null
+++ b/server/lib/cdpmonitor/interaction_cleanup.go
@@ -0,0 +1,97 @@
+package cdpmonitor
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "time"
+
+ "github.com/kernel/kernel-images/server/lib/cdpclient"
+)
+
+const cleanupInteractionJS = `window.__kernelEventCleanup && window.__kernelEventCleanup()`
+
+// Registrations belong to a CDP session, but installed listeners belong to live
+// documents. Target IDs survive socket replacement; session/context IDs do not.
+func (m *Monitor) cleanupAttachedTarget(ctx context.Context, sessionID string, info targetInfo) error {
+ m.sessionsMu.RLock()
+ _, dirty := m.interactionTargets[info.targetID]
+ m.sessionsMu.RUnlock()
+ if !dirty {
+ return nil
+ }
+ ctx, cancel := context.WithTimeout(ctx, telemetryCleanupTimeout)
+ defer cancel()
+ if err := m.cleanupInteraction(ctx, sessionID); err != nil {
+ m.sessionsMu.RLock()
+ _, attached := m.sessions[sessionID]
+ m.sessionsMu.RUnlock()
+ if !attached {
+ return nil
+ }
+ return err
+ }
+ m.sessionsMu.Lock()
+ delete(m.interactionTargets, info.targetID)
+ m.sessionsMu.Unlock()
+ return nil
+}
+
+func (m *Monitor) cleanupInteraction(ctx context.Context, sessionID string) error {
+ // runImmediately covers every existing frame's main world, including
+ // cross-origin frames in the same renderer, without enabling Runtime/Page.
+ raw, err := m.send(ctx, "Page.addScriptToEvaluateOnNewDocument", map[string]any{
+ "source": cleanupInteractionJS, "runImmediately": true,
+ }, sessionID)
+ if err != nil {
+ return err
+ }
+ var result struct {
+ Identifier string `json:"identifier"`
+ }
+ if err := json.Unmarshal(raw, &result); err != nil {
+ return err
+ }
+ if result.Identifier == "" {
+ return errors.New("cleanup script registration returned no identifier")
+ }
+ _, err = m.send(ctx, "Page.removeScriptToEvaluateOnNewDocument", map[string]any{"identifier": result.Identifier}, sessionID)
+ return err
+}
+
+// Before starting discovery on a replacement connection, discard obligations
+// only for targets confirmed gone (including targets from an old Chrome process).
+func (m *Monitor) pruneInteractionTargets(ctx context.Context, protocol *cdpclient.Client) error {
+ m.sessionsMu.RLock()
+ dirty := len(m.interactionTargets) != 0
+ m.sessionsMu.RUnlock()
+ if !dirty {
+ return nil
+ }
+ ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ defer cancel()
+ raw, err := protocol.Send(ctx, "Target.getTargets", nil, "")
+ if err != nil {
+ return err
+ }
+ var result struct {
+ TargetInfos []struct {
+ TargetID string `json:"targetId"`
+ } `json:"targetInfos"`
+ }
+ if err := json.Unmarshal(raw, &result); err != nil {
+ return err
+ }
+ live := make(map[string]bool, len(result.TargetInfos))
+ for _, target := range result.TargetInfos {
+ live[target.TargetID] = true
+ }
+ m.sessionsMu.Lock()
+ for target := range m.interactionTargets {
+ if !live[target] {
+ delete(m.interactionTargets, target)
+ }
+ }
+ m.sessionsMu.Unlock()
+ return nil
+}
diff --git a/server/lib/cdpmonitor/monitor.go b/server/lib/cdpmonitor/monitor.go
index e31939941..4e2cfa63d 100644
--- a/server/lib/cdpmonitor/monitor.go
+++ b/server/lib/cdpmonitor/monitor.go
@@ -31,6 +31,7 @@ type monitorConnection struct {
ctx context.Context
cancel context.CancelFunc
done chan struct{}
+ ready atomic.Bool
}
// Monitor owns its CDP connection and browser-surface tracker independently of
@@ -41,6 +42,25 @@ type Monitor struct {
displayNum int
log *slog.Logger
+ controlMu sync.Mutex // serializes Start and Stop
+ desiredMu sync.RWMutex // fences publication against desired-state changes; no CDP work
+ desiredTelemetry atomic.Uint64 // revision in high bits, enabled in low bit
+ appliedTelemetry atomic.Uint64
+ telemetryChanged chan struct{}
+ telemetryMu sync.RWMutex
+ telemetryEnabled bool
+ telemetryChanging atomic.Bool
+ telemetryCtx context.Context
+ telemetryCancel context.CancelFunc
+ telemetryWg sync.WaitGroup
+ computedPublishMu sync.Mutex
+ optionalSessions map[string]string // session -> injected script identifier
+ interactionTargets map[string]struct{} // cleanup obligations by target ID; survives reconnect; sessionsMu
+ network *networkCounters
+ networkReady map[string]bool // sessionsMu
+
+ pendingInjections map[*interactionInjection]struct{} // sessionsMu; only in-flight registrations
+
lifeMu sync.Mutex
conn *monitorConnection
@@ -79,52 +99,80 @@ type Monitor struct {
// screenshotEnabled gates screenshot capture; a nil predicate always captures.
func New(upstreamMgr UpstreamProvider, publish PublishFunc, displayNum int, log *slog.Logger, screenshotEnabled func() bool) *Monitor {
m := &Monitor{
- upstreamMgr: upstreamMgr,
- publish: publish,
- displayNum: displayNum,
- log: log,
- screenshotEnabled: screenshotEnabled,
- sessions: make(map[string]targetInfo),
- computedStates: make(map[string]*computedState),
- pendingRequests: make(map[networkRequestKey]networkReqState),
- bindingLastSeen: make(map[string]time.Time),
- proxyLastEmit: make(map[string]time.Time),
- lifecycleCtx: context.Background(),
+ telemetryEnabled: true,
+ telemetryCtx: context.Background(),
+ optionalSessions: make(map[string]string),
+ interactionTargets: make(map[string]struct{}),
+ pendingInjections: make(map[*interactionInjection]struct{}),
+ telemetryChanged: make(chan struct{}, 1),
+ network: newNetworkCounters(),
+ networkReady: make(map[string]bool),
+ upstreamMgr: upstreamMgr,
+ displayNum: displayNum,
+ log: log,
+ screenshotEnabled: screenshotEnabled,
+ sessions: make(map[string]targetInfo),
+ computedStates: make(map[string]*computedState),
+ pendingRequests: make(map[networkRequestKey]networkReqState),
+ bindingLastSeen: make(map[string]time.Time),
+ proxyLastEmit: make(map[string]time.Time),
+ lifecycleCtx: context.Background(),
+ }
+ m.desiredTelemetry.Store(1)
+ m.appliedTelemetry.Store(1)
+ m.publish = func(ev events.Event) (events.Envelope, bool) {
+ m.desiredMu.RLock()
+ defer m.desiredMu.RUnlock()
+ if ev.Category != events.Monitor && !m.captureEnabled() {
+ return events.Envelope{}, false
+ }
+ return publish(ev)
}
m.mainSessionID.Store(mainSessionUnset)
return m
}
-// IsRunning reports whether the monitor is actively capturing.
+// IsRunning reports whether the monitor lifecycle is running (including retries).
func (m *Monitor) IsRunning() bool {
return m.running.Load()
}
-// Start begins CDP capture. Restarts if already running.
-// Not concurrency-safe; callers must serialize Start calls.
+// Start starts the lifecycle even if Chrome is not available yet. Capture
+// readiness is reported separately by NetworkSnapshot().Up.
func (m *Monitor) Start(ctx context.Context) error {
- m.Stop()
- devtoolsURL := m.upstreamMgr.Current()
- if devtoolsURL == "" {
- return fmt.Errorf("cdpmonitor: no DevTools URL available")
+ m.controlMu.Lock()
+ defer m.controlMu.Unlock()
+ m.stop()
+ if err := ctx.Err(); err != nil {
+ return err
}
ctx, cancel := context.WithCancel(ctx)
m.lifeMu.Lock()
m.lifecycleCtx, m.cancel = ctx, cancel
m.lifeMu.Unlock()
- if err := m.openConnection(ctx, devtoolsURL); err != nil {
- cancel()
- return err
+ // Subscribe before reading Current so a restart during dialing cannot be lost.
+ ch, unsubscribe := m.upstreamMgr.Subscribe()
+ url := m.upstreamMgr.Current()
+ if url != "" {
+ if err := m.openConnection(ctx, url); err != nil {
+ m.log.Warn("cdpmonitor: initial connection failed", "err", err)
+ }
}
m.running.Store(true)
- m.log.Info("cdpmonitor: started", "url", devtoolsURL)
- m.asyncWg.Go(func() { m.subscribeToUpstream(ctx) })
+ m.asyncWg.Go(func() { defer unsubscribe(); m.supervise(ctx, ch) })
m.asyncWg.Go(func() { m.sweepPendingRequests(ctx) })
+ m.asyncWg.Go(func() { m.reconcileTelemetry(ctx) })
return nil
}
// Stop cancels the lifecycle and waits for both connection and capture work.
func (m *Monitor) Stop() {
+ m.controlMu.Lock()
+ defer m.controlMu.Unlock()
+ m.stop()
+}
+
+func (m *Monitor) stop() {
wasRunning := m.running.Swap(false)
if wasRunning {
m.log.Info("cdpmonitor: stopping")
@@ -145,10 +193,16 @@ func (m *Monitor) Stop() {
}
func (m *Monitor) openConnection(ctx context.Context, devtoolsURL string) error {
- protocol, err := cdpclient.DialWithEvents(ctx, devtoolsURL)
+ dialCtx, dialCancel := context.WithTimeout(ctx, 5*time.Second)
+ defer dialCancel()
+ protocol, err := cdpclient.DialWithEvents(dialCtx, devtoolsURL)
if err != nil {
return fmt.Errorf("cdpmonitor: dial %s: %w", devtoolsURL, err)
}
+ if err := m.pruneInteractionTargets(ctx, protocol); err != nil {
+ _ = protocol.Close()
+ return err
+ }
ctx, cancel := context.WithCancel(ctx)
conn := &monitorConnection{
protocol: protocol,
@@ -162,6 +216,12 @@ func (m *Monitor) openConnection(ctx context.Context, devtoolsURL string) error
m.lifeMu.Lock()
m.conn = conn
m.lifeMu.Unlock()
+ m.telemetryMu.Lock()
+ state := m.desiredTelemetry.Load()
+ m.telemetryEnabled = state&1 != 0
+ m.appliedTelemetry.Store(state)
+ m.telemetryCtx, m.telemetryCancel = context.WithCancel(ctx)
+ m.telemetryMu.Unlock()
go func() {
defer close(conn.done)
defer unsubscribe()
@@ -179,14 +239,19 @@ func (m *Monitor) openConnection(ctx context.Context, devtoolsURL string) error
}
}()
m.captureWg.Go(func() {
- if err := conn.surface.Start(ctx); err != nil && ctx.Err() == nil {
+ initCtx, initCancel := context.WithTimeout(ctx, sendTimeout)
+ defer initCancel()
+ if err := conn.surface.Start(initCtx); err != nil && ctx.Err() == nil {
m.log.Error("cdpmonitor: browser surface discovery failed", "err", err)
data, _ := json.Marshal(oapi.BrowserMonitorInitFailedEventData{Step: "browsersurface.Start"})
m.publish(events.Event{
Ts: time.Now().UnixMicro(), Type: EventMonitorInitFailed, Category: events.Monitor,
Source: oapi.BrowserEventSource{Kind: oapi.LocalProcess}, Data: data,
})
+ conn.cancel()
+ return
}
+ conn.ready.Store(ctx.Err() == nil)
})
return nil
}
@@ -202,6 +267,7 @@ func (m *Monitor) closeConnection() {
<-conn.surface.Done()
}
m.captureWg.Wait()
+ m.telemetryWg.Wait()
m.lifeMu.Lock()
m.conn = nil
m.lifeMu.Unlock()
@@ -209,6 +275,8 @@ func (m *Monitor) closeConnection() {
func (m *Monitor) handleSurfaceEvent(conn *monitorConnection, event browsersurface.Event) {
switch event.Kind {
+ case browsersurface.EventDiscoveryFailed:
+ conn.cancel()
case browsersurface.EventSessionAttached:
if !conn.surface.SessionExists(event.SessionID) {
return
@@ -224,6 +292,22 @@ func (m *Monitor) handleSurfaceEvent(conn *monitorConnection, event browsersurfa
case browsersurface.EventSessionRemoved:
m.handleDetachedFromTarget(cdpTargetDetachedFromTargetParams{SessionID: event.SessionID})
case browsersurface.EventProtocol:
+ if event.Message.Method == "Target.targetDestroyed" {
+ var p struct {
+ TargetID string `json:"targetId"`
+ }
+ if json.Unmarshal(event.Message.Params, &p) == nil {
+ m.sessionsMu.Lock()
+ delete(m.interactionTargets, p.TargetID)
+ for injection := range m.pendingInjections {
+ if injection.targetID == p.TargetID {
+ injection.destroyed = true
+ delete(m.pendingInjections, injection)
+ }
+ }
+ m.sessionsMu.Unlock()
+ }
+ }
// Attachment lifecycle is emitted once by the tracker, including targets
// discovered through enumeration whose attach response arrived first.
if event.Message.Method == "Target.attachedToTarget" || event.Message.Method == "Target.detachedFromTarget" {
@@ -248,11 +332,16 @@ func (m *Monitor) clearState() {
m.sessionsMu.Lock()
prev := m.computedStates
m.sessions = make(map[string]targetInfo)
+ m.networkReady = make(map[string]bool)
m.computedStates = make(map[string]*computedState)
+ clear(m.optionalSessions)
m.sessionsMu.Unlock()
for _, cs := range prev {
cs.stop()
}
+ m.computedPublishMu.Lock()
+ m.computedPublishMu.Unlock()
+ m.network.newGeneration()
m.mainSessionID.Store(mainSessionUnset)
m.pendReqMu.Lock()
m.pendingRequests = make(map[networkRequestKey]networkReqState)
@@ -319,86 +408,99 @@ func (m *Monitor) send(ctx context.Context, method string, params any, sessionID
return conn.protocol.Send(ctx, method, params, sessionID)
}
-func (m *Monitor) subscribeToUpstream(ctx context.Context) {
- ch, cancel := m.upstreamMgr.Subscribe()
- defer cancel()
- for {
- select {
- case <-ctx.Done():
- return
- case newURL, ok := <-ch:
- if !ok {
+func (m *Monitor) supervise(ctx context.Context, updates <-chan string) {
+ backoff := 250 * time.Millisecond
+ var disconnectedAt time.Time
+ probe := time.NewTicker(5 * time.Second)
+ defer probe.Stop()
+ defer m.running.Store(false)
+ defer func() {
+ m.restartMu.Lock()
+ defer m.restartMu.Unlock()
+ m.closeConnection()
+ m.clearState()
+ }()
+ for ctx.Err() == nil {
+ m.lifeMu.Lock()
+ conn := m.conn
+ m.lifeMu.Unlock()
+ if conn != nil {
+ select {
+ case <-ctx.Done():
return
+ case _, ok := <-updates:
+ if !ok {
+ updates = nil
+ continue
+ }
+ case <-conn.ctx.Done():
+ case <-conn.protocol.Done():
+ case <-probe.C:
+ // A bounded round trip also catches half-open sockets after suspend.
+ probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ _, err := conn.protocol.GetBrowserVersion(probeCtx)
+ cancel()
+ if err == nil {
+ if m.NetworkSnapshot().Up {
+ backoff = 250 * time.Millisecond
+ }
+ continue
+ }
}
- m.handleUpstreamRestart(ctx, newURL)
- }
- }
-}
-
-func (m *Monitor) handleUpstreamRestart(ctx context.Context, newURL string) {
- m.restartMu.Lock()
- defer m.restartMu.Unlock()
- if ctx.Err() != nil {
- return
- }
- data, _ := json.Marshal(oapi.BrowserMonitorDisconnectedEventData{Reason: oapi.ChromeRestarted})
- m.publish(events.Event{
- Ts: time.Now().UnixMicro(), Type: EventMonitorDisconnected, Category: events.Monitor,
- Source: oapi.BrowserEventSource{Kind: oapi.LocalProcess}, Data: data,
- })
- startReconnect := time.Now()
- m.closeConnection()
- m.clearState()
- if !m.reconnectWithBackoff(ctx, newURL) {
- if ctx.Err() == nil {
- m.lifeMu.Lock()
- m.cancel()
- m.lifeMu.Unlock()
- m.running.Store(false)
- data, _ := json.Marshal(oapi.BrowserMonitorReconnectFailedEventData{Reason: oapi.ReconnectExhausted})
+ if ctx.Err() != nil {
+ return
+ }
+ // Invalidate health and unblock capture before waiting for restartMu.
+ conn.cancel()
+ disconnectedAt = time.Now()
+ data, _ := json.Marshal(oapi.BrowserMonitorDisconnectedEventData{Reason: oapi.ChromeRestarted})
m.publish(events.Event{
- Ts: time.Now().UnixMicro(), Type: EventMonitorReconnectFailed, Category: events.Monitor,
+ Ts: time.Now().UnixMicro(), Type: EventMonitorDisconnected, Category: events.Monitor,
Source: oapi.BrowserEventSource{Kind: oapi.LocalProcess}, Data: data,
})
}
- return
- }
- durationMs := time.Since(startReconnect).Milliseconds()
- m.log.Info("cdpmonitor: reconnected", "url", newURL, "duration_ms", durationMs)
- data, _ = json.Marshal(oapi.BrowserMonitorReconnectedEventData{ReconnectDurationMs: durationMs})
- m.publish(events.Event{
- Ts: time.Now().UnixMicro(), Type: EventMonitorReconnected, Category: events.Monitor,
- Source: oapi.BrowserEventSource{Kind: oapi.LocalProcess}, Data: data,
- })
-}
-
-const maxReconnectAttempts = 10
-
-var reconnectBackoffs = []time.Duration{
- 250 * time.Millisecond,
- 500 * time.Millisecond,
- 1 * time.Second,
- 2 * time.Second,
-}
-
-func (m *Monitor) reconnectWithBackoff(ctx context.Context, newURL string) bool {
- for attempt := range maxReconnectAttempts {
- if ctx.Err() != nil {
- return false
+ m.restartMu.Lock()
+ m.closeConnection()
+ m.clearState()
+ m.restartMu.Unlock()
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(backoff):
}
- if attempt > 0 {
- idx := min(attempt-1, len(reconnectBackoffs)-1)
+ backoff = min(2*backoff, 5*time.Second)
+ // Current supersedes updates queued before acquisition. Notifications
+ // arriving during acquisition still trigger replacement afterward.
+ drainUpdates:
+ for {
select {
- case <-ctx.Done():
- return false
- case <-time.After(reconnectBackoffs[idx]):
+ case _, ok := <-updates:
+ if !ok {
+ updates = nil
+ }
+ default:
+ break drainUpdates
}
}
- if err := m.openConnection(ctx, newURL); err != nil {
- m.log.Warn("cdpmonitor: reconnect attempt failed", "attempt", attempt+1, "max_attempts", maxReconnectAttempts, "url", newURL, "err", err)
+ // Always reread, including after failed dials and ordinary socket loss.
+ url := m.upstreamMgr.Current()
+ if url == "" {
+ continue
+ }
+ m.restartMu.Lock()
+ err := m.openConnection(ctx, url)
+ m.restartMu.Unlock()
+ if err != nil {
+ m.log.Warn("cdpmonitor: reconnect failed", "err", err)
continue
}
- return true
+ if !disconnectedAt.IsZero() {
+ data, _ := json.Marshal(oapi.BrowserMonitorReconnectedEventData{ReconnectDurationMs: time.Since(disconnectedAt).Milliseconds()})
+ m.publish(events.Event{
+ Ts: time.Now().UnixMicro(), Type: EventMonitorReconnected, Category: events.Monitor,
+ Source: oapi.BrowserEventSource{Kind: oapi.LocalProcess}, Data: data,
+ })
+ disconnectedAt = time.Time{}
+ }
}
- return false
}
diff --git a/server/lib/cdpmonitor/network_cleanup_test.go b/server/lib/cdpmonitor/network_cleanup_test.go
new file mode 100644
index 000000000..468ffee8e
--- /dev/null
+++ b/server/lib/cdpmonitor/network_cleanup_test.go
@@ -0,0 +1,95 @@
+package cdpmonitor
+
+import (
+ "context"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestTelemetryCleanupAttemptsAllSessionsAfterError(t *testing.T) {
+ srv := newTestServer(t)
+ defer srv.close()
+ var mu sync.Mutex
+ removed := make(map[string]bool)
+ stop := make(chan struct{})
+ defer close(stop)
+ go listenAndRespond(srv, stop, func(msg cdpMessage) any {
+ switch msg.Method {
+ case "Page.addScriptToEvaluateOnNewDocument":
+ return map[string]any{"id": msg.ID, "result": map[string]any{"identifier": "script"}}
+ case "Page.removeScriptToEvaluateOnNewDocument":
+ return map[string]any{"id": msg.ID, "error": map[string]any{"code": -32000, "message": "fixture failure"}}
+ case "Runtime.removeBinding":
+ mu.Lock()
+ removed[msg.SessionID] = true
+ mu.Unlock()
+ }
+ return nil
+ })
+ m := New(newTestUpstream(srv.wsURL()), newEventCollector().publishFn(), 0, discardLogger, nil)
+ require.NoError(t, m.SetTelemetry(false))
+ require.NoError(t, m.Start(context.Background()))
+ defer m.Stop()
+ <-srv.connCh
+ for _, id := range []string{"one", "two"} {
+ srv.sendToMonitor(t, map[string]any{"method": "Target.attachedToTarget", "params": map[string]any{"sessionId": id, "targetInfo": map[string]any{"targetId": id, "type": "page"}}})
+ }
+ require.Eventually(t, func() bool {
+ m.sessionsMu.RLock()
+ defer m.sessionsMu.RUnlock()
+ return len(m.networkReady) == 2
+ }, time.Second, time.Millisecond)
+ require.NoError(t, m.SetTelemetry(true))
+ require.Eventually(t, func() bool {
+ m.sessionsMu.RLock()
+ defer m.sessionsMu.RUnlock()
+ return m.optionalSessions["one"] == "script" && m.optionalSessions["two"] == "script"
+ }, time.Second, time.Millisecond)
+ m.lifeMu.Lock()
+ old := m.conn
+ m.lifeMu.Unlock()
+ require.NoError(t, m.SetTelemetry(false))
+ require.Eventually(t, func() bool { return old.ctx.Err() != nil }, time.Second, time.Millisecond)
+ mu.Lock()
+ one, two := removed["one"], removed["two"]
+ mu.Unlock()
+ require.True(t, one)
+ require.True(t, two)
+ waitForTelemetryReconcile(t, m, false)
+ m.telemetryMu.RLock()
+ enabled := m.telemetryEnabled
+ m.telemetryMu.RUnlock()
+ require.False(t, enabled)
+}
+
+func TestScriptRegistrationTimeoutReplacesConnection(t *testing.T) {
+ srv := newTestServer(t)
+ defer srv.close()
+ stop := make(chan struct{})
+ defer close(stop)
+ go listenAndRespond(srv, stop, func(msg cdpMessage) any {
+ if msg.Method == "Page.addScriptToEvaluateOnNewDocument" {
+ return map[string]any{"method": "Test.unanswered"}
+ }
+ return nil
+ })
+ m := New(newTestUpstream(srv.wsURL()), newEventCollector().publishFn(), 0, discardLogger, nil)
+ require.NoError(t, m.Start(context.Background()))
+ defer m.Stop()
+ m.lifeMu.Lock()
+ old := m.conn
+ m.lifeMu.Unlock()
+ info := targetInfo{targetID: "t", targetType: "page"}
+ m.sessionsMu.Lock()
+ m.sessions["s"] = info
+ m.sessionsMu.Unlock()
+ ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ defer cancel()
+ m.telemetryMu.RLock()
+ m.enableOptionalCapture(ctx, "s", info)
+ m.telemetryMu.RUnlock()
+ require.Error(t, old.ctx.Err(), "unknown script registration must invalidate its connection")
+}
diff --git a/server/lib/cdpmonitor/network_lifecycle_test.go b/server/lib/cdpmonitor/network_lifecycle_test.go
new file mode 100644
index 000000000..825a79620
--- /dev/null
+++ b/server/lib/cdpmonitor/network_lifecycle_test.go
@@ -0,0 +1,162 @@
+package cdpmonitor
+
+import (
+ "context"
+ "slices"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestMetricsOnlyDomainsAndTelemetryTransitions(t *testing.T) {
+ srv := newTestServer(t)
+ defer srv.close()
+ var mu sync.Mutex
+ var methods []string
+ stop := make(chan struct{})
+ defer close(stop)
+ go listenAndRespond(srv, stop, func(msg cdpMessage) any {
+ mu.Lock()
+ methods = append(methods, msg.Method)
+ mu.Unlock()
+ if msg.Method == "Page.addScriptToEvaluateOnNewDocument" {
+ return map[string]any{"id": msg.ID, "result": map[string]any{"identifier": "script"}}
+ }
+ return nil
+ })
+ ec := newEventCollector()
+ m := New(newTestUpstream(srv.wsURL()), ec.publishFn(), 0, discardLogger, nil)
+ require.NoError(t, m.SetTelemetry(false))
+ require.NoError(t, m.Start(context.Background()))
+ defer m.Stop()
+ <-srv.connCh
+ srv.sendToMonitor(t, map[string]any{"method": "Target.attachedToTarget", "params": map[string]any{"sessionId": "s", "targetInfo": map[string]any{"targetId": "t", "type": "page"}}})
+ require.Eventually(t, func() bool {
+ m.sessionsMu.RLock()
+ ready := m.networkReady["s"]
+ m.sessionsMu.RUnlock()
+ return ready && m.NetworkSnapshot().Up
+ }, 3*time.Second, 10*time.Millisecond)
+ mu.Lock()
+ initialMethods := slices.Clone(methods)
+ mu.Unlock()
+ require.Contains(t, initialMethods, "Network.enable")
+ for _, forbidden := range []string{"Runtime.enable", "Page.enable", "PerformanceTimeline.enable", "Runtime.addBinding", "Page.addScriptToEvaluateOnNewDocument", "Network.getResponseBody"} {
+ require.NotContains(t, initialMethods, forbidden)
+ }
+ m.sessionsMu.RLock()
+ computedCount := len(m.computedStates)
+ m.sessionsMu.RUnlock()
+ require.Zero(t, computedCount)
+ require.Zero(t, ec.checkpoint())
+ require.NoError(t, m.SetTelemetry(true))
+ ec.waitFor(t, EventTabOpened, time.Second)
+ require.Eventually(t, func() bool {
+ mu.Lock()
+ defer mu.Unlock()
+ return slices.Contains(methods, "Runtime.evaluate")
+ }, time.Second, time.Millisecond)
+ require.NoError(t, m.SetTelemetry(false))
+ waitForTelemetryReconcile(t, m, false)
+ mu.Lock()
+ finalMethods := slices.Clone(methods)
+ mu.Unlock()
+ require.Contains(t, finalMethods, "Page.removeScriptToEvaluateOnNewDocument")
+ require.Contains(t, finalMethods, "Runtime.removeBinding")
+ require.Contains(t, finalMethods, "Runtime.disable")
+ require.NotContains(t, finalMethods, "Network.disable")
+ srv.sendToMonitor(t, map[string]any{"method": "Network.loadingFailed", "sessionId": "s", "params": map[string]any{"requestId": "r", "errorText": "net::ERR_CONNECTION_RESET"}})
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Resets == 1 }, time.Second, time.Millisecond)
+ require.True(t, m.NetworkSnapshot().Up)
+}
+
+func TestNetworkMonitorRetriesStartupAndSameURLDisconnect(t *testing.T) {
+ srv := newTestServer(t)
+ defer srv.close()
+ var discovers atomic.Int32
+ stop := make(chan struct{})
+ defer close(stop)
+ go listenAndRespond(srv, stop, func(msg cdpMessage) any {
+ if msg.Method == "Target.setDiscoverTargets" {
+ discovers.Add(1)
+ }
+ return nil
+ })
+ upstream := newTestUpstream("")
+ m := New(upstream, newEventCollector().publishFn(), 0, discardLogger, nil)
+ require.NoError(t, m.SetTelemetry(false))
+ require.NoError(t, m.Start(context.Background()))
+ defer m.Stop()
+ require.False(t, m.NetworkSnapshot().Up)
+ upstream.notifyRestart(srv.wsURL())
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Up }, 3*time.Second, 10*time.Millisecond)
+ m.network.terminal("s", "r", "net::ERR_CONNECTION_RESET")
+ srv.connMu.Lock()
+ conn := srv.conn
+ srv.connMu.Unlock()
+ require.NoError(t, conn.CloseNow())
+ require.Eventually(t, func() bool { return !m.NetworkSnapshot().Up }, time.Second, time.Millisecond)
+ require.Eventually(t, func() bool { return discovers.Load() >= 2 && m.NetworkSnapshot().Up }, 4*time.Second, 10*time.Millisecond)
+ require.Equal(t, uint64(1), m.NetworkSnapshot().Resets)
+ m.network.terminal("s", "r", "net::ERR_CONNECTION_RESET")
+ require.Equal(t, uint64(2), m.NetworkSnapshot().Resets)
+ m.Stop()
+ require.False(t, m.NetworkSnapshot().Up)
+}
+
+func TestNetworkMonitorRetriesInitializationFailures(t *testing.T) {
+ for _, method := range []string{"Target.setDiscoverTargets", "Target.getTargets", "Target.attachToTarget", "Target.setAutoAttach", "Network.enable"} {
+ t.Run(method, func(t *testing.T) {
+ srv := newTestServer(t)
+ defer srv.close()
+ var failed atomic.Bool
+ stop := make(chan struct{})
+ defer close(stop)
+ go listenAndRespond(srv, stop, func(msg cdpMessage) any {
+ if msg.Method == method && failed.CompareAndSwap(false, true) {
+ return map[string]any{"id": msg.ID, "error": map[string]any{"code": -32000, "message": "fixture failure"}}
+ }
+ switch msg.Method {
+ case "Target.getTargets":
+ return map[string]any{"id": msg.ID, "result": map[string]any{"targetInfos": []any{map[string]any{"targetId": "t", "type": "page"}}}}
+ case "Target.attachToTarget":
+ return map[string]any{"id": msg.ID, "result": map[string]any{"sessionId": "s"}}
+ }
+ return nil
+ })
+ m := New(newTestUpstream(srv.wsURL()), newEventCollector().publishFn(), 0, discardLogger, nil)
+ require.NoError(t, m.SetTelemetry(false))
+ require.NoError(t, m.Start(context.Background()))
+ defer m.Stop()
+ require.Eventually(t, func() bool { return failed.Load() && m.NetworkSnapshot().Up }, 5*time.Second, 10*time.Millisecond)
+ })
+ }
+}
+
+func TestConcurrentMonitorConfigurationAndShutdown(t *testing.T) {
+ srv := newTestServer(t)
+ defer srv.close()
+ stop := make(chan struct{})
+ defer close(stop)
+ go listenAndRespond(srv, stop, nil)
+ m := New(newTestUpstream(srv.wsURL()), newEventCollector().publishFn(), 0, discardLogger, nil)
+ ctx, cancel := context.WithCancel(context.Background())
+ require.NoError(t, m.Start(ctx))
+ var wg sync.WaitGroup
+ for range 3 {
+ wg.Go(func() {
+ for i := range 20 {
+ _ = m.SetTelemetry(i%2 == 0)
+ _ = m.NetworkSnapshot()
+ }
+ })
+ }
+ cancel()
+ m.Stop()
+ wg.Wait()
+ require.False(t, m.IsRunning())
+ require.False(t, m.NetworkSnapshot().Up)
+}
diff --git a/server/lib/cdpmonitor/network_metrics.go b/server/lib/cdpmonitor/network_metrics.go
new file mode 100644
index 000000000..681b4e18e
--- /dev/null
+++ b/server/lib/cdpmonitor/network_metrics.go
@@ -0,0 +1,85 @@
+package cdpmonitor
+
+import "sync"
+
+const terminalHistorySize = 8192
+
+// NetworkSnapshot contains process-lifetime observed terminal outcomes, not
+// socket counts. Up describes capture readiness, not browser responsiveness.
+type NetworkSnapshot struct {
+ Resets uint64
+ Completed uint64
+ Up bool
+}
+
+type networkCounters struct {
+ mu sync.Mutex
+ resets, completed uint64
+ seen map[networkRequestKey]struct{}
+ history [terminalHistorySize]networkRequestKey
+ next int
+}
+
+func newNetworkCounters() *networkCounters {
+ return &networkCounters{seen: make(map[networkRequestKey]struct{}, terminalHistorySize)}
+}
+
+func (c *networkCounters) terminal(sessionID, requestID, errorText string) {
+ // CDP identities are short opaque strings. Bound retained bytes as well as entries.
+ if sessionID == "" || requestID == "" || len(sessionID) > 256 || len(requestID) > 256 {
+ return
+ }
+ key := networkRequestKey{sessionID, requestID}
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if _, exists := c.seen[key]; exists {
+ return
+ }
+ delete(c.seen, c.history[c.next])
+ c.history[c.next] = key
+ c.next = (c.next + 1) % terminalHistorySize
+ c.seen[key] = struct{}{}
+ c.completed++
+ if errorText == "net::ERR_CONNECTION_RESET" {
+ c.resets++
+ }
+}
+
+// Connection replacement drains old events before clearing identities; this is
+// the generation boundary. Totals intentionally survive it.
+func (c *networkCounters) newGeneration() {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ clear(c.seen)
+ clear(c.history[:])
+ c.next = 0
+}
+
+func (c *networkCounters) snapshot() NetworkSnapshot {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return NetworkSnapshot{Resets: c.resets, Completed: c.completed}
+}
+
+func (m *Monitor) NetworkSnapshot() NetworkSnapshot {
+ s := m.network.snapshot()
+ m.lifeMu.Lock()
+ conn := m.conn
+ m.lifeMu.Unlock()
+ if conn == nil || conn.ctx.Err() != nil || conn.protocol.IsClosed() || !conn.ready.Load() {
+ return s
+ }
+ sessions, ready := conn.surface.CaptureSessions()
+ if !ready {
+ return s
+ }
+ m.sessionsMu.RLock()
+ defer m.sessionsMu.RUnlock()
+ for _, id := range sessions {
+ if !m.networkReady[id] {
+ return s
+ }
+ }
+ s.Up = true
+ return s
+}
diff --git a/server/lib/cdpmonitor/network_metrics_bench_test.go b/server/lib/cdpmonitor/network_metrics_bench_test.go
new file mode 100644
index 000000000..9fcdbc10d
--- /dev/null
+++ b/server/lib/cdpmonitor/network_metrics_bench_test.go
@@ -0,0 +1,20 @@
+package cdpmonitor
+
+import (
+ "fmt"
+ "testing"
+)
+
+func BenchmarkMetricsOnlyTerminalDispatch(b *testing.B) {
+ m := New(newTestUpstream(""), newEventCollector().publishFn(), 0, discardLogger, nil)
+ _ = m.SetTelemetry(false)
+ messages := make([]cdpMessage, 2*terminalHistorySize)
+ for i := range messages {
+ messages[i] = cdpMessage{SessionID: "session", Method: "Network.loadingFailed", Params: []byte(fmt.Sprintf(`{"requestId":"%d","errorText":"net::ERR_CONNECTION_RESET"}`, i))}
+ }
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ m.dispatchEvent(messages[i%len(messages)])
+ }
+}
diff --git a/server/lib/cdpmonitor/network_metrics_chrome_e2e_test.go b/server/lib/cdpmonitor/network_metrics_chrome_e2e_test.go
new file mode 100644
index 000000000..d3628195b
--- /dev/null
+++ b/server/lib/cdpmonitor/network_metrics_chrome_e2e_test.go
@@ -0,0 +1,271 @@
+package cdpmonitor
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/kernel/kernel-images/server/lib/cdpclient"
+ "github.com/kernel/kernel-images/server/lib/events"
+ "github.com/kernel/kernel-images/server/lib/metrics"
+ "github.com/kernel/kernel-images/server/lib/oapi"
+ "github.com/kernel/kernel-images/server/lib/telemetry"
+ "github.com/stretchr/testify/require"
+)
+
+func TestAlwaysOnNetworkMetricsChrome(t *testing.T) {
+ if os.Getenv("KERNEL_CDPMONITOR_CHROME_E2E") == "" {
+ t.Skip("set KERNEL_CDPMONITOR_CHROME_E2E=1")
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+ defer cancel()
+ const workerFetch = `async function reply(path, port) { try { const r = await fetch(path, {method:'POST', body:'fixture'}); await r.text(); port.postMessage(r.status); } catch (e) { port.postMessage(e.name); } }`
+ var crossOrigin string
+ stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Cache-Control", "no-store")
+ switch r.URL.Path {
+ case "/reset":
+ _, _ = io.Copy(io.Discard, r.Body)
+ conn, _, err := w.(http.Hijacker).Hijack()
+ if err != nil {
+ return
+ }
+ _ = conn.(*net.TCPConn).SetLinger(0)
+ _ = conn.Close()
+ case "/ok":
+ fmt.Fprint(w, "ok")
+ case "/500":
+ w.WriteHeader(500)
+ fmt.Fprint(w, "fixture error")
+ case "/cancel":
+ _, _ = io.Copy(io.Discard, r.Body)
+ select {
+ case <-r.Context().Done():
+ case <-ctx.Done():
+ }
+ case "/worker.js":
+ w.Header().Set("Content-Type", "text/javascript")
+ fmt.Fprint(w, workerFetch+`; self.onmessage = e => reply(e.data, self);`)
+ case "/shared.js":
+ w.Header().Set("Content-Type", "text/javascript")
+ fmt.Fprint(w, workerFetch+`; self.onconnect = e => { const p=e.ports[0]; p.onmessage=e=>reply(e.data,p); p.start(); };`)
+ case "/service.js":
+ w.Header().Set("Content-Type", "text/javascript")
+ fmt.Fprint(w, workerFetch+`; self.addEventListener('install',e=>e.waitUntil(self.skipWaiting())); self.addEventListener('activate',e=>e.waitUntil(self.clients.claim())); self.addEventListener('message',e=>e.waitUntil(reply(e.data,e.ports[0])));`)
+ case "/":
+ w.Header().Set("Content-Type", "text/html")
+ fmt.Fprintf(w, ``, crossOrigin)
+ default:
+ w.Header().Set("Content-Type", "text/html")
+ fmt.Fprint(w, ``)
+ }
+ }))
+ defer stub.Close()
+ crossOrigin = strings.Replace(stub.URL, "127.0.0.1", "localhost", 1)
+ browserWS := launchChromium(t, ctx, findChromium(t), "--site-per-process")
+ driver := dialCDP(t, ctx, browserWS)
+ defer driver.close()
+ target := driver.call(t, ctx, "", "Target.createTarget", map[string]any{"url": stub.URL}).targetID(t)
+ session := driver.call(t, ctx, "", "Target.attachToTarget", map[string]any{"targetId": target, "flatten": true}).sessionID(t)
+ require.Eventually(t, func() bool {
+ return driver.evalBool(ctx, session, `document.readyState === 'complete' && document.querySelector('#same') !== null`)
+ }, 10*time.Second, 20*time.Millisecond)
+ crossTarget := findNetworkFrameTarget(t, ctx, driver, crossOrigin+"/frame")
+ crossSession := driver.call(t, ctx, "", "Target.attachToTarget", map[string]any{"targetId": crossTarget, "flatten": true}).sessionID(t)
+ es, err := events.NewEventStream(events.EventStreamConfig{RingCapacity: 128})
+ require.NoError(t, err)
+ ts := telemetry.NewTelemetrySession(es)
+ m := New(newTestUpstream(browserWS), ts.Publish, 0, discardLogger, func() bool { return false })
+ require.NoError(t, m.SetTelemetry(false))
+ require.NoError(t, m.Start(ctx))
+ defer m.Stop()
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Up }, 10*time.Second, 20*time.Millisecond)
+ require.False(t, driver.evalBool(ctx, session, `window.__kernelEventInjected === true`))
+
+ // An independent user connection confirms the browser's actual error text.
+ observer, err := cdpclient.DialWithEvents(ctx, browserWS)
+ require.NoError(t, err)
+ defer observer.Close()
+ attached, err := observer.Send(ctx, "Target.attachToTarget", map[string]any{"targetId": target, "flatten": true}, "")
+ require.NoError(t, err)
+ var observedSession struct {
+ SessionID string `json:"sessionId"`
+ }
+ require.NoError(t, json.Unmarshal(attached, &observedSession))
+ failures := make(chan string, 64)
+ go func() {
+ for message := range observer.Events() {
+ if message.Method == "Network.loadingFailed" {
+ var p cdpNetworkLoadingFailedParams
+ if json.Unmarshal(message.Params, &p) == nil {
+ select {
+ case failures <- p.ErrorText:
+ case <-ctx.Done():
+ return
+ }
+ }
+ }
+ }
+ }()
+ _, err = observer.Send(ctx, "Network.enable", nil, observedSession.SessionID)
+ require.NoError(t, err)
+ scrape := func() NetworkSnapshot {
+ t.Helper()
+ snapshot := m.NetworkSnapshot()
+ h := metrics.Handler(discardLogger, metrics.NewNetworkCollector(func() (uint64, uint64, bool) {
+ s := m.NetworkSnapshot()
+ return s.Resets, s.Completed, s.Up
+ }))
+ rr := httptest.NewRecorder()
+ h.ServeHTTP(rr, httptest.NewRequest("GET", "/metrics", nil))
+ require.Contains(t, rr.Body.String(), fmt.Sprintf("kernel_chromium_connection_resets_total %d\n", snapshot.Resets))
+ require.Contains(t, rr.Body.String(), fmt.Sprintf("kernel_chromium_network_requests_completed_total %d\n", snapshot.Completed))
+ return snapshot
+ }
+ settle := func() NetworkSnapshot {
+ t.Helper()
+ time.Sleep(300 * time.Millisecond)
+ return scrape()
+ }
+ fetch := func(sessionID, realm, path string, aborted bool) json.RawMessage {
+ t.Helper()
+ return evaluateNetworkScript(t, ctx, driver, sessionID, fmt.Sprintf(`(async()=>{const c=new AbortController(); const timer=%t?setTimeout(()=>c.abort(),100):null; try {const r=await %s.fetch(%q,{method:'POST',body:'fixture',signal:c.signal}); await r.text(); return r.status;} catch(e){return e.name;} finally {clearTimeout(timer);} })()`, aborted, realm, path))
+ }
+ before := settle()
+ for range 10 {
+ require.JSONEq(t, `"TypeError"`, string(fetch(session, "window", "/reset", false)))
+ }
+ for range 10 {
+ select {
+ case errorText := <-failures:
+ require.Equal(t, "net::ERR_CONNECTION_RESET", errorText)
+ case <-ctx.Done():
+ t.Fatal("missing confirmed reset")
+ }
+ }
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Completed == before.Completed+10 }, 5*time.Second, 10*time.Millisecond)
+ after := scrape()
+ require.Equal(t, before.Resets+10, after.Resets)
+ require.Equal(t, before.Completed+10, after.Completed)
+ require.Equal(t, after, settle(), "scraping must not increment counters")
+ require.Zero(t, es.Seq(), "telemetry off must not export customer data")
+
+ // HTTP status errors are completed requests, not transport resets.
+ refused, err := net.Listen("tcp", "127.0.0.1:0")
+ require.NoError(t, err)
+ refusedURL := "http://" + refused.Addr().String()
+ require.NoError(t, refused.Close())
+ for _, test := range []struct {
+ path, want string
+ abort bool
+ }{{"/ok", "200", false}, {"/500", "500", false}, {"/cancel", `"AbortError"`, true}, {refusedURL, `"TypeError"`, false}} {
+ require.JSONEq(t, test.want, string(fetch(session, "window", test.path, test.abort)))
+ }
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Completed == after.Completed+4 }, 5*time.Second, 10*time.Millisecond)
+ require.Equal(t, after.Resets, scrape().Resets)
+
+ for _, frame := range []struct{ name, session, realm string }{{"same-origin", session, `document.querySelector('#same').contentWindow`}, {"oopif", crossSession, "window"}} {
+ t.Run(frame.name, func(t *testing.T) {
+ b := settle()
+ require.JSONEq(t, `"TypeError"`, string(fetch(frame.session, frame.realm, "/reset", false)))
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Completed == b.Completed+1 }, 5*time.Second, 10*time.Millisecond)
+ require.Equal(t, b.Resets+1, scrape().Resets)
+ })
+ }
+ for _, worker := range []struct{ kind, create, request string }{
+ {"worker", `window.w=new Worker('/worker.js')`, `w.onmessage=e=>resolve(e.data); w.postMessage('/reset');`},
+ {"shared_worker", `window.sw=new SharedWorker('/shared.js'); sw.port.start()`, `sw.port.onmessage=e=>resolve(e.data); sw.port.postMessage('/reset');`},
+ {"service_worker", `await navigator.serviceWorker.register('/service.js'); window.service=(await navigator.serviceWorker.ready).active`, `const c=new MessageChannel(); c.port1.onmessage=e=>resolve(e.data); service.postMessage('/reset',[c.port2]);`},
+ } {
+ t.Run(worker.kind, func(t *testing.T) {
+ evaluateNetworkScript(t, ctx, driver, session, `(async()=>{`+worker.create+`;return true;})()`)
+ require.Eventually(t, func() bool {
+ m.sessionsMu.RLock()
+ defer m.sessionsMu.RUnlock()
+ for id, info := range m.sessions {
+ if info.targetType == worker.kind && m.networkReady[id] {
+ return true
+ }
+ }
+ return false
+ }, 5*time.Second, 10*time.Millisecond)
+ b := settle()
+ require.JSONEq(t, `"TypeError"`, string(evaluateNetworkScript(t, ctx, driver, session, `new Promise(resolve=>{`+worker.request+`})`)))
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Completed == b.Completed+1 }, 5*time.Second, 10*time.Millisecond)
+ require.Equal(t, b.Resets+1, scrape().Resets)
+ })
+ }
+
+ // Optional capture toggles on/off without replacing the monitor connection.
+ m.lifeMu.Lock()
+ original := m.conn
+ m.lifeMu.Unlock()
+ ts.Start("fixture", telemetry.TelemetryConfig{Categories: []oapi.TelemetryEventCategory{events.Interaction}})
+ require.NoError(t, m.SetTelemetry(true))
+ require.Eventually(t, func() bool { return driver.evalBool(ctx, session, `window.__kernelEventInjected === true`) }, 5*time.Second, 10*time.Millisecond)
+ evaluateNetworkScript(t, ctx, driver, session, `document.querySelector('#same').src = '/frame?telemetry=on'; true`)
+ require.Eventually(t, func() bool {
+ return driver.evalBool(ctx, session, `document.querySelector('#same').contentWindow.__kernelEventInjected === true`)
+ }, 5*time.Second, 10*time.Millisecond)
+ require.Zero(t, es.Seq(), "page/network categories must remain gated while interaction capture is enabled")
+ evaluateNetworkScript(t, ctx, driver, session, `document.querySelector('#go').click(); true`)
+ require.Eventually(t, func() bool { return es.Seq() > 0 }, time.Second, 10*time.Millisecond)
+ ts.Stop()
+ require.NoError(t, m.SetTelemetry(false))
+ waitForTelemetryReconcile(t, m, false)
+ seq := es.Seq()
+ require.False(t, driver.evalBool(ctx, session, `window.__kernelEventInjected === true`))
+ require.False(t, driver.evalBool(ctx, session, `document.querySelector('#same').contentWindow.__kernelEventInjected === true`))
+ require.False(t, driver.evalBool(ctx, crossSession, `window.__kernelEventInjected === true`))
+ m.lifeMu.Lock()
+ current := m.conn
+ m.lifeMu.Unlock()
+ require.Same(t, original, current)
+ b := settle()
+ fetch(session, "window", "/reset", false)
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Resets == b.Resets+1 }, time.Second, 10*time.Millisecond)
+ require.Equal(t, seq, es.Seq())
+ m.sessionsMu.RLock()
+ computedCount := len(m.computedStates)
+ m.sessionsMu.RUnlock()
+ m.pendReqMu.Lock()
+ pendingCount := len(m.pendingRequests)
+ m.pendReqMu.Unlock()
+ require.Zero(t, computedCount)
+ require.Zero(t, pendingCount)
+
+ // Drop only the monitor's socket; the independent user connection still works.
+ m.lifeMu.Lock()
+ old := m.conn
+ m.lifeMu.Unlock()
+ require.NoError(t, old.protocol.Close())
+ require.False(t, m.NetworkSnapshot().Up)
+ require.Eventually(t, func() bool {
+ m.lifeMu.Lock()
+ fresh := m.conn != nil && m.conn != old
+ m.lifeMu.Unlock()
+ return fresh && m.NetworkSnapshot().Up
+ }, 10*time.Second, 10*time.Millisecond)
+ _, err = observer.GetBrowserVersion(ctx)
+ require.NoError(t, err)
+ b = settle()
+ fetch(session, "window", "/reset", false)
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Resets == b.Resets+1 }, time.Second, 10*time.Millisecond)
+ // A fresh API-owned monitor starts with zero counters, even on the same Chrome.
+ m.Stop()
+ fresh := New(newTestUpstream(browserWS), ts.Publish, 0, discardLogger, nil)
+ require.Equal(t, NetworkSnapshot{}, fresh.NetworkSnapshot())
+ require.NoError(t, fresh.SetTelemetry(false))
+ require.NoError(t, fresh.Start(ctx))
+ defer fresh.Stop()
+ require.Eventually(t, func() bool { return fresh.NetworkSnapshot().Up }, 5*time.Second, 10*time.Millisecond)
+ require.Zero(t, fresh.NetworkSnapshot().Resets)
+}
diff --git a/server/lib/cdpmonitor/network_metrics_test.go b/server/lib/cdpmonitor/network_metrics_test.go
new file mode 100644
index 000000000..bf8d9ec0d
--- /dev/null
+++ b/server/lib/cdpmonitor/network_metrics_test.go
@@ -0,0 +1,83 @@
+package cdpmonitor
+
+import (
+ "fmt"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestNetworkCounters(t *testing.T) {
+ c := newNetworkCounters()
+ require.Equal(t, NetworkSnapshot{}, c.snapshot())
+ for i, errorText := range []string{"net::ERR_CONNECTION_RESET", "net::ERR_CONNECTION_REFUSED", "net::ERR_ABORTED", "ERR_CONNECTION_RESET", "net::ERR_CONNECTION_RESET extra", ""} {
+ c.terminal("session", fmt.Sprint(i), errorText)
+ c.terminal("session", fmt.Sprint(i), errorText)
+ }
+ require.Equal(t, uint64(1), c.snapshot().Resets)
+ require.Equal(t, uint64(6), c.snapshot().Completed)
+ // Unknown starts count, and colliding request IDs in different sessions don't deduplicate.
+ c.terminal("worker", "0", "net::ERR_CONNECTION_RESET")
+ require.Equal(t, uint64(2), c.snapshot().Resets)
+ c.newGeneration()
+ c.terminal("session", "0", "net::ERR_CONNECTION_RESET")
+ require.Equal(t, uint64(3), c.snapshot().Resets)
+ require.Equal(t, uint64(8), c.snapshot().Completed)
+ c.terminal("s", "first-wins", "")
+ c.terminal("s", "first-wins", "net::ERR_CONNECTION_RESET")
+ require.Equal(t, uint64(3), c.snapshot().Resets)
+}
+
+func TestNetworkCountersBounded(t *testing.T) {
+ c := newNetworkCounters()
+ for i := range terminalHistorySize + 100 {
+ c.terminal("s", fmt.Sprint(i), "")
+ }
+ require.Len(t, c.seen, terminalHistorySize)
+ // FIFO eviction intentionally ends the deduplication guarantee.
+ c.terminal("s", "0", "")
+ require.Equal(t, uint64(terminalHistorySize+101), c.snapshot().Completed)
+ c.newGeneration()
+ require.Empty(t, c.seen)
+ before := c.snapshot()
+ for _, key := range []networkRequestKey{{"", "r"}, {"s", ""}, {strings.Repeat("s", 257), "r"}, {"s", strings.Repeat("r", 257)}} {
+ c.terminal(key.sessionID, key.requestID, "net::ERR_CONNECTION_RESET")
+ }
+ require.Equal(t, before, c.snapshot())
+}
+
+func TestNetworkCountersConcurrent(t *testing.T) {
+ c := newNetworkCounters()
+ var wg sync.WaitGroup
+ for range 8 {
+ wg.Go(func() {
+ for i := range 1000 {
+ c.terminal("s", fmt.Sprint(i), "net::ERR_CONNECTION_RESET")
+ s := c.snapshot()
+ if s.Resets != s.Completed {
+ t.Errorf("inconsistent counter snapshot: %+v", s)
+ return
+ }
+ }
+ })
+ }
+ wg.Wait()
+ require.Equal(t, uint64(1000), c.snapshot().Completed)
+}
+
+func TestMetricsOnlyDispatch(t *testing.T) {
+ m := New(newTestUpstream(""), newEventCollector().publishFn(), 0, discardLogger, nil)
+ require.NoError(t, m.SetTelemetry(false))
+ for range 3 { // redirect starts do not affect terminal counters
+ m.dispatchEvent(cdpMessage{Method: "Network.requestWillBeSent", SessionID: "s", Params: []byte(`{"requestId":"r","redirectResponse":{}}`)})
+ }
+ for range 2 {
+ m.dispatchEvent(cdpMessage{Method: "Network.loadingFailed", SessionID: "s", Params: []byte(`{"requestId":"r","errorText":"net::ERR_CONNECTION_RESET"}`)})
+ }
+ m.dispatchEvent(cdpMessage{Method: "Network.loadingFinished", SessionID: "s", Params: []byte(`{"requestId":"unknown"}`)})
+ require.Equal(t, NetworkSnapshot{Resets: 1, Completed: 2}, m.NetworkSnapshot())
+ require.Empty(t, m.pendingRequests)
+ require.Empty(t, m.computedStates)
+}
diff --git a/server/lib/cdpmonitor/network_probe_drain_test.go b/server/lib/cdpmonitor/network_probe_drain_test.go
new file mode 100644
index 000000000..619c4a669
--- /dev/null
+++ b/server/lib/cdpmonitor/network_probe_drain_test.go
@@ -0,0 +1,189 @@
+package cdpmonitor
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/coder/websocket"
+ "github.com/coder/websocket/wsjson"
+ "github.com/kernel/kernel-images/server/lib/events"
+ "github.com/stretchr/testify/require"
+)
+
+func TestFailedProbeCancelsTelemetryDrain(t *testing.T) {
+ for _, blockedMethod := range []string{"Runtime.enable", "Network.getResponseBody"} {
+ for _, ending := range []string{"disabled", "enabled", "shutdown"} {
+ t.Run(blockedMethod+"/"+ending, func(t *testing.T) {
+ t.Parallel()
+ testFailedProbeCancelsTelemetryDrain(t, blockedMethod, ending)
+ })
+ }
+ }
+}
+
+func testFailedProbeCancelsTelemetryDrain(t *testing.T, blockedMethod, ending string) {
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+ blocked := make(chan struct{})
+ sockets := make(chan *websocket.Conn, 4)
+ var connections, replacementRuntimeEnables atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := websocket.Accept(w, r, nil)
+ if err != nil {
+ return
+ }
+ defer conn.CloseNow()
+ generation := connections.Add(1)
+ select {
+ case sockets <- conn:
+ case <-ctx.Done():
+ return
+ }
+ blackhole := false
+ for {
+ var command struct {
+ ID int `json:"id"`
+ Method string `json:"method"`
+ }
+ if wsjson.Read(ctx, conn, &command) != nil {
+ return
+ }
+ if generation == 1 && command.Method == blockedMethod && !blackhole {
+ blackhole = true
+ close(blocked)
+ }
+ if blackhole {
+ continue // Keep the socket open, but stop replying to commands, including probes.
+ }
+ result := map[string]any{}
+ switch command.Method {
+ case "Target.getTargets":
+ result["targetInfos"] = []any{map[string]any{"targetId": "target", "type": "page"}}
+ case "Target.attachToTarget":
+ result["sessionId"] = "session"
+ case "Page.addScriptToEvaluateOnNewDocument":
+ result["identifier"] = "script"
+ case "Runtime.enable":
+ if generation > 1 {
+ replacementRuntimeEnables.Add(1)
+ }
+ }
+ if wsjson.Write(ctx, conn, map[string]any{"id": command.ID, "result": result}) != nil {
+ return
+ }
+ }
+ }))
+ defer server.Close()
+
+ // Hold the disconnected publication so a fast reconnect cannot conceal a
+ // stale health value. This also checks cancellation precedes publication.
+ disconnected := make(chan bool, 1)
+ released := make(chan struct{})
+ var once sync.Once
+ release := func() { once.Do(func() { close(released) }) }
+ var m *Monitor
+ m = New(newTestUpstream("ws"+strings.TrimPrefix(server.URL, "http")), func(ev events.Event) (events.Envelope, bool) {
+ if ev.Type == EventMonitorDisconnected {
+ select {
+ case disconnected <- m.NetworkSnapshot().Up:
+ case <-ctx.Done():
+ }
+ select {
+ case <-released:
+ case <-ctx.Done():
+ }
+ }
+ return events.Envelope{Event: ev}, true
+ }, 0, discardLogger, func() bool { return false })
+ require.NoError(t, m.SetTelemetry(false))
+ require.NoError(t, m.Start(ctx))
+ defer m.Stop()
+ defer release()
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Up }, time.Second, time.Millisecond)
+ conn := <-sockets
+ send := func(method string, params any, socket *websocket.Conn) {
+ t.Helper()
+ require.NoError(t, wsjson.Write(ctx, socket, map[string]any{"method": method, "sessionId": "session", "params": params}))
+ }
+ terminal := map[string]any{"requestId": "seed", "errorText": "net::ERR_CONNECTION_RESET"}
+ send("Network.loadingFailed", terminal, conn)
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Resets == 1 }, time.Second, time.Millisecond)
+ require.NoError(t, m.SetTelemetry(true))
+ if blockedMethod == "Network.getResponseBody" {
+ require.Eventually(t, func() bool {
+ m.sessionsMu.RLock()
+ defer m.sessionsMu.RUnlock()
+ return m.optionalSessions["session"] == "script" && !m.telemetryChanging.Load()
+ }, time.Second, time.Millisecond)
+ send("Network.requestWillBeSent", map[string]any{"requestId": "body", "type": "Fetch", "request": map[string]any{"method": "GET", "url": "http://fixture/body"}}, conn)
+ send("Network.responseReceived", map[string]any{"requestId": "body", "response": map[string]any{"status": 200, "mimeType": "text/plain"}}, conn)
+ send("Network.loadingFinished", map[string]any{"requestId": "body"}, conn)
+ }
+ select {
+ case <-blocked:
+ case <-time.After(2 * time.Second):
+ t.Fatal("optional command was not blocked")
+ }
+ before := m.NetworkSnapshot()
+ require.True(t, before.Up)
+ m.lifeMu.Lock()
+ old := m.conn
+ m.lifeMu.Unlock()
+ require.NoError(t, m.SetTelemetry(false))
+ require.Eventually(t, m.telemetryChanging.Load, time.Second, time.Millisecond)
+ if m.restartMu.TryLock() {
+ m.restartMu.Unlock()
+ t.Fatal("telemetry drain did not hold replacement serialization")
+ }
+ wantEnabled := ending == "enabled"
+ if wantEnabled {
+ require.NoError(t, m.SetTelemetry(true))
+ }
+ desired := m.desiredTelemetry.Load()
+ select {
+ case up := <-disconnected:
+ require.False(t, up, "failed probe published disconnected while health was still up")
+ case <-time.After(12 * time.Second):
+ t.Fatal("supervisor did not detect the blackholed connection")
+ }
+ require.ErrorIs(t, old.ctx.Err(), context.Canceled, "failed connection was not canceled before waiting for telemetry drain")
+ require.False(t, m.NetworkSnapshot().Up)
+ release()
+ if ending != "shutdown" {
+ require.Eventually(t, func() bool { return connections.Load() == 2 && m.NetworkSnapshot().Up }, 2*time.Second, time.Millisecond)
+ waitForTelemetryReconcile(t, m, wantEnabled)
+ if wantEnabled {
+ require.Eventually(t, func() bool { return replacementRuntimeEnables.Load() == 1 }, time.Second, time.Millisecond)
+ } else {
+ require.Zero(t, replacementRuntimeEnables.Load())
+ }
+ after := m.NetworkSnapshot()
+ require.Equal(t, before.Resets, after.Resets)
+ require.Equal(t, before.Completed, after.Completed)
+ // Reusing the same IDs on the replacement must not reuse terminal history.
+ fresh := <-sockets
+ send("Network.loadingFailed", terminal, fresh)
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Resets == before.Resets+1 }, time.Second, time.Millisecond)
+ require.Equal(t, before.Completed+1, m.NetworkSnapshot().Completed)
+ }
+ require.Equal(t, desired, m.desiredTelemetry.Load())
+ stopped := make(chan struct{})
+ go func() { m.Stop(); close(stopped) }()
+ select {
+ case <-stopped:
+ case <-time.After(time.Second):
+ t.Fatal("shutdown waited for the optional command timeout")
+ }
+ require.False(t, m.NetworkSnapshot().Up)
+ require.False(t, m.IsRunning())
+ if ending == "shutdown" {
+ require.Equal(t, before.Resets, m.NetworkSnapshot().Resets)
+ require.Equal(t, before.Completed, m.NetworkSnapshot().Completed)
+ }
+}
diff --git a/server/lib/cdpmonitor/network_races_test.go b/server/lib/cdpmonitor/network_races_test.go
new file mode 100644
index 000000000..c5dea83f0
--- /dev/null
+++ b/server/lib/cdpmonitor/network_races_test.go
@@ -0,0 +1,136 @@
+package cdpmonitor
+
+import (
+ "context"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/kernel/kernel-images/server/lib/events"
+ "github.com/stretchr/testify/require"
+)
+
+type changingUpstream struct {
+ *testUpstream
+ once sync.Once
+ next string
+}
+
+func (u *changingUpstream) Current() string {
+ current := u.testUpstream.Current()
+ u.once.Do(func() { u.notifyRestart(u.next) })
+ return current
+}
+
+func TestNetworkSubscriptionPrecedesInitialDial(t *testing.T) {
+ old := newTestServer(t)
+ defer old.close()
+ fresh := newTestServer(t)
+ defer fresh.close()
+ stop := make(chan struct{})
+ defer close(stop)
+ go listenAndRespond(old, stop, nil)
+ go listenAndRespond(fresh, stop, nil)
+ upstream := &changingUpstream{testUpstream: newTestUpstream(old.wsURL()), next: fresh.wsURL()}
+ m := New(upstream, newEventCollector().publishFn(), 0, discardLogger, nil)
+ require.NoError(t, m.SetTelemetry(false))
+ require.NoError(t, m.Start(context.Background()))
+ defer m.Stop()
+ select {
+ case <-fresh.connCh:
+ case <-time.After(3 * time.Second):
+ t.Fatal("lost restart between Current and dial")
+ }
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Up }, time.Second, 10*time.Millisecond)
+}
+
+func TestTelemetryDisableDrainsComputedPublication(t *testing.T) {
+ srv := newTestServer(t)
+ defer srv.close()
+ stop := make(chan struct{})
+ defer close(stop)
+ go listenAndRespond(srv, stop, nil)
+ entered := make(chan struct{})
+ release := make(chan struct{})
+ var once sync.Once
+ unblock := func() { once.Do(func() { close(release) }) }
+ m := New(newTestUpstream(srv.wsURL()), func(ev events.Event) (events.Envelope, bool) {
+ if ev.Type == EventNetworkIdle {
+ close(entered)
+ <-release
+ }
+ return events.Envelope{Event: ev}, true
+ }, 0, discardLogger, nil)
+ require.NoError(t, m.Start(context.Background()))
+ defer m.Stop()
+ defer unblock()
+ <-srv.connCh
+ srv.sendToMonitor(t, map[string]any{"method": "Target.attachedToTarget", "params": map[string]any{"sessionId": "s", "targetInfo": map[string]any{"targetId": "t", "type": "page"}}})
+ require.Eventually(t, func() bool { return m.computedFor("s") != nil }, time.Second, time.Millisecond)
+ state := m.computedFor("s")
+ go state.publish(events.Event{Type: EventNetworkIdle})
+ <-entered
+ done := make(chan error, 1)
+ go func() { done <- m.SetTelemetry(false) }()
+ select {
+ case <-done:
+ t.Fatal("disable returned before old computed publication drained")
+ case <-time.After(50 * time.Millisecond):
+ }
+ unblock()
+ require.NoError(t, <-done)
+ require.NoError(t, m.SetTelemetry(true))
+ // Publishing from an old timer after re-enable must not reach the publisher.
+ _, published := state.publish(events.Event{Type: EventNetworkIdle})
+ require.False(t, published)
+}
+
+func TestTelemetryDisableDrainsSetupWhileCountersContinue(t *testing.T) {
+ srv := newTestServer(t)
+ defer srv.close()
+ blocked := make(chan cdpMessage, 1)
+ stop := make(chan struct{})
+ defer close(stop)
+ go listenAndRespond(srv, stop, func(msg cdpMessage) any {
+ if msg.Method == "Runtime.enable" {
+ blocked <- msg
+ return map[string]any{"method": "Test.unanswered"}
+ }
+ return nil
+ })
+ m := New(newTestUpstream(srv.wsURL()), newEventCollector().publishFn(), 0, discardLogger, nil)
+ ctx, cancel := context.WithCancel(context.Background())
+ require.NoError(t, m.SetTelemetry(false))
+ require.NoError(t, m.Start(ctx))
+ defer m.Stop()
+ defer cancel()
+ <-srv.connCh
+ srv.sendToMonitor(t, map[string]any{"method": "Target.attachedToTarget", "params": map[string]any{"sessionId": "s", "targetInfo": map[string]any{"targetId": "t", "type": "page"}}})
+ require.Eventually(t, func() bool {
+ m.sessionsMu.RLock()
+ defer m.sessionsMu.RUnlock()
+ return m.networkReady["s"]
+ }, time.Second, time.Millisecond)
+ require.NoError(t, m.SetTelemetry(true))
+ var command cdpMessage
+ select {
+ case command = <-blocked:
+ case <-time.After(time.Second):
+ t.Fatal("optional setup did not start")
+ }
+ done := make(chan error, 1)
+ go func() { done <- m.SetTelemetry(false) }()
+ require.Eventually(t, m.telemetryChanging.Load, time.Second, time.Millisecond)
+ srv.sendToMonitor(t, map[string]any{"method": "Network.loadingFailed", "sessionId": "s", "params": map[string]any{"requestId": "r", "errorText": "net::ERR_CONNECTION_RESET"}})
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Resets == 1 }, time.Second, time.Millisecond)
+ select {
+ case err := <-done:
+ require.NoError(t, err)
+ case <-time.After(time.Second):
+ t.Fatal("desired state waited for optional setup")
+ }
+ require.NotEqual(t, m.desiredTelemetry.Load(), m.appliedTelemetry.Load())
+ srv.sendToMonitor(t, map[string]any{"id": command.ID, "result": map[string]any{}})
+ waitForTelemetryReconcile(t, m, false)
+ require.True(t, m.NetworkSnapshot().Up)
+}
diff --git a/server/lib/cdpmonitor/network_restart_chrome_e2e_test.go b/server/lib/cdpmonitor/network_restart_chrome_e2e_test.go
new file mode 100644
index 000000000..26cb08d2c
--- /dev/null
+++ b/server/lib/cdpmonitor/network_restart_chrome_e2e_test.go
@@ -0,0 +1,87 @@
+package cdpmonitor
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/kernel/kernel-images/server/lib/cdpclient"
+ "github.com/stretchr/testify/require"
+)
+
+func TestNetworkMetricsActualChromeRestart(t *testing.T) {
+ if os.Getenv("KERNEL_CDPMONITOR_CHROME_E2E") == "" {
+ t.Skip("set KERNEL_CDPMONITOR_CHROME_E2E=1")
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer cancel()
+ stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method == "POST" {
+ _, _ = io.Copy(io.Discard, r.Body)
+ conn, _, err := w.(http.Hijacker).Hijack()
+ if err != nil {
+ return
+ }
+ _ = conn.(*net.TCPConn).SetLinger(0)
+ _ = conn.Close()
+ return
+ }
+ w.Header().Set("Content-Type", "text/html")
+ fmt.Fprint(w, `fixture`)
+ }))
+ defer stub.Close()
+ chrome := findChromium(t)
+ ws := launchChromium(t, ctx, chrome)
+ upstream := newTestUpstream(ws)
+ m := New(upstream, newEventCollector().publishFn(), 0, discardLogger, nil)
+ require.NoError(t, m.SetTelemetry(false))
+ require.NoError(t, m.Start(ctx))
+ defer m.Stop()
+ reset := func(ws string) {
+ driver := dialCDP(t, ctx, ws)
+ defer driver.close()
+ target := driver.call(t, ctx, "", "Target.createTarget", map[string]any{"url": stub.URL}).targetID(t)
+ session := driver.call(t, ctx, "", "Target.attachToTarget", map[string]any{"targetId": target, "flatten": true}).sessionID(t)
+ require.Eventually(t, func() bool {
+ return m.NetworkSnapshot().Up && driver.evalBool(ctx, session, `document.readyState === 'complete'`)
+ }, 10*time.Second, 20*time.Millisecond)
+ // Require this particular target, not just the previously-known target set.
+ require.Eventually(t, func() bool {
+ m.sessionsMu.RLock()
+ defer m.sessionsMu.RUnlock()
+ for id, info := range m.sessions {
+ if info.targetID == target && m.networkReady[id] {
+ return true
+ }
+ }
+ return false
+ }, 5*time.Second, 10*time.Millisecond)
+ b := m.NetworkSnapshot()
+ value := evaluateNetworkScript(t, ctx, driver, session, `fetch('/',{method:'POST',body:'fixture'}).then(()=>false,()=>true)`)
+ require.JSONEq(t, "true", string(value))
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Resets == b.Resets+1 }, 5*time.Second, 10*time.Millisecond)
+ }
+ reset(ws)
+ before := m.NetworkSnapshot()
+ require.Equal(t, uint64(1), before.Resets)
+ killer, err := cdpclient.Dial(ctx, ws)
+ require.NoError(t, err)
+ _, _ = killer.Send(ctx, "Browser.close", nil, "")
+ require.Eventually(t, killer.IsClosed, 5*time.Second, 10*time.Millisecond)
+ _ = killer.Close()
+ require.Eventually(t, func() bool { return !m.NetworkSnapshot().Up }, 5*time.Second, 10*time.Millisecond)
+ require.Equal(t, before.Resets, m.NetworkSnapshot().Resets)
+ ws = launchChromium(t, ctx, chrome)
+ upstream.notifyRestart(ws)
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Up }, 10*time.Second, 20*time.Millisecond)
+ require.Equal(t, before.Resets, m.NetworkSnapshot().Resets)
+ require.GreaterOrEqual(t, m.NetworkSnapshot().Completed, before.Completed)
+ reset(ws)
+ require.Equal(t, uint64(2), m.NetworkSnapshot().Resets)
+}
diff --git a/server/lib/cdpmonitor/screenshot.go b/server/lib/cdpmonitor/screenshot.go
index 49ca51f16..8931dfa50 100644
--- a/server/lib/cdpmonitor/screenshot.go
+++ b/server/lib/cdpmonitor/screenshot.go
@@ -42,7 +42,7 @@ func (m *Monitor) tryScreenshot(ctx context.Context, sourceEvent, sessionID stri
if cs := m.computedFor(sessionID); cs != nil {
_, navMeta = cs.navSnapshot()
}
- m.captureWg.Go(func() {
+ m.telemetryWg.Go(func() {
defer m.screenshotInFlight.Store(false)
m.captureScreenshot(ctx, sourceEvent, navMeta)
})
diff --git a/server/lib/cdpmonitor/startup_events_test.go b/server/lib/cdpmonitor/startup_events_test.go
new file mode 100644
index 000000000..337c9ba7c
--- /dev/null
+++ b/server/lib/cdpmonitor/startup_events_test.go
@@ -0,0 +1,120 @@
+package cdpmonitor
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/kernel/kernel-images/server/lib/events"
+ "github.com/kernel/kernel-images/server/lib/oapi"
+ "github.com/stretchr/testify/require"
+)
+
+type countedUpstream struct {
+ *testUpstream
+ reads atomic.Int32
+}
+
+func (u *countedUpstream) Current() string {
+ u.reads.Add(1)
+ return u.testUpstream.Current()
+}
+
+func TestInitialAcquisitionIsNotReconnection(t *testing.T) {
+ for _, start := range []string{"no_url", "dial_failure"} {
+ t.Run(start, func(t *testing.T) {
+ t.Parallel()
+ srv := newTestServer(t)
+ defer srv.close()
+ stop := make(chan struct{})
+ defer close(stop)
+ var probes atomic.Int32
+ go listenAndRespond(srv, stop, func(msg cdpMessage) any {
+ if msg.Method == "Browser.getVersion" {
+ probes.Add(1)
+ }
+ return nil
+ })
+ var failures atomic.Int32
+ unavailable := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ failures.Add(1)
+ http.Error(w, "fixture unavailable", http.StatusServiceUnavailable)
+ }))
+ defer unavailable.Close()
+ badURL := "ws" + strings.TrimPrefix(unavailable.URL, "http")
+ u := &countedUpstream{testUpstream: newTestUpstream("")}
+ ec := newEventCollector()
+ m := New(u, ec.publishFn(), 0, discardLogger, nil)
+ require.NoError(t, m.SetTelemetry(false))
+ defer m.Stop()
+ lifecycle := func(since int) []events.Event {
+ ec.mu.Lock()
+ defer ec.mu.Unlock()
+ var found []events.Event
+ for _, event := range ec.events[since:] {
+ if event.Type == EventMonitorDisconnected || event.Type == EventMonitorReconnected {
+ found = append(found, event)
+ }
+ }
+ return found
+ }
+ for cycle := 0; cycle < 2; cycle++ {
+ u.mu.Lock()
+ u.current = ""
+ if start == "dial_failure" {
+ u.current = badURL
+ }
+ u.mu.Unlock()
+ checkpoint, reads, failed, probed := ec.checkpoint(), u.reads.Load(), failures.Load(), probes.Load()
+ require.NoError(t, m.Start(context.Background()))
+ require.Eventually(t, func() bool { return u.reads.Load() >= reads+3 }, 2*time.Second, time.Millisecond)
+ if start == "dial_failure" {
+ require.Eventually(t, func() bool { return failures.Load() >= failed+3 }, time.Second, time.Millisecond)
+ }
+ require.False(t, m.NetworkSnapshot().Up)
+ u.notifyRestart(srv.wsURL())
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Up }, 5*time.Second, time.Millisecond)
+ // A healthy probe resets retry backoff before the genuine loss below.
+ require.Eventually(t, func() bool { return probes.Load() > probed }, 6*time.Second, time.Millisecond)
+ require.Never(t, func() bool { return len(lifecycle(checkpoint)) != 0 }, 300*time.Millisecond, time.Millisecond, "startup retries must not emit restart events")
+ before := m.NetworkSnapshot()
+ m.network.terminal("s", "r", "net::ERR_CONNECTION_RESET")
+ require.Equal(t, before.Resets+1, m.NetworkSnapshot().Resets)
+ // Lose a real connection and keep recovery failing across several retries.
+ u.mu.Lock()
+ u.current = badURL
+ u.mu.Unlock()
+ failed = failures.Load()
+ m.lifeMu.Lock()
+ conn := m.conn
+ m.lifeMu.Unlock()
+ require.NoError(t, conn.protocol.Close())
+ ec.waitForNew(t, EventMonitorDisconnected, checkpoint, time.Second)
+ require.Eventually(t, func() bool { return failures.Load() >= failed+2 }, 2*time.Second, time.Millisecond)
+ require.Len(t, lifecycle(checkpoint), 1)
+ if cycle == 0 {
+ m.Stop() // A new Start must not inherit this unfinished recovery.
+ continue
+ }
+ u.notifyRestart(srv.wsURL())
+ ec.waitForNew(t, EventMonitorReconnected, checkpoint, 3*time.Second)
+ require.Eventually(t, func() bool { return m.NetworkSnapshot().Up }, time.Second, time.Millisecond)
+ require.Never(t, func() bool { return len(lifecycle(checkpoint)) != 2 }, 300*time.Millisecond, time.Millisecond)
+ pair := lifecycle(checkpoint)
+ require.Equal(t, EventMonitorDisconnected, pair[0].Type)
+ require.Equal(t, EventMonitorReconnected, pair[1].Type)
+ var data oapi.BrowserMonitorReconnectedEventData
+ require.NoError(t, json.Unmarshal(pair[1].Data, &data))
+ require.GreaterOrEqual(t, data.ReconnectDurationMs, int64(750))
+ require.InDelta(t, (pair[1].Ts-pair[0].Ts)/1000, data.ReconnectDurationMs, 100)
+ require.Equal(t, before.Resets+1, m.NetworkSnapshot().Resets)
+ require.Equal(t, before.Completed+1, m.NetworkSnapshot().Completed)
+ }
+ })
+ }
+}
diff --git a/server/lib/cdpmonitor/telemetry.go b/server/lib/cdpmonitor/telemetry.go
new file mode 100644
index 000000000..587a4987b
--- /dev/null
+++ b/server/lib/cdpmonitor/telemetry.go
@@ -0,0 +1,216 @@
+package cdpmonitor
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "time"
+
+ "github.com/kernel/kernel-images/server/lib/cdpclient"
+ "github.com/kernel/kernel-images/server/lib/events"
+ oapi "github.com/kernel/kernel-images/server/lib/oapi"
+)
+
+const telemetryCleanupTimeout = 3 * time.Second
+
+// SetTelemetry commits and fences the desired capture revision without waiting
+// for CDP. The API lifecycle worker owns draining, cleanup, and re-enabling.
+func (m *Monitor) SetTelemetry(enabled bool) error {
+ m.desiredMu.Lock()
+ state := m.desiredTelemetry.Load()
+ changed := (state&1 != 0) != enabled
+ if changed {
+ state = (state &^ 1) + 2
+ if enabled {
+ state |= 1
+ }
+ m.desiredTelemetry.Store(state)
+ }
+ m.desiredMu.Unlock()
+ if changed {
+ m.signalTelemetry()
+ }
+ return nil
+}
+
+func (m *Monitor) captureEnabled() bool {
+ state := m.desiredTelemetry.Load()
+ return state&1 != 0 && state == m.appliedTelemetry.Load()
+}
+
+func (m *Monitor) signalTelemetry() {
+ select {
+ case m.telemetryChanged <- struct{}{}:
+ default:
+ }
+}
+
+func (m *Monitor) reconcileTelemetry(ctx context.Context) {
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-m.telemetryChanged:
+ m.restartMu.Lock()
+ if ctx.Err() == nil {
+ if err := m.applyTelemetry(m.desiredTelemetry.Load()); err != nil {
+ m.log.Warn("cdpmonitor: telemetry cleanup failed; reconnecting", "err", err)
+ }
+ }
+ m.restartMu.Unlock()
+ }
+ }
+}
+
+// restartMu excludes connection replacement. A revision change always drains the
+// previous capture, including an off/on pair coalesced before this worker runs.
+func (m *Monitor) applyTelemetry(state uint64) error {
+ m.telemetryChanging.Store(true)
+ defer m.telemetryChanging.Store(false)
+ m.telemetryMu.Lock()
+ defer m.telemetryMu.Unlock()
+ if m.appliedTelemetry.Load() == state {
+ return nil
+ }
+ m.telemetryEnabled = false
+ // Do not cancel an in-flight WebSocket write or lose a registration's removal ID.
+ // This bounded work runs outside the API lock; shutdown cancels its parent.
+ m.telemetryWg.Wait()
+ if m.telemetryCancel != nil {
+ m.telemetryCancel()
+ }
+ m.sessionsMu.Lock()
+ states := m.computedStates
+ m.computedStates = make(map[string]*computedState)
+ scripts := make(map[string]string, len(m.optionalSessions))
+ for id, script := range m.optionalSessions {
+ scripts[id] = script
+ }
+ m.sessionsMu.Unlock()
+ for _, cs := range states {
+ cs.stop()
+ }
+ m.computedPublishMu.Lock()
+ m.computedPublishMu.Unlock()
+ m.pendReqMu.Lock()
+ clear(m.pendingRequests)
+ m.pendReqMu.Unlock()
+ m.mainSessionID.Store(mainSessionUnset)
+ m.bindingRateMu.Lock()
+ clear(m.bindingLastSeen)
+ m.bindingRateMu.Unlock()
+ m.proxyRateMu.Lock()
+ clear(m.proxyLastEmit)
+ m.proxyRateMu.Unlock()
+ m.lifeMu.Lock()
+ conn := m.conn
+ m.lifeMu.Unlock()
+ if conn != nil && conn.ctx.Err() == nil {
+ ctx, cancel := context.WithTimeout(conn.ctx, telemetryCleanupTimeout)
+ defer cancel()
+ var cleanupErr error
+ for sessionID, scriptID := range scripts {
+ cleanupErr = errors.Join(cleanupErr, m.disableOptionalDomains(ctx, sessionID, scriptID))
+ }
+ m.sessionsMu.RLock()
+ pending := len(m.interactionTargets) != 0
+ m.sessionsMu.RUnlock()
+ if cleanupErr == nil && pending {
+ cleanupErr = errors.New("interaction cleanup requires target reattachment")
+ }
+ if cleanupErr != nil {
+ // Target-scoped obligations survive this socket; new sessions retry cleanup.
+ conn.cancel()
+ return cleanupErr
+ }
+ }
+ m.sessionsMu.Lock()
+ clear(m.optionalSessions)
+ m.sessionsMu.Unlock()
+ // Do not resurrect an obsolete enabled revision after slow cleanup.
+ if m.desiredTelemetry.Load() != state {
+ m.signalTelemetry()
+ return nil
+ }
+ if state&1 == 0 {
+ m.appliedTelemetry.Store(state)
+ return nil
+ }
+ if conn == nil || conn.ctx.Err() != nil {
+ return nil // The replacement connection applies the latest desired revision.
+ }
+ m.telemetryCtx, m.telemetryCancel = context.WithCancel(conn.ctx)
+ m.telemetryEnabled = true
+ m.appliedTelemetry.Store(state)
+ m.sessionsMu.RLock()
+ sessions := make(map[string]targetInfo, len(m.sessions))
+ for id, info := range m.sessions {
+ // Pending attachments must finish orphan cleanup before optional setup.
+ if m.networkReady[id] {
+ sessions[id] = info
+ }
+ }
+ m.sessionsMu.RUnlock()
+ for id, info := range sessions {
+ m.captureWg.Go(func() {
+ m.telemetryMu.RLock()
+ defer m.telemetryMu.RUnlock()
+ m.enableOptionalCapture(m.telemetryCtx, id, info)
+ })
+ }
+ return nil
+}
+
+// Caller holds telemetryMu; sessionsMu protects attachment/removal and the
+// per-session initialization marker while optional setup runs asynchronously.
+func (m *Monitor) enableOptionalCapture(ctx context.Context, sessionID string, info targetInfo) {
+ m.sessionsMu.Lock()
+ _, exists := m.sessions[sessionID]
+ _, initialized := m.optionalSessions[sessionID]
+ if !exists || initialized || !m.telemetryEnabled || !m.captureEnabled() || ctx.Err() != nil {
+ m.sessionsMu.Unlock()
+ return
+ }
+ m.optionalSessions[sessionID] = ""
+ m.sessionsMu.Unlock()
+ m.preparePageCapture(sessionID, info)
+ ctx, cancel := context.WithTimeout(ctx, sendTimeout)
+ defer cancel()
+ m.enableDomains(ctx, sessionID, info.targetType)
+ if isPageLikeTarget(info.targetType) {
+ var rejection *cdpclient.Error
+ if err := m.injectScript(ctx, sessionID); err != nil && !errors.As(err, &rejection) {
+ m.lifeMu.Lock()
+ if m.conn != nil {
+ m.conn.cancel()
+ }
+ m.lifeMu.Unlock()
+ }
+ }
+}
+
+func (m *Monitor) preparePageCapture(sessionID string, info targetInfo) {
+ if !m.telemetryEnabled || !m.captureEnabled() || info.targetType != targetTypePage {
+ return
+ }
+ m.sessionsMu.Lock()
+ if _, exists := m.sessions[sessionID]; !exists || m.computedStates[sessionID] != nil {
+ m.sessionsMu.Unlock()
+ return
+ }
+ ctx := m.telemetryCtx
+ m.computedStates[sessionID] = newComputedState(func(ev events.Event) (events.Envelope, bool) {
+ m.computedPublishMu.Lock()
+ defer m.computedPublishMu.Unlock()
+ if ctx.Err() != nil {
+ return events.Envelope{}, false
+ }
+ return m.publish(ev)
+ })
+ m.sessionsMu.Unlock()
+ data, _ := json.Marshal(oapi.BrowserPageTabOpenedEventData{
+ TargetId: info.targetID, TargetType: oapi.BrowserTargetType(info.targetType),
+ Url: info.url, Title: ptrOf(info.title), OpenerId: ptrOf(info.openerID),
+ })
+ m.publishEvent(EventTabOpened, events.Page, oapi.BrowserEventSource{Kind: oapi.Cdp}, "Target.attachedToTarget", data, sessionID)
+}
diff --git a/server/lib/cdpmonitor/telemetry_cleanup_chrome_e2e_test.go b/server/lib/cdpmonitor/telemetry_cleanup_chrome_e2e_test.go
new file mode 100644
index 000000000..ec1dfea8f
--- /dev/null
+++ b/server/lib/cdpmonitor/telemetry_cleanup_chrome_e2e_test.go
@@ -0,0 +1,212 @@
+package cdpmonitor
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/coder/websocket"
+ "github.com/coder/websocket/wsjson"
+ "github.com/kernel/kernel-images/server/lib/events"
+ "github.com/kernel/kernel-images/server/lib/oapi"
+ "github.com/kernel/kernel-images/server/lib/telemetry"
+ "github.com/stretchr/testify/require"
+)
+
+func TestTelemetryCleanupAfterSocketLossChrome(t *testing.T) {
+ if os.Getenv("KERNEL_CDPMONITOR_CHROME_E2E") == "" {
+ t.Skip("set KERNEL_CDPMONITOR_CHROME_E2E=1")
+ }
+ for _, mode := range []string{"disable_during_disconnect", "disable_after_reconnect", "failed_listener_cleanup", "failed_registration_removal"} {
+ t.Run(mode, func(t *testing.T) { testTelemetryCleanupRecovery(t, mode) })
+ }
+}
+
+func testTelemetryCleanupRecovery(t *testing.T, mode string) {
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer cancel()
+ stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/html")
+ fmt.Fprint(w, ``)
+ }))
+ defer stub.Close()
+ ws := launchChromium(t, ctx, findChromium(t), "--site-per-process")
+ driver := dialCDP(t, ctx, ws)
+ defer driver.close()
+ target := driver.call(t, ctx, "", "Target.createTarget", map[string]any{"url": stub.URL}).targetID(t)
+ session := driver.call(t, ctx, "", "Target.attachToTarget", map[string]any{"targetId": target, "flatten": true}).sessionID(t)
+ stream, err := events.NewEventStream(events.EventStreamConfig{RingCapacity: 128})
+ require.NoError(t, err)
+ ts := telemetry.NewTelemetrySession(stream)
+ monitorURL := ws
+ var failed *atomic.Bool
+ if strings.HasPrefix(mode, "failed_") {
+ monitorURL, failed = failCleanupProxy(t, ctx, ws, mode == "failed_registration_removal")
+ }
+ mon := New(newTestUpstream(monitorURL), ts.Publish, 0, discardLogger, func() bool { return false })
+ require.NoError(t, mon.SetTelemetry(false))
+ require.NoError(t, mon.Start(ctx))
+ defer mon.Stop()
+ ts.Start("fixture", telemetry.TelemetryConfig{Categories: []oapi.TelemetryEventCategory{events.Interaction}})
+ require.NoError(t, mon.SetTelemetry(true))
+ require.Eventually(t, func() bool { return driver.evalBool(ctx, session, `window.__kernelEventInjected === true`) }, 5*time.Second, 10*time.Millisecond)
+ cross := strings.Replace(stub.URL, "127.0.0.1", "localhost", 1) + "/frame"
+ evaluateNetworkScript(t, ctx, driver, session, fmt.Sprintf(`document.body.insertAdjacentHTML('beforeend', ''); true`, cross))
+ require.Eventually(t, func() bool {
+ return driver.evalBool(ctx, session, `document.querySelector('#same').contentWindow.__kernelEventInjected === true`)
+ }, 5*time.Second, 10*time.Millisecond)
+ crossTarget := findNetworkFrameTarget(t, ctx, driver, cross)
+ crossSession := driver.call(t, ctx, "", "Target.attachToTarget", map[string]any{"targetId": crossTarget, "flatten": true}).sessionID(t)
+ require.Eventually(t, func() bool { return driver.evalBool(ctx, crossSession, `window.__kernelEventInjected === true`) }, 5*time.Second, 10*time.Millisecond)
+ time.Sleep(300 * time.Millisecond)
+ before := mon.NetworkSnapshot()
+ mon.lifeMu.Lock()
+ old := mon.conn
+ mon.lifeMu.Unlock()
+ if failed == nil {
+ require.NoError(t, old.protocol.Close())
+ }
+ if mode == "disable_after_reconnect" {
+ require.Eventually(t, func() bool {
+ mon.lifeMu.Lock()
+ fresh := mon.conn != nil && mon.conn != old
+ mon.lifeMu.Unlock()
+ return fresh && mon.NetworkSnapshot().Up && driver.evalBool(ctx, session, `window.__kernelEventInjected === true`)
+ }, 10*time.Second, 10*time.Millisecond)
+ }
+ ts.Stop()
+ require.NoError(t, mon.SetTelemetry(false))
+ waitForTelemetryReconcile(t, mon, false)
+ if failed != nil {
+ require.True(t, failed.Load(), "cleanup failure was not injected")
+ }
+ after := mon.NetworkSnapshot()
+ require.Equal(t, before.Resets, after.Resets)
+ require.Equal(t, before.Completed, after.Completed, "cleanup/reconnect must not reset or increment request counters")
+ for _, frame := range []struct{ session, realm string }{{session, "window"}, {session, `document.querySelector('#same').contentWindow`}, {crossSession, "window"}} {
+ raw := driver.call(t, ctx, frame.session, "Runtime.evaluate", map[string]any{
+ "expression": fmt.Sprintf(`({injected:!!%s.__kernelEventInjected, clicks:(getEventListeners(%s.document).click||[]).length})`, frame.realm, frame.realm),
+ "includeCommandLineAPI": true, "returnByValue": true,
+ })
+ var result struct {
+ Error json.RawMessage `json:"error"`
+ Result struct {
+ ExceptionDetails json.RawMessage `json:"exceptionDetails"`
+ Result struct {
+ Value *struct {
+ Injected bool `json:"injected"`
+ Clicks int `json:"clicks"`
+ } `json:"value"`
+ } `json:"result"`
+ } `json:"result"`
+ }
+ require.NoError(t, json.Unmarshal(raw.raw, &result))
+ require.Empty(t, result.Error)
+ require.Empty(t, result.Result.ExceptionDetails)
+ require.NotNil(t, result.Result.Result.Value)
+ require.False(t, result.Result.Result.Value.Injected, "orphan interaction instrumentation in %s", frame.realm)
+ require.Zero(t, result.Result.Result.Value.Clicks, "orphan click listener in %s", frame.realm)
+ }
+ seq := stream.Seq()
+ evaluateNetworkScript(t, ctx, driver, session, `document.querySelector('#go').click(); true`)
+ require.Equal(t, seq, stream.Seq())
+ // The user's sessions must still navigate, without resurrecting registrations
+ // in either existing child frame or the top-level document.
+ evaluateNetworkScript(t, ctx, driver, session, `document.querySelector('#same').src = '/frame-next'; true`)
+ require.Eventually(t, func() bool {
+ return driver.evalBool(ctx, session, `document.querySelector('#same').contentWindow.location.pathname === '/frame-next' && document.querySelector('#same').contentDocument.readyState === 'complete'`)
+ }, 5*time.Second, 10*time.Millisecond)
+ require.False(t, driver.evalBool(ctx, session, `document.querySelector('#same').contentWindow.__kernelEventInjected === true`))
+ driver.call(t, ctx, crossSession, "Page.navigate", map[string]any{"url": cross + "-next"})
+ require.Eventually(t, func() bool {
+ return driver.evalBool(ctx, crossSession, `location.pathname === '/frame-next' && document.readyState === 'complete'`)
+ }, 5*time.Second, 10*time.Millisecond)
+ require.False(t, driver.evalBool(ctx, crossSession, `window.__kernelEventInjected === true`))
+ driver.call(t, ctx, session, "Page.navigate", map[string]any{"url": stub.URL + "/next"})
+ require.Eventually(t, func() bool {
+ return driver.evalBool(ctx, session, `location.pathname === '/next' && document.readyState === 'complete'`)
+ }, 5*time.Second, 10*time.Millisecond)
+ require.False(t, driver.evalBool(ctx, session, `window.__kernelEventInjected === true`))
+}
+
+// Fail registration removal or listener cleanup once, leaving the user CDP client
+// untouched. The replacement monitor socket must retry the retained obligation.
+func failCleanupProxy(t *testing.T, parent context.Context, upstreamURL string, failRemoval bool) (string, *atomic.Bool) {
+ t.Helper()
+ failed := &atomic.Bool{}
+ url := rejectPageCommandProxy(t, parent, upstreamURL, func(method, source string) bool {
+ failCommand := method == "Page.addScriptToEvaluateOnNewDocument" && source == cleanupInteractionJS
+ if failRemoval {
+ failCommand = method == "Page.removeScriptToEvaluateOnNewDocument"
+ }
+ return failCommand && failed.CompareAndSwap(false, true)
+ })
+ return url, failed
+}
+
+func rejectPageCommandProxy(t *testing.T, parent context.Context, upstreamURL string, reject func(method, source string) bool) string {
+ t.Helper()
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ client, err := websocket.Accept(w, r, nil)
+ if err != nil {
+ return
+ }
+ defer client.CloseNow()
+ ctx, cancel := context.WithCancel(parent)
+ defer cancel()
+ upstream, _, err := websocket.Dial(ctx, upstreamURL, nil)
+ if err != nil {
+ return
+ }
+ defer upstream.CloseNow()
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ defer cancel()
+ for {
+ kind, data, err := upstream.Read(ctx)
+ if err != nil {
+ return
+ }
+ if client.Write(ctx, kind, data) != nil {
+ return
+ }
+ }
+ }()
+ defer func() { cancel(); <-done }()
+ for {
+ kind, data, err := client.Read(ctx)
+ if err != nil {
+ return
+ }
+ var command struct {
+ ID int `json:"id"`
+ Method string `json:"method"`
+ Params struct {
+ Source string `json:"source"`
+ } `json:"params"`
+ }
+ if json.Unmarshal(data, &command) != nil {
+ return
+ }
+ if reject(command.Method, command.Params.Source) {
+ if wsjson.Write(ctx, client, map[string]any{"id": command.ID, "error": map[string]any{"code": -32000, "message": "fixture cleanup failure"}}) != nil {
+ return
+ }
+ continue
+ }
+ if upstream.Write(ctx, kind, data) != nil {
+ return
+ }
+ }
+ }))
+ t.Cleanup(server.Close)
+ return "ws" + strings.TrimPrefix(server.URL, "http")
+}
diff --git a/server/lib/cdpmonitor/telemetry_reconcile_test.go b/server/lib/cdpmonitor/telemetry_reconcile_test.go
new file mode 100644
index 000000000..67a7c38ba
--- /dev/null
+++ b/server/lib/cdpmonitor/telemetry_reconcile_test.go
@@ -0,0 +1,124 @@
+package cdpmonitor
+
+import (
+ "context"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/kernel/kernel-images/server/lib/browsersurface"
+ "github.com/kernel/kernel-images/server/lib/cdpclient"
+ "github.com/stretchr/testify/require"
+)
+
+func TestInteractionObligationsUseTargetIdentity(t *testing.T) {
+ m := New(newTestUpstream(""), newEventCollector().publishFn(), 0, discardLogger, nil)
+ m.sessions["old-session"] = targetInfo{targetID: "live-target", targetType: "page"}
+ m.optionalSessions["old-session"] = "script"
+ m.interactionTargets["live-target"] = struct{}{}
+ m.handleDetachedFromTarget(cdpTargetDetachedFromTargetParams{SessionID: "old-session"})
+ m.clearState()
+ require.Contains(t, m.interactionTargets, "live-target")
+ require.NoError(t, m.cleanupAttachedTarget(context.Background(), "old-session", targetInfo{targetID: "different-target", targetType: "page"}))
+ require.Contains(t, m.interactionTargets, "live-target", "a reused session ID cannot discharge another target's obligation")
+ m.handleSurfaceEvent(nil, browsersurface.Event{Kind: browsersurface.EventProtocol, Message: cdpclient.Message{Method: "Target.targetDestroyed", Params: []byte(`{"targetId":"live-target"}`)}})
+ require.Empty(t, m.interactionTargets)
+}
+
+func TestTelemetryRevisionFencesPendingBody(t *testing.T) {
+ srv := newTestServer(t)
+ defer srv.close()
+ blocked := make(chan cdpMessage, 1)
+ var first atomic.Bool
+ m, ec, cleanup := startMonitor(t, srv, func(msg cdpMessage) any {
+ if msg.Method == "Network.getResponseBody" {
+ if first.CompareAndSwap(false, true) {
+ blocked <- msg
+ return map[string]any{"method": "Test.unanswered"}
+ }
+ return map[string]any{"id": msg.ID, "result": map[string]any{"body": "new-body"}}
+ }
+ return nil
+ })
+ defer cleanup()
+ srv.sendToMonitor(t, map[string]any{"method": "Target.attachedToTarget", "params": map[string]any{"sessionId": "s", "targetInfo": map[string]any{"targetId": "t", "type": "page"}}})
+ ec.waitFor(t, EventTabOpened, time.Second)
+ request := func(id string) {
+ for _, event := range []map[string]any{
+ {"method": "Network.requestWillBeSent", "params": map[string]any{"requestId": id, "type": "Fetch", "request": map[string]any{"method": "GET", "url": "http://fixture/" + id}}},
+ {"method": "Network.responseReceived", "params": map[string]any{"requestId": id, "response": map[string]any{"status": 200, "mimeType": "text/plain"}}},
+ {"method": "Network.loadingFinished", "params": map[string]any{"requestId": id}},
+ } {
+ event["sessionId"] = "s"
+ srv.sendToMonitor(t, event)
+ }
+ }
+ request("old")
+ var command cdpMessage
+ select {
+ case command = <-blocked:
+ case <-time.After(time.Second):
+ t.Fatal("body capture did not start")
+ }
+ require.NoError(t, m.SetTelemetry(false))
+ require.NoError(t, m.SetTelemetry(true))
+ srv.sendToMonitor(t, map[string]any{"id": command.ID, "result": map[string]any{"body": "old-body"}})
+ waitForTelemetryReconcile(t, m, true)
+ ec.mu.Lock()
+ responses := 0
+ for _, event := range ec.events {
+ if event.Type == EventNetworkResponse {
+ responses++
+ }
+ }
+ ec.mu.Unlock()
+ require.Zero(t, responses, "old body escaped into the new capture revision")
+ request("new")
+ ec.waitFor(t, EventNetworkResponse, time.Second)
+ require.Equal(t, uint64(2), m.NetworkSnapshot().Completed)
+}
+
+func TestTelemetrySchedulesOnlyReadyAttachments(t *testing.T) {
+ srv := newTestServer(t)
+ defer srv.close()
+ m, _, cleanup := startMonitor(t, srv, nil)
+ defer cleanup()
+ require.NoError(t, m.SetTelemetry(false))
+ waitForTelemetryReconcile(t, m, false)
+ m.sessionsMu.Lock()
+ m.sessions["pending"] = targetInfo{targetID: "pending-target", targetType: "page"}
+ m.sessions["ready"] = targetInfo{targetID: "ready-target", targetType: "page"}
+ m.networkReady["ready"] = true
+ m.sessionsMu.Unlock()
+
+ // Run reconciliation synchronously so all optional tasks have been scheduled
+ // before joining them. The pending attachment is deliberately not advanced.
+ m.restartMu.Lock()
+ require.NoError(t, m.SetTelemetry(true))
+ err := m.applyTelemetry(m.desiredTelemetry.Load())
+ m.captureWg.Wait()
+ m.restartMu.Unlock()
+ require.NoError(t, err)
+ m.sessionsMu.RLock()
+ _, pending := m.optionalSessions["pending"]
+ _, ready := m.optionalSessions["ready"]
+ _, dirtyPending := m.interactionTargets["pending-target"]
+ m.sessionsMu.RUnlock()
+ require.False(t, pending, "optional setup ran before attachment recovery")
+ require.False(t, dirtyPending)
+ require.True(t, ready, "ready attachments must still enable capture")
+}
+
+func waitForTelemetryReconcile(t *testing.T, m *Monitor, enabled bool) {
+ t.Helper()
+ require.Eventually(t, func() bool {
+ desired := m.desiredTelemetry.Load()
+ if (desired&1 != 0) != enabled || m.appliedTelemetry.Load() != desired || m.telemetryChanging.Load() {
+ return false
+ }
+ m.sessionsMu.RLock()
+ cleaned := len(m.optionalSessions) == 0 && len(m.interactionTargets) == 0
+ m.sessionsMu.RUnlock()
+ return (enabled || cleaned) && m.NetworkSnapshot().Up
+ }, 10*time.Second, 10*time.Millisecond, "telemetry reconciliation did not settle")
+}
diff --git a/server/lib/cdpmonitor/types.go b/server/lib/cdpmonitor/types.go
index 20147f531..98826012c 100644
--- a/server/lib/cdpmonitor/types.go
+++ b/server/lib/cdpmonitor/types.go
@@ -75,6 +75,8 @@ const targetTypePage = "page"
// targetInfo holds metadata about an attached CDP target/session.
type targetInfo struct {
+ title string
+ openerID string
targetID string
url string
targetType string
diff --git a/server/lib/metrics/network.go b/server/lib/metrics/network.go
new file mode 100644
index 000000000..776aa901d
--- /dev/null
+++ b/server/lib/metrics/network.go
@@ -0,0 +1,33 @@
+package metrics
+
+import "context"
+
+// NetworkCollector exposes in-memory monitor counters without dialing Chrome.
+// It remains available when the independent Chrome/UMA collector fails.
+type NetworkCollector struct {
+ snapshot func() (resets, completed uint64, up bool)
+}
+
+func NewNetworkCollector(snapshot func() (resets, completed uint64, up bool)) *NetworkCollector {
+ return &NetworkCollector{snapshot: snapshot}
+}
+
+func (*NetworkCollector) Name() string { return "chromium_network" }
+
+func (c *NetworkCollector) Collect(_ context.Context, w *Writer) error {
+ resetCount, completedCount, healthy := c.snapshot()
+ const resets = "kernel_chromium_connection_resets_total"
+ const completed = "kernel_chromium_network_requests_completed_total"
+ const up = "kernel_chromium_network_monitor_up"
+ w.Metric(resets, "Observed request terminal outcomes with exact net::ERR_CONNECTION_RESET.", "counter")
+ w.Sample(resets, nil, float64(resetCount))
+ w.Metric(completed, "Observed Network.loadingFinished or Network.loadingFailed outcomes.", "counter")
+ w.Sample(completed, nil, float64(completedCount))
+ w.Metric(up, "Whether CDP network capture is initialized for known targets.", "gauge")
+ value := float64(0)
+ if healthy {
+ value = 1
+ }
+ w.Sample(up, nil, value)
+ return nil
+}
diff --git a/server/lib/metrics/network_test.go b/server/lib/metrics/network_test.go
new file mode 100644
index 000000000..c2427ffd8
--- /dev/null
+++ b/server/lib/metrics/network_test.go
@@ -0,0 +1,33 @@
+package metrics
+
+import (
+ "context"
+ "log/slog"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestNetworkCollectorZerosAndIndependentChromeFailure(t *testing.T) {
+ c := NewNetworkCollector(func() (uint64, uint64, bool) { return 0, 0, false })
+ w := &Writer{}
+ require.NoError(t, c.Collect(context.Background(), w))
+ for _, name := range []string{"kernel_chromium_connection_resets_total", "kernel_chromium_network_requests_completed_total", "kernel_chromium_network_monitor_up"} {
+ require.Contains(t, string(w.Bytes()), name+" 0\n")
+ }
+ require.NotContains(t, string(w.Bytes()), "{")
+ srv := fakeCDP(t)
+ defer srv.Close()
+ chrome := NewChromeCollector(staticUpstream("ws" + strings.TrimPrefix(srv.URL, "http")))
+ chrome.histograms = []UMAHistogram{{Name: "Fail.Me"}}
+ require.Error(t, chrome.Collect(context.Background(), &Writer{}))
+ h := Handler(slog.Default(), chrome, c)
+ for range 2 {
+ rr := httptest.NewRecorder()
+ h.ServeHTTP(rr, httptest.NewRequest("GET", "/metrics", nil))
+ require.Contains(t, rr.Body.String(), string(w.Bytes()))
+ require.NotContains(t, rr.Body.String(), "kernel_chromium_up")
+ }
+}