From 3184e67d55f563cfed42d539e76dc44fb0c9755a Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Fri, 10 Jul 2026 12:39:51 -0400 Subject: [PATCH 1/9] test: add kill-operator-pod and kill-primary-pod chaos operations Add the first two chaos fault-injection operations to the long-haul operation scheduler: - kill-operator-pod: deletes the operator pod via its Deployment selector and waits for the Deployment to become Available again. Asserts the CNPG data plane is unaffected by an operator restart (small write-failure budget). - kill-primary-pod: deletes the CNPG primary pod to exercise the automatic failover path, guarded by an HA precondition (instancesPerNode>=2). Both plug into the existing Operation interface and are selected by the weighted scheduler (one disruptive op at a time, cooldown + steady-state gate). Recovery is judged by the health monitor and workload verifier. Adds ClusterClient.GetPrimaryInstance (reads CNPG Cluster status.currentPrimary) and DeletePod, a LONGHAUL_OPERATOR_NAMESPACE config knob (default documentdb-operator), unit tests, and README operations + RBAC notes. Controlled failover was intentionally excluded: DocumentDB exposes no single-cluster manual switchover, so it would only test upstream CNPG. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- test/longhaul/README.md | 23 +++ test/longhaul/cmd/longhaul/main.go | 2 + test/longhaul/config/config.go | 38 +++-- test/longhaul/config/config_test.go | 16 ++ test/longhaul/go.mod | 2 +- test/longhaul/monitor/health.go | 9 + test/longhaul/monitor/health_test.go | 8 +- test/longhaul/monitor/k8sclient.go | 27 ++- test/longhaul/operations/kill_operator.go | 156 ++++++++++++++++++ .../longhaul/operations/kill_operator_test.go | 118 +++++++++++++ test/longhaul/operations/kill_primary.go | 76 +++++++++ test/longhaul/operations/kill_primary_test.go | 75 +++++++++ test/longhaul/operations/scale_test.go | 18 ++ 13 files changed, 552 insertions(+), 16 deletions(-) create mode 100644 test/longhaul/operations/kill_operator.go create mode 100644 test/longhaul/operations/kill_operator_test.go create mode 100644 test/longhaul/operations/kill_primary.go create mode 100644 test/longhaul/operations/kill_primary_test.go diff --git a/test/longhaul/README.md b/test/longhaul/README.md index d87f153f1..0985bc0d1 100644 --- a/test/longhaul/README.md +++ b/test/longhaul/README.md @@ -125,6 +125,7 @@ All configuration is via environment variables. | `LONGHAUL_DOCUMENTDB_URI` | Yes | — | Connection string to the DocumentDB gateway. | | `LONGHAUL_CLUSTER_NAME` | Yes | — | Name of the target DocumentDB cluster CR. | | `LONGHAUL_NAMESPACE` | No | `default` | Kubernetes namespace of the target cluster. | +| `LONGHAUL_OPERATOR_NAMESPACE` | No | `documentdb-operator` | Namespace of the DocumentDB operator Deployment (target of the `kill-operator-pod` chaos op). | | `LONGHAUL_MAX_DURATION` | No | `30m` | Max test duration. Use `0s` for run-until-failure. | | `LONGHAUL_NUM_WRITERS` | No | `5` | Number of concurrent writers. | | `LONGHAUL_OP_COOLDOWN` | No | `5m` | Cooldown between management operations. | @@ -134,6 +135,28 @@ All configuration is via environment variables. | `LONGHAUL_REPORT_INTERVAL` | No | `1h` | How often to write checkpoint reports to ConfigMap. | | `LONGHAUL_RESET_DATA` | No | `false` | If `true`, drop the workload collection on startup. Off by default so a Deployment pod restart preserves durability history. | +## Operations + +The scheduler runs one disruptive operation at a time, gated by steady state and +a global cooldown. Current operations: + +| Operation | Kind | Notes | +|-----------|------|-------| +| `scale-up` / `scale-down` | Topology | Adjusts `spec.instancesPerNode` within `[MIN, MAX]`. | +| `upgrade-documentdb` | Topology | In-place version upgrade; requires HA (`instancesPerNode>=2`). | +| `kill-operator-pod` | Chaos | Deletes the operator pod; asserts the data plane keeps serving (small write-failure budget). | +| `kill-primary-pod` | Chaos | Deletes the CNPG primary pod to exercise automatic failover; requires HA (`instancesPerNode>=2`). | + +### RBAC for chaos operations + +Beyond the base RBAC the driver already needs, the chaos operations require the +driver ServiceAccount to be granted (in `deploy/rbac.yaml`): + +- **`kill-primary-pod`** — `get`/`list` on `clusters.postgresql.cnpg.io` (to read + `status.currentPrimary`) and `delete` on `pods` in the cluster namespace. +- **`kill-operator-pod`** — `get` on `deployments` and `get`/`list`/`delete` on + `pods` in the operator namespace (`LONGHAUL_OPERATOR_NAMESPACE`). + ## CI Safety The long haul test binary is deployed as a Kubernetes Deployment on a dedicated AKS diff --git a/test/longhaul/cmd/longhaul/main.go b/test/longhaul/cmd/longhaul/main.go index c7f375bd8..5195d49d7 100644 --- a/test/longhaul/cmd/longhaul/main.go +++ b/test/longhaul/cmd/longhaul/main.go @@ -140,6 +140,8 @@ func run(cfg config.Config) int { operations.NewScaleUp(clusterClient, healthMon, cfg.MaxInstances, cfg.RecoveryTimeout), operations.NewScaleDown(clusterClient, healthMon, cfg.MinInstances, cfg.RecoveryTimeout), operations.NewUpgradeDocumentDB(clusterClient, k8sClientset, healthMon, j, cfg.Namespace, cfg.RecoveryTimeout), + operations.NewKillOperatorPod(k8sClientset, cfg.OperatorNamespace, cfg.RecoveryTimeout), + operations.NewKillPrimaryPod(clusterClient, healthMon, cfg.RecoveryTimeout), } // Start operation scheduler. diff --git a/test/longhaul/config/config.go b/test/longhaul/config/config.go index 5665bcd77..918431c35 100644 --- a/test/longhaul/config/config.go +++ b/test/longhaul/config/config.go @@ -18,6 +18,10 @@ const ( EnvNamespace = "LONGHAUL_NAMESPACE" EnvClusterName = "LONGHAUL_CLUSTER_NAME" + // EnvOperatorNamespace is the namespace where the DocumentDB operator + // Deployment runs (target of the kill-operator-pod chaos op). + EnvOperatorNamespace = "LONGHAUL_OPERATOR_NAMESPACE" + // Workload and operation tuning. EnvDocumentDBURI = "LONGHAUL_DOCUMENTDB_URI" EnvNumWriters = "LONGHAUL_NUM_WRITERS" @@ -47,6 +51,10 @@ type Config struct { // ClusterName is the name of the target DocumentDB cluster CR. ClusterName string + // OperatorNamespace is the namespace of the DocumentDB operator Deployment, + // targeted by the kill-operator-pod chaos operation. + OperatorNamespace string + // DocumentDBURI is the DocumentDB connection string for data-plane workload. DocumentDBURI string @@ -82,17 +90,18 @@ type Config struct { // DefaultConfig returns a Config with safe defaults for local development. func DefaultConfig() Config { return Config{ - MaxDuration: 30 * time.Minute, - Namespace: "default", - ClusterName: "", - DocumentDBURI: "", - NumWriters: 5, - OpCooldown: 5 * time.Minute, - RecoveryTimeout: 5 * time.Minute, - SteadyStateWait: 60 * time.Second, - MinInstances: 1, - MaxInstances: 3, - ReportInterval: 1 * time.Hour, + MaxDuration: 30 * time.Minute, + Namespace: "default", + ClusterName: "", + OperatorNamespace: "documentdb-operator", + DocumentDBURI: "", + NumWriters: 5, + OpCooldown: 5 * time.Minute, + RecoveryTimeout: 5 * time.Minute, + SteadyStateWait: 60 * time.Second, + MinInstances: 1, + MaxInstances: 3, + ReportInterval: 1 * time.Hour, } } @@ -117,6 +126,10 @@ func LoadFromEnv() (Config, error) { cfg.ClusterName = v } + if v := os.Getenv(EnvOperatorNamespace); v != "" { + cfg.OperatorNamespace = v + } + if v := os.Getenv(EnvDocumentDBURI); v != "" { cfg.DocumentDBURI = v } @@ -195,6 +208,9 @@ func (c *Config) Validate() error { if c.ClusterName == "" { return fmt.Errorf("cluster name must not be empty") } + if c.OperatorNamespace == "" { + return fmt.Errorf("operator namespace must not be empty") + } if c.NumWriters < 1 { return fmt.Errorf("num writers must be at least 1, got %d", c.NumWriters) } diff --git a/test/longhaul/config/config_test.go b/test/longhaul/config/config_test.go index 506803f52..c02196b87 100644 --- a/test/longhaul/config/config_test.go +++ b/test/longhaul/config/config_test.go @@ -17,6 +17,7 @@ var _ = Describe("Config", func() { Expect(cfg.MaxDuration).To(Equal(30 * time.Minute)) Expect(cfg.Namespace).To(Equal("default")) Expect(cfg.ClusterName).To(BeEmpty()) + Expect(cfg.OperatorNamespace).To(Equal("documentdb-operator")) Expect(cfg.NumWriters).To(Equal(5)) Expect(cfg.OpCooldown).To(Equal(5 * time.Minute)) Expect(cfg.RecoveryTimeout).To(Equal(5 * time.Minute)) @@ -32,6 +33,7 @@ var _ = Describe("Config", func() { BeforeEach(func() { for _, k := range []string{ EnvEnabled, EnvMaxDuration, EnvNamespace, EnvClusterName, + EnvOperatorNamespace, EnvDocumentDBURI, EnvNumWriters, EnvOpCooldown, EnvRecoveryTimeout, EnvSteadyStateWait, EnvMinInstances, EnvMaxInstances, EnvReportInterval, @@ -69,6 +71,13 @@ var _ = Describe("Config", func() { Expect(cfg.ClusterName).To(Equal("my-cluster")) }) + It("parses OperatorNamespace from env", func() { + GinkgoT().Setenv(EnvOperatorNamespace, "custom-operator-ns") + cfg, err := LoadFromEnv() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.OperatorNamespace).To(Equal("custom-operator-ns")) + }) + It("returns error for invalid MaxDuration", func() { GinkgoT().Setenv(EnvMaxDuration, "not-a-duration") _, err := LoadFromEnv() @@ -124,6 +133,13 @@ var _ = Describe("Config", func() { Expect(cfg.Validate()).To(MatchError(ContainSubstring("cluster name"))) }) + It("fails when OperatorNamespace is empty", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperatorNamespace = "" + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operator namespace"))) + }) + It("fails when MaxDuration is negative", func() { cfg := DefaultConfig() cfg.ClusterName = "test" diff --git a/test/longhaul/go.mod b/test/longhaul/go.mod index 04a245693..bb803e443 100644 --- a/test/longhaul/go.mod +++ b/test/longhaul/go.mod @@ -3,6 +3,7 @@ module github.com/documentdb/documentdb-operator/test/longhaul go 1.26.5 require ( + github.com/cloudnative-pg/cloudnative-pg v1.29.2 github.com/documentdb/documentdb-operator v0.0.0-00010101000000-000000000000 github.com/documentdb/documentdb-operator/test/shared v0.0.0-00010101000000-000000000000 github.com/onsi/ginkgo/v2 v2.32.0 @@ -25,7 +26,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudnative-pg/barman-cloud v0.5.1 // indirect - github.com/cloudnative-pg/cloudnative-pg v1.29.2 // indirect github.com/cloudnative-pg/cnpg-i v0.5.0 // indirect github.com/cloudnative-pg/machinery v0.5.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/test/longhaul/monitor/health.go b/test/longhaul/monitor/health.go index 0c5e95163..939f48056 100644 --- a/test/longhaul/monitor/health.go +++ b/test/longhaul/monitor/health.go @@ -49,6 +49,15 @@ type ClusterClient interface { // UpgradeDocumentDB patches spec.documentDBVersion and spec.schemaVersion="auto". UpgradeDocumentDB(ctx context.Context, version string) error + + // GetPrimaryInstance returns the name of the pod currently serving as the + // CNPG primary (from Cluster.status.currentPrimary). The pod name equals + // the CNPG instance name. Returns an error if no primary is known yet. + GetPrimaryInstance(ctx context.Context) (string, error) + + // DeletePod deletes the named pod in the cluster namespace. Used by chaos + // operations to inject pod-loss faults. + DeletePod(ctx context.Context, name string) error } // HealthMonitor continuously monitors cluster health and tracks steady-state. diff --git a/test/longhaul/monitor/health_test.go b/test/longhaul/monitor/health_test.go index 1ad903fd9..4d5385e1e 100644 --- a/test/longhaul/monitor/health_test.go +++ b/test/longhaul/monitor/health_test.go @@ -41,9 +41,11 @@ func (f *fakeClusterClient) GetClusterHealth(_ context.Context) (ClusterHealth, func (f *fakeClusterClient) GetCurrentDocumentDBImageTag(_ context.Context) (string, error) { return "", nil } -func (f *fakeClusterClient) GetInstancesPerNode(_ context.Context) (int, error) { return 1, nil } -func (f *fakeClusterClient) ScaleCluster(_ context.Context, _ int) error { return nil } -func (f *fakeClusterClient) UpgradeDocumentDB(_ context.Context, _ string) error { return nil } +func (f *fakeClusterClient) GetInstancesPerNode(_ context.Context) (int, error) { return 1, nil } +func (f *fakeClusterClient) ScaleCluster(_ context.Context, _ int) error { return nil } +func (f *fakeClusterClient) UpgradeDocumentDB(_ context.Context, _ string) error { return nil } +func (f *fakeClusterClient) GetPrimaryInstance(_ context.Context) (string, error) { return "", nil } +func (f *fakeClusterClient) DeletePod(_ context.Context, _ string) error { return nil } var _ = Describe("HealthMonitor", func() { Describe("IsSteadyState", func() { diff --git a/test/longhaul/monitor/k8sclient.go b/test/longhaul/monitor/k8sclient.go index b51ffdd82..f5b9807df 100644 --- a/test/longhaul/monitor/k8sclient.go +++ b/test/longhaul/monitor/k8sclient.go @@ -19,6 +19,8 @@ import ( metricsv "k8s.io/metrics/pkg/client/clientset/versioned" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + previewv1 "github.com/documentdb/documentdb-operator/api/preview" shareddb "github.com/documentdb/documentdb-operator/test/shared/documentdb" sharedk8s "github.com/documentdb/documentdb-operator/test/shared/k8s" @@ -61,7 +63,7 @@ func NewK8sClusterClient(cfg K8sClientConfig) (*K8sClusterClient, error) { return nil, fmt.Errorf("failed to create clientset: %w", err) } - scheme, err := shareddb.NewScheme() + scheme, err := shareddb.NewScheme(cnpgv1.AddToScheme) if err != nil { return nil, fmt.Errorf("failed to build scheme: %w", err) } @@ -227,6 +229,29 @@ func (k *K8sClusterClient) UpgradeDocumentDB(ctx context.Context, version string return nil } +// GetPrimaryInstance reads status.currentPrimary from the CNPG Cluster that +// backs this DocumentDB. The CNPG Cluster name equals the DocumentDB CR name, +// and the returned instance name equals the primary pod name. +func (k *K8sClusterClient) GetPrimaryInstance(ctx context.Context) (string, error) { + var cluster cnpgv1.Cluster + key := types.NamespacedName{Namespace: k.namespace, Name: k.clusterName} + if err := k.crClient.Get(ctx, key, &cluster); err != nil { + return "", fmt.Errorf("failed to get CNPG Cluster: %w", err) + } + if cluster.Status.CurrentPrimary == "" { + return "", fmt.Errorf("CNPG Cluster %s has no current primary yet", k.clusterName) + } + return cluster.Status.CurrentPrimary, nil +} + +// DeletePod deletes the named pod in the cluster namespace. +func (k *K8sClusterClient) DeletePod(ctx context.Context, name string) error { + if err := k.clientset.CoreV1().Pods(k.namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("failed to delete pod %s/%s: %w", k.namespace, name, err) + } + return nil +} + // GetPodMetrics queries metrics-server for pod resource usage. // Returns nil, nil if metrics-server is not available. func (k *K8sClusterClient) GetPodMetrics(ctx context.Context) ([]PodMetrics, error) { diff --git a/test/longhaul/operations/kill_operator.go b/test/longhaul/operations/kill_operator.go new file mode 100644 index 000000000..306503f63 --- /dev/null +++ b/test/longhaul/operations/kill_operator.go @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "fmt" + "time" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" +) + +// OperatorDeploymentName is the fixed name of the operator Deployment. The +// operator is a cluster singleton, so this name is stable across installs. +const OperatorDeploymentName = "documentdb-operator" + +// KillOperatorPod deletes the running operator pod to verify that an operator +// restart does not disrupt the data plane. The CNPG-managed database keeps +// serving reads and writes while the Deployment reschedules the control plane, +// so the workload verifier should observe (near) zero write failures. Recovery +// is asserted by the Deployment returning to Available. +type KillOperatorPod struct { + clientset kubernetes.Interface + namespace string + deployment string + recovery time.Duration +} + +// NewKillOperatorPod creates a KillOperatorPod operation targeting the operator +// Deployment in the given namespace. +func NewKillOperatorPod(clientset kubernetes.Interface, namespace string, recovery time.Duration) *KillOperatorPod { + return &KillOperatorPod{ + clientset: clientset, + namespace: namespace, + deployment: OperatorDeploymentName, + recovery: recovery, + } +} + +func (k *KillOperatorPod) Name() string { return "kill-operator-pod" } + +func (k *KillOperatorPod) Weight() int { return 2 } + +// Precondition requires the operator Deployment to exist and currently be +// Available, so the fault isn't stacked on an already-restarting operator. +func (k *KillOperatorPod) Precondition(ctx context.Context) (bool, string) { + dep, err := k.getDeployment(ctx) + if err != nil { + return false, fmt.Sprintf("cannot get operator deployment: %v", err) + } + if !isDeploymentAvailable(dep) { + return false, "operator deployment not currently available" + } + return true, "" +} + +func (k *KillOperatorPod) Execute(ctx context.Context) error { + dep, err := k.getDeployment(ctx) + if err != nil { + return fmt.Errorf("get operator deployment: %w", err) + } + + // Resolve the pod set from the Deployment's own selector so we don't + // depend on the release-name-derived "app" label value. + selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels).String() + pods, err := k.clientset.CoreV1().Pods(k.namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return fmt.Errorf("list operator pods: %w", err) + } + + target := oldestRunningPod(pods.Items) + if target == "" { + return fmt.Errorf("no running operator pod found for selector %q", selector) + } + if err := k.clientset.CoreV1().Pods(k.namespace).Delete(ctx, target, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("delete operator pod %s: %w", target, err) + } + + // Wait for the Deployment to reschedule and become Available again. + recoveryCtx, cancel := context.WithTimeout(ctx, k.recovery) + defer cancel() + return k.waitForDeploymentAvailable(recoveryCtx) +} + +// OutagePolicy tolerates only a small number of write failures: an operator +// restart must not take down the data plane. A non-zero budget absorbs +// coincidental blips (e.g. a scrape or client reconnect) without flagging a +// false policy violation. +func (k *KillOperatorPod) OutagePolicy() journal.OutagePolicy { + return journal.OutagePolicy{ + AllowedWriteFailures: 5, + MustRecoverWithin: k.recovery, + } +} + +func (k *KillOperatorPod) getDeployment(ctx context.Context) (*appsv1.Deployment, error) { + return k.clientset.AppsV1().Deployments(k.namespace).Get(ctx, k.deployment, metav1.GetOptions{}) +} + +func (k *KillOperatorPod) waitForDeploymentAvailable(ctx context.Context) error { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + if dep, err := k.getDeployment(ctx); err == nil && isDeploymentAvailable(dep) { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("timed out waiting for operator deployment to become available: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +// isDeploymentAvailable reports whether the Deployment has its full desired +// replica count ready with none unavailable and the observed generation caught +// up to the latest spec. +func isDeploymentAvailable(dep *appsv1.Deployment) bool { + if dep == nil { + return false + } + desired := int32(1) + if dep.Spec.Replicas != nil { + desired = *dep.Spec.Replicas + } + if dep.Status.ObservedGeneration < dep.Generation { + return false + } + return dep.Status.ReadyReplicas >= desired && dep.Status.UnavailableReplicas == 0 +} + +// oldestRunningPod returns the name of the oldest pod in the Running phase, or +// "" if none are running. Targeting the oldest makes the choice deterministic. +func oldestRunningPod(pods []corev1.Pod) string { + name := "" + var oldest time.Time + for i := range pods { + p := &pods[i] + if p.Status.Phase != corev1.PodRunning || p.DeletionTimestamp != nil { + continue + } + ts := p.CreationTimestamp.Time + if name == "" || ts.Before(oldest) { + name = p.Name + oldest = ts + } + } + return name +} diff --git a/test/longhaul/operations/kill_operator_test.go b/test/longhaul/operations/kill_operator_test.go new file mode 100644 index 000000000..f05b04b9b --- /dev/null +++ b/test/longhaul/operations/kill_operator_test.go @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +const opNS = "documentdb-operator" + +func operatorDeployment(desired, ready, unavailable int32, gen, observed int64) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: OperatorDeploymentName, + Namespace: opNS, + Generation: gen, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &desired, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "documentdb-operator"}}, + }, + Status: appsv1.DeploymentStatus{ + ReadyReplicas: ready, + UnavailableReplicas: unavailable, + ObservedGeneration: observed, + }, + } +} + +func operatorPod(name string, phase corev1.PodPhase, ageSeconds int) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: opNS, + Labels: map[string]string{"app": "documentdb-operator"}, + CreationTimestamp: metav1.NewTime(time.Now().Add(-time.Duration(ageSeconds) * time.Second)), + }, + Status: corev1.PodStatus{Phase: phase}, + } +} + +var _ = Describe("KillOperatorPod", func() { + It("Name is kill-operator-pod and Weight is 2", func() { + k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, time.Minute) + Expect(k.Name()).To(Equal("kill-operator-pod")) + Expect(k.Weight()).To(Equal(2)) + }) + + It("OutagePolicy tolerates only a small write-failure budget", func() { + k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, 2*time.Minute) + p := k.OutagePolicy() + Expect(p.AllowedWriteFailures).To(Equal(int64(5))) + Expect(p.MustRecoverWithin).To(Equal(2 * time.Minute)) + }) + + Describe("Precondition", func() { + It("skips when the deployment is missing", func() { + k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, time.Minute) + ok, reason := k.Precondition(context.Background()) + Expect(ok).To(BeFalse()) + Expect(reason).To(ContainSubstring("cannot get operator deployment")) + }) + + It("skips when the deployment is not available", func() { + dep := operatorDeployment(1, 0, 1, 1, 1) + k := NewKillOperatorPod(fake.NewSimpleClientset(dep), opNS, time.Minute) + ok, reason := k.Precondition(context.Background()) + Expect(ok).To(BeFalse()) + Expect(reason).To(ContainSubstring("not currently available")) + }) + + It("is eligible when the deployment is available", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + k := NewKillOperatorPod(fake.NewSimpleClientset(dep), opNS, time.Minute) + ok, _ := k.Precondition(context.Background()) + Expect(ok).To(BeTrue()) + }) + }) + + Describe("Execute", func() { + It("deletes the oldest running operator pod and returns once available", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + newer := operatorPod("op-new", corev1.PodRunning, 10) + older := operatorPod("op-old", corev1.PodRunning, 100) + cs := fake.NewSimpleClientset(dep, newer, older) + k := NewKillOperatorPod(cs, opNS, time.Minute) + + err := k.Execute(context.Background()) + Expect(err).NotTo(HaveOccurred()) + + _, getErr := cs.CoreV1().Pods(opNS).Get(context.Background(), "op-old", metav1.GetOptions{}) + Expect(getErr).To(HaveOccurred(), "oldest pod should have been deleted") + _, getErr = cs.CoreV1().Pods(opNS).Get(context.Background(), "op-new", metav1.GetOptions{}) + Expect(getErr).NotTo(HaveOccurred(), "newer pod should be untouched") + }) + + It("fails when no running pod matches the selector", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + pending := operatorPod("op-pending", corev1.PodPending, 10) + cs := fake.NewSimpleClientset(dep, pending) + k := NewKillOperatorPod(cs, opNS, time.Minute) + + err := k.Execute(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no running operator pod")) + }) + }) +}) diff --git a/test/longhaul/operations/kill_primary.go b/test/longhaul/operations/kill_primary.go new file mode 100644 index 000000000..aa2199db8 --- /dev/null +++ b/test/longhaul/operations/kill_primary.go @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "fmt" + "time" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + "github.com/documentdb/documentdb-operator/test/longhaul/monitor" +) + +// KillPrimaryPod deletes the CNPG primary pod to exercise the automatic +// failover path: CNPG must promote a standby, and the cluster must return to +// steady state within the recovery budget. The continuous workload verifier +// independently catches any data loss caused by the failover. +type KillPrimaryPod struct { + client monitor.ClusterClient + healthMon *monitor.HealthMonitor + recovery time.Duration +} + +// NewKillPrimaryPod creates a KillPrimaryPod operation. +func NewKillPrimaryPod(client monitor.ClusterClient, health *monitor.HealthMonitor, recovery time.Duration) *KillPrimaryPod { + return &KillPrimaryPod{ + client: client, + healthMon: health, + recovery: recovery, + } +} + +func (k *KillPrimaryPod) Name() string { return "kill-primary-pod" } + +func (k *KillPrimaryPod) Weight() int { return 2 } + +// Precondition requires at least one standby (instancesPerNode>=2). Killing the +// sole instance of a single-instance cluster would cause guaranteed downtime +// with no failover target — a true-but-useless policy violation. The same guard +// (and rationale) is used by UpgradeDocumentDB; skips don't consume the +// scheduler cooldown, so this is free to re-evaluate on the next tick. +func (k *KillPrimaryPod) Precondition(ctx context.Context) (bool, string) { + ipn, err := k.client.GetInstancesPerNode(ctx) + if err != nil { + return false, fmt.Sprintf("cannot read instancesPerNode: %v", err) + } + if ipn < 2 { + return false, fmt.Sprintf("instancesPerNode=%d (no HA standby) — killing primary would cause real downtime; skipping", ipn) + } + return true, "" +} + +func (k *KillPrimaryPod) Execute(ctx context.Context) error { + primary, err := k.client.GetPrimaryInstance(ctx) + if err != nil { + return fmt.Errorf("get primary instance: %w", err) + } + if err := k.client.DeletePod(ctx, primary); err != nil { + return fmt.Errorf("delete primary pod %s: %w", primary, err) + } + + // Wait for CNPG to elect a new primary and the cluster to settle. + recoveryCtx, cancel := context.WithTimeout(ctx, k.recovery) + defer cancel() + return k.healthMon.WaitForSteadyState(recoveryCtx) +} + +// OutagePolicy allows a moderate write-failure budget: failover briefly +// interrupts writes while a standby is promoted, similar to scale-down. +func (k *KillPrimaryPod) OutagePolicy() journal.OutagePolicy { + return journal.OutagePolicy{ + AllowedWriteFailures: 50, + MustRecoverWithin: k.recovery, + } +} diff --git a/test/longhaul/operations/kill_primary_test.go b/test/longhaul/operations/kill_primary_test.go new file mode 100644 index 000000000..aab917591 --- /dev/null +++ b/test/longhaul/operations/kill_primary_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "errors" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + "github.com/documentdb/documentdb-operator/test/longhaul/monitor" +) + +var _ = Describe("KillPrimaryPod", func() { + It("Name is kill-primary-pod and Weight is 2", func() { + k := NewKillPrimaryPod(&fakeClient{}, nil, time.Minute) + Expect(k.Name()).To(Equal("kill-primary-pod")) + Expect(k.Weight()).To(Equal(2)) + }) + + It("OutagePolicy allows a moderate failover write-failure budget", func() { + k := NewKillPrimaryPod(&fakeClient{}, nil, 3*time.Minute) + p := k.OutagePolicy() + Expect(p.AllowedWriteFailures).To(Equal(int64(50))) + Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute)) + }) + + DescribeTable("Precondition", + func(ipn int, ipnErr error, wantOK bool, wantReasonHas string) { + c := &fakeClient{instancesPerNode: ipn, ipnErr: ipnErr} + k := NewKillPrimaryPod(c, nil, time.Minute) + + ok, reason := k.Precondition(context.Background()) + Expect(ok).To(Equal(wantOK), "reason=%q", reason) + if wantReasonHas != "" { + Expect(reason).To(ContainSubstring(wantReasonHas)) + } + }, + Entry("single-instance: ipn=1 -> skip", 1, nil, false, "no HA standby"), + Entry("read error -> skip", 0, errors.New("boom"), false, "cannot read instancesPerNode"), + Entry("HA: ipn=2 -> eligible", 2, nil, true, ""), + Entry("HA: ipn=3 -> eligible", 3, nil, true, ""), + ) + + It("Execute deletes the reported primary pod", func() { + c := &fakeClient{instancesPerNode: 2, primary: "cluster-1"} + // The health monitor never reaches steady state here (its Run loop + // isn't started), so Execute times out on WaitForSteadyState — but the + // primary delete side-effect has already happened, which is what we + // assert. A short recovery keeps the test fast. + hm := monitor.NewHealthMonitor(c, journal.New(), time.Hour) + k := NewKillPrimaryPod(c, hm, 500*time.Millisecond) + + _ = k.Execute(context.Background()) + c.mu.Lock() + defer c.mu.Unlock() + Expect(c.deletedPods).To(ConsistOf("cluster-1")) + }) + + It("Execute fails without deleting when the primary is unknown", func() { + c := &fakeClient{instancesPerNode: 2, primaryErr: errors.New("no primary")} + k := NewKillPrimaryPod(c, nil, time.Second) + + err := k.Execute(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("get primary instance")) + c.mu.Lock() + defer c.mu.Unlock() + Expect(c.deletedPods).To(BeEmpty()) + }) +}) diff --git a/test/longhaul/operations/scale_test.go b/test/longhaul/operations/scale_test.go index 3df032c05..a0d46381e 100644 --- a/test/longhaul/operations/scale_test.go +++ b/test/longhaul/operations/scale_test.go @@ -23,6 +23,10 @@ type fakeClient struct { imageTag string scaleCalls []int upgradeCalls []string + primary string + primaryErr error + deleteErr error + deletedPods []string } func (f *fakeClient) GetClusterHealth(_ context.Context) (monitor.ClusterHealth, error) { @@ -51,6 +55,20 @@ func (f *fakeClient) UpgradeDocumentDB(_ context.Context, v string) error { f.upgradeCalls = append(f.upgradeCalls, v) return nil } +func (f *fakeClient) GetPrimaryInstance(_ context.Context) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.primary, f.primaryErr +} +func (f *fakeClient) DeletePod(_ context.Context, name string) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.deleteErr != nil { + return f.deleteErr + } + f.deletedPods = append(f.deletedPods, name) + return nil +} var _ = Describe("ScaleUp", func() { DescribeTable("clamps maxInstances to the CRD upper bound", From bbe1b53125775b8c0d8ff9d0f0a596c3e69f82c2 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Fri, 10 Jul 2026 13:10:49 -0400 Subject: [PATCH 2/9] test: add shared NoOutagePolicy for no-write-failure operations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce journal.NoOutagePolicy (and the NoOutageWriteFailureCushion constant) as the single budget for operations that keep the write path (client -> gateway -> primary) up throughout, so they must not cause write failures. Apply it to: - kill-operator-pod (control-plane fault; was an inline budget of 5), and - scale-up / scale-down, which only add/remove a standby replica — the primary is never touched, so their previous ad-hoc budgets (20 / 50) were too lenient and could mask a regression that disrupted writes. The cushion is a small non-zero value (5) that absorbs unrelated background noise without tolerating a real outage: at the default workload rate (~50 writes/s) it is well under a second of stray errors. Centralizing it lets the value be recalibrated against real long-haul runs in one place. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- test/longhaul/README.md | 9 +++++-- test/longhaul/journal/policy.go | 26 +++++++++++++++++++ test/longhaul/journal/policy_test.go | 10 +++++++ test/longhaul/operations/kill_operator.go | 11 +++----- .../longhaul/operations/kill_operator_test.go | 6 +++-- test/longhaul/operations/scale.go | 17 ++++++------ test/longhaul/operations/scale_test.go | 9 ++++--- 7 files changed, 64 insertions(+), 24 deletions(-) diff --git a/test/longhaul/README.md b/test/longhaul/README.md index 0985bc0d1..ab3b49a67 100644 --- a/test/longhaul/README.md +++ b/test/longhaul/README.md @@ -142,11 +142,16 @@ a global cooldown. Current operations: | Operation | Kind | Notes | |-----------|------|-------| -| `scale-up` / `scale-down` | Topology | Adjusts `spec.instancesPerNode` within `[MIN, MAX]`. | +| `scale-up` / `scale-down` | Topology | Adjusts `spec.instancesPerNode` within `[MIN, MAX]`. Only adds/removes a standby, so the primary write path is untouched (near-zero outage budget). | | `upgrade-documentdb` | Topology | In-place version upgrade; requires HA (`instancesPerNode>=2`). | -| `kill-operator-pod` | Chaos | Deletes the operator pod; asserts the data plane keeps serving (small write-failure budget). | +| `kill-operator-pod` | Chaos | Deletes the operator pod; asserts the data plane keeps serving (near-zero outage budget). | | `kill-primary-pod` | Chaos | Deletes the CNPG primary pod to exercise automatic failover; requires HA (`instancesPerNode>=2`). | +Operations that keep the write path up throughout — the scale ops and +`kill-operator-pod` — share the near-zero `journal.NoOutagePolicy` budget instead +of ad-hoc per-op numbers, so a regression that unexpectedly disrupts writes +during a "safe" operation trips the policy. + ### RBAC for chaos operations Beyond the base RBAC the driver already needs, the chaos operations require the diff --git a/test/longhaul/journal/policy.go b/test/longhaul/journal/policy.go index 25d28e826..8ac106b0b 100644 --- a/test/longhaul/journal/policy.go +++ b/test/longhaul/journal/policy.go @@ -22,6 +22,32 @@ func DefaultOutagePolicy() OutagePolicy { } } +// NoOutageWriteFailureCushion is the small write-failure budget granted to +// operations that are expected NOT to disrupt the data plane. It is not a +// tolerance for real outages: at the default workload rate (~50 writes/s +// aggregate, 5 writers x 100ms) this corresponds to well under a second of +// stray errors, so any genuine primary disruption still trips the policy. The +// non-zero value only absorbs unrelated background noise (a client reconnect, +// service-endpoint churn) that would otherwise cause flaky false positives +// against a strict 0. Centralized here so the single value can be recalibrated +// against real long-haul runs. +const NoOutageWriteFailureCushion int64 = 5 + +// NoOutagePolicy is the outage budget for operations that keep the write path +// (client -> gateway -> primary) up throughout and therefore must not cause +// write failures. It is shared by every "no data-plane impact" operation: +// - control-plane faults, e.g. an operator pod restart, and +// - scaling that only adds or removes a standby replica (the primary, and +// thus the write path, is never touched). +// +// recovery bounds how long the cluster may take to return to steady state. +func NoOutagePolicy(recovery time.Duration) OutagePolicy { + return OutagePolicy{ + AllowedWriteFailures: NoOutageWriteFailureCushion, + MustRecoverWithin: recovery, + } +} + // DisruptionWindow represents an active or closed disruption period. type DisruptionWindow struct { // OperationName identifies which operation opened this window. diff --git a/test/longhaul/journal/policy_test.go b/test/longhaul/journal/policy_test.go index 4fe26b1a4..cac082d45 100644 --- a/test/longhaul/journal/policy_test.go +++ b/test/longhaul/journal/policy_test.go @@ -81,4 +81,14 @@ var _ = Describe("DisruptionWindow", func() { Expect(p.MustRecoverWithin).NotTo(BeZero()) Expect(p.AllowedWriteFailures).NotTo(BeZero()) }) + + It("NoOutagePolicy grants the near-zero cushion and echoes recovery", func() { + p := NoOutagePolicy(3 * time.Minute) + Expect(p.AllowedWriteFailures).To(Equal(NoOutageWriteFailureCushion)) + Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute)) + }) + + It("NoOutagePolicy is far tighter than DefaultOutagePolicy", func() { + Expect(NoOutageWriteFailureCushion).To(BeNumerically("<", DefaultOutagePolicy().AllowedWriteFailures)) + }) }) diff --git a/test/longhaul/operations/kill_operator.go b/test/longhaul/operations/kill_operator.go index 306503f63..6018aab98 100644 --- a/test/longhaul/operations/kill_operator.go +++ b/test/longhaul/operations/kill_operator.go @@ -89,15 +89,10 @@ func (k *KillOperatorPod) Execute(ctx context.Context) error { return k.waitForDeploymentAvailable(recoveryCtx) } -// OutagePolicy tolerates only a small number of write failures: an operator -// restart must not take down the data plane. A non-zero budget absorbs -// coincidental blips (e.g. a scrape or client reconnect) without flagging a -// false policy violation. +// OutagePolicy: an operator restart is a control-plane fault that must not take +// down the data plane, so it shares the near-zero NoOutagePolicy budget. func (k *KillOperatorPod) OutagePolicy() journal.OutagePolicy { - return journal.OutagePolicy{ - AllowedWriteFailures: 5, - MustRecoverWithin: k.recovery, - } + return journal.NoOutagePolicy(k.recovery) } func (k *KillOperatorPod) getDeployment(ctx context.Context) (*appsv1.Deployment, error) { diff --git a/test/longhaul/operations/kill_operator_test.go b/test/longhaul/operations/kill_operator_test.go index f05b04b9b..e690702ec 100644 --- a/test/longhaul/operations/kill_operator_test.go +++ b/test/longhaul/operations/kill_operator_test.go @@ -10,6 +10,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -56,10 +58,10 @@ var _ = Describe("KillOperatorPod", func() { Expect(k.Weight()).To(Equal(2)) }) - It("OutagePolicy tolerates only a small write-failure budget", func() { + It("OutagePolicy uses the near-zero NoOutagePolicy budget", func() { k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, 2*time.Minute) p := k.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(int64(5))) + Expect(p.AllowedWriteFailures).To(Equal(journal.NoOutageWriteFailureCushion)) Expect(p.MustRecoverWithin).To(Equal(2 * time.Minute)) }) diff --git a/test/longhaul/operations/scale.go b/test/longhaul/operations/scale.go index 9a1672959..f002fdc61 100644 --- a/test/longhaul/operations/scale.go +++ b/test/longhaul/operations/scale.go @@ -77,6 +77,9 @@ type ScaleUp struct{ scaleOp } // NewScaleUp creates a ScaleUp operation. maxInstances is clamped to the // CRD upper bound (3) to avoid admission rejections. +// +// Scaling up only adds a standby replica (the primary and thus the write path +// is untouched), so it uses the near-zero NoOutagePolicy budget. func NewScaleUp(client monitor.ClusterClient, health *monitor.HealthMonitor, maxInstances int, recovery time.Duration) *ScaleUp { if maxInstances > 3 { maxInstances = 3 @@ -90,10 +93,7 @@ func NewScaleUp(client monitor.ClusterClient, health *monitor.HealthMonitor, max bound: maxInstances, boundKind: "max", recovery: recovery, - policy: journal.OutagePolicy{ - AllowedWriteFailures: 20, - MustRecoverWithin: recovery, - }, + policy: journal.NoOutagePolicy(recovery), }} } @@ -105,6 +105,10 @@ type ScaleDown struct{ scaleOp } // NewScaleDown creates a ScaleDown operation. minInstances is clamped to the // CRD lower bound (1) to avoid admission rejections. +// +// Scaling down removes the highest-ordinal standby (CNPG never removes the +// primary), so the write path stays up and it uses the same near-zero +// NoOutagePolicy budget as scale-up. func NewScaleDown(client monitor.ClusterClient, health *monitor.HealthMonitor, minInstances int, recovery time.Duration) *ScaleDown { if minInstances < 1 { minInstances = 1 @@ -118,10 +122,7 @@ func NewScaleDown(client monitor.ClusterClient, health *monitor.HealthMonitor, m bound: minInstances, boundKind: "min", recovery: recovery, - policy: journal.OutagePolicy{ - AllowedWriteFailures: 50, - MustRecoverWithin: recovery, - }, + policy: journal.NoOutagePolicy(recovery), }} } diff --git a/test/longhaul/operations/scale_test.go b/test/longhaul/operations/scale_test.go index a0d46381e..222d215a5 100644 --- a/test/longhaul/operations/scale_test.go +++ b/test/longhaul/operations/scale_test.go @@ -12,6 +12,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" "github.com/documentdb/documentdb-operator/test/longhaul/monitor" ) @@ -105,10 +106,10 @@ var _ = Describe("ScaleUp", func() { Entry("blocked: ipn read error", 0, errors.New("apiserver down"), 3, false, "cannot get instancesPerNode"), ) - It("OutagePolicy uses tighter budgets and echoes MustRecoverWithin", func() { + It("OutagePolicy uses the near-zero NoOutagePolicy budget and echoes MustRecoverWithin", func() { s := NewScaleUp(&fakeClient{}, nil, 3, 5*time.Minute) p := s.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(int64(20))) + Expect(p.AllowedWriteFailures).To(Equal(journal.NoOutageWriteFailureCushion)) Expect(p.MustRecoverWithin).To(Equal(5 * time.Minute)) }) }) @@ -148,9 +149,9 @@ var _ = Describe("ScaleDown", func() { Entry("blocked: ipn read error", 0, errors.New("apiserver down"), 1, false, "cannot get instancesPerNode"), ) - It("OutagePolicy is more lenient than scale-up", func() { + It("OutagePolicy shares the near-zero NoOutagePolicy budget with scale-up", func() { s := NewScaleDown(&fakeClient{}, nil, 1, 5*time.Minute) p := s.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(int64(50))) + Expect(p.AllowedWriteFailures).To(Equal(journal.NoOutageWriteFailureCushion)) }) }) From 5c249b84e7f2f5696b19bb47a3bd21811c62c7e2 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Fri, 10 Jul 2026 13:19:58 -0400 Subject: [PATCH 3/9] refactor(longhaul): make outage budgets duration-based Express OutagePolicy as a wall-clock write-outage duration (MaxWriteOutage) instead of a raw write-failure count. The journal converts the observed failure count into an estimated outage using the workload's aggregate write rate, so budgets no longer scale with LONGHAUL_NUM_WRITERS. - policy: OutagePolicy.AllowedWriteFailures -> MaxWriteOutage; DisruptionWindow gains WritesPerSecond + EstimatedWriteOutage(); NoOutageWriteFailureCushion -> NoOutageWriteOutageCushion (300ms). - journal: New() defaults to DefaultWritesPerSecond; SetWriteRate lets main.go supply the real rate; OpenDisruptionWindow stamps the rate. - workload: expose AggregateWriteRate(numWriters). - ops: kill-primary 30s, upgrade 45s (was 50/200 failures). - report: surface Est. Write Outage column. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- test/longhaul/README.md | 8 +++ test/longhaul/cmd/longhaul/main.go | 1 + test/longhaul/journal/journal.go | 35 +++++++++-- test/longhaul/journal/journal_test.go | 10 ++-- test/longhaul/journal/policy.go | 59 +++++++++++++------ test/longhaul/journal/policy_test.go | 59 +++++++++++-------- .../longhaul/operations/kill_operator_test.go | 2 +- test/longhaul/operations/kill_primary.go | 11 ++-- test/longhaul/operations/kill_primary_test.go | 4 +- test/longhaul/operations/scale_test.go | 4 +- test/longhaul/operations/upgrade.go | 11 ++-- test/longhaul/operations/upgrade_test.go | 4 +- test/longhaul/report/report.go | 9 +-- test/longhaul/report/report_test.go | 11 ++-- test/longhaul/workload/writer.go | 12 ++++ 15 files changed, 166 insertions(+), 74 deletions(-) diff --git a/test/longhaul/README.md b/test/longhaul/README.md index ab3b49a67..e95fca627 100644 --- a/test/longhaul/README.md +++ b/test/longhaul/README.md @@ -152,6 +152,14 @@ Operations that keep the write path up throughout — the scale ops and of ad-hoc per-op numbers, so a regression that unexpectedly disrupts writes during a "safe" operation trips the policy. +Outage budgets are expressed as **wall-clock write-outage durations** +(`OutagePolicy.MaxWriteOutage`), not raw write-failure counts. The journal +converts the observed failure count into an estimated outage using the workload's +aggregate write rate (`workload.AggregateWriteRate(NumWriters)`), so the budgets +are independent of `LONGHAUL_NUM_WRITERS`: `NoOutagePolicy` ≈ 300ms (noise +cushion), `kill-primary-pod` = 30s (single automatic failover), and +`upgrade-documentdb` = 45s (primary switchover during the rolling upgrade). + ### RBAC for chaos operations Beyond the base RBAC the driver already needs, the chaos operations require the diff --git a/test/longhaul/cmd/longhaul/main.go b/test/longhaul/cmd/longhaul/main.go index 5195d49d7..a93870dbd 100644 --- a/test/longhaul/cmd/longhaul/main.go +++ b/test/longhaul/cmd/longhaul/main.go @@ -57,6 +57,7 @@ func run(cfg config.Config) int { // Initialize components. j := journal.New() + j.SetWriteRate(workload.AggregateWriteRate(cfg.NumWriters)) metrics := workload.NewMetrics() // Connect to DocumentDB. diff --git a/test/longhaul/journal/journal.go b/test/longhaul/journal/journal.go index bfddebc76..2ad071682 100644 --- a/test/longhaul/journal/journal.go +++ b/test/longhaul/journal/journal.go @@ -52,13 +52,39 @@ type Journal struct { // All closed disruption windows. closedWindows []DisruptionWindow + + // writesPerSecond is the workload's aggregate write rate, stamped onto each + // disruption window so ExceededPolicy can convert write-failure counts into + // an estimated outage duration. Defaults to DefaultWritesPerSecond; override + // with SetWriteRate once the real writer count is known. + writesPerSecond float64 } +// DefaultWritesPerSecond is the assumed aggregate write rate used until +// SetWriteRate is called. It matches the default workload (5 writers at one +// write per 100ms = 50 writes/s) so tests and un-configured journals still +// evaluate write-outage budgets sensibly. +const DefaultWritesPerSecond = 50.0 + // New creates a new empty Journal. func New() *Journal { return &Journal{ - events: make([]Event, 0, 256), + events: make([]Event, 0, 256), + writesPerSecond: DefaultWritesPerSecond, + } +} + +// SetWriteRate records the workload's aggregate write rate (writes/second across +// all writers) so disruption windows can translate write-failure counts into an +// estimated outage duration. Non-positive values are ignored, preserving the +// default. Safe for concurrent use. +func (j *Journal) SetWriteRate(writesPerSecond float64) { + if writesPerSecond <= 0 { + return } + j.mu.Lock() + defer j.mu.Unlock() + j.writesPerSecond = writesPerSecond } // Record appends a new event to the journal. Safe for concurrent use. @@ -112,9 +138,10 @@ func (j *Journal) OpenDisruptionWindow(operationName string, policy OutagePolicy } j.activeWindow = &DisruptionWindow{ - OperationName: operationName, - StartTime: time.Now(), - Policy: policy, + OperationName: operationName, + StartTime: time.Now(), + Policy: policy, + WritesPerSecond: j.writesPerSecond, } j.events = append(j.events, Event{ diff --git a/test/longhaul/journal/journal_test.go b/test/longhaul/journal/journal_test.go index 02e2b3c6c..43c159db6 100644 --- a/test/longhaul/journal/journal_test.go +++ b/test/longhaul/journal/journal_test.go @@ -43,7 +43,7 @@ var _ = Describe("Journal", func() { Describe("DisruptionWindow lifecycle", func() { It("opens, records failures, and closes correctly", func() { j := New() - policy := OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 10} + policy := OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second} Expect(j.ActiveWindow()).To(BeNil()) @@ -89,14 +89,14 @@ var _ = Describe("Journal", func() { It("returns false on a closed window within budget", func() { j := New() - j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 10}) + j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}) j.CloseDisruptionWindow() Expect(j.HasPolicyViolation()).To(BeFalse()) }) - It("returns true on a closed window over write-failure budget", func() { + It("returns true on a closed window over write-outage budget", func() { j := New() - j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 1}) + j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: 10 * time.Millisecond}) j.RecordWriteFailure() j.RecordWriteFailure() j.CloseDisruptionWindow() @@ -105,7 +105,7 @@ var _ = Describe("Journal", func() { It("returns true on an active window over time budget", func() { j := New() - j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Nanosecond, AllowedWriteFailures: 10}) + j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Nanosecond, MaxWriteOutage: time.Second}) time.Sleep(1 * time.Millisecond) Expect(j.HasPolicyViolation()).To(BeTrue()) }) diff --git a/test/longhaul/journal/policy.go b/test/longhaul/journal/policy.go index 8ac106b0b..738e86438 100644 --- a/test/longhaul/journal/policy.go +++ b/test/longhaul/journal/policy.go @@ -7,8 +7,13 @@ import "time" // OutagePolicy defines acceptable disruption bounds for an operation. type OutagePolicy struct { - // AllowedWriteFailures is the maximum number of write failures during the window. - AllowedWriteFailures int64 + // MaxWriteOutage bounds how long the write path (client -> gateway -> + // primary) may be unavailable during the window. It is evaluated from the + // observed write-failure count normalized by the workload's aggregate write + // rate (see DisruptionWindow.EstimatedWriteOutage), so the budget is + // expressed in wall-clock outage time and is independent of how many writer + // goroutines (LONGHAUL_NUM_WRITERS) are configured. + MaxWriteOutage time.Duration // MustRecoverWithin is the maximum time from operation start to full recovery. MustRecoverWithin time.Duration @@ -17,25 +22,24 @@ type OutagePolicy struct { // DefaultOutagePolicy returns a conservative policy suitable for most operations. func DefaultOutagePolicy() OutagePolicy { return OutagePolicy{ - AllowedWriteFailures: 50, - MustRecoverWithin: 5 * time.Minute, + MaxWriteOutage: 5 * time.Second, + MustRecoverWithin: 5 * time.Minute, } } -// NoOutageWriteFailureCushion is the small write-failure budget granted to +// NoOutageWriteOutageCushion is the tiny write-outage budget granted to // operations that are expected NOT to disrupt the data plane. It is not a -// tolerance for real outages: at the default workload rate (~50 writes/s -// aggregate, 5 writers x 100ms) this corresponds to well under a second of -// stray errors, so any genuine primary disruption still trips the policy. The -// non-zero value only absorbs unrelated background noise (a client reconnect, -// service-endpoint churn) that would otherwise cause flaky false positives -// against a strict 0. Centralized here so the single value can be recalibrated -// against real long-haul runs. -const NoOutageWriteFailureCushion int64 = 5 +// tolerance for real outages: one fully-failed write tick (every configured +// writer failing once) maps to exactly one writeInterval of estimated outage +// (~100ms) regardless of writer count, so this ~3-tick cushion absorbs unrelated +// background noise (a client reconnect, service-endpoint churn) without +// tolerating a genuine primary outage. Centralized so it can be recalibrated +// against real long-haul runs in one place. +const NoOutageWriteOutageCushion = 300 * time.Millisecond // NoOutagePolicy is the outage budget for operations that keep the write path -// (client -> gateway -> primary) up throughout and therefore must not cause -// write failures. It is shared by every "no data-plane impact" operation: +// up throughout and therefore must not cause a write outage. It is shared by +// every "no data-plane impact" operation: // - control-plane faults, e.g. an operator pod restart, and // - scaling that only adds or removes a standby replica (the primary, and // thus the write path, is never touched). @@ -43,8 +47,8 @@ const NoOutageWriteFailureCushion int64 = 5 // recovery bounds how long the cluster may take to return to steady state. func NoOutagePolicy(recovery time.Duration) OutagePolicy { return OutagePolicy{ - AllowedWriteFailures: NoOutageWriteFailureCushion, - MustRecoverWithin: recovery, + MaxWriteOutage: NoOutageWriteOutageCushion, + MustRecoverWithin: recovery, } } @@ -64,6 +68,25 @@ type DisruptionWindow struct { // WriteFailures counts failures observed during this window. WriteFailures int64 + + // WritesPerSecond is the workload's aggregate write rate at the time the + // window opened. It is used to convert the raw WriteFailures count into an + // estimated write-outage duration (see EstimatedWriteOutage). A real outage + // makes every writer fail on every tick, so failures accrue at the full + // aggregate rate and count/rate recovers the wall-clock outage duration + // regardless of writer count. Zero disables the write-outage check. + WritesPerSecond float64 +} + +// EstimatedWriteOutage converts the observed write-failure count into an +// approximate duration for which the write path was unavailable, using the +// aggregate write rate captured when the window opened. Returns 0 when the rate +// is unknown (<= 0), which disables the write-outage portion of the policy. +func (w *DisruptionWindow) EstimatedWriteOutage() time.Duration { + if w.WritesPerSecond <= 0 { + return 0 + } + return time.Duration(float64(w.WriteFailures) / w.WritesPerSecond * float64(time.Second)) } // IsActive returns true if the disruption window has not been closed. @@ -85,7 +108,7 @@ func (w *DisruptionWindow) ExceededPolicy() bool { if w.Duration() > w.Policy.MustRecoverWithin { return true } - if w.WriteFailures > w.Policy.AllowedWriteFailures { + if w.EstimatedWriteOutage() > w.Policy.MaxWriteOutage { return true } return false diff --git a/test/longhaul/journal/policy_test.go b/test/longhaul/journal/policy_test.go index cac082d45..dec577337 100644 --- a/test/longhaul/journal/policy_test.go +++ b/test/longhaul/journal/policy_test.go @@ -43,52 +43,65 @@ var _ = Describe("DisruptionWindow", func() { }, Entry("within all budgets", DisruptionWindow{ - StartTime: time.Now().Add(-10 * time.Second), - EndTime: time.Now(), - WriteFailures: 5, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + WriteFailures: 5, // 5/50 = 0.1s < 1s + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, false), Entry("exceeds MustRecoverWithin", DisruptionWindow{ - StartTime: time.Now().Add(-2 * time.Minute), - EndTime: time.Now(), - WriteFailures: 1, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-2 * time.Minute), + EndTime: time.Now(), + WriteFailures: 1, + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, true), - Entry("exceeds AllowedWriteFailures", + Entry("exceeds MaxWriteOutage", DisruptionWindow{ - StartTime: time.Now().Add(-10 * time.Second), - EndTime: time.Now(), - WriteFailures: 100, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + WriteFailures: 100, // 100/50 = 2s > 1s + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, true), - Entry("boundary: equal to write-failure budget is allowed", + Entry("boundary: estimated outage equal to budget is allowed", DisruptionWindow{ - StartTime: time.Now().Add(-10 * time.Second), - EndTime: time.Now(), - WriteFailures: 50, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + WriteFailures: 50, // 50/50 = exactly 1s + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, + }, false), + Entry("unknown write rate disables the write-outage check", + DisruptionWindow{ + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + WriteFailures: 100000, + WritesPerSecond: 0, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, false), Entry("active window also evaluated against MustRecoverWithin", DisruptionWindow{ - StartTime: time.Now().Add(-2 * time.Minute), - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-2 * time.Minute), + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, true), ) It("DefaultOutagePolicy returns no zero-valued field", func() { p := DefaultOutagePolicy() Expect(p.MustRecoverWithin).NotTo(BeZero()) - Expect(p.AllowedWriteFailures).NotTo(BeZero()) + Expect(p.MaxWriteOutage).NotTo(BeZero()) }) It("NoOutagePolicy grants the near-zero cushion and echoes recovery", func() { p := NoOutagePolicy(3 * time.Minute) - Expect(p.AllowedWriteFailures).To(Equal(NoOutageWriteFailureCushion)) + Expect(p.MaxWriteOutage).To(Equal(NoOutageWriteOutageCushion)) Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute)) }) It("NoOutagePolicy is far tighter than DefaultOutagePolicy", func() { - Expect(NoOutageWriteFailureCushion).To(BeNumerically("<", DefaultOutagePolicy().AllowedWriteFailures)) + Expect(NoOutageWriteOutageCushion).To(BeNumerically("<", DefaultOutagePolicy().MaxWriteOutage)) }) }) diff --git a/test/longhaul/operations/kill_operator_test.go b/test/longhaul/operations/kill_operator_test.go index e690702ec..d409b5ab4 100644 --- a/test/longhaul/operations/kill_operator_test.go +++ b/test/longhaul/operations/kill_operator_test.go @@ -61,7 +61,7 @@ var _ = Describe("KillOperatorPod", func() { It("OutagePolicy uses the near-zero NoOutagePolicy budget", func() { k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, 2*time.Minute) p := k.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(journal.NoOutageWriteFailureCushion)) + Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion)) Expect(p.MustRecoverWithin).To(Equal(2 * time.Minute)) }) diff --git a/test/longhaul/operations/kill_primary.go b/test/longhaul/operations/kill_primary.go index aa2199db8..15084085d 100644 --- a/test/longhaul/operations/kill_primary.go +++ b/test/longhaul/operations/kill_primary.go @@ -66,11 +66,14 @@ func (k *KillPrimaryPod) Execute(ctx context.Context) error { return k.healthMon.WaitForSteadyState(recoveryCtx) } -// OutagePolicy allows a moderate write-failure budget: failover briefly -// interrupts writes while a standby is promoted, similar to scale-down. +// OutagePolicy bounds the write outage of an automatic failover. Killing the +// primary interrupts writes until CNPG detects the loss and promotes a standby; +// a healthy single failover should restore the write path well within this +// budget. Expressed as wall-clock outage time, so it is independent of the +// configured writer count. func (k *KillPrimaryPod) OutagePolicy() journal.OutagePolicy { return journal.OutagePolicy{ - AllowedWriteFailures: 50, - MustRecoverWithin: k.recovery, + MaxWriteOutage: 30 * time.Second, + MustRecoverWithin: k.recovery, } } diff --git a/test/longhaul/operations/kill_primary_test.go b/test/longhaul/operations/kill_primary_test.go index aab917591..f35d49b39 100644 --- a/test/longhaul/operations/kill_primary_test.go +++ b/test/longhaul/operations/kill_primary_test.go @@ -22,10 +22,10 @@ var _ = Describe("KillPrimaryPod", func() { Expect(k.Weight()).To(Equal(2)) }) - It("OutagePolicy allows a moderate failover write-failure budget", func() { + It("OutagePolicy bounds the failover write outage in wall-clock time", func() { k := NewKillPrimaryPod(&fakeClient{}, nil, 3*time.Minute) p := k.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(int64(50))) + Expect(p.MaxWriteOutage).To(Equal(30 * time.Second)) Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute)) }) diff --git a/test/longhaul/operations/scale_test.go b/test/longhaul/operations/scale_test.go index 222d215a5..8c9fc953b 100644 --- a/test/longhaul/operations/scale_test.go +++ b/test/longhaul/operations/scale_test.go @@ -109,7 +109,7 @@ var _ = Describe("ScaleUp", func() { It("OutagePolicy uses the near-zero NoOutagePolicy budget and echoes MustRecoverWithin", func() { s := NewScaleUp(&fakeClient{}, nil, 3, 5*time.Minute) p := s.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(journal.NoOutageWriteFailureCushion)) + Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion)) Expect(p.MustRecoverWithin).To(Equal(5 * time.Minute)) }) }) @@ -152,6 +152,6 @@ var _ = Describe("ScaleDown", func() { It("OutagePolicy shares the near-zero NoOutagePolicy budget with scale-up", func() { s := NewScaleDown(&fakeClient{}, nil, 1, 5*time.Minute) p := s.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(journal.NoOutageWriteFailureCushion)) + Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion)) }) }) diff --git a/test/longhaul/operations/upgrade.go b/test/longhaul/operations/upgrade.go index f9a586b6e..6f823c47b 100644 --- a/test/longhaul/operations/upgrade.go +++ b/test/longhaul/operations/upgrade.go @@ -170,11 +170,14 @@ func (u *UpgradeDocumentDB) readDesiredVersion(ctx context.Context) (string, err return cm.Data[VersionConfigMapKey], nil } -// OutagePolicy allows for a longer disruption window during an upgrade -// because rolling restarts touch every pod sequentially. +// OutagePolicy bounds the write outage of a rolling upgrade. Standby restarts +// do not block writes; the write path is only interrupted during the primary +// switchover, so the outage is comparable to a failover with some extra slack +// for the graceful switchover coordination. Expressed as wall-clock outage +// time, independent of the configured writer count. func (u *UpgradeDocumentDB) OutagePolicy() journal.OutagePolicy { return journal.OutagePolicy{ - AllowedWriteFailures: 200, - MustRecoverWithin: u.recovery, + MaxWriteOutage: 45 * time.Second, + MustRecoverWithin: u.recovery, } } diff --git a/test/longhaul/operations/upgrade_test.go b/test/longhaul/operations/upgrade_test.go index f442e5b58..9b9a7a59c 100644 --- a/test/longhaul/operations/upgrade_test.go +++ b/test/longhaul/operations/upgrade_test.go @@ -22,10 +22,10 @@ var _ = Describe("UpgradeDocumentDB", func() { Expect(u.Weight()).To(Equal(1)) }) - It("OutagePolicy gives upgrades a more lenient failure budget", func() { + It("OutagePolicy gives upgrades a more lenient write-outage budget", func() { u := NewUpgradeDocumentDB(&fakeClient{}, fake.NewSimpleClientset(), nil, nil, "ns", 10*time.Minute) p := u.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(int64(200))) + Expect(p.MaxWriteOutage).To(Equal(45 * time.Second)) Expect(p.MustRecoverWithin).To(Equal(10 * time.Minute)) }) diff --git a/test/longhaul/report/report.go b/test/longhaul/report/report.go index e7316a7f7..07576ef58 100644 --- a/test/longhaul/report/report.go +++ b/test/longhaul/report/report.go @@ -93,15 +93,16 @@ func GenerateMarkdown(s Summary) string { // Disruption Windows if len(s.Windows) > 0 { b.WriteString("## Disruption Windows\n\n") - b.WriteString("| Operation | Duration | Write Failures | Policy Exceeded |\n") - b.WriteString("|-----------|----------|----------------|------------------|\n") + b.WriteString("| Operation | Duration | Write Failures | Est. Write Outage | Policy Exceeded |\n") + b.WriteString("|-----------|----------|----------------|-------------------|------------------|\n") for _, w := range s.Windows { exceeded := "No" if w.ExceededPolicy() { exceeded = "**YES**" } - fmt.Fprintf(&b, "| %s | %s | %d | %s |\n", - w.OperationName, w.Duration().Round(time.Second), w.WriteFailures, exceeded) + fmt.Fprintf(&b, "| %s | %s | %d | %s | %s |\n", + w.OperationName, w.Duration().Round(time.Second), w.WriteFailures, + w.EstimatedWriteOutage().Round(time.Millisecond), exceeded) } b.WriteString("\n") } diff --git a/test/longhaul/report/report_test.go b/test/longhaul/report/report_test.go index f8b77297e..33fc844b5 100644 --- a/test/longhaul/report/report_test.go +++ b/test/longhaul/report/report_test.go @@ -74,11 +74,12 @@ var _ = Describe("GenerateMarkdown", func() { md := GenerateMarkdown(Summary{ Result: ResultPass, Windows: []journal.DisruptionWindow{{ - OperationName: "scale-up", - StartTime: now.Add(-30 * time.Second), - EndTime: now, - WriteFailures: 3, - Policy: journal.OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + OperationName: "scale-up", + StartTime: now.Add(-30 * time.Second), + EndTime: now, + WriteFailures: 3, + WritesPerSecond: 50, + Policy: journal.OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }}, }) Expect(md).To(ContainSubstring("Disruption Windows")) diff --git a/test/longhaul/workload/writer.go b/test/longhaul/workload/writer.go index c2fdcc3e2..dddfd0e54 100644 --- a/test/longhaul/workload/writer.go +++ b/test/longhaul/workload/writer.go @@ -28,6 +28,18 @@ const ( writeInterval = 100 * time.Millisecond ) +// AggregateWriteRate returns the workload's aggregate write rate in writes per +// second across all writer goroutines, given the configured writer count. It is +// the reciprocal of the per-writer writeInterval scaled by numWriters, and is +// used to convert observed write-failure counts into an estimated outage +// duration (see journal.DisruptionWindow). Returns 0 for a non-positive count. +func AggregateWriteRate(numWriters int) float64 { + if numWriters <= 0 { + return 0 + } + return float64(numWriters) / writeInterval.Seconds() +} + // WriteDocument is the schema for data-plane durability tracking. type WriteDocument struct { WriterID string `bson:"writer_id"` From 350336874b02373cde9a50682a2e9fe50d18160f Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Fri, 10 Jul 2026 13:29:21 -0400 Subject: [PATCH 4/9] docs(longhaul): clarify OutagePolicy field roles Document that MaxWriteOutage (data plane) and MustRecoverWithin (control plane) assert on orthogonal subsystems and fail independently, and that MustRecoverWithin is the only path that fails the run when the cluster never converges (op errors are logged, not scored). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- test/longhaul/journal/policy.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test/longhaul/journal/policy.go b/test/longhaul/journal/policy.go index 738e86438..94c63a5b0 100644 --- a/test/longhaul/journal/policy.go +++ b/test/longhaul/journal/policy.go @@ -5,7 +5,13 @@ package journal import "time" -// OutagePolicy defines acceptable disruption bounds for an operation. +// OutagePolicy defines acceptable disruption bounds for an operation. Its two +// fields assert on different subsystems and fail independently (ExceededPolicy +// trips if either is exceeded): MaxWriteOutage bounds the client-visible write +// interruption (data plane), while MustRecoverWithin bounds full cluster +// recovery (control plane). Each can be violated while the other is fine — e.g. +// a promoted standby that never rejoins keeps writes flowing yet leaves the +// cluster degraded, and only MustRecoverWithin catches it. type OutagePolicy struct { // MaxWriteOutage bounds how long the write path (client -> gateway -> // primary) may be unavailable during the window. It is evaluated from the @@ -15,7 +21,10 @@ type OutagePolicy struct { // goroutines (LONGHAUL_NUM_WRITERS) are configured. MaxWriteOutage time.Duration - // MustRecoverWithin is the maximum time from operation start to full recovery. + // MustRecoverWithin is the maximum time from operation start to full cluster + // recovery (steady state). Because a failed op is only logged, not counted + // toward the run verdict, this is the sole mechanism that turns a cluster + // that never converges back into a FAIL. MustRecoverWithin time.Duration } From f35b66a6e8cd3e09e7b6cd1730390c1304dc1575 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Fri, 10 Jul 2026 13:39:42 -0400 Subject: [PATCH 5/9] docs(longhaul): correct OutagePolicy field framing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MustRecoverWithin bounds the managed database cluster's return to full topology (all pods Ready, CR Ready) — not the operator/control plane. Reframe as write-availability vs. full-topology recovery to avoid implying scale/kill-primary recovery involves the operator. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- test/longhaul/journal/policy.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/longhaul/journal/policy.go b/test/longhaul/journal/policy.go index 94c63a5b0..1d41ac581 100644 --- a/test/longhaul/journal/policy.go +++ b/test/longhaul/journal/policy.go @@ -6,12 +6,13 @@ package journal import "time" // OutagePolicy defines acceptable disruption bounds for an operation. Its two -// fields assert on different subsystems and fail independently (ExceededPolicy -// trips if either is exceeded): MaxWriteOutage bounds the client-visible write -// interruption (data plane), while MustRecoverWithin bounds full cluster -// recovery (control plane). Each can be violated while the other is fine — e.g. -// a promoted standby that never rejoins keeps writes flowing yet leaves the -// cluster degraded, and only MustRecoverWithin catches it. +// fields assert on different properties of the managed cluster and fail +// independently (ExceededPolicy trips if either is exceeded): MaxWriteOutage +// bounds client-visible write availability, while MustRecoverWithin bounds the +// cluster's return to its full declared topology (all pods Ready, CR Ready). +// Each can be violated while the other is fine — e.g. after a failover writes +// resume quickly (MaxWriteOutage happy) yet the cluster stays degraded until a +// replacement standby rejoins, which only MustRecoverWithin catches. type OutagePolicy struct { // MaxWriteOutage bounds how long the write path (client -> gateway -> // primary) may be unavailable during the window. It is evaluated from the From 40c62ce482d38b8d4794e499df911d5d7ba29167 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Fri, 10 Jul 2026 13:45:38 -0400 Subject: [PATCH 6/9] refactor(longhaul): share one primary-handover write-outage budget kill-primary-pod and upgrade-documentdb both interrupt writes for a single primary handover, so they now share journal.PrimaryHandoverPolicy (30s) instead of diverging (was 30s vs 45s). The 45s was inflated by conflating the rolling upgrade's whole-topology restart with its write-outage window; that longer restart is bounded by MustRecoverWithin instead. A graceful switchover (upgrade) is no worse than an ungraceful failover (kill-primary), which also pays a detection delay. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- test/longhaul/README.md | 7 +++-- test/longhaul/journal/policy.go | 28 +++++++++++++++++++ test/longhaul/operations/kill_primary.go | 12 +++----- test/longhaul/operations/kill_primary_test.go | 4 +-- test/longhaul/operations/upgrade.go | 13 ++++----- test/longhaul/operations/upgrade_test.go | 5 ++-- 6 files changed, 47 insertions(+), 22 deletions(-) diff --git a/test/longhaul/README.md b/test/longhaul/README.md index e95fca627..a9e3be0bb 100644 --- a/test/longhaul/README.md +++ b/test/longhaul/README.md @@ -157,8 +157,11 @@ Outage budgets are expressed as **wall-clock write-outage durations** converts the observed failure count into an estimated outage using the workload's aggregate write rate (`workload.AggregateWriteRate(NumWriters)`), so the budgets are independent of `LONGHAUL_NUM_WRITERS`: `NoOutagePolicy` ≈ 300ms (noise -cushion), `kill-primary-pod` = 30s (single automatic failover), and -`upgrade-documentdb` = 45s (primary switchover during the rolling upgrade). +cushion), while `kill-primary-pod` and `upgrade-documentdb` share the +`journal.PrimaryHandoverPolicy` budget of 30s — both interrupt writes for a +single primary handover (an ungraceful failover vs. a graceful switchover), and +an upgrade's longer whole-topology restart is bounded by `MustRecoverWithin`, +not the write-outage budget. ### RBAC for chaos operations diff --git a/test/longhaul/journal/policy.go b/test/longhaul/journal/policy.go index 1d41ac581..c6226877d 100644 --- a/test/longhaul/journal/policy.go +++ b/test/longhaul/journal/policy.go @@ -62,6 +62,34 @@ func NoOutagePolicy(recovery time.Duration) OutagePolicy { } } +// PrimaryHandoverWriteOutage is the write-outage budget for operations that +// interrupt writes for exactly one primary handover. It is shared so the two +// such operations cannot drift apart: +// - kill-primary-pod — an *ungraceful* failover (detect the lost pod, then +// promote a standby), and +// - upgrade-documentdb — a *graceful* switchover of the primary; the standby +// pod restarts during the rolling upgrade do NOT interrupt writes, so the +// write outage is just the one switchover (and a graceful switchover is +// typically no worse than an ungraceful failover, which pays a detection +// delay). The upgrade's longer, whole-topology restart is bounded by +// MustRecoverWithin, not here. +// +// Sized to comfortably cover a healthy single CNPG failover; heuristic pending +// calibration against real long-haul runs. +const PrimaryHandoverWriteOutage = 30 * time.Second + +// PrimaryHandoverPolicy is the outage budget for operations whose write path is +// interrupted for a single primary handover (see PrimaryHandoverWriteOutage). +// recovery bounds how long the cluster may take to return to full topology, +// which can legitimately differ per operation (a rolling upgrade restarts every +// pod and takes longer than a single failover). +func PrimaryHandoverPolicy(recovery time.Duration) OutagePolicy { + return OutagePolicy{ + MaxWriteOutage: PrimaryHandoverWriteOutage, + MustRecoverWithin: recovery, + } +} + // DisruptionWindow represents an active or closed disruption period. type DisruptionWindow struct { // OperationName identifies which operation opened this window. diff --git a/test/longhaul/operations/kill_primary.go b/test/longhaul/operations/kill_primary.go index 15084085d..dbd960571 100644 --- a/test/longhaul/operations/kill_primary.go +++ b/test/longhaul/operations/kill_primary.go @@ -67,13 +67,9 @@ func (k *KillPrimaryPod) Execute(ctx context.Context) error { } // OutagePolicy bounds the write outage of an automatic failover. Killing the -// primary interrupts writes until CNPG detects the loss and promotes a standby; -// a healthy single failover should restore the write path well within this -// budget. Expressed as wall-clock outage time, so it is independent of the -// configured writer count. +// primary interrupts writes until CNPG detects the loss and promotes a standby. +// It shares the single-primary-handover budget with upgrade-documentdb (see +// journal.PrimaryHandoverPolicy). func (k *KillPrimaryPod) OutagePolicy() journal.OutagePolicy { - return journal.OutagePolicy{ - MaxWriteOutage: 30 * time.Second, - MustRecoverWithin: k.recovery, - } + return journal.PrimaryHandoverPolicy(k.recovery) } diff --git a/test/longhaul/operations/kill_primary_test.go b/test/longhaul/operations/kill_primary_test.go index f35d49b39..36c723006 100644 --- a/test/longhaul/operations/kill_primary_test.go +++ b/test/longhaul/operations/kill_primary_test.go @@ -22,10 +22,10 @@ var _ = Describe("KillPrimaryPod", func() { Expect(k.Weight()).To(Equal(2)) }) - It("OutagePolicy bounds the failover write outage in wall-clock time", func() { + It("OutagePolicy shares the single-primary-handover budget with upgrade", func() { k := NewKillPrimaryPod(&fakeClient{}, nil, 3*time.Minute) p := k.OutagePolicy() - Expect(p.MaxWriteOutage).To(Equal(30 * time.Second)) + Expect(p.MaxWriteOutage).To(Equal(journal.PrimaryHandoverWriteOutage)) Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute)) }) diff --git a/test/longhaul/operations/upgrade.go b/test/longhaul/operations/upgrade.go index 6f823c47b..554356997 100644 --- a/test/longhaul/operations/upgrade.go +++ b/test/longhaul/operations/upgrade.go @@ -171,13 +171,10 @@ func (u *UpgradeDocumentDB) readDesiredVersion(ctx context.Context) (string, err } // OutagePolicy bounds the write outage of a rolling upgrade. Standby restarts -// do not block writes; the write path is only interrupted during the primary -// switchover, so the outage is comparable to a failover with some extra slack -// for the graceful switchover coordination. Expressed as wall-clock outage -// time, independent of the configured writer count. +// do not block writes; the write path is only interrupted during the single +// graceful primary switchover, so it shares the primary-handover budget with +// kill-primary-pod (see journal.PrimaryHandoverPolicy). The upgrade's longer +// whole-topology restart is bounded separately by MustRecoverWithin. func (u *UpgradeDocumentDB) OutagePolicy() journal.OutagePolicy { - return journal.OutagePolicy{ - MaxWriteOutage: 45 * time.Second, - MustRecoverWithin: u.recovery, - } + return journal.PrimaryHandoverPolicy(u.recovery) } diff --git a/test/longhaul/operations/upgrade_test.go b/test/longhaul/operations/upgrade_test.go index 9b9a7a59c..ae5b7450e 100644 --- a/test/longhaul/operations/upgrade_test.go +++ b/test/longhaul/operations/upgrade_test.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -22,10 +23,10 @@ var _ = Describe("UpgradeDocumentDB", func() { Expect(u.Weight()).To(Equal(1)) }) - It("OutagePolicy gives upgrades a more lenient write-outage budget", func() { + It("OutagePolicy shares the single-primary-handover budget with kill-primary", func() { u := NewUpgradeDocumentDB(&fakeClient{}, fake.NewSimpleClientset(), nil, nil, "ns", 10*time.Minute) p := u.OutagePolicy() - Expect(p.MaxWriteOutage).To(Equal(45 * time.Second)) + Expect(p.MaxWriteOutage).To(Equal(journal.PrimaryHandoverWriteOutage)) Expect(p.MustRecoverWithin).To(Equal(10 * time.Minute)) }) From f3231b8d6c865b98d65cbc91b89dbb24e819d83a Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Tue, 14 Jul 2026 15:44:25 -0400 Subject: [PATCH 7/9] test(longhaul): grant chaos-op RBAC to driver deploy/rbac.yaml now exists on main (added by #413), so add the chaos operations' required verbs rather than leaving them as a follow-up: - kill-primary-pod: get/list on clusters.postgresql.cnpg.io and delete on pods, added to the longhaul-test Role in the target namespace. - kill-operator-pod: a separate longhaul-test-operator Role/RoleBinding in the operator namespace (documentdb-operator) granting get on deployments and get/list/delete on pods, since the operator runs outside the driver's own namespace. Update the README RBAC section to reflect that the verbs are now granted. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- test/longhaul/README.md | 10 +++++--- test/longhaul/deploy/rbac.yaml | 46 ++++++++++++++++++++++++++++++++-- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/test/longhaul/README.md b/test/longhaul/README.md index a9e3be0bb..411178fd8 100644 --- a/test/longhaul/README.md +++ b/test/longhaul/README.md @@ -166,12 +166,16 @@ not the write-outage budget. ### RBAC for chaos operations Beyond the base RBAC the driver already needs, the chaos operations require the -driver ServiceAccount to be granted (in `deploy/rbac.yaml`): +driver ServiceAccount to be granted (all present in `deploy/rbac.yaml`): - **`kill-primary-pod`** — `get`/`list` on `clusters.postgresql.cnpg.io` (to read - `status.currentPrimary`) and `delete` on `pods` in the cluster namespace. + `status.currentPrimary`) and `delete` on `pods` in the cluster namespace; both + added to the `longhaul-test` Role. - **`kill-operator-pod`** — `get` on `deployments` and `get`/`list`/`delete` on - `pods` in the operator namespace (`LONGHAUL_OPERATOR_NAMESPACE`). + `pods` in the operator namespace (`LONGHAUL_OPERATOR_NAMESPACE`, default + `documentdb-operator`); granted by a separate `longhaul-test-operator` + Role/RoleBinding in that namespace, since the operator runs outside the + driver's own namespace. ## CI Safety diff --git a/test/longhaul/deploy/rbac.yaml b/test/longhaul/deploy/rbac.yaml index f5ed5abcd..05ebd6923 100644 --- a/test/longhaul/deploy/rbac.yaml +++ b/test/longhaul/deploy/rbac.yaml @@ -21,14 +21,19 @@ metadata: app.kubernetes.io/name: longhaul-test app.kubernetes.io/component: testing rules: - # Read pod status for health monitoring. + # Read pod status for health monitoring; delete pods for the kill-primary-pod + # chaos op (deletes the CNPG primary to exercise automatic failover). - apiGroups: [""] resources: ["pods"] - verbs: ["get", "list", "watch"] + verbs: ["get", "list", "watch", "delete"] # Read and patch DocumentDB CRs for health check and scale operations. - apiGroups: ["documentdb.io"] resources: ["dbs"] verbs: ["get", "list", "patch"] + # Read the CNPG Cluster to resolve the current primary pod (kill-primary-pod). + - apiGroups: ["postgresql.cnpg.io"] + resources: ["clusters"] + verbs: ["get", "list"] # Create/update ConfigMaps for periodic report persistence. - apiGroups: [""] resources: ["configmaps"] @@ -51,6 +56,43 @@ subjects: name: longhaul-test namespace: documentdb-test-ns --- +# Role in the operator namespace for the kill-operator-pod chaos op: read the +# operator Deployment (to build its pod selector and check availability) and +# delete its pod. Namespaced separately because the operator runs outside the +# driver's own namespace (LONGHAUL_OPERATOR_NAMESPACE, default documentdb-operator). +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: longhaul-test-operator + namespace: documentdb-operator + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +rules: + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: longhaul-test-operator + namespace: documentdb-operator + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: longhaul-test-operator +subjects: + - kind: ServiceAccount + name: longhaul-test + namespace: documentdb-test-ns +--- # ClusterRole for metrics-server access (metrics API is cluster-scoped). apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole From 7d370b4c5424f60fdbe2b0532234987111ce0c63 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Thu, 23 Jul 2026 13:13:42 -0400 Subject: [PATCH 8/9] test(longhaul): harden kill-operator/kill-primary chaos ops Address PR #421 review: - kill-operator-pod: fail fast when the operator Deployment has no matchLabels selector, so SelectorFromSet can't produce an everything selector that lists/targets unrelated pods. - kill-operator-pod: after deleting the pod, wait until that specific pod (by UID) is actually gone before checking Deployment availability. Pod deletion doesn't bump ObservedGeneration, so status can stay Available and mask the restart otherwise. - kill-primary-pod: validate GetPrimaryInstance returned a non-empty pod name and guard the health monitor against nil before dereferencing. - rbac.yaml: note that the operator-namespace Role/RoleBinding must be kept in sync with LONGHAUL_OPERATOR_NAMESPACE. Add unit tests for the empty-selector and empty-primary guard paths. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- test/longhaul/deploy/rbac.yaml | 6 ++ test/longhaul/operations/kill_operator.go | 56 +++++++++++++++++-- .../longhaul/operations/kill_operator_test.go | 15 +++++ test/longhaul/operations/kill_primary.go | 6 ++ test/longhaul/operations/kill_primary_test.go | 12 ++++ 5 files changed, 89 insertions(+), 6 deletions(-) diff --git a/test/longhaul/deploy/rbac.yaml b/test/longhaul/deploy/rbac.yaml index 05ebd6923..2a724393b 100644 --- a/test/longhaul/deploy/rbac.yaml +++ b/test/longhaul/deploy/rbac.yaml @@ -64,6 +64,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: longhaul-test-operator + # NOTE: keep in sync with LONGHAUL_OPERATOR_NAMESPACE. If the driver overrides + # that env var, this namespace must be edited to match or kill-operator-pod + # fails with RBAC errors. namespace: documentdb-operator labels: app.kubernetes.io/name: longhaul-test @@ -80,6 +83,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: longhaul-test-operator + # NOTE: must match the Role namespace above (and LONGHAUL_OPERATOR_NAMESPACE). + # Update in lockstep or the binding won't grant permissions in the right + # namespace. namespace: documentdb-operator labels: app.kubernetes.io/name: longhaul-test diff --git a/test/longhaul/operations/kill_operator.go b/test/longhaul/operations/kill_operator.go index 6018aab98..e6a68db79 100644 --- a/test/longhaul/operations/kill_operator.go +++ b/test/longhaul/operations/kill_operator.go @@ -12,8 +12,10 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" ) @@ -67,6 +69,13 @@ func (k *KillOperatorPod) Execute(ctx context.Context) error { return fmt.Errorf("get operator deployment: %w", err) } + // Fail fast if the Deployment has no label selector: SelectorFromSet on an + // empty map yields an "everything" selector, so the List below would match + // (and the delete could target) every pod in the namespace. + if dep.Spec.Selector == nil || len(dep.Spec.Selector.MatchLabels) == 0 { + return fmt.Errorf("operator deployment %s has no matchLabels selector; refusing to list all pods", k.deployment) + } + // Resolve the pod set from the Deployment's own selector so we don't // depend on the release-name-derived "app" label value. selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels).String() @@ -75,7 +84,7 @@ func (k *KillOperatorPod) Execute(ctx context.Context) error { return fmt.Errorf("list operator pods: %w", err) } - target := oldestRunningPod(pods.Items) + target, targetUID := oldestRunningPod(pods.Items) if target == "" { return fmt.Errorf("no running operator pod found for selector %q", selector) } @@ -83,12 +92,44 @@ func (k *KillOperatorPod) Execute(ctx context.Context) error { return fmt.Errorf("delete operator pod %s: %w", target, err) } - // Wait for the Deployment to reschedule and become Available again. recoveryCtx, cancel := context.WithTimeout(ctx, k.recovery) defer cancel() + + // Confirm the targeted pod is actually gone before checking Deployment + // availability. Deleting a pod does not bump the Deployment's + // ObservedGeneration, so its status can still read "Available" from the + // pre-deletion state and let waitForDeploymentAvailable return immediately + // without ever observing the restart. + if err := k.waitForPodGone(recoveryCtx, target, targetUID); err != nil { + return err + } + + // Wait for the Deployment to reschedule and become Available again. return k.waitForDeploymentAvailable(recoveryCtx) } +// waitForPodGone blocks until the pod identified by name/uid is deleted +// (NotFound) or replaced by a new pod with a different UID, guaranteeing the +// disruption has actually landed before we assert recovery. +func (k *KillOperatorPod) waitForPodGone(ctx context.Context, name string, uid types.UID) error { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + pod, err := k.clientset.CoreV1().Pods(k.namespace).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err == nil && pod.UID != uid { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("timed out waiting for operator pod %s to be deleted: %w", name, ctx.Err()) + case <-ticker.C: + } + } +} + // OutagePolicy: an operator restart is a control-plane fault that must not take // down the data plane, so it shares the near-zero NoOutagePolicy budget. func (k *KillOperatorPod) OutagePolicy() journal.OutagePolicy { @@ -131,10 +172,12 @@ func isDeploymentAvailable(dep *appsv1.Deployment) bool { return dep.Status.ReadyReplicas >= desired && dep.Status.UnavailableReplicas == 0 } -// oldestRunningPod returns the name of the oldest pod in the Running phase, or -// "" if none are running. Targeting the oldest makes the choice deterministic. -func oldestRunningPod(pods []corev1.Pod) string { +// oldestRunningPod returns the name and UID of the oldest pod in the Running +// phase, or ("", "") if none are running. Targeting the oldest makes the choice +// deterministic; the UID lets callers confirm that specific pod is later gone. +func oldestRunningPod(pods []corev1.Pod) (string, types.UID) { name := "" + var uid types.UID var oldest time.Time for i := range pods { p := &pods[i] @@ -144,8 +187,9 @@ func oldestRunningPod(pods []corev1.Pod) string { ts := p.CreationTimestamp.Time if name == "" || ts.Before(oldest) { name = p.Name + uid = p.UID oldest = ts } } - return name + return name, uid } diff --git a/test/longhaul/operations/kill_operator_test.go b/test/longhaul/operations/kill_operator_test.go index d409b5ab4..416241096 100644 --- a/test/longhaul/operations/kill_operator_test.go +++ b/test/longhaul/operations/kill_operator_test.go @@ -116,5 +116,20 @@ var _ = Describe("KillOperatorPod", func() { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no running operator pod")) }) + + It("refuses to run when the deployment has no matchLabels selector", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + dep.Spec.Selector = &metav1.LabelSelector{} + running := operatorPod("op-run", corev1.PodRunning, 10) + cs := fake.NewSimpleClientset(dep, running) + k := NewKillOperatorPod(cs, opNS, time.Minute) + + err := k.Execute(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no matchLabels selector")) + + _, getErr := cs.CoreV1().Pods(opNS).Get(context.Background(), "op-run", metav1.GetOptions{}) + Expect(getErr).NotTo(HaveOccurred(), "no pod should be deleted when the selector is empty") + }) }) }) diff --git a/test/longhaul/operations/kill_primary.go b/test/longhaul/operations/kill_primary.go index dbd960571..147219dd2 100644 --- a/test/longhaul/operations/kill_primary.go +++ b/test/longhaul/operations/kill_primary.go @@ -56,6 +56,12 @@ func (k *KillPrimaryPod) Execute(ctx context.Context) error { if err != nil { return fmt.Errorf("get primary instance: %w", err) } + if primary == "" { + return fmt.Errorf("get primary instance: cluster returned an empty primary pod name") + } + if k.healthMon == nil { + return fmt.Errorf("kill-primary-pod: health monitor is nil") + } if err := k.client.DeletePod(ctx, primary); err != nil { return fmt.Errorf("delete primary pod %s: %w", primary, err) } diff --git a/test/longhaul/operations/kill_primary_test.go b/test/longhaul/operations/kill_primary_test.go index 36c723006..cb72db4a9 100644 --- a/test/longhaul/operations/kill_primary_test.go +++ b/test/longhaul/operations/kill_primary_test.go @@ -72,4 +72,16 @@ var _ = Describe("KillPrimaryPod", func() { defer c.mu.Unlock() Expect(c.deletedPods).To(BeEmpty()) }) + + It("Execute fails without deleting when the primary name is empty", func() { + c := &fakeClient{instancesPerNode: 2, primary: ""} + k := NewKillPrimaryPod(c, nil, time.Second) + + err := k.Execute(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("empty primary pod name")) + c.mu.Lock() + defer c.mu.Unlock() + Expect(c.deletedPods).To(BeEmpty()) + }) }) From fb0a4a0a1eb401697af700a6051cd5a459a3fec9 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Fri, 21 Aug 2026 15:49:20 -0400 Subject: [PATCH 9/9] test(longhaul): add seeded coverage mode for random scheduler smoke gate Introduce a without-replacement "coverage mode" so the CI smoke gate exercises the production random scheduler path while still guaranteeing every operation runs at least once. Adds LONGHAUL_OPERATION_COVERAGE and LONGHAUL_OPERATION_SEED config, seeded RNG + completion-driven Run loop in the scheduler, and switches longhaul-smoke.yml from sequence to random+coverage with a jq assertion on operation-aggregates. Includes the operations registry/sequence/runner refactor, journal and report updates, and accompanying unit tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: dceb76cc-e7c1-40f2-92ad-3b6de007281c Signed-off-by: Wenting Wu --- .github/workflows/longhaul-smoke.yml | 139 ++++++++-- docs/designs/long-haul-test-design.md | 34 ++- test/longhaul/README.md | 42 ++- test/longhaul/cmd/longhaul/main.go | 156 ++++++++--- test/longhaul/cmd/longhaul/main_test.go | 89 +++++++ test/longhaul/cmd/longhaul/suite_test.go | 16 ++ test/longhaul/config/config.go | 103 ++++++++ test/longhaul/config/config_test.go | 135 ++++++++-- test/longhaul/deploy/deployment.yaml | 12 +- test/longhaul/journal/journal.go | 25 +- test/longhaul/journal/journal_test.go | 14 + test/longhaul/journal/policy.go | 5 +- test/longhaul/operations/kill_primary.go | 75 +++++- test/longhaul/operations/kill_primary_test.go | 64 ++++- test/longhaul/operations/registry.go | 88 +++++++ test/longhaul/operations/registry_test.go | 62 +++++ test/longhaul/operations/runner.go | 152 +++++++++++ test/longhaul/operations/scale_test.go | 24 +- test/longhaul/operations/scheduler.go | 196 +++++++++++++- test/longhaul/operations/scheduler_test.go | 88 +++++++ test/longhaul/operations/sequence.go | 247 ++++++++++++++++++ test/longhaul/operations/sequence_test.go | 217 +++++++++++++++ test/longhaul/report/checkpoint.go | 80 ++++-- test/longhaul/report/checkpoint_test.go | 97 ++++++- test/longhaul/report/report.go | 47 +++- test/longhaul/report/report_test.go | 37 +++ 26 files changed, 2048 insertions(+), 196 deletions(-) create mode 100644 test/longhaul/cmd/longhaul/main_test.go create mode 100644 test/longhaul/cmd/longhaul/suite_test.go create mode 100644 test/longhaul/operations/registry.go create mode 100644 test/longhaul/operations/registry_test.go create mode 100644 test/longhaul/operations/runner.go create mode 100644 test/longhaul/operations/sequence.go create mode 100644 test/longhaul/operations/sequence_test.go diff --git a/.github/workflows/longhaul-smoke.yml b/.github/workflows/longhaul-smoke.yml index 8d9e08df6..07ad69ee0 100644 --- a/.github/workflows/longhaul-smoke.yml +++ b/.github/workflows/longhaul-smoke.yml @@ -42,23 +42,33 @@ # longhaul-report ConfigMap's result field. # # CI budget -# Scale/upgrade disruption ops are disabled for the smoke run -# (LONGHAUL_MIN_INSTANCES == LONGHAUL_MAX_INSTANCES) so the gate is a fast, -# deterministic data-durability check (writers + verifier) that fits a -# GitHub-hosted runner and finishes in a few minutes. +# Scale ops run for real (MIN=2, MAX=3). The gate exercises every registered +# operation once and finishes within a GitHub-hosted runner's budget. +# +# Operation coverage (random mode) +# The smoke runs the real random scheduler — the exact path the multi-day +# long-haul run uses — but in coverage mode (LONGHAUL_OPERATION_COVERAGE) with +# a pinned seed (LONGHAUL_OPERATION_SEED). Coverage mode draws each operation +# without replacement and completes once every operation has run at least once, +# so the gate exercises scheduler.go's weighted selection, cooldown, and +# steady-state gates while still guaranteeing per-op coverage and a +# deterministic PASS/FAIL verdict. The upgrade uses a second tag for the same +# database image payload: this gates the rolling-update mechanics without +# turning this smoke test into a cross-version compatibility suite. MAX_DURATION +# is only the completion watchdog. # # Data-protection gate # The backup verifier is exercised for real, not just compiled in. The kind # cluster already has CSI VolumeSnapshot support (setup-test-environment runs # deploy-csi-driver.sh: external-snapshotter + a default csi-hostpath -# VolumeSnapshotClass), so a single-instance cluster can complete snapshot -# backups — exactly as the e2e scheduled-backup test proves. The smoke run -# sets a per-minute backup schedule and a 30s verify interval (vs the 5m -# default, via LONGHAUL_BACKUP_VERIFY_INTERVAL) so the verifier's periodic -# loop fires many times within the window and reliably observes a -# scheduled+completed backup. It then asserts scheduled AND completed >= 1 -# (with no retention leak or completion stall), so a broken backup path fails -# the PR rather than passing silently as a no-op. +# VolumeSnapshotClass), so the cluster can complete snapshot backups — +# exactly as the e2e scheduled-backup test proves. The smoke run sets a +# per-minute backup schedule and a 30s verify interval (vs the 5m default, +# via LONGHAUL_BACKUP_VERIFY_INTERVAL) so the verifier's periodic loop fires +# many times within the window and reliably observes a scheduled+completed +# backup. It then asserts scheduled AND completed >= 1 (with no retention leak +# or completion stall), so a broken backup path fails the PR rather than +# passing silently as a no-op. name: Long-Haul Smoke Gate @@ -74,9 +84,9 @@ on: workflow_dispatch: inputs: max_duration: - description: "Bounded driver run length (Go duration). Keep >= 6m so at least one per-minute backup is scheduled and completed within the window." + description: "Coverage watchdog duration (Go duration, e.g. 20m). Keep >= 6m so at least one per-minute backup is scheduled and completed within the window." required: false - default: "6m" + default: "20m" permissions: contents: read @@ -119,7 +129,7 @@ jobs: needs: build if: always() && needs.build.result == 'success' runs-on: ubuntu-22.04 - timeout-minutes: 40 + timeout-minutes: 50 env: IMAGE_TAG: ${{ needs.build.outputs.image_tag }} EXT_IMAGE_TAG: ${{ needs.build.outputs.ext_image_tag }} @@ -128,8 +138,10 @@ jobs: # Must match the cluster name the composite action derives: # documentdb--- KIND_CLUSTER: documentdb-longhaul-amd64-smoke - # The pruner's first tick is at 5m; the default must run beyond it. - MAX_DURATION: ${{ github.event.inputs.max_duration || '6m' }} + # Coverage watchdog; also kept beyond the 5m pruner tick and long enough + # for at least one per-minute backup to schedule and complete. + MAX_DURATION: ${{ github.event.inputs.max_duration || '20m' }} + UPGRADE_IMAGE_TAG: ${{ needs.build.outputs.ext_image_tag }}-smoke-upgrade steps: - name: Checkout uses: actions/checkout@v4 @@ -167,7 +179,7 @@ jobs: runner: "ubuntu-22.04" test-scenario-name: "smoke" node-count: "1" - instances-per-node: "1" + instances-per-node: "2" cert-manager-namespace: ${{ env.CERT_MANAGER_NS }} operator-namespace: ${{ env.OPERATOR_NS }} db-namespace: ${{ env.DB_NS }} @@ -198,6 +210,48 @@ jobs: --from-literal=uri="${URI}" \ --dry-run=client -o yaml | kubectl apply -f - + - name: Prepare deterministic upgrade target + run: | + set -euo pipefail + owner=$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]') + source_tag="${EXT_IMAGE_TAG}-amd64" + + # The operator resolves documentDBVersion against the canonical image + # repositories. Give the existing payload a second local tag so the + # upgrade operation performs a real rolling image-reference update + # without depending on a second database release. + for component in documentdb gateway; do + source="ghcr.io/${owner}/documentdb-kubernetes-operator/${component}:${source_tag}" + base_target="ghcr.io/documentdb/documentdb-kubernetes-operator/${component}:${source_tag}" + upgrade_target="ghcr.io/documentdb/documentdb-kubernetes-operator/${component}:${UPGRADE_IMAGE_TAG}" + docker image inspect "${source}" >/dev/null + docker tag "${source}" "${base_target}" + docker tag "${source}" "${upgrade_target}" + kind load docker-image "${base_target}" --name "${KIND_CLUSTER}" + kind load docker-image "${upgrade_target}" --name "${KIND_CLUSTER}" + done + + # setup-test-environment pins explicit image fields, which take + # precedence over documentDBVersion. Move to the equivalent + # version-based reference before starting the driver so its upgrade + # operation can change both database component image tags. + version_patch=$(BASE_VERSION="${source_tag}" jq -nc '{ + spec: { + documentDBVersion: env.BASE_VERSION, + image: { + documentDB: null, + gateway: null + } + } + }') + kubectl patch documentdb "${DB_NAME}" -n "${DB_NS}" \ + --type merge -p "${version_patch}" + + kubectl create configmap longhaul-versions \ + -n "${DB_NS}" \ + --from-literal="desired-documentdb-version=${UPGRADE_IMAGE_TAG}" \ + --dry-run=client -o yaml | kubectl apply -f - + - name: Deploy long-haul driver (real manifests, bounded override) run: | # RBAC applies unmodified (namespace matches DB_NS). @@ -215,11 +269,17 @@ jobs: # Bounded, deterministic smoke override — patch ONLY runtime knobs on # the shipped ConfigMap; the manifest structure is unchanged. - # - MAX_DURATION: finite run + # - MAX_DURATION: finite run (coverage-completion watchdog) # - RESET_DATA: fresh collection each CI run # - RETAIN_PER_WRITER: low enough to force a real prune at 5m - # - MIN==MAX instances: disable disruptive scale ops (fast + stable) + # - OPERATION_MODE=random + COVERAGE: run the real scheduler but draw + # each operation without replacement and finish once every operation + # (scale-up/-down, kill-operator-pod, kill-primary-pod, + # upgrade-documentdb) has run once — deterministic coverage of the + # production path. + # - OPERATION_SEED: pin selection so the run is reproducible. # - short cadences so the verifier gets several cycles in the window + # - short steady-state gate with a bounded recovery budget # - BACKUP_*: exercise the data-protection verifier for real — a # per-minute schedule so at least one backup is scheduled and # completed within the bounded window, plus a 30s verify interval @@ -233,12 +293,15 @@ jobs: LONGHAUL_RESET_DATA: "true", LONGHAUL_NUM_WRITERS: "2", LONGHAUL_RETAIN_PER_WRITER: "100", + LONGHAUL_OPERATION_MODE: "random", + LONGHAUL_OPERATION_COVERAGE: "true", + LONGHAUL_OPERATION_SEED: "1", LONGHAUL_OP_COOLDOWN: "30s", - LONGHAUL_STEADY_STATE_WAIT: "10s", - LONGHAUL_RECOVERY_TIMEOUT: "2m", + LONGHAUL_STEADY_STATE_WAIT: "5s", + LONGHAUL_RECOVERY_TIMEOUT: "5m", LONGHAUL_REPORT_INTERVAL: "30s", - LONGHAUL_MIN_INSTANCES: "1", - LONGHAUL_MAX_INSTANCES: "1", + LONGHAUL_MIN_INSTANCES: "2", + LONGHAUL_MAX_INSTANCES: "3", LONGHAUL_BACKUP_ENABLED: "true", LONGHAUL_BACKUP_SCHEDULE: "*/1 * * * *", LONGHAUL_BACKUP_RETENTION_DAYS: "1", @@ -255,7 +318,7 @@ jobs: id: wait run: | set -euo pipefail - deadline=$(( $(date +%s) + 900 )) # 15 min hard cap + deadline=$(( $(date +%s) + 1800 )) # 30 min hard cap exit_code="" while [[ $(date +%s) -lt ${deadline} ]]; do pod=$(kubectl get pods -n "${DB_NS}" \ @@ -301,8 +364,14 @@ jobs: -o jsonpath='{.data.result}' 2>/dev/null || echo "MISSING") report=$(kubectl get configmap longhaul-report -n "${DB_NS}" \ -o jsonpath='{.data.latest-report}' 2>/dev/null || echo "MISSING") - echo "Driver exit code : ${exit_code}" - echo "Report result : ${result}" + operation_status=$(kubectl get configmap longhaul-report -n "${DB_NS}" \ + -o json | jq -r '.data["operation-status"] // "MISSING"') + operation_aggregates=$(kubectl get configmap longhaul-report -n "${DB_NS}" \ + -o json | jq -r '.data["operation-aggregates"] // "MISSING"') + echo "Driver exit code : ${exit_code}" + echo "Report result : ${result}" + echo "Operation status : ${operation_status}" + echo "Operation aggregates: ${operation_aggregates}" if [[ "${exit_code}" != "0" ]]; then echo "::error::Driver exited non-zero (${exit_code})." @@ -312,11 +381,25 @@ jobs: echo "::error::longhaul-report result is '${result}', expected PASS." exit 1 fi + if [[ "${operation_status}" != "COMPLETE" ]]; then + echo "::error::operation-status is '${operation_status}', expected COMPLETE." + exit 1 + fi + # Coverage mode: assert every registered operation ran at least once + # (passed >= 1) and none failed. Order is not asserted — coverage, + # not sequence, is the guarantee. + if ! jq -e ' + (map(.name) | sort) == (["scale-up","scale-down","kill-operator-pod","kill-primary-pod","upgrade-documentdb"] | sort) and + all(.[]; .passed >= 1 and .failed == 0) + ' <<<"${operation_aggregates}" >/dev/null; then + echo "::error::operation-aggregates did not show every operation covered (passed>=1, failed==0)." + exit 1 + fi if ! grep -Eq 'pruner: pruned [1-9][0-9]* docs' <<<"${report}"; then echo "::error::Retention pruner did not report deleting any documents." exit 1 fi - echo "✅ Long-haul smoke gate passed (exit 0, report PASS, retention pruned documents)." + echo "✅ Long-haul smoke gate passed (random coverage COMPLETE, all operations covered, report PASS, retention pruned documents)." - name: Assert data-protection verifier ran run: | diff --git a/docs/designs/long-haul-test-design.md b/docs/designs/long-haul-test-design.md index 0fc0aea82..e2b0ad0e7 100644 --- a/docs/designs/long-haul-test-design.md +++ b/docs/designs/long-haul-test-design.md @@ -44,7 +44,7 @@ flowchart LR | Component | Role | Output | |---|---|---| | **Writer/Verifier** | Data-plane workload. Connects via `mongodb://` only — no k8s imports. Writers insert monotonic sequences with checksums under majority write concern; verifiers scan for gaps and bad checksums. | Counters (acked, failed, verify passes, gaps, checksum errors); errors to journal. | -| **Operation Scheduler** | Control plane. Applies weighted-random ops (scale, kill, failover, backup, upgrade) with preconditions and cooldowns. | Operation start/end events to journal. | +| **Operation Runner** | Control plane. Applies weighted-random ops for production long-haul runs, a deterministic named sequence for smoke/reproduction, or no ops when disabled. | Bounded per-operation results/aggregates plus operation events to journal. | | **Monitor** | Polls pod RSS/CPU and checks readiness of operator + DB pods. | Periodic samples + readiness events to journal. | | **Journal** | In-process append-only event log shared by all components. | Reproducible event stream for the report. | | **Report** | Aggregates the journal into a markdown summary at a configurable interval; raises alerts on threshold breaches. | Markdown report; alert lines. | @@ -77,7 +77,11 @@ The test runs **continuously** — no cycles, no scheduled resets. Workload, met ## Operations -The scheduler picks operations from these categories with weighted randomization: +Production runs use weighted randomization. Deterministic smoke and reproduction +runs can instead request a comma-separated sequence of stable operation names; +each operation runs exactly once in order and the driver exits as soon as the +sequence completes or fails. A disabled mode leaves the workload running without +management operations. | Category | Examples | |---|---| @@ -87,16 +91,34 @@ The scheduler picks operations from these categories with weighted randomization | **Chaos** | kill primary pod, drain node, kill operator pod | | **Data protection** | trigger backup, verify backup | -**Sequencing invariants** (enforced by the scheduler — exact values live in code): +**Operation invariants** (exact values live in code): -- One disruptive op at a time. Overlapping disruptions are non-diagnosable. -- Per-category cooldown between ops. Lets the cluster stabilize. -- Steady-state gate — health check must pass before the next op fires. +- One disruptive op at a time in every mode. Overlapping disruptions are + non-diagnosable. +- Random mode applies the global cooldown between attempts. +- The steady-state gate must pass before each operation. Sequence mode also + requires each named precondition to become true within the recovery timeout. **Backup is not isolated.** It runs concurrently with topology changes and chaos so that backup-vs-topology serialization bugs surface here rather than in production — that serialization is the backup feature's job, not the harness's. Each operation declares an **outage policy**: tolerated write failures during its disruption window and a max recovery time. Breaching the policy is recorded as a Tier-1 failure (see Failure Tiers). +Operation state is intentionally bounded for multi-day runs. Random mode keeps +only passed/failed counters per registered operation type; sequence mode keeps +one mutable `PENDING`/`RUNNING`/`PASSED`/`FAILED` result per requested item. +Execution errors, precondition timeouts, outage-policy violations, and an +incomplete sequence at shutdown all produce a failing final verdict. + +**Random coverage mode** (used by the PR smoke gate) is a variant of random +mode: the scheduler draws each operation *without replacement* and completes +once every registered operation has run at least once, rather than running for +the full duration. A fixed seed (`LONGHAUL_OPERATION_SEED`) makes selection +reproducible. This lets the smoke gate exercise the production scheduler path — +weighted selection, cooldown, and steady-state gates — while guaranteeing per-op +coverage and a deterministic PASS/FAIL verdict; `MAX_DURATION` becomes the +completion watchdog, and a run that stops before covering every operation is a +failing `INCOMPLETE` verdict. + --- ## Data Plane Workload diff --git a/test/longhaul/README.md b/test/longhaul/README.md index 8337dff89..bbf82e1b7 100644 --- a/test/longhaul/README.md +++ b/test/longhaul/README.md @@ -128,8 +128,13 @@ All configuration is via environment variables. | `LONGHAUL_OPERATOR_NAMESPACE` | No | `documentdb-operator` | Namespace of the DocumentDB operator Deployment (target of the `kill-operator-pod` chaos op). | | `LONGHAUL_MAX_DURATION` | No | `30m` | Max test duration. Use `0s` for run-until-failure. | | `LONGHAUL_NUM_WRITERS` | No | `5` | Number of concurrent writers. | +| `LONGHAUL_OPERATION_MODE` | No | `random` | Operation runner: `random`, `sequence`, or `disabled`. | +| `LONGHAUL_OPERATION_SEQUENCE` | No | empty | Comma-separated stable operation names. Required and used only in `sequence` mode; rejected in `random`/`disabled` mode. Whitespace is trimmed, and duplicate or unknown names are rejected. | +| `LONGHAUL_OPERATION_COVERAGE` | No | `false` | Only valid in `random` mode. When true, the scheduler draws each operation without replacement and completes once every operation has run at least once (instead of running until `MAX_DURATION`, which becomes a watchdog). Used by the smoke gate to exercise the real scheduler while guaranteeing per-op coverage. | +| `LONGHAUL_OPERATION_SEED` | No | unset | Only valid in `random` mode. Pins weighted-random selection to a fixed seed for reproducible runs. Unset uses the process-global generator (production behavior). | | `LONGHAUL_OP_COOLDOWN` | No | `5m` | Cooldown between management operations. | | `LONGHAUL_RECOVERY_TIMEOUT` | No | `5m` | Max wait for cluster recovery after an operation. | +| `LONGHAUL_STEADY_STATE_WAIT` | No | `60s` | Continuous healthy duration required by the steady-state gate. | | `LONGHAUL_MIN_INSTANCES` | No | `1` | Minimum `spec.instancesPerNode` for scale-down operations (CRD lower bound: 1). | | `LONGHAUL_MAX_INSTANCES` | No | `3` | Maximum `spec.instancesPerNode` for scale-up operations (CRD upper bound: 3). | | `LONGHAUL_REPORT_INTERVAL` | No | `1h` | How often to write checkpoint reports to ConfigMap. | @@ -184,8 +189,16 @@ accumulation window long-haul exists to cover. ## Operations -The scheduler runs one disruptive operation at a time, gated by steady state and -a global cooldown. Current operations: +`random` mode preserves the production long-haul behavior: the scheduler picks +weighted eligible operations every 10 seconds, runs one disruptive operation at +a time, and applies the global cooldown. `sequence` mode runs each configured +operation exactly once and in order, stopping on the first execution, +precondition, recovery, or policy failure; a successful or failed sequence +emits its final report and exits immediately instead of waiting for +`LONGHAUL_MAX_DURATION`. `disabled` mode runs no operations. All modes keep the +continuous writer/verifier workload active. + +Current stable operation names: | Operation | Kind | Notes | |-----------|------|-------| @@ -210,6 +223,13 @@ single primary handover (an ungraceful failover vs. a graceful switchover), and an upgrade's longer whole-topology restart is bounded by `MustRecoverWithin`, not the write-outage budget. +Operation execution failures are terminal verdict failures in both `random` and +`sequence` modes. Reports keep bounded operation state: one mutable result per +requested sequence item, or aggregate passed/failed counters per operation name +in random mode. The `longhaul-report` ConfigMap exposes `operation-status`, +`operation-results` JSON, and random-mode `operation-aggregates` JSON alongside +the existing `result` and `latest-report` fields. + ### RBAC for chaos operations Beyond the base RBAC the driver already needs, the chaos operations require the @@ -226,12 +246,24 @@ driver ServiceAccount to be granted (all present in `deploy/rbac.yaml`): ## CI Safety -The long haul test binary is deployed as a Kubernetes Deployment on a dedicated AKS -cluster. It does **not** run in any PR-gated CI workflow. Because a Deployment -auto-restarts crashed pods, the source of truth for "did the test pass?" is the +The production long haul test binary is deployed as a Kubernetes Deployment on +a dedicated AKS cluster. A short PR smoke workflow runs the same driver and +manifests against kind in **random coverage mode**. Because a Deployment +auto-restarts exited pods, the source of truth for "did the test pass?" is the `longhaul-report` ConfigMap and the GitHub Actions annotations, not the pod status. +The smoke gate runs the real random scheduler — the exact path the multi-day +run uses — but with `LONGHAUL_OPERATION_COVERAGE=true` and a pinned +`LONGHAUL_OPERATION_SEED`, so it draws each operation without replacement and +completes once every registered operation has run at least once: scale up, +scale down, kill the operator pod, kill the primary pod, and upgrade DocumentDB. +This exercises `scheduler.go`'s weighted selection, cooldown, and steady-state +gates while still guaranteeing per-op coverage and a deterministic verdict. The +upgrade gives the existing database images a second local tag, exercising the +rolling-update mechanics without conflating this gate with cross-version +compatibility testing. + The config unit tests (`test/longhaul/config/`) run unconditionally and are included in normal CI test runs — they are fast (~0.002s) and require no cluster. diff --git a/test/longhaul/cmd/longhaul/main.go b/test/longhaul/cmd/longhaul/main.go index 0bcbfe15c..3c69c549d 100644 --- a/test/longhaul/cmd/longhaul/main.go +++ b/test/longhaul/cmd/longhaul/main.go @@ -148,18 +148,16 @@ func run(cfg config.Config) int { j.Info("main", "retention pruning disabled (LONGHAUL_RETAIN_PER_WRITER=0)") } - // Configure operations. - ops := []operations.Operation{ - operations.NewScaleUp(clusterClient, healthMon, cfg.MaxInstances, cfg.RecoveryTimeout), - operations.NewScaleDown(clusterClient, healthMon, cfg.MinInstances, cfg.RecoveryTimeout), - operations.NewUpgradeDocumentDB(clusterClient, k8sClientset, healthMon, j, cfg.Namespace, cfg.RecoveryTimeout), - operations.NewKillOperatorPod(k8sClientset, cfg.OperatorNamespace, cfg.RecoveryTimeout), - operations.NewKillPrimaryPod(clusterClient, healthMon, cfg.RecoveryTimeout), + // Build the operation registry once, then select the configured runner. + registry, err := operations.NewDefaultRegistry(cfg, clusterClient, k8sClientset, healthMon, j) + if err != nil { + log.Fatalf("failed to build operation registry: %v", err) } - - // Start operation scheduler. - scheduler := operations.NewScheduler(ops, healthMon, j, cfg.OpCooldown) - go scheduler.Run(ctx) + opRunner, err := newOperationRunner(cfg, registry, healthMon, j) + if err != nil { + log.Fatalf("failed to configure operation runner: %v", err) + } + go opRunner.Run(ctx) // Start data-protection verifier (ScheduledBackup + retention). Runs // concurrently with the scheduler by design — backup is deliberately not @@ -187,32 +185,48 @@ func run(cfg config.Config) int { go runMetricsSampling(ctx, clusterClient, leakDetector, j) // Start periodic checkpoint reporter. - summaryFunc := func() report.Summary { - return buildSummary(metrics, backupMetrics, leakDetector, scheduler, j) + summaryFunc := func(final bool) report.Summary { + return buildSummary(metrics, backupMetrics, leakDetector, opRunner, j, final) } reporter := report.NewCheckpointReporter(k8sClientset, cfg.Namespace, cfg.ReportInterval, summaryFunc) go reporter.Run(ctx) j.Info("main", "all components started, entering main loop") - // Main loop: wait for context expiry. - <-ctx.Done() - j.Info("main", fmt.Sprintf("test ending: %v", ctx.Err())) + // Sequence mode and random coverage mode are completion-driven: they exit as + // soon as their operations have finished (or a failure occurs); MaxDuration + // is only their watchdog. Plain random and disabled modes are duration-driven. + completionDriven := cfg.OperationMode == config.OperationModeSequence || + (cfg.OperationMode == config.OperationModeRandom && cfg.OperationCoverage) + if completionDriven { + select { + case <-opRunner.Done(): + j.Info("main", "operations finished") + case <-ctx.Done(): + j.Info("main", fmt.Sprintf("operations watchdog fired: %v", ctx.Err())) + if sr, ok := opRunner.(*operations.SequenceRunner); ok { + sr.MarkIncomplete( + fmt.Sprintf("operation sequence incomplete: watchdog fired: %v", ctx.Err()), + ) + } + // Coverage runners publish their own terminal (incomplete) snapshot + // once cancellation unwinds their loop; wait for that to land. + <-opRunner.Done() + } + } else { + <-ctx.Done() + j.Info("main", fmt.Sprintf("test ending: %v", ctx.Err())) + <-opRunner.Done() + } + cancel() // Allow goroutines to flush. time.Sleep(500 * time.Millisecond) - // Generate final report. Persist to the report ConfigMap synchronously - // here (before os.Exit) so the authoritative verdict reaches the source - // of truth that operators consult — the Run() goroutine cannot do this - // reliably because os.Exit can kill it mid-Update. - summary := buildSummary(metrics, backupMetrics, leakDetector, scheduler, j) - markdown := report.GenerateMarkdown(summary) - fmt.Println("\n" + markdown) - reporter.EmitFinal() - - // Emit final GitHub Actions annotation. - report.EmitAnnotation(summary) + // Emit exactly one terminal report synchronously before os.Exit. EmitFinal + // prints the markdown, emits the GitHub Actions annotation, and persists the + // authoritative verdict to the report ConfigMap. + summary := reporter.EmitFinal() if summary.Result == report.ResultFail { log.Printf("TEST FAILED: %s", summary.FailReason) @@ -223,36 +237,84 @@ func run(cfg config.Config) int { return 0 } +func newOperationRunner( + cfg config.Config, + registry *operations.Registry, + health *monitor.HealthMonitor, + j *journal.Journal, +) (operations.Runner, error) { + switch cfg.OperationMode { + case config.OperationModeRandom: + opts := make([]operations.SchedulerOption, 0, 2) + if cfg.OperationCoverage { + opts = append(opts, operations.WithCoverage()) + } + if cfg.OperationSeedSet { + opts = append(opts, operations.WithSeed(cfg.OperationSeed)) + } + return operations.NewScheduler(registry.All(), health, j, cfg.OpCooldown, opts...), nil + case config.OperationModeSequence: + ops, err := registry.Resolve(cfg.OperationSequence) + if err != nil { + return nil, err + } + return operations.NewSequenceRunner(ops, health, j, cfg.RecoveryTimeout), nil + case config.OperationModeDisabled: + return operations.NewDisabledRunner(), nil + default: + return nil, fmt.Errorf("unsupported operation mode %q", cfg.OperationMode) + } +} + // buildSummary constructs a report.Summary from current state. -func buildSummary(metrics *workload.Metrics, backupMetrics *backup.Metrics, leakDetector *monitor.LeakDetector, scheduler *operations.Scheduler, j *journal.Journal) report.Summary { +func buildSummary( + metrics *workload.Metrics, + backupMetrics *backup.Metrics, + leakDetector *monitor.LeakDetector, + opRunner operations.Runner, + j *journal.Journal, + final bool, +) report.Summary { snap := metrics.Snapshot() backupSnap := backupMetrics.Snapshot() leakAnalysis := leakDetector.Analyze() + operationRun := opRunner.Snapshot() result := report.ResultPass failReason := "" - appendReason := func(msg string) { + if snap.HasDataLoss() { + result = report.ResultFail + failReason = appendFailReason(failReason, fmt.Sprintf("data loss: %d gaps, %d checksum errors", + snap.GapsDetected, snap.ChecksumErrors)) + } + if operationRun.HasFailure() { result = report.ResultFail - if failReason != "" { - failReason += "; " + failReason = appendFailReason(failReason, operationRun.FailureReason) + if operationRun.FailureReason == "" { + failReason = appendFailReason(failReason, "operation execution failed") } - failReason += msg } - - if snap.HasDataLoss() { - appendReason(fmt.Sprintf("data loss: %d gaps, %d checksum errors", - snap.GapsDetected, snap.ChecksumErrors)) + if final && + operationRun.Mode == config.OperationModeSequence && + operationRun.Status != operations.RunStatusComplete && + !operationRun.HasFailure() { + result = report.ResultFail + failReason = appendFailReason(failReason, + fmt.Sprintf("operation sequence incomplete (status %s)", operationRun.Status)) } if j.HasPolicyViolation() { - appendReason("outage policy violated") + result = report.ResultFail + failReason = appendFailReason(failReason, "outage policy violated") } if backupSnap.HasRetentionLeak() { - appendReason(fmt.Sprintf("backup retention leak: %d expired backups not collected", + result = report.ResultFail + failReason = appendFailReason(failReason, fmt.Sprintf("backup retention leak: %d expired backups not collected", backupSnap.RetentionLeaks)) } if backupSnap.HasCompletionStall() { - appendReason(fmt.Sprintf("backup completion stalled: %d backups scheduled with no completion", + result = report.ResultFail + failReason = appendFailReason(failReason, fmt.Sprintf("backup completion stalled: %d backups scheduled with no completion", backupSnap.MaxScheduledWithoutCompletion)) } @@ -262,13 +324,27 @@ func buildSummary(metrics *workload.Metrics, backupMetrics *backup.Metrics, leak Metrics: snap, Backup: backupSnap, LeakAnalysis: leakAnalysis, - OpsExecuted: scheduler.OpsExecuted(), + OpsExecuted: operationRun.OpsExecuted(), + OperationRun: operationRun, Windows: j.DisruptionWindows(), Events: j.Events(), FailReason: failReason, } } +func appendFailReason(existing, reason string) string { + if reason == "" { + return existing + } + if existing == "" { + return reason + } + if existing == reason { + return existing + } + return existing + "; " + reason +} + // runMetricsSampling periodically collects pod resource metrics and feeds the leak detector. func runMetricsSampling(ctx context.Context, client *monitor.K8sClusterClient, ld *monitor.LeakDetector, j *journal.Journal) { if !client.MetricsAvailable() { diff --git a/test/longhaul/cmd/longhaul/main_test.go b/test/longhaul/cmd/longhaul/main_test.go new file mode 100644 index 000000000..921efacb2 --- /dev/null +++ b/test/longhaul/cmd/longhaul/main_test.go @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/backup" + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + "github.com/documentdb/documentdb-operator/test/longhaul/monitor" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" + "github.com/documentdb/documentdb-operator/test/longhaul/report" + "github.com/documentdb/documentdb-operator/test/longhaul/workload" +) + +type snapshotRunner struct { + snapshot operations.RunSnapshot + done chan struct{} +} + +func (r *snapshotRunner) Run(context.Context) {} +func (r *snapshotRunner) Snapshot() operations.RunSnapshot { return r.snapshot } +func (r *snapshotRunner) Done() <-chan struct{} { return r.done } + +func summaryFor(snapshot operations.RunSnapshot, final bool) report.Summary { + j := journal.New() + return buildSummary( + workload.NewMetrics(), + backup.NewMetrics(), + monitor.NewLeakDetector(j, 10, 10), + &snapshotRunner{snapshot: snapshot, done: make(chan struct{})}, + j, + final, + ) +} + +var _ = Describe("buildSummary operation verdicts", func() { + It("fails random mode when any execution failed", func() { + summary := summaryFor(operations.RunSnapshot{ + Mode: config.OperationModeRandom, + Status: operations.RunStatusFailed, + FailureReason: "operation scale-up execute failed: boom", + Aggregates: []operations.OperationAggregate{ + {Name: "scale-up", Failed: 1}, + }, + }, true) + + Expect(summary.Result).To(Equal(report.ResultFail)) + Expect(summary.FailReason).To(ContainSubstring("execute failed")) + }) + + It("allows an in-progress sequence at a checkpoint", func() { + summary := summaryFor(operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusRunning, + Results: []operations.OperationResult{ + {Name: "kill-operator-pod", Status: operations.OperationPending}, + }, + }, false) + Expect(summary.Result).To(Equal(report.ResultPass)) + }) + + It("fails an incomplete requested sequence at final shutdown", func() { + summary := summaryFor(operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusRunning, + Results: []operations.OperationResult{ + {Name: "kill-operator-pod", Status: operations.OperationRunning}, + {Name: "kill-primary-pod", Status: operations.OperationPending}, + }, + }, true) + + Expect(summary.Result).To(Equal(report.ResultFail)) + Expect(summary.FailReason).To(ContainSubstring("operation sequence incomplete")) + }) + + It("does not impose an operation completion requirement in disabled mode", func() { + summary := summaryFor(operations.RunSnapshot{ + Mode: config.OperationModeDisabled, + Status: operations.RunStatusDisabled, + }, true) + Expect(summary.Result).To(Equal(report.ResultPass)) + }) +}) diff --git a/test/longhaul/cmd/longhaul/suite_test.go b/test/longhaul/cmd/longhaul/suite_test.go new file mode 100644 index 000000000..27e6e2bdf --- /dev/null +++ b/test/longhaul/cmd/longhaul/suite_test.go @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestLonghaulMain(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Long Haul Main Suite") +} diff --git a/test/longhaul/config/config.go b/test/longhaul/config/config.go index 2738192f1..1bfe5c871 100644 --- a/test/longhaul/config/config.go +++ b/test/longhaul/config/config.go @@ -28,6 +28,15 @@ const ( EnvOpCooldown = "LONGHAUL_OP_COOLDOWN" EnvRecoveryTimeout = "LONGHAUL_RECOVERY_TIMEOUT" EnvSteadyStateWait = "LONGHAUL_STEADY_STATE_WAIT" + EnvOperationMode = "LONGHAUL_OPERATION_MODE" + EnvOperationSeq = "LONGHAUL_OPERATION_SEQUENCE" + // EnvOperationCoverage enables coverage mode for random operation mode: the + // scheduler draws each operation without replacement and completes once every + // operation has run at least once (instead of running until MaxDuration). + EnvOperationCoverage = "LONGHAUL_OPERATION_COVERAGE" + // EnvOperationSeed pins the scheduler's weighted-random selection to a fixed + // seed so a random-mode run is reproducible. Only valid in random mode. + EnvOperationSeed = "LONGHAUL_OPERATION_SEED" // Scale operation bounds. The DocumentDB CRD hard-caps spec.nodeCount=1, // so the scale dimension actually exercised is spec.instancesPerNode (1-3). EnvMinInstances = "LONGHAUL_MIN_INSTANCES" @@ -55,6 +64,15 @@ const ( // roughly 55 hours of history per writer while bounding steady-state disk use. const DefaultRetainPerWriter = 2_000_000 +// OperationMode controls how disruptive operations are run. +type OperationMode string + +const ( + OperationModeRandom OperationMode = "random" + OperationModeSequence OperationMode = "sequence" + OperationModeDisabled OperationMode = "disabled" +) + // Config holds all configuration for a long haul test run. type Config struct { // MaxDuration is the maximum test duration. Zero means run until failure. @@ -85,6 +103,24 @@ type Config struct { // SteadyStateWait is how long the cluster must be healthy before an operation fires. SteadyStateWait time.Duration + // OperationMode selects weighted-random, deterministic sequence, or no operations. + OperationMode OperationMode + + // OperationSequence is the ordered list used only in sequence mode. + OperationSequence []string + + // OperationCoverage, valid only in random mode, makes the scheduler draw each + // operation without replacement and finish once every operation has run at + // least once. This gates the real scheduler path while guaranteeing per-op + // coverage for the smoke gate; MaxDuration becomes a watchdog. + OperationCoverage bool + + // OperationSeed pins weighted-random selection for reproducibility. Only used + // in random mode. OperationSeedSet distinguishes an explicit 0 from "unset" + // (unset uses the process-global generator, i.e. production behavior). + OperationSeed int64 + OperationSeedSet bool + // MinInstances is the minimum spec.instancesPerNode for scale-down. // CRD lower bound is 1. MinInstances int @@ -135,6 +171,7 @@ func DefaultConfig() Config { OpCooldown: 5 * time.Minute, RecoveryTimeout: 5 * time.Minute, SteadyStateWait: 60 * time.Second, + OperationMode: OperationModeRandom, MinInstances: 1, MaxInstances: 3, ReportInterval: 1 * time.Hour, @@ -209,6 +246,31 @@ func LoadFromEnv() (Config, error) { cfg.SteadyStateWait = d } + if v := strings.TrimSpace(os.Getenv(EnvOperationMode)); v != "" { + cfg.OperationMode = OperationMode(strings.ToLower(v)) + } + + if v := strings.TrimSpace(os.Getenv(EnvOperationSeq)); v != "" { + sequence, err := parseOperationSequence(v) + if err != nil { + return cfg, fmt.Errorf("invalid %s=%q: %w", EnvOperationSeq, v, err) + } + cfg.OperationSequence = sequence + } + + if v := strings.TrimSpace(strings.ToLower(os.Getenv(EnvOperationCoverage))); v != "" { + cfg.OperationCoverage = v == "true" || v == "1" || v == "yes" + } + + if v := strings.TrimSpace(os.Getenv(EnvOperationSeed)); v != "" { + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return cfg, fmt.Errorf("invalid %s=%q: %w", EnvOperationSeed, v, err) + } + cfg.OperationSeed = n + cfg.OperationSeedSet = true + } + if v := os.Getenv(EnvMinInstances); v != "" { n, err := strconv.Atoi(v) if err != nil { @@ -295,6 +357,34 @@ func (c *Config) Validate() error { if c.RecoveryTimeout <= 0 { return fmt.Errorf("recovery timeout must be positive, got %s", c.RecoveryTimeout) } + switch c.OperationMode { + case OperationModeRandom, OperationModeDisabled: + if len(c.OperationSequence) > 0 { + return fmt.Errorf("operation sequence must be empty when operation mode is %q", c.OperationMode) + } + case OperationModeSequence: + if len(c.OperationSequence) == 0 { + return fmt.Errorf("operation sequence must not be empty when operation mode is %q", c.OperationMode) + } + seen := make(map[string]struct{}, len(c.OperationSequence)) + for _, name := range c.OperationSequence { + if _, ok := seen[name]; ok { + return fmt.Errorf("operation sequence contains duplicate name %q", name) + } + seen[name] = struct{}{} + } + default: + return fmt.Errorf("operation mode must be one of %q, %q, or %q, got %q", + OperationModeRandom, OperationModeSequence, OperationModeDisabled, c.OperationMode) + } + if c.OperationCoverage && c.OperationMode != OperationModeRandom { + return fmt.Errorf("operation coverage is only supported in %q mode, got %q", + OperationModeRandom, c.OperationMode) + } + if c.OperationSeedSet && c.OperationMode != OperationModeRandom { + return fmt.Errorf("operation seed is only supported in %q mode, got %q", + OperationModeRandom, c.OperationMode) + } if c.MinInstances < 1 { return fmt.Errorf("min instances must be at least 1, got %d", c.MinInstances) } @@ -321,6 +411,19 @@ func (c *Config) Validate() error { return nil } +func parseOperationSequence(value string) ([]string, error) { + parts := strings.Split(value, ",") + sequence := make([]string, 0, len(parts)) + for _, part := range parts { + name := strings.TrimSpace(part) + if name == "" { + return nil, fmt.Errorf("operation names must not be empty") + } + sequence = append(sequence, name) + } + return sequence, nil +} + // IsEnabled returns true if the long haul test is explicitly enabled // via the LONGHAUL_ENABLED environment variable. func IsEnabled() bool { diff --git a/test/longhaul/config/config_test.go b/test/longhaul/config/config_test.go index fac762cab..78ed0c3ea 100644 --- a/test/longhaul/config/config_test.go +++ b/test/longhaul/config/config_test.go @@ -22,6 +22,8 @@ var _ = Describe("Config", func() { Expect(cfg.OpCooldown).To(Equal(5 * time.Minute)) Expect(cfg.RecoveryTimeout).To(Equal(5 * time.Minute)) Expect(cfg.SteadyStateWait).To(Equal(60 * time.Second)) + Expect(cfg.OperationMode).To(Equal(OperationModeRandom)) + Expect(cfg.OperationSequence).To(BeEmpty()) Expect(cfg.MinInstances).To(Equal(1)) Expect(cfg.MaxInstances).To(Equal(3)) Expect(cfg.RetainPerWriter).To(Equal(int64(DefaultRetainPerWriter))) @@ -37,6 +39,8 @@ var _ = Describe("Config", func() { EnvOperatorNamespace, EnvDocumentDBURI, EnvNumWriters, EnvOpCooldown, EnvRecoveryTimeout, EnvSteadyStateWait, + EnvOperationMode, EnvOperationSeq, + EnvOperationCoverage, EnvOperationSeed, EnvMinInstances, EnvMaxInstances, EnvReportInterval, EnvBackupEnabled, EnvBackupSchedule, EnvBackupRetentionDays, EnvBackupVerifyInterval, @@ -118,51 +122,48 @@ var _ = Describe("Config", func() { Expect(cfg.DocumentDBURI).To(Equal("mongodb://localhost:27017")) }) - It("parses the backup env knobs", func() { - GinkgoT().Setenv(EnvBackupEnabled, "true") - GinkgoT().Setenv(EnvBackupSchedule, "0 */6 * * *") - GinkgoT().Setenv(EnvBackupRetentionDays, "7") - GinkgoT().Setenv(EnvBackupVerifyInterval, "30s") + It("returns error for invalid RetainPerWriter", func() { + GinkgoT().Setenv(EnvRetainPerWriter, "not-a-number") + _, err := LoadFromEnv() + Expect(err).To(MatchError(ContainSubstring(EnvRetainPerWriter))) + }) + + It("normalizes operation mode and trims sequence names", func() { + GinkgoT().Setenv(EnvOperationMode, " Sequence ") + GinkgoT().Setenv(EnvOperationSeq, " kill-operator-pod, kill-primary-pod ") cfg, err := LoadFromEnv() Expect(err).NotTo(HaveOccurred()) - Expect(cfg.BackupEnabled).To(BeTrue()) - Expect(cfg.BackupSchedule).To(Equal("0 */6 * * *")) - Expect(cfg.BackupRetentionDays).To(Equal(7)) - Expect(cfg.BackupVerifyInterval).To(Equal(30 * time.Second)) + Expect(cfg.OperationMode).To(Equal(OperationModeSequence)) + Expect(cfg.OperationSequence).To(Equal([]string{"kill-operator-pod", "kill-primary-pod"})) }) - It("returns error for invalid BackupRetentionDays", func() { - GinkgoT().Setenv(EnvBackupRetentionDays, "abc") + It("rejects empty names in a non-empty sequence", func() { + GinkgoT().Setenv(EnvOperationSeq, "scale-up, ,scale-down") _, err := LoadFromEnv() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring(EnvBackupRetentionDays)) + Expect(err).To(MatchError(ContainSubstring("operation names must not be empty"))) }) - It("returns error for invalid BackupVerifyInterval", func() { - GinkgoT().Setenv(EnvBackupVerifyInterval, "not-a-duration") - _, err := LoadFromEnv() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring(EnvBackupVerifyInterval)) - }) - - It("parses RetainPerWriter from env", func() { - GinkgoT().Setenv(EnvRetainPerWriter, "500000") + It("parses operation coverage and seed in random mode", func() { + GinkgoT().Setenv(EnvOperationMode, "random") + GinkgoT().Setenv(EnvOperationCoverage, "true") + GinkgoT().Setenv(EnvOperationSeed, "-7") cfg, err := LoadFromEnv() Expect(err).NotTo(HaveOccurred()) - Expect(cfg.RetainPerWriter).To(Equal(int64(500_000))) + Expect(cfg.OperationCoverage).To(BeTrue()) + Expect(cfg.OperationSeed).To(Equal(int64(-7))) + Expect(cfg.OperationSeedSet).To(BeTrue()) }) - It("parses RetainPerWriter=0 to disable pruning", func() { - GinkgoT().Setenv(EnvRetainPerWriter, "0") + It("leaves OperationSeedSet false when the seed env is unset", func() { cfg, err := LoadFromEnv() Expect(err).NotTo(HaveOccurred()) - Expect(cfg.RetainPerWriter).To(BeZero()) + Expect(cfg.OperationSeedSet).To(BeFalse()) }) - It("returns error for invalid RetainPerWriter", func() { - GinkgoT().Setenv(EnvRetainPerWriter, "not-a-number") + It("returns error for an invalid operation seed", func() { + GinkgoT().Setenv(EnvOperationSeed, "not-a-number") _, err := LoadFromEnv() - Expect(err).To(MatchError(ContainSubstring(EnvRetainPerWriter))) + Expect(err).To(MatchError(ContainSubstring(EnvOperationSeed))) }) }) @@ -213,6 +214,82 @@ var _ = Describe("Config", func() { Expect(cfg.Validate()).To(MatchError(ContainSubstring("recovery timeout"))) }) + It("fails for an unknown operation mode", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = "roulette" + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operation mode must be one of"))) + }) + + It("requires a non-empty sequence in sequence mode", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = OperationModeSequence + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operation sequence must not be empty"))) + }) + + It("rejects duplicate sequence names", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = OperationModeSequence + cfg.OperationSequence = []string{"scale-up", "scale-up"} + Expect(cfg.Validate()).To(MatchError(ContainSubstring(`duplicate name "scale-up"`))) + }) + + DescribeTable("rejects a sequence outside sequence mode", + func(mode OperationMode) { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = mode + cfg.OperationSequence = []string{"scale-up"} + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operation sequence must be empty"))) + }, + Entry("random", OperationModeRandom), + Entry("disabled", OperationModeDisabled), + ) + + It("accepts coverage and seed in random mode", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = OperationModeRandom + cfg.OperationCoverage = true + cfg.OperationSeed = 99 + cfg.OperationSeedSet = true + Expect(cfg.Validate()).To(Succeed()) + }) + + DescribeTable("rejects coverage outside random mode", + func(mode OperationMode) { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = mode + if mode == OperationModeSequence { + cfg.OperationSequence = []string{"scale-up"} + } + cfg.OperationCoverage = true + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operation coverage is only supported"))) + }, + Entry("sequence", OperationModeSequence), + Entry("disabled", OperationModeDisabled), + ) + + It("rejects a seed outside random mode", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = OperationModeSequence + cfg.OperationSequence = []string{"scale-up"} + cfg.OperationSeedSet = true + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operation seed is only supported"))) + }) + + It("accepts a valid sequence configuration", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = OperationModeSequence + cfg.OperationSequence = []string{"kill-operator-pod", "kill-primary-pod"} + Expect(cfg.Validate()).To(Succeed()) + }) + It("fails when MaxInstances < MinInstances", func() { cfg := DefaultConfig() cfg.ClusterName = "test" diff --git a/test/longhaul/deploy/deployment.yaml b/test/longhaul/deploy/deployment.yaml index 835e577fc..fecc7cdf2 100644 --- a/test/longhaul/deploy/deployment.yaml +++ b/test/longhaul/deploy/deployment.yaml @@ -15,10 +15,10 @@ # driver pods concurrently against the same DocumentDB cluster / # workload collection). # -# Failure semantics: on critical failure (data loss, policy violation) -# the driver exits non-zero. The Deployment auto-restarts the pod, which -# gives MTBF data; the alert workflow polls the report ConfigMap and -# pages on incident-count thresholds. +# Failure semantics: on critical failure (data loss, operation failure, or +# policy violation) the driver exits non-zero. The Deployment auto-restarts +# the pod, which gives MTBF data; the alert workflow polls the report ConfigMap +# and pages on incident-count thresholds. # # Image refs are templated; the longhaul-deploy workflow substitutes: # __OWNER__ -> lowercased ${{ github.repository_owner }} @@ -42,6 +42,10 @@ data: # Writer/verifier counts. LONGHAUL_NUM_WRITERS: "5" # Operation scheduling. + # random preserves the production long-haul behavior. sequence executes + # LONGHAUL_OPERATION_SEQUENCE exactly once in order; disabled runs no ops. + LONGHAUL_OPERATION_MODE: "random" + LONGHAUL_OPERATION_SEQUENCE: "" LONGHAUL_OP_COOLDOWN: "10m" LONGHAUL_RECOVERY_TIMEOUT: "5m" # How long the cluster must be observed healthy before the next diff --git a/test/longhaul/journal/journal.go b/test/longhaul/journal/journal.go index 2ad071682..57b886b18 100644 --- a/test/longhaul/journal/journal.go +++ b/test/longhaul/journal/journal.go @@ -24,8 +24,9 @@ const ( // trim cost is amortized over many appends (one copy every trimHeadroom // events), not paid on every append once we hit the cap. const ( - maxEvents = 10000 - trimHeadroom = 1000 + maxEvents = 10000 + trimHeadroom = 1000 + maxDisruptionWindows = 1000 ) // Event represents a single journal entry. @@ -134,7 +135,7 @@ func (j *Journal) OpenDisruptionWindow(operationName string, policy OutagePolicy // Close any existing window first. if j.activeWindow != nil { j.activeWindow.EndTime = time.Now() - j.closedWindows = append(j.closedWindows, *j.activeWindow) + j.appendClosedWindow(*j.activeWindow) } j.activeWindow = &DisruptionWindow{ @@ -152,17 +153,18 @@ func (j *Journal) OpenDisruptionWindow(operationName string, policy OutagePolicy }) } -// CloseDisruptionWindow ends the active disruption period. -func (j *Journal) CloseDisruptionWindow() { +// CloseDisruptionWindow ends the active disruption period and returns a copy. +func (j *Journal) CloseDisruptionWindow() *DisruptionWindow { j.mu.Lock() defer j.mu.Unlock() if j.activeWindow == nil { - return + return nil } j.activeWindow.EndTime = time.Now() - j.closedWindows = append(j.closedWindows, *j.activeWindow) + j.appendClosedWindow(*j.activeWindow) + closed := *j.activeWindow j.events = append(j.events, Event{ Timestamp: time.Now(), @@ -173,6 +175,15 @@ func (j *Journal) CloseDisruptionWindow() { }) j.activeWindow = nil + return &closed +} + +func (j *Journal) appendClosedWindow(window DisruptionWindow) { + j.closedWindows = append(j.closedWindows, window) + if len(j.closedWindows) > maxDisruptionWindows { + copy(j.closedWindows, j.closedWindows[len(j.closedWindows)-maxDisruptionWindows:]) + j.closedWindows = j.closedWindows[:maxDisruptionWindows] + } } // RecordWriteFailure increments the failure count for the active disruption window. diff --git a/test/longhaul/journal/journal_test.go b/test/longhaul/journal/journal_test.go index 43c159db6..d70277020 100644 --- a/test/longhaul/journal/journal_test.go +++ b/test/longhaul/journal/journal_test.go @@ -80,6 +80,20 @@ var _ = Describe("Journal", func() { j := New() Expect(func() { j.RecordWriteFailure() }).NotTo(Panic()) }) + + It("bounds closed disruption-window diagnostics to the newest entries", func() { + j := New() + total := maxDisruptionWindows + 5 + for i := 0; i < total; i++ { + j.OpenDisruptionWindow(fmt.Sprintf("op-%d", i), DefaultOutagePolicy()) + j.CloseDisruptionWindow() + } + + windows := j.DisruptionWindows() + Expect(windows).To(HaveLen(maxDisruptionWindows)) + Expect(windows[0].OperationName).To(Equal("op-5")) + Expect(windows[len(windows)-1].OperationName).To(Equal(fmt.Sprintf("op-%d", total-1))) + }) }) Describe("HasPolicyViolation", func() { diff --git a/test/longhaul/journal/policy.go b/test/longhaul/journal/policy.go index c6226877d..9c985c162 100644 --- a/test/longhaul/journal/policy.go +++ b/test/longhaul/journal/policy.go @@ -10,6 +10,7 @@ import "time" // independently (ExceededPolicy trips if either is exceeded): MaxWriteOutage // bounds client-visible write availability, while MustRecoverWithin bounds the // cluster's return to its full declared topology (all pods Ready, CR Ready). +// Operation execution errors also fail the run independently of this policy. // Each can be violated while the other is fine — e.g. after a failover writes // resume quickly (MaxWriteOutage happy) yet the cluster stays degraded until a // replacement standby rejoins, which only MustRecoverWithin catches. @@ -23,9 +24,7 @@ type OutagePolicy struct { MaxWriteOutage time.Duration // MustRecoverWithin is the maximum time from operation start to full cluster - // recovery (steady state). Because a failed op is only logged, not counted - // toward the run verdict, this is the sole mechanism that turns a cluster - // that never converges back into a FAIL. + // recovery (steady state). MustRecoverWithin time.Duration } diff --git a/test/longhaul/operations/kill_primary.go b/test/longhaul/operations/kill_primary.go index 147219dd2..ec4c4f39e 100644 --- a/test/longhaul/operations/kill_primary.go +++ b/test/longhaul/operations/kill_primary.go @@ -17,17 +17,19 @@ import ( // steady state within the recovery budget. The continuous workload verifier // independently catches any data loss caused by the failover. type KillPrimaryPod struct { - client monitor.ClusterClient - healthMon *monitor.HealthMonitor - recovery time.Duration + client monitor.ClusterClient + healthMon SteadyStateGate + recovery time.Duration + primaryPollInterval time.Duration } // NewKillPrimaryPod creates a KillPrimaryPod operation. -func NewKillPrimaryPod(client monitor.ClusterClient, health *monitor.HealthMonitor, recovery time.Duration) *KillPrimaryPod { +func NewKillPrimaryPod(client monitor.ClusterClient, health SteadyStateGate, recovery time.Duration) *KillPrimaryPod { return &KillPrimaryPod{ - client: client, - healthMon: health, - recovery: recovery, + client: client, + healthMon: health, + recovery: recovery, + primaryPollInterval: time.Second, } } @@ -62,14 +64,63 @@ func (k *KillPrimaryPod) Execute(ctx context.Context) error { if k.healthMon == nil { return fmt.Errorf("kill-primary-pod: health monitor is nil") } - if err := k.client.DeletePod(ctx, primary); err != nil { - return fmt.Errorf("delete primary pod %s: %w", primary, err) - } - // Wait for CNPG to elect a new primary and the cluster to settle. recoveryCtx, cancel := context.WithTimeout(ctx, k.recovery) defer cancel() - return k.healthMon.WaitForSteadyState(recoveryCtx) + + if err := k.client.DeletePod(recoveryCtx, primary); err != nil { + return fmt.Errorf("delete primary pod %s: %w", primary, err) + } + + if err := k.waitForPrimaryChange(recoveryCtx, primary); err != nil { + return err + } + + // A changed primary proves CNPG promoted a standby rather than merely + // recreating the deleted pod and reporting the old primary again. + if err := k.healthMon.WaitForSteadyState(recoveryCtx); err != nil { + return fmt.Errorf("wait for steady-state recovery: %w", err) + } + + current, err := k.client.GetPrimaryInstance(recoveryCtx) + if err != nil { + return fmt.Errorf("verify primary after steady-state recovery: %w", err) + } + if current == "" || current == primary { + return fmt.Errorf("verify primary after steady-state recovery: expected a non-empty primary different from %q, got %q", + primary, current) + } + return nil +} + +func (k *KillPrimaryPod) waitForPrimaryChange(ctx context.Context, original string) error { + ticker := time.NewTicker(k.primaryPollInterval) + defer ticker.Stop() + + lastObserved := original + var lastErr error + for { + current, err := k.client.GetPrimaryInstance(ctx) + if err == nil { + lastObserved = current + if current != "" && current != original { + return nil + } + } else { + lastErr = err + } + + select { + case <-ctx.Done(): + if lastErr != nil { + return fmt.Errorf("primary did not change from %q before recovery timeout (last read error: %v): %w", + original, lastErr, ctx.Err()) + } + return fmt.Errorf("primary did not change from %q before recovery timeout (last observed %q): %w", + original, lastObserved, ctx.Err()) + case <-ticker.C: + } + } } // OutagePolicy bounds the write outage of an automatic failover. Killing the diff --git a/test/longhaul/operations/kill_primary_test.go b/test/longhaul/operations/kill_primary_test.go index cb72db4a9..cfda9043f 100644 --- a/test/longhaul/operations/kill_primary_test.go +++ b/test/longhaul/operations/kill_primary_test.go @@ -12,9 +12,21 @@ import ( . "github.com/onsi/gomega" "github.com/documentdb/documentdb-operator/test/longhaul/journal" - "github.com/documentdb/documentdb-operator/test/longhaul/monitor" ) +type successfulSteadyGate struct { + calls int + onWait func() +} + +func (g *successfulSteadyGate) WaitForSteadyState(context.Context) error { + g.calls++ + if g.onWait != nil { + g.onWait() + } + return nil +} + var _ = Describe("KillPrimaryPod", func() { It("Name is kill-primary-pod and Weight is 2", func() { k := NewKillPrimaryPod(&fakeClient{}, nil, time.Minute) @@ -46,19 +58,49 @@ var _ = Describe("KillPrimaryPod", func() { Entry("HA: ipn=3 -> eligible", 3, nil, true, ""), ) - It("Execute deletes the reported primary pod", func() { - c := &fakeClient{instancesPerNode: 2, primary: "cluster-1"} - // The health monitor never reaches steady state here (its Run loop - // isn't started), so Execute times out on WaitForSteadyState — but the - // primary delete side-effect has already happened, which is what we - // assert. A short recovery keeps the test fast. - hm := monitor.NewHealthMonitor(c, journal.New(), time.Hour) - k := NewKillPrimaryPod(c, hm, 500*time.Millisecond) - - _ = k.Execute(context.Background()) + It("Execute deletes the original primary and verifies a different primary", func() { + c := &fakeClient{ + instancesPerNode: 2, + primary: "cluster-1", + replacementPrimary: "cluster-2", + } + gate := &successfulSteadyGate{} + k := NewKillPrimaryPod(c, gate, time.Second) + + Expect(k.Execute(context.Background())).To(Succeed()) c.mu.Lock() defer c.mu.Unlock() Expect(c.deletedPods).To(ConsistOf("cluster-1")) + Expect(c.primary).To(Equal("cluster-2")) + Expect(gate.calls).To(Equal(1)) + }) + + It("fails when CNPG keeps reporting the deleted primary", func() { + c := &fakeClient{instancesPerNode: 2, primary: "cluster-1"} + gate := &successfulSteadyGate{} + k := NewKillPrimaryPod(c, gate, 20*time.Millisecond) + k.primaryPollInterval = time.Millisecond + + err := k.Execute(context.Background()) + Expect(err).To(MatchError(ContainSubstring(`primary did not change from "cluster-1"`))) + Expect(gate.calls).To(Equal(0), "steady-state recovery must wait until primary change is proven") + }) + + It("fails if the recovered cluster reports the original primary again", func() { + c := &fakeClient{ + instancesPerNode: 2, + primary: "cluster-1", + replacementPrimary: "cluster-2", + } + gate := &successfulSteadyGate{onWait: func() { + c.mu.Lock() + defer c.mu.Unlock() + c.primary = "cluster-1" + }} + k := NewKillPrimaryPod(c, gate, time.Second) + + err := k.Execute(context.Background()) + Expect(err).To(MatchError(ContainSubstring("expected a non-empty primary different"))) }) It("Execute fails without deleting when the primary is unknown", func() { diff --git a/test/longhaul/operations/registry.go b/test/longhaul/operations/registry.go new file mode 100644 index 000000000..f85c3f843 --- /dev/null +++ b/test/longhaul/operations/registry.go @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "fmt" + + "k8s.io/client-go/kubernetes" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + "github.com/documentdb/documentdb-operator/test/longhaul/monitor" +) + +// Registry stores operations by their stable Name() values while preserving +// registration order for deterministic snapshots and random-mode summaries. +type Registry struct { + order []string + operations map[string]Operation +} + +// NewRegistry builds a validated operation registry. +func NewRegistry(ops ...Operation) (*Registry, error) { + registry := &Registry{ + order: make([]string, 0, len(ops)), + operations: make(map[string]Operation, len(ops)), + } + for _, op := range ops { + if op == nil { + return nil, fmt.Errorf("operation registry contains a nil operation") + } + name := op.Name() + if name == "" { + return nil, fmt.Errorf("operation registry contains an operation with an empty name") + } + if _, exists := registry.operations[name]; exists { + return nil, fmt.Errorf("operation registry contains duplicate name %q", name) + } + registry.order = append(registry.order, name) + registry.operations[name] = op + } + return registry, nil +} + +// NewDefaultRegistry centralizes construction of every supported operation. +func NewDefaultRegistry( + cfg config.Config, + clusterClient monitor.ClusterClient, + clientset kubernetes.Interface, + health *monitor.HealthMonitor, + j *journal.Journal, +) (*Registry, error) { + return NewRegistry( + NewScaleUp(clusterClient, health, cfg.MaxInstances, cfg.RecoveryTimeout), + NewScaleDown(clusterClient, health, cfg.MinInstances, cfg.RecoveryTimeout), + NewUpgradeDocumentDB(clusterClient, clientset, health, j, cfg.Namespace, cfg.RecoveryTimeout), + NewKillOperatorPod(clientset, cfg.OperatorNamespace, cfg.RecoveryTimeout), + NewKillPrimaryPod(clusterClient, health, cfg.RecoveryTimeout), + ) +} + +// All returns all registered operations in stable registration order. +func (r *Registry) All() []Operation { + ops := make([]Operation, 0, len(r.order)) + for _, name := range r.order { + ops = append(ops, r.operations[name]) + } + return ops +} + +// Resolve returns the named operations in exactly the requested order. +func (r *Registry) Resolve(names []string) ([]Operation, error) { + resolved := make([]Operation, 0, len(names)) + seen := make(map[string]struct{}, len(names)) + for _, name := range names { + if _, duplicate := seen[name]; duplicate { + return nil, fmt.Errorf("operation sequence contains duplicate name %q", name) + } + op, ok := r.operations[name] + if !ok { + return nil, fmt.Errorf("operation sequence contains unknown name %q", name) + } + seen[name] = struct{}{} + resolved = append(resolved, op) + } + return resolved, nil +} diff --git a/test/longhaul/operations/registry_test.go b/test/longhaul/operations/registry_test.go new file mode 100644 index 000000000..03493269a --- /dev/null +++ b/test/longhaul/operations/registry_test.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" +) + +var _ = Describe("Registry", func() { + It("resolves exact stable names in requested order", func() { + a := &fakeOp{name: "a"} + b := &fakeOp{name: "b"} + registry, err := NewRegistry(a, b) + Expect(err).NotTo(HaveOccurred()) + + resolved, err := registry.Resolve([]string{"b", "a"}) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved).To(Equal([]Operation{b, a})) + Expect(registry.All()).To(Equal([]Operation{a, b})) + }) + + It("rejects unknown requested names", func() { + registry, err := NewRegistry(&fakeOp{name: "known"}) + Expect(err).NotTo(HaveOccurred()) + _, err = registry.Resolve([]string{"unknown"}) + Expect(err).To(MatchError(ContainSubstring(`unknown name "unknown"`))) + }) + + It("rejects duplicate requested names", func() { + registry, err := NewRegistry(&fakeOp{name: "known"}) + Expect(err).NotTo(HaveOccurred()) + _, err = registry.Resolve([]string{"known", "known"}) + Expect(err).To(MatchError(ContainSubstring(`duplicate name "known"`))) + }) + + It("rejects duplicate registered operation names", func() { + _, err := NewRegistry(&fakeOp{name: "same"}, &fakeOp{name: "same"}) + Expect(err).To(MatchError(ContainSubstring(`duplicate name "same"`))) + }) + + It("constructs the default registry with the stable operation names", func() { + registry, err := NewDefaultRegistry(config.DefaultConfig(), nil, nil, nil, journal.New()) + Expect(err).NotTo(HaveOccurred()) + + names := make([]string, 0) + for _, op := range registry.All() { + names = append(names, op.Name()) + } + Expect(names).To(Equal([]string{ + "scale-up", + "scale-down", + "upgrade-documentdb", + "kill-operator-pod", + "kill-primary-pod", + })) + }) +}) diff --git a/test/longhaul/operations/runner.go b/test/longhaul/operations/runner.go new file mode 100644 index 000000000..1884290a4 --- /dev/null +++ b/test/longhaul/operations/runner.go @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "sync" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" +) + +// RunStatus is the bounded lifecycle state of the operation runner. +type RunStatus string + +const ( + RunStatusPending RunStatus = "PENDING" + RunStatusRunning RunStatus = "RUNNING" + RunStatusComplete RunStatus = "COMPLETE" + RunStatusFailed RunStatus = "FAILED" + RunStatusIncomplete RunStatus = "INCOMPLETE" + RunStatusDisabled RunStatus = "DISABLED" +) + +// OperationResultStatus is the state of one requested sequence operation. +type OperationResultStatus string + +const ( + OperationPending OperationResultStatus = "PENDING" + OperationRunning OperationResultStatus = "RUNNING" + OperationPassed OperationResultStatus = "PASSED" + OperationFailed OperationResultStatus = "FAILED" +) + +// OperationResult is the single mutable result for one requested sequence item. +type OperationResult struct { + Name string `json:"name"` + Status OperationResultStatus `json:"status"` + Error string `json:"error,omitempty"` +} + +// OperationAggregate bounds random-mode history to counters per operation type. +type OperationAggregate struct { + Name string `json:"name"` + Passed int `json:"passed"` + Failed int `json:"failed"` +} + +// RunSnapshot is a concurrency-safe value snapshot of operation execution. +type RunSnapshot struct { + Mode config.OperationMode `json:"mode"` + Status RunStatus `json:"status"` + Results []OperationResult `json:"results,omitempty"` + Aggregates []OperationAggregate `json:"aggregates,omitempty"` + FailureReason string `json:"failureReason,omitempty"` +} + +// OpsExecuted returns the number of terminal operation attempts. +func (s RunSnapshot) OpsExecuted() int { + if s.Mode == config.OperationModeSequence { + count := 0 + for _, result := range s.Results { + if result.Status == OperationPassed || result.Status == OperationFailed { + count++ + } + } + return count + } + + count := 0 + for _, aggregate := range s.Aggregates { + count += aggregate.Passed + aggregate.Failed + } + return count +} + +// HasFailure reports whether an operation attempt or sequence lifecycle failed. +func (s RunSnapshot) HasFailure() bool { + if s.Status == RunStatusFailed || s.Status == RunStatusIncomplete { + return true + } + for _, aggregate := range s.Aggregates { + if aggregate.Failed > 0 { + return true + } + } + return false +} + +// Runner is the common reporting and lifecycle surface for every operation mode. +type Runner interface { + Run(ctx context.Context) + Snapshot() RunSnapshot + Done() <-chan struct{} +} + +type runnerState struct { + mu sync.RWMutex + snapshot RunSnapshot + done chan struct{} + doneOnce sync.Once +} + +func newRunnerState(snapshot RunSnapshot) runnerState { + return runnerState{snapshot: snapshot, done: make(chan struct{})} +} + +func (s *runnerState) Snapshot() RunSnapshot { + s.mu.RLock() + defer s.mu.RUnlock() + snapshot := s.snapshot + snapshot.Results = append([]OperationResult(nil), s.snapshot.Results...) + snapshot.Aggregates = append([]OperationAggregate(nil), s.snapshot.Aggregates...) + return snapshot +} + +func (s *runnerState) Done() <-chan struct{} { + return s.done +} + +func (s *runnerState) closeDone() { + s.doneOnce.Do(func() { close(s.done) }) +} + +// DisabledRunner performs no operations and has no completion requirement. +type DisabledRunner struct { + state runnerState +} + +// NewDisabledRunner creates a runner for disabled operation mode. +func NewDisabledRunner() *DisabledRunner { + return &DisabledRunner{state: newRunnerState(RunSnapshot{ + Mode: config.OperationModeDisabled, + Status: RunStatusDisabled, + })} +} + +// Run waits for shutdown without scheduling operations. +func (r *DisabledRunner) Run(ctx context.Context) { + <-ctx.Done() + r.state.closeDone() +} + +// Snapshot returns the disabled runner state. +func (r *DisabledRunner) Snapshot() RunSnapshot { + return r.state.Snapshot() +} + +// Done closes when Run returns after cancellation. +func (r *DisabledRunner) Done() <-chan struct{} { + return r.state.Done() +} diff --git a/test/longhaul/operations/scale_test.go b/test/longhaul/operations/scale_test.go index 8c9fc953b..e929033d6 100644 --- a/test/longhaul/operations/scale_test.go +++ b/test/longhaul/operations/scale_test.go @@ -18,16 +18,17 @@ import ( // fakeClient is a minimal monitor.ClusterClient stub for unit tests. type fakeClient struct { - mu sync.Mutex - instancesPerNode int - ipnErr error - imageTag string - scaleCalls []int - upgradeCalls []string - primary string - primaryErr error - deleteErr error - deletedPods []string + mu sync.Mutex + instancesPerNode int + ipnErr error + imageTag string + scaleCalls []int + upgradeCalls []string + primary string + primaryErr error + replacementPrimary string + deleteErr error + deletedPods []string } func (f *fakeClient) GetClusterHealth(_ context.Context) (monitor.ClusterHealth, error) { @@ -68,6 +69,9 @@ func (f *fakeClient) DeletePod(_ context.Context, name string) error { return f.deleteErr } f.deletedPods = append(f.deletedPods, name) + if f.replacementPrimary != "" { + f.primary = f.replacementPrimary + } return nil } diff --git a/test/longhaul/operations/scheduler.go b/test/longhaul/operations/scheduler.go index a9351d18d..a486febaa 100644 --- a/test/longhaul/operations/scheduler.go +++ b/test/longhaul/operations/scheduler.go @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// Package operations implements the operation scheduler and individual -// disruptive operations for long haul tests. +// Package operations implements operation runners and individual disruptive +// operations for long haul tests. package operations import ( @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/documentdb/documentdb-operator/test/longhaul/config" "github.com/documentdb/documentdb-operator/test/longhaul/journal" "github.com/documentdb/documentdb-operator/test/longhaul/monitor" ) @@ -42,10 +43,38 @@ type Scheduler struct { journal *journal.Journal cooldown time.Duration + // rng, when non-nil, pins weighted-random selection for reproducibility. + // When nil the process-global generator is used (production behavior). + rng *rand.Rand + // coverage draws each operation without replacement and completes the run + // once every operation has run at least once. + coverage bool + mu sync.Mutex lastOpTime time.Time opsExecuted int inProgress bool + + state runnerState + aggregateIndex map[string]int +} + +// SchedulerOption configures optional Scheduler behavior. +type SchedulerOption func(*Scheduler) + +// WithSeed pins weighted-random selection to a fixed seed so the run is +// reproducible. Without it the scheduler uses the process-global generator. +func WithSeed(seed int64) SchedulerOption { + return func(s *Scheduler) { + s.rng = rand.New(rand.NewPCG(uint64(seed), uint64(seed))) + } +} + +// WithCoverage enables coverage mode: the scheduler draws each operation +// without replacement and completes once every operation has run at least once, +// rather than running until context cancellation. +func WithCoverage() SchedulerOption { + return func(s *Scheduler) { s.coverage = true } } // NewScheduler creates an operation scheduler. @@ -54,19 +83,68 @@ func NewScheduler( health *monitor.HealthMonitor, j *journal.Journal, cooldown time.Duration, + opts ...SchedulerOption, ) *Scheduler { - return &Scheduler{ + aggregates := make([]OperationAggregate, 0, len(ops)) + aggregateIndex := make(map[string]int, len(ops)) + for _, op := range ops { + if _, exists := aggregateIndex[op.Name()]; exists { + continue + } + aggregateIndex[op.Name()] = len(aggregates) + aggregates = append(aggregates, OperationAggregate{Name: op.Name()}) + } + s := &Scheduler{ operations: ops, healthMonitor: health, journal: j, cooldown: cooldown, + state: newRunnerState(RunSnapshot{ + Mode: config.OperationModeRandom, + Status: RunStatusPending, + Aggregates: aggregates, + }), + aggregateIndex: aggregateIndex, + } + for _, opt := range opts { + opt(s) + } + return s +} + +// intn returns a non-negative pseudo-random int in [0,n) from the scheduler's +// seeded generator when present, otherwise the process-global generator. +func (s *Scheduler) intn(n int) int { + if s.rng != nil { + return s.rng.IntN(n) } + return rand.IntN(n) } // Run starts the scheduler loop. It blocks until context is cancelled. func (s *Scheduler) Run(ctx context.Context) { s.journal.Info("scheduler", "operation scheduler started") - defer s.journal.Info("scheduler", "operation scheduler stopped") + s.state.mu.Lock() + s.state.snapshot.Status = RunStatusRunning + s.state.mu.Unlock() + defer func() { + s.state.mu.Lock() + if s.state.snapshot.Status == RunStatusRunning { + // Coverage runs that stop before covering every operation (watchdog + // or shutdown) are terminally incomplete, not complete. + if s.coverage && !s.allCoveredLocked() { + s.state.snapshot.Status = RunStatusIncomplete + if s.state.snapshot.FailureReason == "" { + s.state.snapshot.FailureReason = "operation coverage incomplete: run stopped before every operation ran" + } + } else { + s.state.snapshot.Status = RunStatusComplete + } + } + s.state.mu.Unlock() + s.state.closeDone() + s.journal.Info("scheduler", "operation scheduler stopped") + }() ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() @@ -77,8 +155,49 @@ func (s *Scheduler) Run(ctx context.Context) { return case <-ticker.C: s.tryExecute(ctx) + // Coverage mode is completion-driven: stop as soon as the run + // reaches a terminal state (all operations covered, or a failure). + if s.coverage && s.coverageTerminalReached() { + return + } + } + } +} + +// coverageTerminalReached reports whether a coverage run has reached a terminal +// state and its loop should return. +func (s *Scheduler) coverageTerminalReached() bool { + s.state.mu.RLock() + defer s.state.mu.RUnlock() + return s.state.snapshot.Status == RunStatusComplete || + s.state.snapshot.Status == RunStatusFailed +} + +// allCoveredLocked reports whether every registered operation has run at least +// once. Callers must hold s.state.mu. +func (s *Scheduler) allCoveredLocked() bool { + if len(s.state.snapshot.Aggregates) == 0 { + return false + } + for _, a := range s.state.snapshot.Aggregates { + if a.Passed+a.Failed == 0 { + return false } } + return true +} + +// coveredSet returns the set of operation names that have run at least once. +func (s *Scheduler) coveredSet() map[string]bool { + s.state.mu.RLock() + defer s.state.mu.RUnlock() + covered := make(map[string]bool, len(s.state.snapshot.Aggregates)) + for _, a := range s.state.snapshot.Aggregates { + if a.Passed+a.Failed > 0 { + covered[a.Name] = true + } + } + return covered } func (s *Scheduler) tryExecute(ctx context.Context) { @@ -111,16 +230,25 @@ func (s *Scheduler) tryExecute(ctx context.Context) { s.inProgress = true s.mu.Unlock() - s.executeOp(ctx, op) + err := s.executeOp(ctx, op) s.mu.Lock() s.inProgress = false s.lastOpTime = time.Now() s.opsExecuted++ s.mu.Unlock() + + s.recordExecution(op.Name(), err) } func (s *Scheduler) selectOperation(ctx context.Context) Operation { + // In coverage mode, exclude operations that have already run so each is + // drawn without replacement until every operation has been covered. + var covered map[string]bool + if s.coverage { + covered = s.coveredSet() + } + // Filter by preconditions and build weighted list. type candidate struct { op Operation @@ -130,6 +258,9 @@ func (s *Scheduler) selectOperation(ctx context.Context) Operation { totalWeight := 0 for _, op := range s.operations { + if s.coverage && covered[op.Name()] { + continue + } ok, _ := op.Precondition(ctx) if ok { w := op.Weight() @@ -143,7 +274,7 @@ func (s *Scheduler) selectOperation(ctx context.Context) Operation { } // Weighted random selection. - r := rand.IntN(totalWeight) + r := s.intn(totalWeight) for _, c := range candidates { r -= c.weight if r < 0 { @@ -153,18 +284,51 @@ func (s *Scheduler) selectOperation(ctx context.Context) Operation { return candidates[len(candidates)-1].op } -func (s *Scheduler) executeOp(ctx context.Context, op Operation) { +func (s *Scheduler) executeOp(ctx context.Context, op Operation) error { s.journal.Info("scheduler", fmt.Sprintf("executing operation: %s", op.Name())) s.journal.OpenDisruptionWindow(op.Name(), op.OutagePolicy()) err := op.Execute(ctx) - - s.journal.CloseDisruptionWindow() + window := s.journal.CloseDisruptionWindow() if err != nil { s.journal.Error("scheduler", fmt.Sprintf("operation %s failed: %v", op.Name(), err)) - } else { - s.journal.Info("scheduler", fmt.Sprintf("operation %s completed successfully", op.Name())) + return fmt.Errorf("operation %s execute failed: %w", op.Name(), err) + } + if window == nil { + err = fmt.Errorf("operation %s closed without a disruption window", op.Name()) + s.journal.Error("scheduler", err.Error()) + return err + } + if window.ExceededPolicy() { + err = fmt.Errorf("operation %s exceeded its outage policy", op.Name()) + s.journal.Error("scheduler", err.Error()) + return err + } + + s.journal.Info("scheduler", fmt.Sprintf("operation %s completed successfully", op.Name())) + return nil +} + +func (s *Scheduler) recordExecution(name string, err error) { + s.state.mu.Lock() + defer s.state.mu.Unlock() + index, ok := s.aggregateIndex[name] + if !ok { + return + } + if err != nil { + s.state.snapshot.Aggregates[index].Failed++ + s.state.snapshot.Status = RunStatusFailed + if s.state.snapshot.FailureReason == "" { + s.state.snapshot.FailureReason = err.Error() + } + return + } + s.state.snapshot.Aggregates[index].Passed++ + // Coverage mode completes once every operation has run at least once. + if s.coverage && s.state.snapshot.Status == RunStatusRunning && s.allCoveredLocked() { + s.state.snapshot.Status = RunStatusComplete } } @@ -174,3 +338,13 @@ func (s *Scheduler) OpsExecuted() int { defer s.mu.Unlock() return s.opsExecuted } + +// Snapshot returns bounded aggregate counters in registration order. +func (s *Scheduler) Snapshot() RunSnapshot { + return s.state.Snapshot() +} + +// Done closes when the scheduler stops after context cancellation. +func (s *Scheduler) Done() <-chan struct{} { + return s.state.Done() +} diff --git a/test/longhaul/operations/scheduler_test.go b/test/longhaul/operations/scheduler_test.go index cd351eb9c..a12baf7a6 100644 --- a/test/longhaul/operations/scheduler_test.go +++ b/test/longhaul/operations/scheduler_test.go @@ -120,4 +120,92 @@ var _ = Describe("Scheduler", func() { s.opsExecuted = 7 Expect(s.OpsExecuted()).To(Equal(7)) }) + + It("keeps bounded aggregate counters and exposes failures", func() { + a := &fakeOp{name: "a"} + b := &fakeOp{name: "b"} + s := NewScheduler([]Operation{a, b}, nil, journal.New(), time.Hour) + + for i := 0; i < 1000; i++ { + s.recordExecution("a", nil) + } + s.recordExecution("b", errors.New("first failure")) + s.recordExecution("b", errors.New("second failure")) + + snapshot := s.Snapshot() + Expect(snapshot.Aggregates).To(Equal([]OperationAggregate{ + {Name: "a", Passed: 1000}, + {Name: "b", Failed: 2}, + })) + Expect(snapshot.Status).To(Equal(RunStatusFailed)) + Expect(snapshot.HasFailure()).To(BeTrue()) + Expect(snapshot.FailureReason).To(ContainSubstring("first failure")) + }) + + Describe("WithSeed", func() { + It("makes weighted selection reproducible across schedulers", func() { + mk := func() *Scheduler { + return NewScheduler([]Operation{ + &fakeOp{name: "a", weight: 1, available: true}, + &fakeOp{name: "b", weight: 1, available: true}, + &fakeOp{name: "c", weight: 1, available: true}, + }, nil, journal.New(), time.Hour, WithSeed(42)) + } + s1, s2 := mk(), mk() + var seq1, seq2 []string + for i := 0; i < 25; i++ { + seq1 = append(seq1, s1.selectOperation(context.Background()).Name()) + seq2 = append(seq2, s2.selectOperation(context.Background()).Name()) + } + Expect(seq1).To(Equal(seq2)) + }) + }) + + Describe("WithCoverage", func() { + It("draws each operation without replacement", func() { + a := &fakeOp{name: "a", weight: 1, available: true} + b := &fakeOp{name: "b", weight: 1, available: true} + s := NewScheduler([]Operation{a, b}, nil, journal.New(), time.Hour, WithCoverage()) + + // Once "a" is covered, selection must never return it again. + s.recordExecution("a", nil) + for i := 0; i < 50; i++ { + got := s.selectOperation(context.Background()) + Expect(got).NotTo(BeNil(), "iter %d", i) + Expect(got.Name()).To(Equal("b")) + } + + // Once every operation is covered there are no candidates left. + s.recordExecution("b", nil) + Expect(s.selectOperation(context.Background())).To(BeNil()) + }) + + It("completes the run once every operation has run at least once", func() { + a := &fakeOp{name: "a"} + b := &fakeOp{name: "b"} + s := NewScheduler([]Operation{a, b}, nil, journal.New(), time.Hour, WithCoverage()) + s.state.snapshot.Status = RunStatusRunning + + s.recordExecution("a", nil) + Expect(s.Snapshot().Status).To(Equal(RunStatusRunning)) + + s.recordExecution("b", nil) + Expect(s.Snapshot().Status).To(Equal(RunStatusComplete)) + }) + + It("does not complete a partially covered run", func() { + a := &fakeOp{name: "a"} + b := &fakeOp{name: "b"} + s := NewScheduler([]Operation{a, b}, nil, journal.New(), time.Hour, WithCoverage()) + s.state.snapshot.Status = RunStatusRunning + + s.recordExecution("a", nil) + + s.state.mu.RLock() + covered := s.allCoveredLocked() + s.state.mu.RUnlock() + Expect(covered).To(BeFalse()) + Expect(s.Snapshot().Status).To(Equal(RunStatusRunning)) + }) + }) }) diff --git a/test/longhaul/operations/sequence.go b/test/longhaul/operations/sequence.go new file mode 100644 index 000000000..2426f1e64 --- /dev/null +++ b/test/longhaul/operations/sequence.go @@ -0,0 +1,247 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "fmt" + "time" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" +) + +const defaultPreconditionPollInterval = time.Second + +// SteadyStateGate is the health-monitor surface needed by sequence mode. +type SteadyStateGate interface { + WaitForSteadyState(ctx context.Context) error +} + +type preconditionWaitFunc func(context.Context, Operation) error + +// SequenceRunner executes each requested operation exactly once and in order. +type SequenceRunner struct { + operations []Operation + steadyStateGate SteadyStateGate + journal *journal.Journal + recoveryTimeout time.Duration + state runnerState + + waitForPrecondition preconditionWaitFunc + terminal bool +} + +// NewSequenceRunner creates a deterministic sequential operation runner. +func NewSequenceRunner( + ops []Operation, + gate SteadyStateGate, + j *journal.Journal, + recoveryTimeout time.Duration, +) *SequenceRunner { + results := make([]OperationResult, len(ops)) + for i, op := range ops { + results[i] = OperationResult{Name: op.Name(), Status: OperationPending} + } + runner := &SequenceRunner{ + operations: append([]Operation(nil), ops...), + steadyStateGate: gate, + journal: j, + recoveryTimeout: recoveryTimeout, + state: newRunnerState(RunSnapshot{ + Mode: config.OperationModeSequence, + Status: RunStatusPending, + Results: results, + }), + } + runner.waitForPrecondition = runner.pollPrecondition + return runner +} + +// Run executes the configured sequence and stops on the first failure. +func (r *SequenceRunner) Run(ctx context.Context) { + defer r.state.closeDone() + + r.state.mu.Lock() + if r.terminal { + r.state.mu.Unlock() + return + } + r.state.snapshot.Status = RunStatusRunning + r.state.mu.Unlock() + + for i, op := range r.operations { + if !r.setResult(i, OperationRunning, "") { + return + } + if err := r.runOne(ctx, op); err != nil { + status := RunStatusFailed + reason := err.Error() + if ctx.Err() != nil { + status = RunStatusIncomplete + reason = fmt.Sprintf("operation sequence incomplete during %s: %v", op.Name(), ctx.Err()) + } + r.setFailure(i, status, reason) + return + } + if !r.setResult(i, OperationPassed, "") { + return + } + } + + r.state.mu.Lock() + if !r.terminal { + r.terminal = true + r.state.snapshot.Status = RunStatusComplete + } + r.state.mu.Unlock() +} + +func (r *SequenceRunner) runOne(ctx context.Context, op Operation) error { + if r.steadyStateGate == nil { + return fmt.Errorf("operation %s steady-state gate is nil", op.Name()) + } + + steadyCtx, cancelSteady := context.WithTimeout(ctx, r.recoveryTimeout) + err := r.steadyStateGate.WaitForSteadyState(steadyCtx) + cancelSteady() + if err != nil { + return fmt.Errorf("operation %s initial steady-state gate failed: %w", op.Name(), err) + } + + preconditionCtx, cancelPrecondition := context.WithTimeout(ctx, r.recoveryTimeout) + err = r.waitForPrecondition(preconditionCtx, op) + cancelPrecondition() + if err != nil { + return fmt.Errorf("operation %s precondition timeout: %w", op.Name(), err) + } + + r.journal.Info("sequence", fmt.Sprintf("executing operation: %s", op.Name())) + r.journal.OpenDisruptionWindow(op.Name(), op.OutagePolicy()) + + executeCtx, cancelExecute := context.WithTimeout(ctx, r.recoveryTimeout) + executeErr := op.Execute(executeCtx) + cancelExecute() + window := r.journal.CloseDisruptionWindow() + + if executeErr != nil { + r.journal.Error("sequence", fmt.Sprintf("operation %s failed: %v", op.Name(), executeErr)) + return fmt.Errorf("operation %s execute failed: %w", op.Name(), executeErr) + } + if window == nil { + return fmt.Errorf("operation %s closed without a disruption window", op.Name()) + } + if window.ExceededPolicy() { + err := fmt.Errorf("operation %s exceeded its outage policy", op.Name()) + r.journal.Error("sequence", err.Error()) + return err + } + + recoveryCtx, cancelRecovery := context.WithTimeout(ctx, r.recoveryTimeout) + err = r.steadyStateGate.WaitForSteadyState(recoveryCtx) + cancelRecovery() + if err != nil { + return fmt.Errorf("operation %s post-recovery steady-state gate failed: %w", op.Name(), err) + } + + r.journal.Info("sequence", fmt.Sprintf("operation %s completed successfully", op.Name())) + return nil +} + +func (r *SequenceRunner) pollPrecondition(ctx context.Context, op Operation) error { + ticker := time.NewTicker(defaultPreconditionPollInterval) + defer ticker.Stop() + + lastReason := "precondition not met" + for { + ok, reason := op.Precondition(ctx) + if ok { + return nil + } + if reason != "" { + lastReason = reason + } + + select { + case <-ctx.Done(): + return fmt.Errorf("%s: %w", lastReason, ctx.Err()) + case <-ticker.C: + } + } +} + +func (r *SequenceRunner) setResult(index int, status OperationResultStatus, reason string) bool { + r.state.mu.Lock() + defer r.state.mu.Unlock() + if r.terminal { + return false + } + r.state.snapshot.Results[index].Status = status + r.state.snapshot.Results[index].Error = reason + return true +} + +func (r *SequenceRunner) setFailure(index int, status RunStatus, reason string) { + r.state.mu.Lock() + defer r.state.mu.Unlock() + if r.terminal { + return + } + r.terminal = true + r.state.snapshot.Status = status + r.state.snapshot.FailureReason = reason + r.state.snapshot.Results[index].Status = OperationFailed + r.state.snapshot.Results[index].Error = reason +} + +// MarkIncomplete terminally fails a sequence whose watchdog or shutdown +// cancellation fired before Run could publish its own terminal snapshot. +func (r *SequenceRunner) MarkIncomplete(reason string) { + r.state.mu.Lock() + if r.terminal { + r.state.mu.Unlock() + return + } + allPassed := len(r.state.snapshot.Results) > 0 + for _, result := range r.state.snapshot.Results { + if result.Status != OperationPassed { + allPassed = false + break + } + } + if allPassed { + r.terminal = true + r.state.snapshot.Status = RunStatusComplete + r.state.mu.Unlock() + r.state.closeDone() + return + } + r.terminal = true + r.state.snapshot.Status = RunStatusIncomplete + r.state.snapshot.FailureReason = reason + for i := range r.state.snapshot.Results { + if r.state.snapshot.Results[i].Status == OperationRunning { + r.state.snapshot.Results[i].Status = OperationFailed + r.state.snapshot.Results[i].Error = reason + break + } + if r.state.snapshot.Results[i].Status == OperationPending { + r.state.snapshot.Results[i].Status = OperationFailed + r.state.snapshot.Results[i].Error = reason + break + } + } + r.state.mu.Unlock() + r.state.closeDone() +} + +// Snapshot returns a deterministic copy ordered by the configured sequence. +func (r *SequenceRunner) Snapshot() RunSnapshot { + return r.state.Snapshot() +} + +// Done closes when the sequence completes or stops on failure/cancellation. +func (r *SequenceRunner) Done() <-chan struct{} { + return r.state.Done() +} diff --git a/test/longhaul/operations/sequence_test.go b/test/longhaul/operations/sequence_test.go new file mode 100644 index 000000000..f2e8541da --- /dev/null +++ b/test/longhaul/operations/sequence_test.go @@ -0,0 +1,217 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "errors" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" +) + +type sequenceTestOp struct { + name string + execute func(context.Context) error + precondition func(context.Context) (bool, string) + policy journal.OutagePolicy +} + +func (o *sequenceTestOp) Name() string { return o.name } +func (o *sequenceTestOp) Weight() int { return 1 } +func (o *sequenceTestOp) Precondition(ctx context.Context) (bool, string) { + if o.precondition != nil { + return o.precondition(ctx) + } + return true, "" +} +func (o *sequenceTestOp) Execute(ctx context.Context) error { + if o.execute != nil { + return o.execute(ctx) + } + return nil +} +func (o *sequenceTestOp) OutagePolicy() journal.OutagePolicy { + if o.policy.MustRecoverWithin != 0 || o.policy.MaxWriteOutage != 0 { + return o.policy + } + return journal.DefaultOutagePolicy() +} + +type sequenceTestGate struct { + mu sync.Mutex + calls int + err error +} + +func (g *sequenceTestGate) WaitForSteadyState(context.Context) error { + g.mu.Lock() + defer g.mu.Unlock() + g.calls++ + return g.err +} + +func runSequence(runner *SequenceRunner, ctx context.Context) RunSnapshot { + go runner.Run(ctx) + Eventually(runner.Done()).Should(BeClosed()) + return runner.Snapshot() +} + +var _ = Describe("SequenceRunner", func() { + It("executes each operation exactly once in exact order", func() { + var order []string + ops := []Operation{ + &sequenceTestOp{name: "first", execute: func(context.Context) error { + order = append(order, "first") + return nil + }}, + &sequenceTestOp{name: "second", execute: func(context.Context) error { + order = append(order, "second") + return nil + }}, + } + gate := &sequenceTestGate{} + snapshot := runSequence(NewSequenceRunner(ops, gate, journal.New(), time.Second), context.Background()) + + Expect(order).To(Equal([]string{"first", "second"})) + Expect(snapshot.Status).To(Equal(RunStatusComplete)) + Expect(snapshot.Results).To(Equal([]OperationResult{ + {Name: "first", Status: OperationPassed}, + {Name: "second", Status: OperationPassed}, + })) + Expect(gate.calls).To(Equal(4), "initial and post-recovery gate for each operation") + }) + + It("records an execute error and stops before later operations", func() { + var order []string + ops := []Operation{ + &sequenceTestOp{name: "first", execute: func(context.Context) error { + order = append(order, "first") + return nil + }}, + &sequenceTestOp{name: "broken", execute: func(context.Context) error { + order = append(order, "broken") + return errors.New("kaboom") + }}, + &sequenceTestOp{name: "never", execute: func(context.Context) error { + order = append(order, "never") + return nil + }}, + } + snapshot := runSequence( + NewSequenceRunner(ops, &sequenceTestGate{}, journal.New(), time.Second), + context.Background(), + ) + + Expect(order).To(Equal([]string{"first", "broken"})) + Expect(snapshot.Status).To(Equal(RunStatusFailed)) + Expect(snapshot.FailureReason).To(ContainSubstring("kaboom")) + Expect(snapshot.Results).To(Equal([]OperationResult{ + {Name: "first", Status: OperationPassed}, + {Name: "broken", Status: OperationFailed, Error: snapshot.FailureReason}, + {Name: "never", Status: OperationPending}, + })) + }) + + It("fails deterministically when a precondition times out", func() { + op := &sequenceTestOp{name: "blocked"} + runner := NewSequenceRunner([]Operation{op}, &sequenceTestGate{}, journal.New(), time.Second) + runner.waitForPrecondition = func(context.Context, Operation) error { + return context.DeadlineExceeded + } + + snapshot := runSequence(runner, context.Background()) + Expect(snapshot.Status).To(Equal(RunStatusFailed)) + Expect(snapshot.Results[0].Status).To(Equal(OperationFailed)) + Expect(snapshot.FailureReason).To(ContainSubstring("precondition timeout")) + }) + + It("marks cancellation as incomplete and leaves later operations pending", func() { + started := make(chan struct{}) + op := &sequenceTestOp{name: "cancelled", execute: func(ctx context.Context) error { + close(started) + <-ctx.Done() + return ctx.Err() + }} + runner := NewSequenceRunner( + []Operation{op, &sequenceTestOp{name: "never"}}, + &sequenceTestGate{}, + journal.New(), + time.Minute, + ) + ctx, cancel := context.WithCancel(context.Background()) + go runner.Run(ctx) + Eventually(started).Should(BeClosed()) + cancel() + Eventually(runner.Done()).Should(BeClosed()) + + snapshot := runner.Snapshot() + Expect(snapshot.Status).To(Equal(RunStatusIncomplete)) + Expect(snapshot.Results[0].Status).To(Equal(OperationFailed)) + Expect(snapshot.Results[1].Status).To(Equal(OperationPending)) + Expect(snapshot.FailureReason).To(ContainSubstring("incomplete")) + }) + + It("publishes a terminal incomplete snapshot when the watchdog wins", func() { + runner := NewSequenceRunner( + []Operation{ + &sequenceTestOp{name: "first"}, + &sequenceTestOp{name: "second"}, + }, + &sequenceTestGate{}, + journal.New(), + time.Minute, + ) + + runner.MarkIncomplete("watchdog fired") + Expect(runner.Done()).To(BeClosed()) + snapshot := runner.Snapshot() + Expect(snapshot.Status).To(Equal(RunStatusIncomplete)) + Expect(snapshot.FailureReason).To(Equal("watchdog fired")) + Expect(snapshot.Results).To(Equal([]OperationResult{ + {Name: "first", Status: OperationFailed, Error: "watchdog fired"}, + {Name: "second", Status: OperationPending}, + })) + }) + + It("preserves completion when the watchdog races after every operation passed", func() { + runner := NewSequenceRunner( + []Operation{&sequenceTestOp{name: "done"}}, + &sequenceTestGate{}, + journal.New(), + time.Second, + ) + runner.state.snapshot.Results[0].Status = OperationPassed + + runner.MarkIncomplete("watchdog fired") + + snapshot := runner.Snapshot() + Expect(snapshot.Status).To(Equal(RunStatusComplete)) + Expect(snapshot.Results).To(Equal([]OperationResult{{ + Name: "done", + Status: OperationPassed, + }})) + }) + + It("fails when the closed disruption window exceeds policy", func() { + op := &sequenceTestOp{ + name: "policy", + policy: journal.OutagePolicy{ + MaxWriteOutage: time.Hour, + MustRecoverWithin: -time.Nanosecond, + }, + } + snapshot := runSequence( + NewSequenceRunner([]Operation{op}, &sequenceTestGate{}, journal.New(), time.Second), + context.Background(), + ) + + Expect(snapshot.Status).To(Equal(RunStatusFailed)) + Expect(snapshot.FailureReason).To(ContainSubstring("exceeded its outage policy")) + }) +}) diff --git a/test/longhaul/report/checkpoint.go b/test/longhaul/report/checkpoint.go index 5875412c7..d6d28130f 100644 --- a/test/longhaul/report/checkpoint.go +++ b/test/longhaul/report/checkpoint.go @@ -8,8 +8,11 @@ import ( "encoding/json" "fmt" "log" + "sync" "time" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,8 +24,9 @@ const ( ConfigMapName = "longhaul-report" ) -// SummaryFunc is called to generate the current test summary. -type SummaryFunc func() Summary +// SummaryFunc is called to generate the current test summary. final is true +// only for the terminal emit, when incomplete sequence execution must fail. +type SummaryFunc func(final bool) Summary // CheckpointReporter periodically generates and persists reports. type CheckpointReporter struct { @@ -30,6 +34,10 @@ type CheckpointReporter struct { namespace string interval time.Duration summaryFunc SummaryFunc + + emitMu sync.Mutex + finalEmitted bool + finalSummary Summary } // NewCheckpointReporter creates a periodic reporter that writes to stdout and ConfigMap. @@ -67,18 +75,31 @@ func (r *CheckpointReporter) Run(ctx context.Context) { // not as RUNNING) using a bounded context. Safe to call after the main // context has been cancelled. Intended to be called synchronously from main // just before exit so the verdict is durable in the ConfigMap. -func (r *CheckpointReporter) EmitFinal() { +func (r *CheckpointReporter) EmitFinal() Summary { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - r.emit(ctx, true) + return r.emit(ctx, true) } // emit writes the current summary to stdout, GH Actions annotations, and the // status ConfigMap. final=true means this is the shutdown emit, in which case // PASS is persisted as "PASS" (not "RUNNING") so consumers can distinguish a // finished clean run from an in-flight checkpoint. -func (r *CheckpointReporter) emit(ctx context.Context, final bool) { - summary := r.summaryFunc() +func (r *CheckpointReporter) emit(ctx context.Context, final bool) Summary { + r.emitMu.Lock() + defer r.emitMu.Unlock() + if final && r.finalEmitted { + return r.finalSummary + } + if !final && r.finalEmitted { + return Summary{} + } + + summary := r.summaryFunc(final) + if final { + r.finalEmitted = true + r.finalSummary = summary + } // Intermediate PASS checkpoints surface as RUNNING; the final emit // preserves the true PASS/FAIL outcome. @@ -99,13 +120,18 @@ func (r *CheckpointReporter) emit(ctx context.Context, final bool) { // Persist to ConfigMap. if r.clientset == nil { - return + return summary } data := map[string]string{ - "latest-report": markdown, - "last-updated": time.Now().UTC().Format(time.RFC3339), - "result": resultStr, + "latest-report": markdown, + "last-updated": time.Now().UTC().Format(time.RFC3339), + "result": resultStr, + "operation-status": string(summary.OperationRun.Status), + "operation-results": marshalOperationResults(summary.OperationRun.Results), + } + if len(summary.OperationRun.Aggregates) > 0 { + data["operation-aggregates"] = marshalOperationAggregates(summary.OperationRun.Aggregates) } cm := &corev1.ConfigMap{ @@ -142,14 +168,32 @@ func (r *CheckpointReporter) emit(ctx context.Context, final bool) { // Also log the summary as JSON for structured log consumers. summaryJSON, _ := json.Marshal(map[string]any{ - "result": resultStr, - "elapsed": summary.Duration.String(), - "writes": summary.Metrics.WriteAttempted, - "gaps": summary.Metrics.GapsDetected, - "ops": summary.OpsExecuted, - "memory_leak": summary.LeakAnalysis.HasLeak, - "memory_slope": fmt.Sprintf("%.2f MB/h", summary.LeakAnalysis.MemorySlopeMB), - "checkpoint_time": time.Now().UTC().Format(time.RFC3339), + "result": resultStr, + "elapsed": summary.Duration.String(), + "writes": summary.Metrics.WriteAttempted, + "gaps": summary.Metrics.GapsDetected, + "ops": summary.OpsExecuted, + "memory_leak": summary.LeakAnalysis.HasLeak, + "memory_slope": fmt.Sprintf("%.2f MB/h", summary.LeakAnalysis.MemorySlopeMB), + "operation_status": summary.OperationRun.Status, + "checkpoint_time": time.Now().UTC().Format(time.RFC3339), }) log.Printf("[checkpoint] %s", string(summaryJSON)) + return summary +} + +func marshalOperationResults(results []operations.OperationResult) string { + if results == nil { + results = []operations.OperationResult{} + } + data, _ := json.Marshal(results) + return string(data) +} + +func marshalOperationAggregates(aggregates []operations.OperationAggregate) string { + if aggregates == nil { + aggregates = []operations.OperationAggregate{} + } + data, _ := json.Marshal(aggregates) + return string(data) } diff --git a/test/longhaul/report/checkpoint_test.go b/test/longhaul/report/checkpoint_test.go index 891e90b4d..d9c80f7ef 100644 --- a/test/longhaul/report/checkpoint_test.go +++ b/test/longhaul/report/checkpoint_test.go @@ -5,6 +5,7 @@ package report import ( "context" + "encoding/json" "time" . "github.com/onsi/ginkgo/v2" @@ -12,11 +13,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" ) var _ = Describe("CheckpointReporter", func() { It("emit() is safe with a nil clientset (logs to stdout, does not panic)", func() { - r := NewCheckpointReporter(nil, "ns", time.Second, func() Summary { + r := NewCheckpointReporter(nil, "ns", time.Second, func(bool) Summary { return Summary{Result: ResultPass, Duration: time.Minute} }) Expect(func() { r.emit(context.Background(), false) }).NotTo(Panic()) @@ -24,7 +28,7 @@ var _ = Describe("CheckpointReporter", func() { It("creates the ConfigMap on first emit and labels it identifiably", func() { cs := fake.NewSimpleClientset() - r := NewCheckpointReporter(cs, "ns", time.Second, func() Summary { + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { return Summary{Result: ResultPass, Duration: 2 * time.Hour, OpsExecuted: 5} }) @@ -35,6 +39,8 @@ var _ = Describe("CheckpointReporter", func() { Expect(cm.Data).To(HaveKey("latest-report")) Expect(cm.Data).To(HaveKey("last-updated")) Expect(cm.Data).To(HaveKey("result")) + Expect(cm.Data).To(HaveKeyWithValue("operation-status", "")) + Expect(cm.Data).To(HaveKeyWithValue("operation-results", "[]")) // PASS at intermediate checkpoint is persisted as RUNNING so consumers // can distinguish in-flight from final state. Expect(cm.Data["result"]).To(Equal("RUNNING")) @@ -43,7 +49,7 @@ var _ = Describe("CheckpointReporter", func() { It("persists FAIL results as FAIL", func() { cs := fake.NewSimpleClientset() - r := NewCheckpointReporter(cs, "ns", time.Second, func() Summary { + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { return Summary{Result: ResultFail, FailReason: "data loss"} }) @@ -58,7 +64,7 @@ var _ = Describe("CheckpointReporter", func() { cs := fake.NewSimpleClientset() calls := 0 - r := NewCheckpointReporter(cs, "ns", time.Second, func() Summary { + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { calls++ return Summary{Result: ResultPass, Duration: time.Duration(calls) * time.Hour, OpsExecuted: calls * 10} }) @@ -76,4 +82,87 @@ var _ = Describe("CheckpointReporter", func() { Expect(cm2.Data["latest-report"]).NotTo(Equal(report1)) Expect(calls).To(Equal(2)) }) + + It("persists ordered sequence results as bounded JSON", func() { + cs := fake.NewSimpleClientset() + results := []operations.OperationResult{ + {Name: "kill-operator-pod", Status: operations.OperationPassed}, + {Name: "kill-primary-pod", Status: operations.OperationPassed}, + } + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { + return Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusComplete, + Results: results, + }, + } + }) + + r.emit(context.Background(), true) + cm, err := cs.CoreV1().ConfigMaps("ns").Get(context.Background(), ConfigMapName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data["operation-status"]).To(Equal("COMPLETE")) + + var persisted []operations.OperationResult + Expect(json.Unmarshal([]byte(cm.Data["operation-results"]), &persisted)).To(Succeed()) + Expect(persisted).To(Equal(results)) + Expect(cm.Data).NotTo(HaveKey("operation-aggregates")) + }) + + It("overwrites mode-specific fields instead of retaining stale aggregates", func() { + cs := fake.NewSimpleClientset() + random := true + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { + if random { + return Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeRandom, + Status: operations.RunStatusRunning, + Aggregates: []operations.OperationAggregate{{Name: "scale-up", Passed: 3}}, + }, + } + } + return Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusComplete, + Results: []operations.OperationResult{{Name: "scale-up", Status: operations.OperationPassed}}, + }, + } + }) + + r.emit(context.Background(), false) + random = false + r.emit(context.Background(), true) + + cm, err := cs.CoreV1().ConfigMaps("ns").Get(context.Background(), ConfigMapName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).NotTo(HaveKey("operation-aggregates")) + Expect(cm.Data["operation-results"]).To(MatchJSON(`[{"name":"scale-up","status":"PASSED"}]`)) + }) + + It("emits the final report exactly once and rejects later checkpoints", func() { + cs := fake.NewSimpleClientset() + calls := 0 + r := NewCheckpointReporter(cs, "ns", time.Second, func(final bool) Summary { + calls++ + Expect(final).To(BeTrue()) + return Summary{Result: ResultPass, Duration: time.Duration(calls) * time.Minute} + }) + + first := r.EmitFinal() + second := r.EmitFinal() + r.emit(context.Background(), false) + + Expect(calls).To(Equal(1)) + Expect(second).To(Equal(first)) + cm, err := cs.CoreV1().ConfigMaps("ns").Get(context.Background(), ConfigMapName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data["result"]).To(Equal("PASS")) + Expect(cm.Data["latest-report"]).To(ContainSubstring("**Duration:** 1m0s")) + }) }) diff --git a/test/longhaul/report/report.go b/test/longhaul/report/report.go index 1a449b407..f31abf87a 100644 --- a/test/longhaul/report/report.go +++ b/test/longhaul/report/report.go @@ -9,8 +9,10 @@ import ( "time" "github.com/documentdb/documentdb-operator/test/longhaul/backup" + "github.com/documentdb/documentdb-operator/test/longhaul/config" "github.com/documentdb/documentdb-operator/test/longhaul/journal" "github.com/documentdb/documentdb-operator/test/longhaul/monitor" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" "github.com/documentdb/documentdb-operator/test/longhaul/workload" ) @@ -26,9 +28,8 @@ const ( // It is a pure value snapshot — no live counters, no channels — so it can be // passed across goroutines and re-rendered offline. type Summary struct { - // Result is the current verdict. PASS while data-loss counters stay zero, - // flipped to FAIL when the durability oracle detects gaps/checksum errors - // or a disruption window blows its policy budget. + // Result is the current verdict. It flips to FAIL for durability errors, + // operation failures/incomplete sequences, or outage-policy violations. Result Result // Duration is wall-clock time since the run started (process StartTime), @@ -49,13 +50,14 @@ type Summary struct { // only emits a warning annotation. LeakAnalysis monitor.LeakAnalysis - // OpsExecuted is the count of operations (scale up/down, restart, etc.) - // the operations scheduler has run since startup. + // OpsExecuted is the count of terminal operation attempts since startup. OpsExecuted int - // Windows is every disruption window opened during the run, in start - // order. Each window records its op, duration, write-failure count, and - // whether it exceeded its policy budget. + // OperationRun is the bounded sequence result or random aggregate snapshot. + OperationRun operations.RunSnapshot + + // Windows is the journal's bounded set of recent closed disruption windows, + // in start order. Windows []journal.DisruptionWindow // Events is the journal's full event ring (info/warn/error log lines). @@ -83,6 +85,27 @@ func GenerateMarkdown(s Summary) string { } b.WriteString("\n") + switch s.OperationRun.Mode { + case config.OperationModeSequence: + b.WriteString("## Operation Results\n\n") + b.WriteString("| # | Operation | Status | Error |\n") + b.WriteString("|---|-----------|--------|-------|\n") + for i, result := range s.OperationRun.Results { + fmt.Fprintf(&b, "| %d | %s | %s | %s |\n", + i+1, result.Name, result.Status, markdownCell(result.Error)) + } + b.WriteString("\n") + case config.OperationModeRandom: + b.WriteString("## Operation Summary\n\n") + b.WriteString("| Operation | Passed | Failed |\n") + b.WriteString("|-----------|--------|--------|\n") + for _, aggregate := range s.OperationRun.Aggregates { + fmt.Fprintf(&b, "| %s | %d | %d |\n", + aggregate.Name, aggregate.Passed, aggregate.Failed) + } + b.WriteString("\n") + } + // Data Plane Metrics b.WriteString("## Data Plane Metrics\n\n") b.WriteString("| Metric | Value |\n") @@ -157,3 +180,11 @@ func GenerateMarkdown(s Summary) string { return b.String() } + +func markdownCell(value string) string { + if value == "" { + return "—" + } + value = strings.ReplaceAll(value, "|", "\\|") + return strings.ReplaceAll(value, "\n", " ") +} diff --git a/test/longhaul/report/report_test.go b/test/longhaul/report/report_test.go index 33fc844b5..bf3e3dc62 100644 --- a/test/longhaul/report/report_test.go +++ b/test/longhaul/report/report_test.go @@ -10,8 +10,10 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/documentdb/documentdb-operator/test/longhaul/config" "github.com/documentdb/documentdb-operator/test/longhaul/journal" "github.com/documentdb/documentdb-operator/test/longhaul/monitor" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" "github.com/documentdb/documentdb-operator/test/longhaul/workload" ) @@ -69,6 +71,41 @@ var _ = Describe("GenerateMarkdown", func() { Expect(md).NotTo(ContainSubstring("Disruption Windows")) }) + It("renders ordered sequence operation results", func() { + md := GenerateMarkdown(Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusComplete, + Results: []operations.OperationResult{ + {Name: "kill-operator-pod", Status: operations.OperationPassed}, + {Name: "kill-primary-pod", Status: operations.OperationFailed, Error: "primary unchanged"}, + }, + }, + }) + Expect(md).To(ContainSubstring("## Operation Results")) + Expect(md).To(ContainSubstring("| 1 | kill-operator-pod | PASSED |")) + Expect(md).To(ContainSubstring("| 2 | kill-primary-pod | FAILED | primary unchanged |")) + Expect(strings.Index(md, "kill-operator-pod")).To(BeNumerically("<", strings.Index(md, "kill-primary-pod"))) + }) + + It("renders bounded random aggregate counters", func() { + md := GenerateMarkdown(Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeRandom, + Status: operations.RunStatusRunning, + Aggregates: []operations.OperationAggregate{ + {Name: "scale-up", Passed: 12, Failed: 1}, + {Name: "scale-down", Passed: 9}, + }, + }, + }) + Expect(md).To(ContainSubstring("## Operation Summary")) + Expect(md).To(ContainSubstring("| scale-up | 12 | 1 |")) + Expect(md).To(ContainSubstring("| scale-down | 9 | 0 |")) + }) + It("appears with the operation name when at least one window exists", func() { now := time.Now() md := GenerateMarkdown(Summary{