diff --git a/.github/workflows/longhaul-smoke.yml b/.github/workflows/longhaul-smoke.yml index 8d9e08df6..07ad69ee0 100644 --- a/.github/workflows/longhaul-smoke.yml +++ b/.github/workflows/longhaul-smoke.yml @@ -42,23 +42,33 @@ # longhaul-report ConfigMap's result field. # # CI budget -# Scale/upgrade disruption ops are disabled for the smoke run -# (LONGHAUL_MIN_INSTANCES == LONGHAUL_MAX_INSTANCES) so the gate is a fast, -# deterministic data-durability check (writers + verifier) that fits a -# GitHub-hosted runner and finishes in a few minutes. +# Scale ops run for real (MIN=2, MAX=3). The gate exercises every registered +# operation once and finishes within a GitHub-hosted runner's budget. +# +# Operation coverage (random mode) +# The smoke runs the real random scheduler — the exact path the multi-day +# long-haul run uses — but in coverage mode (LONGHAUL_OPERATION_COVERAGE) with +# a pinned seed (LONGHAUL_OPERATION_SEED). Coverage mode draws each operation +# without replacement and completes once every operation has run at least once, +# so the gate exercises scheduler.go's weighted selection, cooldown, and +# steady-state gates while still guaranteeing per-op coverage and a +# deterministic PASS/FAIL verdict. The upgrade uses a second tag for the same +# database image payload: this gates the rolling-update mechanics without +# turning this smoke test into a cross-version compatibility suite. MAX_DURATION +# is only the completion watchdog. # # Data-protection gate # The backup verifier is exercised for real, not just compiled in. The kind # cluster already has CSI VolumeSnapshot support (setup-test-environment runs # deploy-csi-driver.sh: external-snapshotter + a default csi-hostpath -# VolumeSnapshotClass), so a single-instance cluster can complete snapshot -# backups — exactly as the e2e scheduled-backup test proves. The smoke run -# sets a per-minute backup schedule and a 30s verify interval (vs the 5m -# default, via LONGHAUL_BACKUP_VERIFY_INTERVAL) so the verifier's periodic -# loop fires many times within the window and reliably observes a -# scheduled+completed backup. It then asserts scheduled AND completed >= 1 -# (with no retention leak or completion stall), so a broken backup path fails -# the PR rather than passing silently as a no-op. +# VolumeSnapshotClass), so the cluster can complete snapshot backups — +# exactly as the e2e scheduled-backup test proves. The smoke run sets a +# per-minute backup schedule and a 30s verify interval (vs the 5m default, +# via LONGHAUL_BACKUP_VERIFY_INTERVAL) so the verifier's periodic loop fires +# many times within the window and reliably observes a scheduled+completed +# backup. It then asserts scheduled AND completed >= 1 (with no retention leak +# or completion stall), so a broken backup path fails the PR rather than +# passing silently as a no-op. name: Long-Haul Smoke Gate @@ -74,9 +84,9 @@ on: workflow_dispatch: inputs: max_duration: - description: "Bounded driver run length (Go duration). Keep >= 6m so at least one per-minute backup is scheduled and completed within the window." + description: "Coverage watchdog duration (Go duration, e.g. 20m). Keep >= 6m so at least one per-minute backup is scheduled and completed within the window." required: false - default: "6m" + default: "20m" permissions: contents: read @@ -119,7 +129,7 @@ jobs: needs: build if: always() && needs.build.result == 'success' runs-on: ubuntu-22.04 - timeout-minutes: 40 + timeout-minutes: 50 env: IMAGE_TAG: ${{ needs.build.outputs.image_tag }} EXT_IMAGE_TAG: ${{ needs.build.outputs.ext_image_tag }} @@ -128,8 +138,10 @@ jobs: # Must match the cluster name the composite action derives: # documentdb--- KIND_CLUSTER: documentdb-longhaul-amd64-smoke - # The pruner's first tick is at 5m; the default must run beyond it. - MAX_DURATION: ${{ github.event.inputs.max_duration || '6m' }} + # Coverage watchdog; also kept beyond the 5m pruner tick and long enough + # for at least one per-minute backup to schedule and complete. + MAX_DURATION: ${{ github.event.inputs.max_duration || '20m' }} + UPGRADE_IMAGE_TAG: ${{ needs.build.outputs.ext_image_tag }}-smoke-upgrade steps: - name: Checkout uses: actions/checkout@v4 @@ -167,7 +179,7 @@ jobs: runner: "ubuntu-22.04" test-scenario-name: "smoke" node-count: "1" - instances-per-node: "1" + instances-per-node: "2" cert-manager-namespace: ${{ env.CERT_MANAGER_NS }} operator-namespace: ${{ env.OPERATOR_NS }} db-namespace: ${{ env.DB_NS }} @@ -198,6 +210,48 @@ jobs: --from-literal=uri="${URI}" \ --dry-run=client -o yaml | kubectl apply -f - + - name: Prepare deterministic upgrade target + run: | + set -euo pipefail + owner=$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]') + source_tag="${EXT_IMAGE_TAG}-amd64" + + # The operator resolves documentDBVersion against the canonical image + # repositories. Give the existing payload a second local tag so the + # upgrade operation performs a real rolling image-reference update + # without depending on a second database release. + for component in documentdb gateway; do + source="ghcr.io/${owner}/documentdb-kubernetes-operator/${component}:${source_tag}" + base_target="ghcr.io/documentdb/documentdb-kubernetes-operator/${component}:${source_tag}" + upgrade_target="ghcr.io/documentdb/documentdb-kubernetes-operator/${component}:${UPGRADE_IMAGE_TAG}" + docker image inspect "${source}" >/dev/null + docker tag "${source}" "${base_target}" + docker tag "${source}" "${upgrade_target}" + kind load docker-image "${base_target}" --name "${KIND_CLUSTER}" + kind load docker-image "${upgrade_target}" --name "${KIND_CLUSTER}" + done + + # setup-test-environment pins explicit image fields, which take + # precedence over documentDBVersion. Move to the equivalent + # version-based reference before starting the driver so its upgrade + # operation can change both database component image tags. + version_patch=$(BASE_VERSION="${source_tag}" jq -nc '{ + spec: { + documentDBVersion: env.BASE_VERSION, + image: { + documentDB: null, + gateway: null + } + } + }') + kubectl patch documentdb "${DB_NAME}" -n "${DB_NS}" \ + --type merge -p "${version_patch}" + + kubectl create configmap longhaul-versions \ + -n "${DB_NS}" \ + --from-literal="desired-documentdb-version=${UPGRADE_IMAGE_TAG}" \ + --dry-run=client -o yaml | kubectl apply -f - + - name: Deploy long-haul driver (real manifests, bounded override) run: | # RBAC applies unmodified (namespace matches DB_NS). @@ -215,11 +269,17 @@ jobs: # Bounded, deterministic smoke override — patch ONLY runtime knobs on # the shipped ConfigMap; the manifest structure is unchanged. - # - MAX_DURATION: finite run + # - MAX_DURATION: finite run (coverage-completion watchdog) # - RESET_DATA: fresh collection each CI run # - RETAIN_PER_WRITER: low enough to force a real prune at 5m - # - MIN==MAX instances: disable disruptive scale ops (fast + stable) + # - OPERATION_MODE=random + COVERAGE: run the real scheduler but draw + # each operation without replacement and finish once every operation + # (scale-up/-down, kill-operator-pod, kill-primary-pod, + # upgrade-documentdb) has run once — deterministic coverage of the + # production path. + # - OPERATION_SEED: pin selection so the run is reproducible. # - short cadences so the verifier gets several cycles in the window + # - short steady-state gate with a bounded recovery budget # - BACKUP_*: exercise the data-protection verifier for real — a # per-minute schedule so at least one backup is scheduled and # completed within the bounded window, plus a 30s verify interval @@ -233,12 +293,15 @@ jobs: LONGHAUL_RESET_DATA: "true", LONGHAUL_NUM_WRITERS: "2", LONGHAUL_RETAIN_PER_WRITER: "100", + LONGHAUL_OPERATION_MODE: "random", + LONGHAUL_OPERATION_COVERAGE: "true", + LONGHAUL_OPERATION_SEED: "1", LONGHAUL_OP_COOLDOWN: "30s", - LONGHAUL_STEADY_STATE_WAIT: "10s", - LONGHAUL_RECOVERY_TIMEOUT: "2m", + LONGHAUL_STEADY_STATE_WAIT: "5s", + LONGHAUL_RECOVERY_TIMEOUT: "5m", LONGHAUL_REPORT_INTERVAL: "30s", - LONGHAUL_MIN_INSTANCES: "1", - LONGHAUL_MAX_INSTANCES: "1", + LONGHAUL_MIN_INSTANCES: "2", + LONGHAUL_MAX_INSTANCES: "3", LONGHAUL_BACKUP_ENABLED: "true", LONGHAUL_BACKUP_SCHEDULE: "*/1 * * * *", LONGHAUL_BACKUP_RETENTION_DAYS: "1", @@ -255,7 +318,7 @@ jobs: id: wait run: | set -euo pipefail - deadline=$(( $(date +%s) + 900 )) # 15 min hard cap + deadline=$(( $(date +%s) + 1800 )) # 30 min hard cap exit_code="" while [[ $(date +%s) -lt ${deadline} ]]; do pod=$(kubectl get pods -n "${DB_NS}" \ @@ -301,8 +364,14 @@ jobs: -o jsonpath='{.data.result}' 2>/dev/null || echo "MISSING") report=$(kubectl get configmap longhaul-report -n "${DB_NS}" \ -o jsonpath='{.data.latest-report}' 2>/dev/null || echo "MISSING") - echo "Driver exit code : ${exit_code}" - echo "Report result : ${result}" + operation_status=$(kubectl get configmap longhaul-report -n "${DB_NS}" \ + -o json | jq -r '.data["operation-status"] // "MISSING"') + operation_aggregates=$(kubectl get configmap longhaul-report -n "${DB_NS}" \ + -o json | jq -r '.data["operation-aggregates"] // "MISSING"') + echo "Driver exit code : ${exit_code}" + echo "Report result : ${result}" + echo "Operation status : ${operation_status}" + echo "Operation aggregates: ${operation_aggregates}" if [[ "${exit_code}" != "0" ]]; then echo "::error::Driver exited non-zero (${exit_code})." @@ -312,11 +381,25 @@ jobs: echo "::error::longhaul-report result is '${result}', expected PASS." exit 1 fi + if [[ "${operation_status}" != "COMPLETE" ]]; then + echo "::error::operation-status is '${operation_status}', expected COMPLETE." + exit 1 + fi + # Coverage mode: assert every registered operation ran at least once + # (passed >= 1) and none failed. Order is not asserted — coverage, + # not sequence, is the guarantee. + if ! jq -e ' + (map(.name) | sort) == (["scale-up","scale-down","kill-operator-pod","kill-primary-pod","upgrade-documentdb"] | sort) and + all(.[]; .passed >= 1 and .failed == 0) + ' <<<"${operation_aggregates}" >/dev/null; then + echo "::error::operation-aggregates did not show every operation covered (passed>=1, failed==0)." + exit 1 + fi if ! grep -Eq 'pruner: pruned [1-9][0-9]* docs' <<<"${report}"; then echo "::error::Retention pruner did not report deleting any documents." exit 1 fi - echo "✅ Long-haul smoke gate passed (exit 0, report PASS, retention pruned documents)." + echo "✅ Long-haul smoke gate passed (random coverage COMPLETE, all operations covered, report PASS, retention pruned documents)." - name: Assert data-protection verifier ran run: | diff --git a/docs/designs/long-haul-test-design.md b/docs/designs/long-haul-test-design.md index 0fc0aea82..e2b0ad0e7 100644 --- a/docs/designs/long-haul-test-design.md +++ b/docs/designs/long-haul-test-design.md @@ -44,7 +44,7 @@ flowchart LR | Component | Role | Output | |---|---|---| | **Writer/Verifier** | Data-plane workload. Connects via `mongodb://` only — no k8s imports. Writers insert monotonic sequences with checksums under majority write concern; verifiers scan for gaps and bad checksums. | Counters (acked, failed, verify passes, gaps, checksum errors); errors to journal. | -| **Operation Scheduler** | Control plane. Applies weighted-random ops (scale, kill, failover, backup, upgrade) with preconditions and cooldowns. | Operation start/end events to journal. | +| **Operation Runner** | Control plane. Applies weighted-random ops for production long-haul runs, a deterministic named sequence for smoke/reproduction, or no ops when disabled. | Bounded per-operation results/aggregates plus operation events to journal. | | **Monitor** | Polls pod RSS/CPU and checks readiness of operator + DB pods. | Periodic samples + readiness events to journal. | | **Journal** | In-process append-only event log shared by all components. | Reproducible event stream for the report. | | **Report** | Aggregates the journal into a markdown summary at a configurable interval; raises alerts on threshold breaches. | Markdown report; alert lines. | @@ -77,7 +77,11 @@ The test runs **continuously** — no cycles, no scheduled resets. Workload, met ## Operations -The scheduler picks operations from these categories with weighted randomization: +Production runs use weighted randomization. Deterministic smoke and reproduction +runs can instead request a comma-separated sequence of stable operation names; +each operation runs exactly once in order and the driver exits as soon as the +sequence completes or fails. A disabled mode leaves the workload running without +management operations. | Category | Examples | |---|---| @@ -87,16 +91,34 @@ The scheduler picks operations from these categories with weighted randomization | **Chaos** | kill primary pod, drain node, kill operator pod | | **Data protection** | trigger backup, verify backup | -**Sequencing invariants** (enforced by the scheduler — exact values live in code): +**Operation invariants** (exact values live in code): -- One disruptive op at a time. Overlapping disruptions are non-diagnosable. -- Per-category cooldown between ops. Lets the cluster stabilize. -- Steady-state gate — health check must pass before the next op fires. +- One disruptive op at a time in every mode. Overlapping disruptions are + non-diagnosable. +- Random mode applies the global cooldown between attempts. +- The steady-state gate must pass before each operation. Sequence mode also + requires each named precondition to become true within the recovery timeout. **Backup is not isolated.** It runs concurrently with topology changes and chaos so that backup-vs-topology serialization bugs surface here rather than in production — that serialization is the backup feature's job, not the harness's. Each operation declares an **outage policy**: tolerated write failures during its disruption window and a max recovery time. Breaching the policy is recorded as a Tier-1 failure (see Failure Tiers). +Operation state is intentionally bounded for multi-day runs. Random mode keeps +only passed/failed counters per registered operation type; sequence mode keeps +one mutable `PENDING`/`RUNNING`/`PASSED`/`FAILED` result per requested item. +Execution errors, precondition timeouts, outage-policy violations, and an +incomplete sequence at shutdown all produce a failing final verdict. + +**Random coverage mode** (used by the PR smoke gate) is a variant of random +mode: the scheduler draws each operation *without replacement* and completes +once every registered operation has run at least once, rather than running for +the full duration. A fixed seed (`LONGHAUL_OPERATION_SEED`) makes selection +reproducible. This lets the smoke gate exercise the production scheduler path — +weighted selection, cooldown, and steady-state gates — while guaranteeing per-op +coverage and a deterministic PASS/FAIL verdict; `MAX_DURATION` becomes the +completion watchdog, and a run that stops before covering every operation is a +failing `INCOMPLETE` verdict. + --- ## Data Plane Workload diff --git a/test/longhaul/README.md b/test/longhaul/README.md index 61ea8478e..bbf82e1b7 100644 --- a/test/longhaul/README.md +++ b/test/longhaul/README.md @@ -125,10 +125,16 @@ All configuration is via environment variables. | `LONGHAUL_DOCUMENTDB_URI` | Yes | — | Connection string to the DocumentDB gateway. | | `LONGHAUL_CLUSTER_NAME` | Yes | — | Name of the target DocumentDB cluster CR. | | `LONGHAUL_NAMESPACE` | No | `default` | Kubernetes namespace of the target cluster. | +| `LONGHAUL_OPERATOR_NAMESPACE` | No | `documentdb-operator` | Namespace of the DocumentDB operator Deployment (target of the `kill-operator-pod` chaos op). | | `LONGHAUL_MAX_DURATION` | No | `30m` | Max test duration. Use `0s` for run-until-failure. | | `LONGHAUL_NUM_WRITERS` | No | `5` | Number of concurrent writers. | +| `LONGHAUL_OPERATION_MODE` | No | `random` | Operation runner: `random`, `sequence`, or `disabled`. | +| `LONGHAUL_OPERATION_SEQUENCE` | No | empty | Comma-separated stable operation names. Required and used only in `sequence` mode; rejected in `random`/`disabled` mode. Whitespace is trimmed, and duplicate or unknown names are rejected. | +| `LONGHAUL_OPERATION_COVERAGE` | No | `false` | Only valid in `random` mode. When true, the scheduler draws each operation without replacement and completes once every operation has run at least once (instead of running until `MAX_DURATION`, which becomes a watchdog). Used by the smoke gate to exercise the real scheduler while guaranteeing per-op coverage. | +| `LONGHAUL_OPERATION_SEED` | No | unset | Only valid in `random` mode. Pins weighted-random selection to a fixed seed for reproducible runs. Unset uses the process-global generator (production behavior). | | `LONGHAUL_OP_COOLDOWN` | No | `5m` | Cooldown between management operations. | | `LONGHAUL_RECOVERY_TIMEOUT` | No | `5m` | Max wait for cluster recovery after an operation. | +| `LONGHAUL_STEADY_STATE_WAIT` | No | `60s` | Continuous healthy duration required by the steady-state gate. | | `LONGHAUL_MIN_INSTANCES` | No | `1` | Minimum `spec.instancesPerNode` for scale-down operations (CRD lower bound: 1). | | `LONGHAUL_MAX_INSTANCES` | No | `3` | Maximum `spec.instancesPerNode` for scale-up operations (CRD upper bound: 3). | | `LONGHAUL_REPORT_INTERVAL` | No | `1h` | How often to write checkpoint reports to ConfigMap. | @@ -181,14 +187,83 @@ accumulation window long-haul exists to cover. > verbs are granted by the `longhaul-test` Role in `deploy/rbac.yaml`; without > them the backup verifier logs an error and the rest of the run continues. +## Operations + +`random` mode preserves the production long-haul behavior: the scheduler picks +weighted eligible operations every 10 seconds, runs one disruptive operation at +a time, and applies the global cooldown. `sequence` mode runs each configured +operation exactly once and in order, stopping on the first execution, +precondition, recovery, or policy failure; a successful or failed sequence +emits its final report and exits immediately instead of waiting for +`LONGHAUL_MAX_DURATION`. `disabled` mode runs no operations. All modes keep the +continuous writer/verifier workload active. + +Current stable operation names: + +| Operation | Kind | Notes | +|-----------|------|-------| +| `scale-up` / `scale-down` | Topology | Adjusts `spec.instancesPerNode` within `[MIN, MAX]`. Only adds/removes a standby, so the primary write path is untouched (near-zero outage budget). | +| `upgrade-documentdb` | Topology | In-place version upgrade; requires HA (`instancesPerNode>=2`). | +| `kill-operator-pod` | Chaos | Deletes the operator pod; asserts the data plane keeps serving (near-zero outage budget). | +| `kill-primary-pod` | Chaos | Deletes the CNPG primary pod to exercise automatic failover; requires HA (`instancesPerNode>=2`). | + +Operations that keep the write path up throughout — the scale ops and +`kill-operator-pod` — share the near-zero `journal.NoOutagePolicy` budget instead +of ad-hoc per-op numbers, so a regression that unexpectedly disrupts writes +during a "safe" operation trips the policy. + +Outage budgets are expressed as **wall-clock write-outage durations** +(`OutagePolicy.MaxWriteOutage`), not raw write-failure counts. The journal +converts the observed failure count into an estimated outage using the workload's +aggregate write rate (`workload.AggregateWriteRate(NumWriters)`), so the budgets +are independent of `LONGHAUL_NUM_WRITERS`: `NoOutagePolicy` ≈ 300ms (noise +cushion), while `kill-primary-pod` and `upgrade-documentdb` share the +`journal.PrimaryHandoverPolicy` budget of 30s — both interrupt writes for a +single primary handover (an ungraceful failover vs. a graceful switchover), and +an upgrade's longer whole-topology restart is bounded by `MustRecoverWithin`, +not the write-outage budget. + +Operation execution failures are terminal verdict failures in both `random` and +`sequence` modes. Reports keep bounded operation state: one mutable result per +requested sequence item, or aggregate passed/failed counters per operation name +in random mode. The `longhaul-report` ConfigMap exposes `operation-status`, +`operation-results` JSON, and random-mode `operation-aggregates` JSON alongside +the existing `result` and `latest-report` fields. + +### RBAC for chaos operations + +Beyond the base RBAC the driver already needs, the chaos operations require the +driver ServiceAccount to be granted (all present in `deploy/rbac.yaml`): + +- **`kill-primary-pod`** — `get`/`list` on `clusters.postgresql.cnpg.io` (to read + `status.currentPrimary`) and `delete` on `pods` in the cluster namespace; both + added to the `longhaul-test` Role. +- **`kill-operator-pod`** — `get` on `deployments` and `get`/`list`/`delete` on + `pods` in the operator namespace (`LONGHAUL_OPERATOR_NAMESPACE`, default + `documentdb-operator`); granted by a separate `longhaul-test-operator` + Role/RoleBinding in that namespace, since the operator runs outside the + driver's own namespace. + ## CI Safety -The long haul test binary is deployed as a Kubernetes Deployment on a dedicated AKS -cluster. It does **not** run in any PR-gated CI workflow. Because a Deployment -auto-restarts crashed pods, the source of truth for "did the test pass?" is the +The production long haul test binary is deployed as a Kubernetes Deployment on +a dedicated AKS cluster. A short PR smoke workflow runs the same driver and +manifests against kind in **random coverage mode**. Because a Deployment +auto-restarts exited pods, the source of truth for "did the test pass?" is the `longhaul-report` ConfigMap and the GitHub Actions annotations, not the pod status. +The smoke gate runs the real random scheduler — the exact path the multi-day +run uses — but with `LONGHAUL_OPERATION_COVERAGE=true` and a pinned +`LONGHAUL_OPERATION_SEED`, so it draws each operation without replacement and +completes once every registered operation has run at least once: scale up, +scale down, kill the operator pod, kill the primary pod, and upgrade DocumentDB. +This exercises `scheduler.go`'s weighted selection, cooldown, and steady-state +gates while still guaranteeing per-op coverage and a deterministic verdict. The +upgrade gives the existing database images a second local tag, exercising the +rolling-update mechanics without conflating this gate with cross-version +compatibility testing. + The config unit tests (`test/longhaul/config/`) run unconditionally and are included in normal CI test runs — they are fast (~0.002s) and require no cluster. diff --git a/test/longhaul/cmd/longhaul/main.go b/test/longhaul/cmd/longhaul/main.go index 47733402c..3c69c549d 100644 --- a/test/longhaul/cmd/longhaul/main.go +++ b/test/longhaul/cmd/longhaul/main.go @@ -58,6 +58,7 @@ func run(cfg config.Config) int { // Initialize components. j := journal.New() + j.SetWriteRate(workload.AggregateWriteRate(cfg.NumWriters)) metrics := workload.NewMetrics() // Connect to DocumentDB. @@ -147,16 +148,16 @@ func run(cfg config.Config) int { j.Info("main", "retention pruning disabled (LONGHAUL_RETAIN_PER_WRITER=0)") } - // Configure operations. - ops := []operations.Operation{ - operations.NewScaleUp(clusterClient, healthMon, cfg.MaxInstances, cfg.RecoveryTimeout), - operations.NewScaleDown(clusterClient, healthMon, cfg.MinInstances, cfg.RecoveryTimeout), - operations.NewUpgradeDocumentDB(clusterClient, k8sClientset, healthMon, j, cfg.Namespace, cfg.RecoveryTimeout), + // Build the operation registry once, then select the configured runner. + registry, err := operations.NewDefaultRegistry(cfg, clusterClient, k8sClientset, healthMon, j) + if err != nil { + log.Fatalf("failed to build operation registry: %v", err) } - - // Start operation scheduler. - scheduler := operations.NewScheduler(ops, healthMon, j, cfg.OpCooldown) - go scheduler.Run(ctx) + opRunner, err := newOperationRunner(cfg, registry, healthMon, j) + if err != nil { + log.Fatalf("failed to configure operation runner: %v", err) + } + go opRunner.Run(ctx) // Start data-protection verifier (ScheduledBackup + retention). Runs // concurrently with the scheduler by design — backup is deliberately not @@ -184,32 +185,48 @@ func run(cfg config.Config) int { go runMetricsSampling(ctx, clusterClient, leakDetector, j) // Start periodic checkpoint reporter. - summaryFunc := func() report.Summary { - return buildSummary(metrics, backupMetrics, leakDetector, scheduler, j) + summaryFunc := func(final bool) report.Summary { + return buildSummary(metrics, backupMetrics, leakDetector, opRunner, j, final) } reporter := report.NewCheckpointReporter(k8sClientset, cfg.Namespace, cfg.ReportInterval, summaryFunc) go reporter.Run(ctx) j.Info("main", "all components started, entering main loop") - // Main loop: wait for context expiry. - <-ctx.Done() - j.Info("main", fmt.Sprintf("test ending: %v", ctx.Err())) + // Sequence mode and random coverage mode are completion-driven: they exit as + // soon as their operations have finished (or a failure occurs); MaxDuration + // is only their watchdog. Plain random and disabled modes are duration-driven. + completionDriven := cfg.OperationMode == config.OperationModeSequence || + (cfg.OperationMode == config.OperationModeRandom && cfg.OperationCoverage) + if completionDriven { + select { + case <-opRunner.Done(): + j.Info("main", "operations finished") + case <-ctx.Done(): + j.Info("main", fmt.Sprintf("operations watchdog fired: %v", ctx.Err())) + if sr, ok := opRunner.(*operations.SequenceRunner); ok { + sr.MarkIncomplete( + fmt.Sprintf("operation sequence incomplete: watchdog fired: %v", ctx.Err()), + ) + } + // Coverage runners publish their own terminal (incomplete) snapshot + // once cancellation unwinds their loop; wait for that to land. + <-opRunner.Done() + } + } else { + <-ctx.Done() + j.Info("main", fmt.Sprintf("test ending: %v", ctx.Err())) + <-opRunner.Done() + } + cancel() // Allow goroutines to flush. time.Sleep(500 * time.Millisecond) - // Generate final report. Persist to the report ConfigMap synchronously - // here (before os.Exit) so the authoritative verdict reaches the source - // of truth that operators consult — the Run() goroutine cannot do this - // reliably because os.Exit can kill it mid-Update. - summary := buildSummary(metrics, backupMetrics, leakDetector, scheduler, j) - markdown := report.GenerateMarkdown(summary) - fmt.Println("\n" + markdown) - reporter.EmitFinal() - - // Emit final GitHub Actions annotation. - report.EmitAnnotation(summary) + // Emit exactly one terminal report synchronously before os.Exit. EmitFinal + // prints the markdown, emits the GitHub Actions annotation, and persists the + // authoritative verdict to the report ConfigMap. + summary := reporter.EmitFinal() if summary.Result == report.ResultFail { log.Printf("TEST FAILED: %s", summary.FailReason) @@ -220,36 +237,84 @@ func run(cfg config.Config) int { return 0 } +func newOperationRunner( + cfg config.Config, + registry *operations.Registry, + health *monitor.HealthMonitor, + j *journal.Journal, +) (operations.Runner, error) { + switch cfg.OperationMode { + case config.OperationModeRandom: + opts := make([]operations.SchedulerOption, 0, 2) + if cfg.OperationCoverage { + opts = append(opts, operations.WithCoverage()) + } + if cfg.OperationSeedSet { + opts = append(opts, operations.WithSeed(cfg.OperationSeed)) + } + return operations.NewScheduler(registry.All(), health, j, cfg.OpCooldown, opts...), nil + case config.OperationModeSequence: + ops, err := registry.Resolve(cfg.OperationSequence) + if err != nil { + return nil, err + } + return operations.NewSequenceRunner(ops, health, j, cfg.RecoveryTimeout), nil + case config.OperationModeDisabled: + return operations.NewDisabledRunner(), nil + default: + return nil, fmt.Errorf("unsupported operation mode %q", cfg.OperationMode) + } +} + // buildSummary constructs a report.Summary from current state. -func buildSummary(metrics *workload.Metrics, backupMetrics *backup.Metrics, leakDetector *monitor.LeakDetector, scheduler *operations.Scheduler, j *journal.Journal) report.Summary { +func buildSummary( + metrics *workload.Metrics, + backupMetrics *backup.Metrics, + leakDetector *monitor.LeakDetector, + opRunner operations.Runner, + j *journal.Journal, + final bool, +) report.Summary { snap := metrics.Snapshot() backupSnap := backupMetrics.Snapshot() leakAnalysis := leakDetector.Analyze() + operationRun := opRunner.Snapshot() result := report.ResultPass failReason := "" - appendReason := func(msg string) { + if snap.HasDataLoss() { + result = report.ResultFail + failReason = appendFailReason(failReason, fmt.Sprintf("data loss: %d gaps, %d checksum errors", + snap.GapsDetected, snap.ChecksumErrors)) + } + if operationRun.HasFailure() { result = report.ResultFail - if failReason != "" { - failReason += "; " + failReason = appendFailReason(failReason, operationRun.FailureReason) + if operationRun.FailureReason == "" { + failReason = appendFailReason(failReason, "operation execution failed") } - failReason += msg } - - if snap.HasDataLoss() { - appendReason(fmt.Sprintf("data loss: %d gaps, %d checksum errors", - snap.GapsDetected, snap.ChecksumErrors)) + if final && + operationRun.Mode == config.OperationModeSequence && + operationRun.Status != operations.RunStatusComplete && + !operationRun.HasFailure() { + result = report.ResultFail + failReason = appendFailReason(failReason, + fmt.Sprintf("operation sequence incomplete (status %s)", operationRun.Status)) } if j.HasPolicyViolation() { - appendReason("outage policy violated") + result = report.ResultFail + failReason = appendFailReason(failReason, "outage policy violated") } if backupSnap.HasRetentionLeak() { - appendReason(fmt.Sprintf("backup retention leak: %d expired backups not collected", + result = report.ResultFail + failReason = appendFailReason(failReason, fmt.Sprintf("backup retention leak: %d expired backups not collected", backupSnap.RetentionLeaks)) } if backupSnap.HasCompletionStall() { - appendReason(fmt.Sprintf("backup completion stalled: %d backups scheduled with no completion", + result = report.ResultFail + failReason = appendFailReason(failReason, fmt.Sprintf("backup completion stalled: %d backups scheduled with no completion", backupSnap.MaxScheduledWithoutCompletion)) } @@ -259,13 +324,27 @@ func buildSummary(metrics *workload.Metrics, backupMetrics *backup.Metrics, leak Metrics: snap, Backup: backupSnap, LeakAnalysis: leakAnalysis, - OpsExecuted: scheduler.OpsExecuted(), + OpsExecuted: operationRun.OpsExecuted(), + OperationRun: operationRun, Windows: j.DisruptionWindows(), Events: j.Events(), FailReason: failReason, } } +func appendFailReason(existing, reason string) string { + if reason == "" { + return existing + } + if existing == "" { + return reason + } + if existing == reason { + return existing + } + return existing + "; " + reason +} + // runMetricsSampling periodically collects pod resource metrics and feeds the leak detector. func runMetricsSampling(ctx context.Context, client *monitor.K8sClusterClient, ld *monitor.LeakDetector, j *journal.Journal) { if !client.MetricsAvailable() { diff --git a/test/longhaul/cmd/longhaul/main_test.go b/test/longhaul/cmd/longhaul/main_test.go new file mode 100644 index 000000000..921efacb2 --- /dev/null +++ b/test/longhaul/cmd/longhaul/main_test.go @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/backup" + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + "github.com/documentdb/documentdb-operator/test/longhaul/monitor" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" + "github.com/documentdb/documentdb-operator/test/longhaul/report" + "github.com/documentdb/documentdb-operator/test/longhaul/workload" +) + +type snapshotRunner struct { + snapshot operations.RunSnapshot + done chan struct{} +} + +func (r *snapshotRunner) Run(context.Context) {} +func (r *snapshotRunner) Snapshot() operations.RunSnapshot { return r.snapshot } +func (r *snapshotRunner) Done() <-chan struct{} { return r.done } + +func summaryFor(snapshot operations.RunSnapshot, final bool) report.Summary { + j := journal.New() + return buildSummary( + workload.NewMetrics(), + backup.NewMetrics(), + monitor.NewLeakDetector(j, 10, 10), + &snapshotRunner{snapshot: snapshot, done: make(chan struct{})}, + j, + final, + ) +} + +var _ = Describe("buildSummary operation verdicts", func() { + It("fails random mode when any execution failed", func() { + summary := summaryFor(operations.RunSnapshot{ + Mode: config.OperationModeRandom, + Status: operations.RunStatusFailed, + FailureReason: "operation scale-up execute failed: boom", + Aggregates: []operations.OperationAggregate{ + {Name: "scale-up", Failed: 1}, + }, + }, true) + + Expect(summary.Result).To(Equal(report.ResultFail)) + Expect(summary.FailReason).To(ContainSubstring("execute failed")) + }) + + It("allows an in-progress sequence at a checkpoint", func() { + summary := summaryFor(operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusRunning, + Results: []operations.OperationResult{ + {Name: "kill-operator-pod", Status: operations.OperationPending}, + }, + }, false) + Expect(summary.Result).To(Equal(report.ResultPass)) + }) + + It("fails an incomplete requested sequence at final shutdown", func() { + summary := summaryFor(operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusRunning, + Results: []operations.OperationResult{ + {Name: "kill-operator-pod", Status: operations.OperationRunning}, + {Name: "kill-primary-pod", Status: operations.OperationPending}, + }, + }, true) + + Expect(summary.Result).To(Equal(report.ResultFail)) + Expect(summary.FailReason).To(ContainSubstring("operation sequence incomplete")) + }) + + It("does not impose an operation completion requirement in disabled mode", func() { + summary := summaryFor(operations.RunSnapshot{ + Mode: config.OperationModeDisabled, + Status: operations.RunStatusDisabled, + }, true) + Expect(summary.Result).To(Equal(report.ResultPass)) + }) +}) diff --git a/test/longhaul/cmd/longhaul/suite_test.go b/test/longhaul/cmd/longhaul/suite_test.go new file mode 100644 index 000000000..27e6e2bdf --- /dev/null +++ b/test/longhaul/cmd/longhaul/suite_test.go @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestLonghaulMain(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Long Haul Main Suite") +} diff --git a/test/longhaul/config/config.go b/test/longhaul/config/config.go index d2e640fd8..1bfe5c871 100644 --- a/test/longhaul/config/config.go +++ b/test/longhaul/config/config.go @@ -18,12 +18,25 @@ const ( EnvNamespace = "LONGHAUL_NAMESPACE" EnvClusterName = "LONGHAUL_CLUSTER_NAME" + // EnvOperatorNamespace is the namespace where the DocumentDB operator + // Deployment runs (target of the kill-operator-pod chaos op). + EnvOperatorNamespace = "LONGHAUL_OPERATOR_NAMESPACE" + // Workload and operation tuning. EnvDocumentDBURI = "LONGHAUL_DOCUMENTDB_URI" EnvNumWriters = "LONGHAUL_NUM_WRITERS" EnvOpCooldown = "LONGHAUL_OP_COOLDOWN" EnvRecoveryTimeout = "LONGHAUL_RECOVERY_TIMEOUT" EnvSteadyStateWait = "LONGHAUL_STEADY_STATE_WAIT" + EnvOperationMode = "LONGHAUL_OPERATION_MODE" + EnvOperationSeq = "LONGHAUL_OPERATION_SEQUENCE" + // EnvOperationCoverage enables coverage mode for random operation mode: the + // scheduler draws each operation without replacement and completes once every + // operation has run at least once (instead of running until MaxDuration). + EnvOperationCoverage = "LONGHAUL_OPERATION_COVERAGE" + // EnvOperationSeed pins the scheduler's weighted-random selection to a fixed + // seed so a random-mode run is reproducible. Only valid in random mode. + EnvOperationSeed = "LONGHAUL_OPERATION_SEED" // Scale operation bounds. The DocumentDB CRD hard-caps spec.nodeCount=1, // so the scale dimension actually exercised is spec.instancesPerNode (1-3). EnvMinInstances = "LONGHAUL_MIN_INSTANCES" @@ -51,6 +64,15 @@ const ( // roughly 55 hours of history per writer while bounding steady-state disk use. const DefaultRetainPerWriter = 2_000_000 +// OperationMode controls how disruptive operations are run. +type OperationMode string + +const ( + OperationModeRandom OperationMode = "random" + OperationModeSequence OperationMode = "sequence" + OperationModeDisabled OperationMode = "disabled" +) + // Config holds all configuration for a long haul test run. type Config struct { // MaxDuration is the maximum test duration. Zero means run until failure. @@ -62,6 +84,10 @@ type Config struct { // ClusterName is the name of the target DocumentDB cluster CR. ClusterName string + // OperatorNamespace is the namespace of the DocumentDB operator Deployment, + // targeted by the kill-operator-pod chaos operation. + OperatorNamespace string + // DocumentDBURI is the DocumentDB connection string for data-plane workload. DocumentDBURI string @@ -77,6 +103,24 @@ type Config struct { // SteadyStateWait is how long the cluster must be healthy before an operation fires. SteadyStateWait time.Duration + // OperationMode selects weighted-random, deterministic sequence, or no operations. + OperationMode OperationMode + + // OperationSequence is the ordered list used only in sequence mode. + OperationSequence []string + + // OperationCoverage, valid only in random mode, makes the scheduler draw each + // operation without replacement and finish once every operation has run at + // least once. This gates the real scheduler path while guaranteeing per-op + // coverage for the smoke gate; MaxDuration becomes a watchdog. + OperationCoverage bool + + // OperationSeed pins weighted-random selection for reproducibility. Only used + // in random mode. OperationSeedSet distinguishes an explicit 0 from "unset" + // (unset uses the process-global generator, i.e. production behavior). + OperationSeed int64 + OperationSeedSet bool + // MinInstances is the minimum spec.instancesPerNode for scale-down. // CRD lower bound is 1. MinInstances int @@ -118,17 +162,19 @@ type Config struct { // DefaultConfig returns a Config with safe defaults for local development. func DefaultConfig() Config { return Config{ - MaxDuration: 30 * time.Minute, - Namespace: "default", - ClusterName: "", - DocumentDBURI: "", - NumWriters: 5, - OpCooldown: 5 * time.Minute, - RecoveryTimeout: 5 * time.Minute, - SteadyStateWait: 60 * time.Second, - MinInstances: 1, - MaxInstances: 3, - ReportInterval: 1 * time.Hour, + MaxDuration: 30 * time.Minute, + Namespace: "default", + ClusterName: "", + OperatorNamespace: "documentdb-operator", + DocumentDBURI: "", + NumWriters: 5, + OpCooldown: 5 * time.Minute, + RecoveryTimeout: 5 * time.Minute, + SteadyStateWait: 60 * time.Second, + OperationMode: OperationModeRandom, + MinInstances: 1, + MaxInstances: 3, + ReportInterval: 1 * time.Hour, BackupEnabled: true, BackupSchedule: "0 */6 * * *", @@ -160,6 +206,10 @@ func LoadFromEnv() (Config, error) { cfg.ClusterName = v } + if v := os.Getenv(EnvOperatorNamespace); v != "" { + cfg.OperatorNamespace = v + } + if v := os.Getenv(EnvDocumentDBURI); v != "" { cfg.DocumentDBURI = v } @@ -196,6 +246,31 @@ func LoadFromEnv() (Config, error) { cfg.SteadyStateWait = d } + if v := strings.TrimSpace(os.Getenv(EnvOperationMode)); v != "" { + cfg.OperationMode = OperationMode(strings.ToLower(v)) + } + + if v := strings.TrimSpace(os.Getenv(EnvOperationSeq)); v != "" { + sequence, err := parseOperationSequence(v) + if err != nil { + return cfg, fmt.Errorf("invalid %s=%q: %w", EnvOperationSeq, v, err) + } + cfg.OperationSequence = sequence + } + + if v := strings.TrimSpace(strings.ToLower(os.Getenv(EnvOperationCoverage))); v != "" { + cfg.OperationCoverage = v == "true" || v == "1" || v == "yes" + } + + if v := strings.TrimSpace(os.Getenv(EnvOperationSeed)); v != "" { + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return cfg, fmt.Errorf("invalid %s=%q: %w", EnvOperationSeed, v, err) + } + cfg.OperationSeed = n + cfg.OperationSeedSet = true + } + if v := os.Getenv(EnvMinInstances); v != "" { n, err := strconv.Atoi(v) if err != nil { @@ -270,6 +345,9 @@ func (c *Config) Validate() error { if c.ClusterName == "" { return fmt.Errorf("cluster name must not be empty") } + if c.OperatorNamespace == "" { + return fmt.Errorf("operator namespace must not be empty") + } if c.NumWriters < 1 { return fmt.Errorf("num writers must be at least 1, got %d", c.NumWriters) } @@ -279,6 +357,34 @@ func (c *Config) Validate() error { if c.RecoveryTimeout <= 0 { return fmt.Errorf("recovery timeout must be positive, got %s", c.RecoveryTimeout) } + switch c.OperationMode { + case OperationModeRandom, OperationModeDisabled: + if len(c.OperationSequence) > 0 { + return fmt.Errorf("operation sequence must be empty when operation mode is %q", c.OperationMode) + } + case OperationModeSequence: + if len(c.OperationSequence) == 0 { + return fmt.Errorf("operation sequence must not be empty when operation mode is %q", c.OperationMode) + } + seen := make(map[string]struct{}, len(c.OperationSequence)) + for _, name := range c.OperationSequence { + if _, ok := seen[name]; ok { + return fmt.Errorf("operation sequence contains duplicate name %q", name) + } + seen[name] = struct{}{} + } + default: + return fmt.Errorf("operation mode must be one of %q, %q, or %q, got %q", + OperationModeRandom, OperationModeSequence, OperationModeDisabled, c.OperationMode) + } + if c.OperationCoverage && c.OperationMode != OperationModeRandom { + return fmt.Errorf("operation coverage is only supported in %q mode, got %q", + OperationModeRandom, c.OperationMode) + } + if c.OperationSeedSet && c.OperationMode != OperationModeRandom { + return fmt.Errorf("operation seed is only supported in %q mode, got %q", + OperationModeRandom, c.OperationMode) + } if c.MinInstances < 1 { return fmt.Errorf("min instances must be at least 1, got %d", c.MinInstances) } @@ -305,6 +411,19 @@ func (c *Config) Validate() error { return nil } +func parseOperationSequence(value string) ([]string, error) { + parts := strings.Split(value, ",") + sequence := make([]string, 0, len(parts)) + for _, part := range parts { + name := strings.TrimSpace(part) + if name == "" { + return nil, fmt.Errorf("operation names must not be empty") + } + sequence = append(sequence, name) + } + return sequence, nil +} + // IsEnabled returns true if the long haul test is explicitly enabled // via the LONGHAUL_ENABLED environment variable. func IsEnabled() bool { diff --git a/test/longhaul/config/config_test.go b/test/longhaul/config/config_test.go index 69054b73f..78ed0c3ea 100644 --- a/test/longhaul/config/config_test.go +++ b/test/longhaul/config/config_test.go @@ -17,10 +17,13 @@ var _ = Describe("Config", func() { Expect(cfg.MaxDuration).To(Equal(30 * time.Minute)) Expect(cfg.Namespace).To(Equal("default")) Expect(cfg.ClusterName).To(BeEmpty()) + Expect(cfg.OperatorNamespace).To(Equal("documentdb-operator")) Expect(cfg.NumWriters).To(Equal(5)) Expect(cfg.OpCooldown).To(Equal(5 * time.Minute)) Expect(cfg.RecoveryTimeout).To(Equal(5 * time.Minute)) Expect(cfg.SteadyStateWait).To(Equal(60 * time.Second)) + Expect(cfg.OperationMode).To(Equal(OperationModeRandom)) + Expect(cfg.OperationSequence).To(BeEmpty()) Expect(cfg.MinInstances).To(Equal(1)) Expect(cfg.MaxInstances).To(Equal(3)) Expect(cfg.RetainPerWriter).To(Equal(int64(DefaultRetainPerWriter))) @@ -33,8 +36,11 @@ var _ = Describe("Config", func() { BeforeEach(func() { for _, k := range []string{ EnvEnabled, EnvMaxDuration, EnvNamespace, EnvClusterName, + EnvOperatorNamespace, EnvDocumentDBURI, EnvNumWriters, EnvOpCooldown, EnvRecoveryTimeout, EnvSteadyStateWait, + EnvOperationMode, EnvOperationSeq, + EnvOperationCoverage, EnvOperationSeed, EnvMinInstances, EnvMaxInstances, EnvReportInterval, EnvBackupEnabled, EnvBackupSchedule, EnvBackupRetentionDays, EnvBackupVerifyInterval, @@ -74,6 +80,13 @@ var _ = Describe("Config", func() { Expect(cfg.ClusterName).To(Equal("my-cluster")) }) + It("parses OperatorNamespace from env", func() { + GinkgoT().Setenv(EnvOperatorNamespace, "custom-operator-ns") + cfg, err := LoadFromEnv() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.OperatorNamespace).To(Equal("custom-operator-ns")) + }) + It("returns error for invalid MaxDuration", func() { GinkgoT().Setenv(EnvMaxDuration, "not-a-duration") _, err := LoadFromEnv() @@ -109,51 +122,48 @@ var _ = Describe("Config", func() { Expect(cfg.DocumentDBURI).To(Equal("mongodb://localhost:27017")) }) - It("parses the backup env knobs", func() { - GinkgoT().Setenv(EnvBackupEnabled, "true") - GinkgoT().Setenv(EnvBackupSchedule, "0 */6 * * *") - GinkgoT().Setenv(EnvBackupRetentionDays, "7") - GinkgoT().Setenv(EnvBackupVerifyInterval, "30s") - cfg, err := LoadFromEnv() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg.BackupEnabled).To(BeTrue()) - Expect(cfg.BackupSchedule).To(Equal("0 */6 * * *")) - Expect(cfg.BackupRetentionDays).To(Equal(7)) - Expect(cfg.BackupVerifyInterval).To(Equal(30 * time.Second)) + It("returns error for invalid RetainPerWriter", func() { + GinkgoT().Setenv(EnvRetainPerWriter, "not-a-number") + _, err := LoadFromEnv() + Expect(err).To(MatchError(ContainSubstring(EnvRetainPerWriter))) }) - It("returns error for invalid BackupRetentionDays", func() { - GinkgoT().Setenv(EnvBackupRetentionDays, "abc") - _, err := LoadFromEnv() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring(EnvBackupRetentionDays)) + It("normalizes operation mode and trims sequence names", func() { + GinkgoT().Setenv(EnvOperationMode, " Sequence ") + GinkgoT().Setenv(EnvOperationSeq, " kill-operator-pod, kill-primary-pod ") + cfg, err := LoadFromEnv() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.OperationMode).To(Equal(OperationModeSequence)) + Expect(cfg.OperationSequence).To(Equal([]string{"kill-operator-pod", "kill-primary-pod"})) }) - It("returns error for invalid BackupVerifyInterval", func() { - GinkgoT().Setenv(EnvBackupVerifyInterval, "not-a-duration") + It("rejects empty names in a non-empty sequence", func() { + GinkgoT().Setenv(EnvOperationSeq, "scale-up, ,scale-down") _, err := LoadFromEnv() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring(EnvBackupVerifyInterval)) + Expect(err).To(MatchError(ContainSubstring("operation names must not be empty"))) }) - It("parses RetainPerWriter from env", func() { - GinkgoT().Setenv(EnvRetainPerWriter, "500000") + It("parses operation coverage and seed in random mode", func() { + GinkgoT().Setenv(EnvOperationMode, "random") + GinkgoT().Setenv(EnvOperationCoverage, "true") + GinkgoT().Setenv(EnvOperationSeed, "-7") cfg, err := LoadFromEnv() Expect(err).NotTo(HaveOccurred()) - Expect(cfg.RetainPerWriter).To(Equal(int64(500_000))) + Expect(cfg.OperationCoverage).To(BeTrue()) + Expect(cfg.OperationSeed).To(Equal(int64(-7))) + Expect(cfg.OperationSeedSet).To(BeTrue()) }) - It("parses RetainPerWriter=0 to disable pruning", func() { - GinkgoT().Setenv(EnvRetainPerWriter, "0") + It("leaves OperationSeedSet false when the seed env is unset", func() { cfg, err := LoadFromEnv() Expect(err).NotTo(HaveOccurred()) - Expect(cfg.RetainPerWriter).To(BeZero()) + Expect(cfg.OperationSeedSet).To(BeFalse()) }) - It("returns error for invalid RetainPerWriter", func() { - GinkgoT().Setenv(EnvRetainPerWriter, "not-a-number") + It("returns error for an invalid operation seed", func() { + GinkgoT().Setenv(EnvOperationSeed, "not-a-number") _, err := LoadFromEnv() - Expect(err).To(MatchError(ContainSubstring(EnvRetainPerWriter))) + Expect(err).To(MatchError(ContainSubstring(EnvOperationSeed))) }) }) @@ -176,6 +186,13 @@ var _ = Describe("Config", func() { Expect(cfg.Validate()).To(MatchError(ContainSubstring("cluster name"))) }) + It("fails when OperatorNamespace is empty", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperatorNamespace = "" + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operator namespace"))) + }) + It("fails when MaxDuration is negative", func() { cfg := DefaultConfig() cfg.ClusterName = "test" @@ -197,6 +214,82 @@ var _ = Describe("Config", func() { Expect(cfg.Validate()).To(MatchError(ContainSubstring("recovery timeout"))) }) + It("fails for an unknown operation mode", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = "roulette" + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operation mode must be one of"))) + }) + + It("requires a non-empty sequence in sequence mode", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = OperationModeSequence + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operation sequence must not be empty"))) + }) + + It("rejects duplicate sequence names", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = OperationModeSequence + cfg.OperationSequence = []string{"scale-up", "scale-up"} + Expect(cfg.Validate()).To(MatchError(ContainSubstring(`duplicate name "scale-up"`))) + }) + + DescribeTable("rejects a sequence outside sequence mode", + func(mode OperationMode) { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = mode + cfg.OperationSequence = []string{"scale-up"} + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operation sequence must be empty"))) + }, + Entry("random", OperationModeRandom), + Entry("disabled", OperationModeDisabled), + ) + + It("accepts coverage and seed in random mode", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = OperationModeRandom + cfg.OperationCoverage = true + cfg.OperationSeed = 99 + cfg.OperationSeedSet = true + Expect(cfg.Validate()).To(Succeed()) + }) + + DescribeTable("rejects coverage outside random mode", + func(mode OperationMode) { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = mode + if mode == OperationModeSequence { + cfg.OperationSequence = []string{"scale-up"} + } + cfg.OperationCoverage = true + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operation coverage is only supported"))) + }, + Entry("sequence", OperationModeSequence), + Entry("disabled", OperationModeDisabled), + ) + + It("rejects a seed outside random mode", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = OperationModeSequence + cfg.OperationSequence = []string{"scale-up"} + cfg.OperationSeedSet = true + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operation seed is only supported"))) + }) + + It("accepts a valid sequence configuration", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = OperationModeSequence + cfg.OperationSequence = []string{"kill-operator-pod", "kill-primary-pod"} + Expect(cfg.Validate()).To(Succeed()) + }) + It("fails when MaxInstances < MinInstances", func() { cfg := DefaultConfig() cfg.ClusterName = "test" diff --git a/test/longhaul/deploy/deployment.yaml b/test/longhaul/deploy/deployment.yaml index 835e577fc..fecc7cdf2 100644 --- a/test/longhaul/deploy/deployment.yaml +++ b/test/longhaul/deploy/deployment.yaml @@ -15,10 +15,10 @@ # driver pods concurrently against the same DocumentDB cluster / # workload collection). # -# Failure semantics: on critical failure (data loss, policy violation) -# the driver exits non-zero. The Deployment auto-restarts the pod, which -# gives MTBF data; the alert workflow polls the report ConfigMap and -# pages on incident-count thresholds. +# Failure semantics: on critical failure (data loss, operation failure, or +# policy violation) the driver exits non-zero. The Deployment auto-restarts +# the pod, which gives MTBF data; the alert workflow polls the report ConfigMap +# and pages on incident-count thresholds. # # Image refs are templated; the longhaul-deploy workflow substitutes: # __OWNER__ -> lowercased ${{ github.repository_owner }} @@ -42,6 +42,10 @@ data: # Writer/verifier counts. LONGHAUL_NUM_WRITERS: "5" # Operation scheduling. + # random preserves the production long-haul behavior. sequence executes + # LONGHAUL_OPERATION_SEQUENCE exactly once in order; disabled runs no ops. + LONGHAUL_OPERATION_MODE: "random" + LONGHAUL_OPERATION_SEQUENCE: "" LONGHAUL_OP_COOLDOWN: "10m" LONGHAUL_RECOVERY_TIMEOUT: "5m" # How long the cluster must be observed healthy before the next diff --git a/test/longhaul/deploy/rbac.yaml b/test/longhaul/deploy/rbac.yaml index 0f4395546..6029fbea0 100644 --- a/test/longhaul/deploy/rbac.yaml +++ b/test/longhaul/deploy/rbac.yaml @@ -21,14 +21,19 @@ metadata: app.kubernetes.io/name: longhaul-test app.kubernetes.io/component: testing rules: - # Read pod status for health monitoring. + # Read pod status for health monitoring; delete pods for the kill-primary-pod + # chaos op (deletes the CNPG primary to exercise automatic failover). - apiGroups: [""] resources: ["pods"] - verbs: ["get", "list", "watch"] + verbs: ["get", "list", "watch", "delete"] # Read and patch DocumentDB CRs for health check and scale operations. - apiGroups: ["documentdb.io"] resources: ["dbs"] verbs: ["get", "list", "patch"] + # Read the CNPG Cluster to resolve the current primary pod (kill-primary-pod). + - apiGroups: ["postgresql.cnpg.io"] + resources: ["clusters"] + verbs: ["get", "list"] # Manage ScheduledBackups and read their child Backups for the data-protection # verifier (ensure a ScheduledBackup, then watch child Backup CRs). - apiGroups: ["documentdb.io"] @@ -59,6 +64,49 @@ subjects: name: longhaul-test namespace: documentdb-test-ns --- +# Role in the operator namespace for the kill-operator-pod chaos op: read the +# operator Deployment (to build its pod selector and check availability) and +# delete its pod. Namespaced separately because the operator runs outside the +# driver's own namespace (LONGHAUL_OPERATOR_NAMESPACE, default documentdb-operator). +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: longhaul-test-operator + # NOTE: keep in sync with LONGHAUL_OPERATOR_NAMESPACE. If the driver overrides + # that env var, this namespace must be edited to match or kill-operator-pod + # fails with RBAC errors. + namespace: documentdb-operator + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +rules: + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: longhaul-test-operator + # NOTE: must match the Role namespace above (and LONGHAUL_OPERATOR_NAMESPACE). + # Update in lockstep or the binding won't grant permissions in the right + # namespace. + namespace: documentdb-operator + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: longhaul-test-operator +subjects: + - kind: ServiceAccount + name: longhaul-test + namespace: documentdb-test-ns +--- # ClusterRole for metrics-server access (metrics API is cluster-scoped). apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/test/longhaul/go.mod b/test/longhaul/go.mod index 5d3fa4687..ebde56ba3 100644 --- a/test/longhaul/go.mod +++ b/test/longhaul/go.mod @@ -3,6 +3,7 @@ module github.com/documentdb/documentdb-operator/test/longhaul go 1.26.6 require ( + github.com/cloudnative-pg/cloudnative-pg v1.29.2 github.com/documentdb/documentdb-operator v0.0.0-00010101000000-000000000000 github.com/documentdb/documentdb-operator/test/shared v0.0.0-00010101000000-000000000000 github.com/onsi/ginkgo/v2 v2.32.0 @@ -25,7 +26,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudnative-pg/barman-cloud v0.5.1 // indirect - github.com/cloudnative-pg/cloudnative-pg v1.29.2 // indirect github.com/cloudnative-pg/cnpg-i v0.5.0 // indirect github.com/cloudnative-pg/machinery v0.5.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/test/longhaul/journal/journal.go b/test/longhaul/journal/journal.go index bfddebc76..57b886b18 100644 --- a/test/longhaul/journal/journal.go +++ b/test/longhaul/journal/journal.go @@ -24,8 +24,9 @@ const ( // trim cost is amortized over many appends (one copy every trimHeadroom // events), not paid on every append once we hit the cap. const ( - maxEvents = 10000 - trimHeadroom = 1000 + maxEvents = 10000 + trimHeadroom = 1000 + maxDisruptionWindows = 1000 ) // Event represents a single journal entry. @@ -52,15 +53,41 @@ type Journal struct { // All closed disruption windows. closedWindows []DisruptionWindow + + // writesPerSecond is the workload's aggregate write rate, stamped onto each + // disruption window so ExceededPolicy can convert write-failure counts into + // an estimated outage duration. Defaults to DefaultWritesPerSecond; override + // with SetWriteRate once the real writer count is known. + writesPerSecond float64 } +// DefaultWritesPerSecond is the assumed aggregate write rate used until +// SetWriteRate is called. It matches the default workload (5 writers at one +// write per 100ms = 50 writes/s) so tests and un-configured journals still +// evaluate write-outage budgets sensibly. +const DefaultWritesPerSecond = 50.0 + // New creates a new empty Journal. func New() *Journal { return &Journal{ - events: make([]Event, 0, 256), + events: make([]Event, 0, 256), + writesPerSecond: DefaultWritesPerSecond, } } +// SetWriteRate records the workload's aggregate write rate (writes/second across +// all writers) so disruption windows can translate write-failure counts into an +// estimated outage duration. Non-positive values are ignored, preserving the +// default. Safe for concurrent use. +func (j *Journal) SetWriteRate(writesPerSecond float64) { + if writesPerSecond <= 0 { + return + } + j.mu.Lock() + defer j.mu.Unlock() + j.writesPerSecond = writesPerSecond +} + // Record appends a new event to the journal. Safe for concurrent use. // // To bound memory on multi-day runs the in-memory ring is capped at @@ -108,13 +135,14 @@ func (j *Journal) OpenDisruptionWindow(operationName string, policy OutagePolicy // Close any existing window first. if j.activeWindow != nil { j.activeWindow.EndTime = time.Now() - j.closedWindows = append(j.closedWindows, *j.activeWindow) + j.appendClosedWindow(*j.activeWindow) } j.activeWindow = &DisruptionWindow{ - OperationName: operationName, - StartTime: time.Now(), - Policy: policy, + OperationName: operationName, + StartTime: time.Now(), + Policy: policy, + WritesPerSecond: j.writesPerSecond, } j.events = append(j.events, Event{ @@ -125,17 +153,18 @@ func (j *Journal) OpenDisruptionWindow(operationName string, policy OutagePolicy }) } -// CloseDisruptionWindow ends the active disruption period. -func (j *Journal) CloseDisruptionWindow() { +// CloseDisruptionWindow ends the active disruption period and returns a copy. +func (j *Journal) CloseDisruptionWindow() *DisruptionWindow { j.mu.Lock() defer j.mu.Unlock() if j.activeWindow == nil { - return + return nil } j.activeWindow.EndTime = time.Now() - j.closedWindows = append(j.closedWindows, *j.activeWindow) + j.appendClosedWindow(*j.activeWindow) + closed := *j.activeWindow j.events = append(j.events, Event{ Timestamp: time.Now(), @@ -146,6 +175,15 @@ func (j *Journal) CloseDisruptionWindow() { }) j.activeWindow = nil + return &closed +} + +func (j *Journal) appendClosedWindow(window DisruptionWindow) { + j.closedWindows = append(j.closedWindows, window) + if len(j.closedWindows) > maxDisruptionWindows { + copy(j.closedWindows, j.closedWindows[len(j.closedWindows)-maxDisruptionWindows:]) + j.closedWindows = j.closedWindows[:maxDisruptionWindows] + } } // RecordWriteFailure increments the failure count for the active disruption window. diff --git a/test/longhaul/journal/journal_test.go b/test/longhaul/journal/journal_test.go index 02e2b3c6c..d70277020 100644 --- a/test/longhaul/journal/journal_test.go +++ b/test/longhaul/journal/journal_test.go @@ -43,7 +43,7 @@ var _ = Describe("Journal", func() { Describe("DisruptionWindow lifecycle", func() { It("opens, records failures, and closes correctly", func() { j := New() - policy := OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 10} + policy := OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second} Expect(j.ActiveWindow()).To(BeNil()) @@ -80,6 +80,20 @@ var _ = Describe("Journal", func() { j := New() Expect(func() { j.RecordWriteFailure() }).NotTo(Panic()) }) + + It("bounds closed disruption-window diagnostics to the newest entries", func() { + j := New() + total := maxDisruptionWindows + 5 + for i := 0; i < total; i++ { + j.OpenDisruptionWindow(fmt.Sprintf("op-%d", i), DefaultOutagePolicy()) + j.CloseDisruptionWindow() + } + + windows := j.DisruptionWindows() + Expect(windows).To(HaveLen(maxDisruptionWindows)) + Expect(windows[0].OperationName).To(Equal("op-5")) + Expect(windows[len(windows)-1].OperationName).To(Equal(fmt.Sprintf("op-%d", total-1))) + }) }) Describe("HasPolicyViolation", func() { @@ -89,14 +103,14 @@ var _ = Describe("Journal", func() { It("returns false on a closed window within budget", func() { j := New() - j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 10}) + j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}) j.CloseDisruptionWindow() Expect(j.HasPolicyViolation()).To(BeFalse()) }) - It("returns true on a closed window over write-failure budget", func() { + It("returns true on a closed window over write-outage budget", func() { j := New() - j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 1}) + j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: 10 * time.Millisecond}) j.RecordWriteFailure() j.RecordWriteFailure() j.CloseDisruptionWindow() @@ -105,7 +119,7 @@ var _ = Describe("Journal", func() { It("returns true on an active window over time budget", func() { j := New() - j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Nanosecond, AllowedWriteFailures: 10}) + j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Nanosecond, MaxWriteOutage: time.Second}) time.Sleep(1 * time.Millisecond) Expect(j.HasPolicyViolation()).To(BeTrue()) }) diff --git a/test/longhaul/journal/policy.go b/test/longhaul/journal/policy.go index 25d28e826..9c985c162 100644 --- a/test/longhaul/journal/policy.go +++ b/test/longhaul/journal/policy.go @@ -5,20 +5,87 @@ package journal import "time" -// OutagePolicy defines acceptable disruption bounds for an operation. +// OutagePolicy defines acceptable disruption bounds for an operation. Its two +// fields assert on different properties of the managed cluster and fail +// independently (ExceededPolicy trips if either is exceeded): MaxWriteOutage +// bounds client-visible write availability, while MustRecoverWithin bounds the +// cluster's return to its full declared topology (all pods Ready, CR Ready). +// Operation execution errors also fail the run independently of this policy. +// Each can be violated while the other is fine — e.g. after a failover writes +// resume quickly (MaxWriteOutage happy) yet the cluster stays degraded until a +// replacement standby rejoins, which only MustRecoverWithin catches. type OutagePolicy struct { - // AllowedWriteFailures is the maximum number of write failures during the window. - AllowedWriteFailures int64 + // MaxWriteOutage bounds how long the write path (client -> gateway -> + // primary) may be unavailable during the window. It is evaluated from the + // observed write-failure count normalized by the workload's aggregate write + // rate (see DisruptionWindow.EstimatedWriteOutage), so the budget is + // expressed in wall-clock outage time and is independent of how many writer + // goroutines (LONGHAUL_NUM_WRITERS) are configured. + MaxWriteOutage time.Duration - // MustRecoverWithin is the maximum time from operation start to full recovery. + // MustRecoverWithin is the maximum time from operation start to full cluster + // recovery (steady state). MustRecoverWithin time.Duration } // DefaultOutagePolicy returns a conservative policy suitable for most operations. func DefaultOutagePolicy() OutagePolicy { return OutagePolicy{ - AllowedWriteFailures: 50, - MustRecoverWithin: 5 * time.Minute, + MaxWriteOutage: 5 * time.Second, + MustRecoverWithin: 5 * time.Minute, + } +} + +// NoOutageWriteOutageCushion is the tiny write-outage budget granted to +// operations that are expected NOT to disrupt the data plane. It is not a +// tolerance for real outages: one fully-failed write tick (every configured +// writer failing once) maps to exactly one writeInterval of estimated outage +// (~100ms) regardless of writer count, so this ~3-tick cushion absorbs unrelated +// background noise (a client reconnect, service-endpoint churn) without +// tolerating a genuine primary outage. Centralized so it can be recalibrated +// against real long-haul runs in one place. +const NoOutageWriteOutageCushion = 300 * time.Millisecond + +// NoOutagePolicy is the outage budget for operations that keep the write path +// up throughout and therefore must not cause a write outage. It is shared by +// every "no data-plane impact" operation: +// - control-plane faults, e.g. an operator pod restart, and +// - scaling that only adds or removes a standby replica (the primary, and +// thus the write path, is never touched). +// +// recovery bounds how long the cluster may take to return to steady state. +func NoOutagePolicy(recovery time.Duration) OutagePolicy { + return OutagePolicy{ + MaxWriteOutage: NoOutageWriteOutageCushion, + MustRecoverWithin: recovery, + } +} + +// PrimaryHandoverWriteOutage is the write-outage budget for operations that +// interrupt writes for exactly one primary handover. It is shared so the two +// such operations cannot drift apart: +// - kill-primary-pod — an *ungraceful* failover (detect the lost pod, then +// promote a standby), and +// - upgrade-documentdb — a *graceful* switchover of the primary; the standby +// pod restarts during the rolling upgrade do NOT interrupt writes, so the +// write outage is just the one switchover (and a graceful switchover is +// typically no worse than an ungraceful failover, which pays a detection +// delay). The upgrade's longer, whole-topology restart is bounded by +// MustRecoverWithin, not here. +// +// Sized to comfortably cover a healthy single CNPG failover; heuristic pending +// calibration against real long-haul runs. +const PrimaryHandoverWriteOutage = 30 * time.Second + +// PrimaryHandoverPolicy is the outage budget for operations whose write path is +// interrupted for a single primary handover (see PrimaryHandoverWriteOutage). +// recovery bounds how long the cluster may take to return to full topology, +// which can legitimately differ per operation (a rolling upgrade restarts every +// pod and takes longer than a single failover). +func PrimaryHandoverPolicy(recovery time.Duration) OutagePolicy { + return OutagePolicy{ + MaxWriteOutage: PrimaryHandoverWriteOutage, + MustRecoverWithin: recovery, } } @@ -38,6 +105,25 @@ type DisruptionWindow struct { // WriteFailures counts failures observed during this window. WriteFailures int64 + + // WritesPerSecond is the workload's aggregate write rate at the time the + // window opened. It is used to convert the raw WriteFailures count into an + // estimated write-outage duration (see EstimatedWriteOutage). A real outage + // makes every writer fail on every tick, so failures accrue at the full + // aggregate rate and count/rate recovers the wall-clock outage duration + // regardless of writer count. Zero disables the write-outage check. + WritesPerSecond float64 +} + +// EstimatedWriteOutage converts the observed write-failure count into an +// approximate duration for which the write path was unavailable, using the +// aggregate write rate captured when the window opened. Returns 0 when the rate +// is unknown (<= 0), which disables the write-outage portion of the policy. +func (w *DisruptionWindow) EstimatedWriteOutage() time.Duration { + if w.WritesPerSecond <= 0 { + return 0 + } + return time.Duration(float64(w.WriteFailures) / w.WritesPerSecond * float64(time.Second)) } // IsActive returns true if the disruption window has not been closed. @@ -59,7 +145,7 @@ func (w *DisruptionWindow) ExceededPolicy() bool { if w.Duration() > w.Policy.MustRecoverWithin { return true } - if w.WriteFailures > w.Policy.AllowedWriteFailures { + if w.EstimatedWriteOutage() > w.Policy.MaxWriteOutage { return true } return false diff --git a/test/longhaul/journal/policy_test.go b/test/longhaul/journal/policy_test.go index 4fe26b1a4..dec577337 100644 --- a/test/longhaul/journal/policy_test.go +++ b/test/longhaul/journal/policy_test.go @@ -43,42 +43,65 @@ var _ = Describe("DisruptionWindow", func() { }, Entry("within all budgets", DisruptionWindow{ - StartTime: time.Now().Add(-10 * time.Second), - EndTime: time.Now(), - WriteFailures: 5, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + WriteFailures: 5, // 5/50 = 0.1s < 1s + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, false), Entry("exceeds MustRecoverWithin", DisruptionWindow{ - StartTime: time.Now().Add(-2 * time.Minute), - EndTime: time.Now(), - WriteFailures: 1, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-2 * time.Minute), + EndTime: time.Now(), + WriteFailures: 1, + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, true), - Entry("exceeds AllowedWriteFailures", + Entry("exceeds MaxWriteOutage", DisruptionWindow{ - StartTime: time.Now().Add(-10 * time.Second), - EndTime: time.Now(), - WriteFailures: 100, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + WriteFailures: 100, // 100/50 = 2s > 1s + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, true), - Entry("boundary: equal to write-failure budget is allowed", + Entry("boundary: estimated outage equal to budget is allowed", DisruptionWindow{ - StartTime: time.Now().Add(-10 * time.Second), - EndTime: time.Now(), - WriteFailures: 50, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + WriteFailures: 50, // 50/50 = exactly 1s + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, + }, false), + Entry("unknown write rate disables the write-outage check", + DisruptionWindow{ + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + WriteFailures: 100000, + WritesPerSecond: 0, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, false), Entry("active window also evaluated against MustRecoverWithin", DisruptionWindow{ - StartTime: time.Now().Add(-2 * time.Minute), - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-2 * time.Minute), + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, true), ) It("DefaultOutagePolicy returns no zero-valued field", func() { p := DefaultOutagePolicy() Expect(p.MustRecoverWithin).NotTo(BeZero()) - Expect(p.AllowedWriteFailures).NotTo(BeZero()) + Expect(p.MaxWriteOutage).NotTo(BeZero()) + }) + + It("NoOutagePolicy grants the near-zero cushion and echoes recovery", func() { + p := NoOutagePolicy(3 * time.Minute) + Expect(p.MaxWriteOutage).To(Equal(NoOutageWriteOutageCushion)) + Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute)) + }) + + It("NoOutagePolicy is far tighter than DefaultOutagePolicy", func() { + Expect(NoOutageWriteOutageCushion).To(BeNumerically("<", DefaultOutagePolicy().MaxWriteOutage)) }) }) diff --git a/test/longhaul/monitor/health.go b/test/longhaul/monitor/health.go index 0c5e95163..939f48056 100644 --- a/test/longhaul/monitor/health.go +++ b/test/longhaul/monitor/health.go @@ -49,6 +49,15 @@ type ClusterClient interface { // UpgradeDocumentDB patches spec.documentDBVersion and spec.schemaVersion="auto". UpgradeDocumentDB(ctx context.Context, version string) error + + // GetPrimaryInstance returns the name of the pod currently serving as the + // CNPG primary (from Cluster.status.currentPrimary). The pod name equals + // the CNPG instance name. Returns an error if no primary is known yet. + GetPrimaryInstance(ctx context.Context) (string, error) + + // DeletePod deletes the named pod in the cluster namespace. Used by chaos + // operations to inject pod-loss faults. + DeletePod(ctx context.Context, name string) error } // HealthMonitor continuously monitors cluster health and tracks steady-state. diff --git a/test/longhaul/monitor/health_test.go b/test/longhaul/monitor/health_test.go index 1ad903fd9..4d5385e1e 100644 --- a/test/longhaul/monitor/health_test.go +++ b/test/longhaul/monitor/health_test.go @@ -41,9 +41,11 @@ func (f *fakeClusterClient) GetClusterHealth(_ context.Context) (ClusterHealth, func (f *fakeClusterClient) GetCurrentDocumentDBImageTag(_ context.Context) (string, error) { return "", nil } -func (f *fakeClusterClient) GetInstancesPerNode(_ context.Context) (int, error) { return 1, nil } -func (f *fakeClusterClient) ScaleCluster(_ context.Context, _ int) error { return nil } -func (f *fakeClusterClient) UpgradeDocumentDB(_ context.Context, _ string) error { return nil } +func (f *fakeClusterClient) GetInstancesPerNode(_ context.Context) (int, error) { return 1, nil } +func (f *fakeClusterClient) ScaleCluster(_ context.Context, _ int) error { return nil } +func (f *fakeClusterClient) UpgradeDocumentDB(_ context.Context, _ string) error { return nil } +func (f *fakeClusterClient) GetPrimaryInstance(_ context.Context) (string, error) { return "", nil } +func (f *fakeClusterClient) DeletePod(_ context.Context, _ string) error { return nil } var _ = Describe("HealthMonitor", func() { Describe("IsSteadyState", func() { diff --git a/test/longhaul/monitor/k8sclient.go b/test/longhaul/monitor/k8sclient.go index f843f960b..f1bc91b13 100644 --- a/test/longhaul/monitor/k8sclient.go +++ b/test/longhaul/monitor/k8sclient.go @@ -63,7 +63,7 @@ func NewK8sClusterClient(cfg K8sClientConfig) (*K8sClusterClient, error) { return nil, fmt.Errorf("failed to create clientset: %w", err) } - scheme, err := shareddb.NewScheme() + scheme, err := shareddb.NewScheme(cnpgv1.AddToScheme) if err != nil { return nil, fmt.Errorf("failed to build scheme: %w", err) } @@ -229,6 +229,29 @@ func (k *K8sClusterClient) UpgradeDocumentDB(ctx context.Context, version string return nil } +// GetPrimaryInstance reads status.currentPrimary from the CNPG Cluster that +// backs this DocumentDB. The CNPG Cluster name equals the DocumentDB CR name, +// and the returned instance name equals the primary pod name. +func (k *K8sClusterClient) GetPrimaryInstance(ctx context.Context) (string, error) { + var cluster cnpgv1.Cluster + key := types.NamespacedName{Namespace: k.namespace, Name: k.clusterName} + if err := k.crClient.Get(ctx, key, &cluster); err != nil { + return "", fmt.Errorf("failed to get CNPG Cluster: %w", err) + } + if cluster.Status.CurrentPrimary == "" { + return "", fmt.Errorf("CNPG Cluster %s has no current primary yet", k.clusterName) + } + return cluster.Status.CurrentPrimary, nil +} + +// DeletePod deletes the named pod in the cluster namespace. +func (k *K8sClusterClient) DeletePod(ctx context.Context, name string) error { + if err := k.clientset.CoreV1().Pods(k.namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("failed to delete pod %s/%s: %w", k.namespace, name, err) + } + return nil +} + // GetPodMetrics queries metrics-server for pod resource usage. // Returns nil, nil if metrics-server is not available. func (k *K8sClusterClient) GetPodMetrics(ctx context.Context) ([]PodMetrics, error) { diff --git a/test/longhaul/operations/kill_operator.go b/test/longhaul/operations/kill_operator.go new file mode 100644 index 000000000..e6a68db79 --- /dev/null +++ b/test/longhaul/operations/kill_operator.go @@ -0,0 +1,195 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "fmt" + "time" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" +) + +// OperatorDeploymentName is the fixed name of the operator Deployment. The +// operator is a cluster singleton, so this name is stable across installs. +const OperatorDeploymentName = "documentdb-operator" + +// KillOperatorPod deletes the running operator pod to verify that an operator +// restart does not disrupt the data plane. The CNPG-managed database keeps +// serving reads and writes while the Deployment reschedules the control plane, +// so the workload verifier should observe (near) zero write failures. Recovery +// is asserted by the Deployment returning to Available. +type KillOperatorPod struct { + clientset kubernetes.Interface + namespace string + deployment string + recovery time.Duration +} + +// NewKillOperatorPod creates a KillOperatorPod operation targeting the operator +// Deployment in the given namespace. +func NewKillOperatorPod(clientset kubernetes.Interface, namespace string, recovery time.Duration) *KillOperatorPod { + return &KillOperatorPod{ + clientset: clientset, + namespace: namespace, + deployment: OperatorDeploymentName, + recovery: recovery, + } +} + +func (k *KillOperatorPod) Name() string { return "kill-operator-pod" } + +func (k *KillOperatorPod) Weight() int { return 2 } + +// Precondition requires the operator Deployment to exist and currently be +// Available, so the fault isn't stacked on an already-restarting operator. +func (k *KillOperatorPod) Precondition(ctx context.Context) (bool, string) { + dep, err := k.getDeployment(ctx) + if err != nil { + return false, fmt.Sprintf("cannot get operator deployment: %v", err) + } + if !isDeploymentAvailable(dep) { + return false, "operator deployment not currently available" + } + return true, "" +} + +func (k *KillOperatorPod) Execute(ctx context.Context) error { + dep, err := k.getDeployment(ctx) + if err != nil { + return fmt.Errorf("get operator deployment: %w", err) + } + + // Fail fast if the Deployment has no label selector: SelectorFromSet on an + // empty map yields an "everything" selector, so the List below would match + // (and the delete could target) every pod in the namespace. + if dep.Spec.Selector == nil || len(dep.Spec.Selector.MatchLabels) == 0 { + return fmt.Errorf("operator deployment %s has no matchLabels selector; refusing to list all pods", k.deployment) + } + + // Resolve the pod set from the Deployment's own selector so we don't + // depend on the release-name-derived "app" label value. + selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels).String() + pods, err := k.clientset.CoreV1().Pods(k.namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return fmt.Errorf("list operator pods: %w", err) + } + + target, targetUID := oldestRunningPod(pods.Items) + if target == "" { + return fmt.Errorf("no running operator pod found for selector %q", selector) + } + if err := k.clientset.CoreV1().Pods(k.namespace).Delete(ctx, target, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("delete operator pod %s: %w", target, err) + } + + recoveryCtx, cancel := context.WithTimeout(ctx, k.recovery) + defer cancel() + + // Confirm the targeted pod is actually gone before checking Deployment + // availability. Deleting a pod does not bump the Deployment's + // ObservedGeneration, so its status can still read "Available" from the + // pre-deletion state and let waitForDeploymentAvailable return immediately + // without ever observing the restart. + if err := k.waitForPodGone(recoveryCtx, target, targetUID); err != nil { + return err + } + + // Wait for the Deployment to reschedule and become Available again. + return k.waitForDeploymentAvailable(recoveryCtx) +} + +// waitForPodGone blocks until the pod identified by name/uid is deleted +// (NotFound) or replaced by a new pod with a different UID, guaranteeing the +// disruption has actually landed before we assert recovery. +func (k *KillOperatorPod) waitForPodGone(ctx context.Context, name string, uid types.UID) error { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + pod, err := k.clientset.CoreV1().Pods(k.namespace).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err == nil && pod.UID != uid { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("timed out waiting for operator pod %s to be deleted: %w", name, ctx.Err()) + case <-ticker.C: + } + } +} + +// OutagePolicy: an operator restart is a control-plane fault that must not take +// down the data plane, so it shares the near-zero NoOutagePolicy budget. +func (k *KillOperatorPod) OutagePolicy() journal.OutagePolicy { + return journal.NoOutagePolicy(k.recovery) +} + +func (k *KillOperatorPod) getDeployment(ctx context.Context) (*appsv1.Deployment, error) { + return k.clientset.AppsV1().Deployments(k.namespace).Get(ctx, k.deployment, metav1.GetOptions{}) +} + +func (k *KillOperatorPod) waitForDeploymentAvailable(ctx context.Context) error { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + if dep, err := k.getDeployment(ctx); err == nil && isDeploymentAvailable(dep) { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("timed out waiting for operator deployment to become available: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +// isDeploymentAvailable reports whether the Deployment has its full desired +// replica count ready with none unavailable and the observed generation caught +// up to the latest spec. +func isDeploymentAvailable(dep *appsv1.Deployment) bool { + if dep == nil { + return false + } + desired := int32(1) + if dep.Spec.Replicas != nil { + desired = *dep.Spec.Replicas + } + if dep.Status.ObservedGeneration < dep.Generation { + return false + } + return dep.Status.ReadyReplicas >= desired && dep.Status.UnavailableReplicas == 0 +} + +// oldestRunningPod returns the name and UID of the oldest pod in the Running +// phase, or ("", "") if none are running. Targeting the oldest makes the choice +// deterministic; the UID lets callers confirm that specific pod is later gone. +func oldestRunningPod(pods []corev1.Pod) (string, types.UID) { + name := "" + var uid types.UID + var oldest time.Time + for i := range pods { + p := &pods[i] + if p.Status.Phase != corev1.PodRunning || p.DeletionTimestamp != nil { + continue + } + ts := p.CreationTimestamp.Time + if name == "" || ts.Before(oldest) { + name = p.Name + uid = p.UID + oldest = ts + } + } + return name, uid +} diff --git a/test/longhaul/operations/kill_operator_test.go b/test/longhaul/operations/kill_operator_test.go new file mode 100644 index 000000000..416241096 --- /dev/null +++ b/test/longhaul/operations/kill_operator_test.go @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +const opNS = "documentdb-operator" + +func operatorDeployment(desired, ready, unavailable int32, gen, observed int64) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: OperatorDeploymentName, + Namespace: opNS, + Generation: gen, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &desired, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "documentdb-operator"}}, + }, + Status: appsv1.DeploymentStatus{ + ReadyReplicas: ready, + UnavailableReplicas: unavailable, + ObservedGeneration: observed, + }, + } +} + +func operatorPod(name string, phase corev1.PodPhase, ageSeconds int) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: opNS, + Labels: map[string]string{"app": "documentdb-operator"}, + CreationTimestamp: metav1.NewTime(time.Now().Add(-time.Duration(ageSeconds) * time.Second)), + }, + Status: corev1.PodStatus{Phase: phase}, + } +} + +var _ = Describe("KillOperatorPod", func() { + It("Name is kill-operator-pod and Weight is 2", func() { + k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, time.Minute) + Expect(k.Name()).To(Equal("kill-operator-pod")) + Expect(k.Weight()).To(Equal(2)) + }) + + It("OutagePolicy uses the near-zero NoOutagePolicy budget", func() { + k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, 2*time.Minute) + p := k.OutagePolicy() + Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion)) + Expect(p.MustRecoverWithin).To(Equal(2 * time.Minute)) + }) + + Describe("Precondition", func() { + It("skips when the deployment is missing", func() { + k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, time.Minute) + ok, reason := k.Precondition(context.Background()) + Expect(ok).To(BeFalse()) + Expect(reason).To(ContainSubstring("cannot get operator deployment")) + }) + + It("skips when the deployment is not available", func() { + dep := operatorDeployment(1, 0, 1, 1, 1) + k := NewKillOperatorPod(fake.NewSimpleClientset(dep), opNS, time.Minute) + ok, reason := k.Precondition(context.Background()) + Expect(ok).To(BeFalse()) + Expect(reason).To(ContainSubstring("not currently available")) + }) + + It("is eligible when the deployment is available", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + k := NewKillOperatorPod(fake.NewSimpleClientset(dep), opNS, time.Minute) + ok, _ := k.Precondition(context.Background()) + Expect(ok).To(BeTrue()) + }) + }) + + Describe("Execute", func() { + It("deletes the oldest running operator pod and returns once available", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + newer := operatorPod("op-new", corev1.PodRunning, 10) + older := operatorPod("op-old", corev1.PodRunning, 100) + cs := fake.NewSimpleClientset(dep, newer, older) + k := NewKillOperatorPod(cs, opNS, time.Minute) + + err := k.Execute(context.Background()) + Expect(err).NotTo(HaveOccurred()) + + _, getErr := cs.CoreV1().Pods(opNS).Get(context.Background(), "op-old", metav1.GetOptions{}) + Expect(getErr).To(HaveOccurred(), "oldest pod should have been deleted") + _, getErr = cs.CoreV1().Pods(opNS).Get(context.Background(), "op-new", metav1.GetOptions{}) + Expect(getErr).NotTo(HaveOccurred(), "newer pod should be untouched") + }) + + It("fails when no running pod matches the selector", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + pending := operatorPod("op-pending", corev1.PodPending, 10) + cs := fake.NewSimpleClientset(dep, pending) + k := NewKillOperatorPod(cs, opNS, time.Minute) + + err := k.Execute(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no running operator pod")) + }) + + It("refuses to run when the deployment has no matchLabels selector", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + dep.Spec.Selector = &metav1.LabelSelector{} + running := operatorPod("op-run", corev1.PodRunning, 10) + cs := fake.NewSimpleClientset(dep, running) + k := NewKillOperatorPod(cs, opNS, time.Minute) + + err := k.Execute(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no matchLabels selector")) + + _, getErr := cs.CoreV1().Pods(opNS).Get(context.Background(), "op-run", metav1.GetOptions{}) + Expect(getErr).NotTo(HaveOccurred(), "no pod should be deleted when the selector is empty") + }) + }) +}) diff --git a/test/longhaul/operations/kill_primary.go b/test/longhaul/operations/kill_primary.go new file mode 100644 index 000000000..ec4c4f39e --- /dev/null +++ b/test/longhaul/operations/kill_primary.go @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "fmt" + "time" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + "github.com/documentdb/documentdb-operator/test/longhaul/monitor" +) + +// KillPrimaryPod deletes the CNPG primary pod to exercise the automatic +// failover path: CNPG must promote a standby, and the cluster must return to +// steady state within the recovery budget. The continuous workload verifier +// independently catches any data loss caused by the failover. +type KillPrimaryPod struct { + client monitor.ClusterClient + healthMon SteadyStateGate + recovery time.Duration + primaryPollInterval time.Duration +} + +// NewKillPrimaryPod creates a KillPrimaryPod operation. +func NewKillPrimaryPod(client monitor.ClusterClient, health SteadyStateGate, recovery time.Duration) *KillPrimaryPod { + return &KillPrimaryPod{ + client: client, + healthMon: health, + recovery: recovery, + primaryPollInterval: time.Second, + } +} + +func (k *KillPrimaryPod) Name() string { return "kill-primary-pod" } + +func (k *KillPrimaryPod) Weight() int { return 2 } + +// Precondition requires at least one standby (instancesPerNode>=2). Killing the +// sole instance of a single-instance cluster would cause guaranteed downtime +// with no failover target — a true-but-useless policy violation. The same guard +// (and rationale) is used by UpgradeDocumentDB; skips don't consume the +// scheduler cooldown, so this is free to re-evaluate on the next tick. +func (k *KillPrimaryPod) Precondition(ctx context.Context) (bool, string) { + ipn, err := k.client.GetInstancesPerNode(ctx) + if err != nil { + return false, fmt.Sprintf("cannot read instancesPerNode: %v", err) + } + if ipn < 2 { + return false, fmt.Sprintf("instancesPerNode=%d (no HA standby) — killing primary would cause real downtime; skipping", ipn) + } + return true, "" +} + +func (k *KillPrimaryPod) Execute(ctx context.Context) error { + primary, err := k.client.GetPrimaryInstance(ctx) + if err != nil { + return fmt.Errorf("get primary instance: %w", err) + } + if primary == "" { + return fmt.Errorf("get primary instance: cluster returned an empty primary pod name") + } + if k.healthMon == nil { + return fmt.Errorf("kill-primary-pod: health monitor is nil") + } + + recoveryCtx, cancel := context.WithTimeout(ctx, k.recovery) + defer cancel() + + if err := k.client.DeletePod(recoveryCtx, primary); err != nil { + return fmt.Errorf("delete primary pod %s: %w", primary, err) + } + + if err := k.waitForPrimaryChange(recoveryCtx, primary); err != nil { + return err + } + + // A changed primary proves CNPG promoted a standby rather than merely + // recreating the deleted pod and reporting the old primary again. + if err := k.healthMon.WaitForSteadyState(recoveryCtx); err != nil { + return fmt.Errorf("wait for steady-state recovery: %w", err) + } + + current, err := k.client.GetPrimaryInstance(recoveryCtx) + if err != nil { + return fmt.Errorf("verify primary after steady-state recovery: %w", err) + } + if current == "" || current == primary { + return fmt.Errorf("verify primary after steady-state recovery: expected a non-empty primary different from %q, got %q", + primary, current) + } + return nil +} + +func (k *KillPrimaryPod) waitForPrimaryChange(ctx context.Context, original string) error { + ticker := time.NewTicker(k.primaryPollInterval) + defer ticker.Stop() + + lastObserved := original + var lastErr error + for { + current, err := k.client.GetPrimaryInstance(ctx) + if err == nil { + lastObserved = current + if current != "" && current != original { + return nil + } + } else { + lastErr = err + } + + select { + case <-ctx.Done(): + if lastErr != nil { + return fmt.Errorf("primary did not change from %q before recovery timeout (last read error: %v): %w", + original, lastErr, ctx.Err()) + } + return fmt.Errorf("primary did not change from %q before recovery timeout (last observed %q): %w", + original, lastObserved, ctx.Err()) + case <-ticker.C: + } + } +} + +// OutagePolicy bounds the write outage of an automatic failover. Killing the +// primary interrupts writes until CNPG detects the loss and promotes a standby. +// It shares the single-primary-handover budget with upgrade-documentdb (see +// journal.PrimaryHandoverPolicy). +func (k *KillPrimaryPod) OutagePolicy() journal.OutagePolicy { + return journal.PrimaryHandoverPolicy(k.recovery) +} diff --git a/test/longhaul/operations/kill_primary_test.go b/test/longhaul/operations/kill_primary_test.go new file mode 100644 index 000000000..cfda9043f --- /dev/null +++ b/test/longhaul/operations/kill_primary_test.go @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "errors" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" +) + +type successfulSteadyGate struct { + calls int + onWait func() +} + +func (g *successfulSteadyGate) WaitForSteadyState(context.Context) error { + g.calls++ + if g.onWait != nil { + g.onWait() + } + return nil +} + +var _ = Describe("KillPrimaryPod", func() { + It("Name is kill-primary-pod and Weight is 2", func() { + k := NewKillPrimaryPod(&fakeClient{}, nil, time.Minute) + Expect(k.Name()).To(Equal("kill-primary-pod")) + Expect(k.Weight()).To(Equal(2)) + }) + + It("OutagePolicy shares the single-primary-handover budget with upgrade", func() { + k := NewKillPrimaryPod(&fakeClient{}, nil, 3*time.Minute) + p := k.OutagePolicy() + Expect(p.MaxWriteOutage).To(Equal(journal.PrimaryHandoverWriteOutage)) + Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute)) + }) + + DescribeTable("Precondition", + func(ipn int, ipnErr error, wantOK bool, wantReasonHas string) { + c := &fakeClient{instancesPerNode: ipn, ipnErr: ipnErr} + k := NewKillPrimaryPod(c, nil, time.Minute) + + ok, reason := k.Precondition(context.Background()) + Expect(ok).To(Equal(wantOK), "reason=%q", reason) + if wantReasonHas != "" { + Expect(reason).To(ContainSubstring(wantReasonHas)) + } + }, + Entry("single-instance: ipn=1 -> skip", 1, nil, false, "no HA standby"), + Entry("read error -> skip", 0, errors.New("boom"), false, "cannot read instancesPerNode"), + Entry("HA: ipn=2 -> eligible", 2, nil, true, ""), + Entry("HA: ipn=3 -> eligible", 3, nil, true, ""), + ) + + It("Execute deletes the original primary and verifies a different primary", func() { + c := &fakeClient{ + instancesPerNode: 2, + primary: "cluster-1", + replacementPrimary: "cluster-2", + } + gate := &successfulSteadyGate{} + k := NewKillPrimaryPod(c, gate, time.Second) + + Expect(k.Execute(context.Background())).To(Succeed()) + c.mu.Lock() + defer c.mu.Unlock() + Expect(c.deletedPods).To(ConsistOf("cluster-1")) + Expect(c.primary).To(Equal("cluster-2")) + Expect(gate.calls).To(Equal(1)) + }) + + It("fails when CNPG keeps reporting the deleted primary", func() { + c := &fakeClient{instancesPerNode: 2, primary: "cluster-1"} + gate := &successfulSteadyGate{} + k := NewKillPrimaryPod(c, gate, 20*time.Millisecond) + k.primaryPollInterval = time.Millisecond + + err := k.Execute(context.Background()) + Expect(err).To(MatchError(ContainSubstring(`primary did not change from "cluster-1"`))) + Expect(gate.calls).To(Equal(0), "steady-state recovery must wait until primary change is proven") + }) + + It("fails if the recovered cluster reports the original primary again", func() { + c := &fakeClient{ + instancesPerNode: 2, + primary: "cluster-1", + replacementPrimary: "cluster-2", + } + gate := &successfulSteadyGate{onWait: func() { + c.mu.Lock() + defer c.mu.Unlock() + c.primary = "cluster-1" + }} + k := NewKillPrimaryPod(c, gate, time.Second) + + err := k.Execute(context.Background()) + Expect(err).To(MatchError(ContainSubstring("expected a non-empty primary different"))) + }) + + It("Execute fails without deleting when the primary is unknown", func() { + c := &fakeClient{instancesPerNode: 2, primaryErr: errors.New("no primary")} + k := NewKillPrimaryPod(c, nil, time.Second) + + err := k.Execute(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("get primary instance")) + c.mu.Lock() + defer c.mu.Unlock() + Expect(c.deletedPods).To(BeEmpty()) + }) + + It("Execute fails without deleting when the primary name is empty", func() { + c := &fakeClient{instancesPerNode: 2, primary: ""} + k := NewKillPrimaryPod(c, nil, time.Second) + + err := k.Execute(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("empty primary pod name")) + c.mu.Lock() + defer c.mu.Unlock() + Expect(c.deletedPods).To(BeEmpty()) + }) +}) diff --git a/test/longhaul/operations/registry.go b/test/longhaul/operations/registry.go new file mode 100644 index 000000000..f85c3f843 --- /dev/null +++ b/test/longhaul/operations/registry.go @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "fmt" + + "k8s.io/client-go/kubernetes" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + "github.com/documentdb/documentdb-operator/test/longhaul/monitor" +) + +// Registry stores operations by their stable Name() values while preserving +// registration order for deterministic snapshots and random-mode summaries. +type Registry struct { + order []string + operations map[string]Operation +} + +// NewRegistry builds a validated operation registry. +func NewRegistry(ops ...Operation) (*Registry, error) { + registry := &Registry{ + order: make([]string, 0, len(ops)), + operations: make(map[string]Operation, len(ops)), + } + for _, op := range ops { + if op == nil { + return nil, fmt.Errorf("operation registry contains a nil operation") + } + name := op.Name() + if name == "" { + return nil, fmt.Errorf("operation registry contains an operation with an empty name") + } + if _, exists := registry.operations[name]; exists { + return nil, fmt.Errorf("operation registry contains duplicate name %q", name) + } + registry.order = append(registry.order, name) + registry.operations[name] = op + } + return registry, nil +} + +// NewDefaultRegistry centralizes construction of every supported operation. +func NewDefaultRegistry( + cfg config.Config, + clusterClient monitor.ClusterClient, + clientset kubernetes.Interface, + health *monitor.HealthMonitor, + j *journal.Journal, +) (*Registry, error) { + return NewRegistry( + NewScaleUp(clusterClient, health, cfg.MaxInstances, cfg.RecoveryTimeout), + NewScaleDown(clusterClient, health, cfg.MinInstances, cfg.RecoveryTimeout), + NewUpgradeDocumentDB(clusterClient, clientset, health, j, cfg.Namespace, cfg.RecoveryTimeout), + NewKillOperatorPod(clientset, cfg.OperatorNamespace, cfg.RecoveryTimeout), + NewKillPrimaryPod(clusterClient, health, cfg.RecoveryTimeout), + ) +} + +// All returns all registered operations in stable registration order. +func (r *Registry) All() []Operation { + ops := make([]Operation, 0, len(r.order)) + for _, name := range r.order { + ops = append(ops, r.operations[name]) + } + return ops +} + +// Resolve returns the named operations in exactly the requested order. +func (r *Registry) Resolve(names []string) ([]Operation, error) { + resolved := make([]Operation, 0, len(names)) + seen := make(map[string]struct{}, len(names)) + for _, name := range names { + if _, duplicate := seen[name]; duplicate { + return nil, fmt.Errorf("operation sequence contains duplicate name %q", name) + } + op, ok := r.operations[name] + if !ok { + return nil, fmt.Errorf("operation sequence contains unknown name %q", name) + } + seen[name] = struct{}{} + resolved = append(resolved, op) + } + return resolved, nil +} diff --git a/test/longhaul/operations/registry_test.go b/test/longhaul/operations/registry_test.go new file mode 100644 index 000000000..03493269a --- /dev/null +++ b/test/longhaul/operations/registry_test.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" +) + +var _ = Describe("Registry", func() { + It("resolves exact stable names in requested order", func() { + a := &fakeOp{name: "a"} + b := &fakeOp{name: "b"} + registry, err := NewRegistry(a, b) + Expect(err).NotTo(HaveOccurred()) + + resolved, err := registry.Resolve([]string{"b", "a"}) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved).To(Equal([]Operation{b, a})) + Expect(registry.All()).To(Equal([]Operation{a, b})) + }) + + It("rejects unknown requested names", func() { + registry, err := NewRegistry(&fakeOp{name: "known"}) + Expect(err).NotTo(HaveOccurred()) + _, err = registry.Resolve([]string{"unknown"}) + Expect(err).To(MatchError(ContainSubstring(`unknown name "unknown"`))) + }) + + It("rejects duplicate requested names", func() { + registry, err := NewRegistry(&fakeOp{name: "known"}) + Expect(err).NotTo(HaveOccurred()) + _, err = registry.Resolve([]string{"known", "known"}) + Expect(err).To(MatchError(ContainSubstring(`duplicate name "known"`))) + }) + + It("rejects duplicate registered operation names", func() { + _, err := NewRegistry(&fakeOp{name: "same"}, &fakeOp{name: "same"}) + Expect(err).To(MatchError(ContainSubstring(`duplicate name "same"`))) + }) + + It("constructs the default registry with the stable operation names", func() { + registry, err := NewDefaultRegistry(config.DefaultConfig(), nil, nil, nil, journal.New()) + Expect(err).NotTo(HaveOccurred()) + + names := make([]string, 0) + for _, op := range registry.All() { + names = append(names, op.Name()) + } + Expect(names).To(Equal([]string{ + "scale-up", + "scale-down", + "upgrade-documentdb", + "kill-operator-pod", + "kill-primary-pod", + })) + }) +}) diff --git a/test/longhaul/operations/runner.go b/test/longhaul/operations/runner.go new file mode 100644 index 000000000..1884290a4 --- /dev/null +++ b/test/longhaul/operations/runner.go @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "sync" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" +) + +// RunStatus is the bounded lifecycle state of the operation runner. +type RunStatus string + +const ( + RunStatusPending RunStatus = "PENDING" + RunStatusRunning RunStatus = "RUNNING" + RunStatusComplete RunStatus = "COMPLETE" + RunStatusFailed RunStatus = "FAILED" + RunStatusIncomplete RunStatus = "INCOMPLETE" + RunStatusDisabled RunStatus = "DISABLED" +) + +// OperationResultStatus is the state of one requested sequence operation. +type OperationResultStatus string + +const ( + OperationPending OperationResultStatus = "PENDING" + OperationRunning OperationResultStatus = "RUNNING" + OperationPassed OperationResultStatus = "PASSED" + OperationFailed OperationResultStatus = "FAILED" +) + +// OperationResult is the single mutable result for one requested sequence item. +type OperationResult struct { + Name string `json:"name"` + Status OperationResultStatus `json:"status"` + Error string `json:"error,omitempty"` +} + +// OperationAggregate bounds random-mode history to counters per operation type. +type OperationAggregate struct { + Name string `json:"name"` + Passed int `json:"passed"` + Failed int `json:"failed"` +} + +// RunSnapshot is a concurrency-safe value snapshot of operation execution. +type RunSnapshot struct { + Mode config.OperationMode `json:"mode"` + Status RunStatus `json:"status"` + Results []OperationResult `json:"results,omitempty"` + Aggregates []OperationAggregate `json:"aggregates,omitempty"` + FailureReason string `json:"failureReason,omitempty"` +} + +// OpsExecuted returns the number of terminal operation attempts. +func (s RunSnapshot) OpsExecuted() int { + if s.Mode == config.OperationModeSequence { + count := 0 + for _, result := range s.Results { + if result.Status == OperationPassed || result.Status == OperationFailed { + count++ + } + } + return count + } + + count := 0 + for _, aggregate := range s.Aggregates { + count += aggregate.Passed + aggregate.Failed + } + return count +} + +// HasFailure reports whether an operation attempt or sequence lifecycle failed. +func (s RunSnapshot) HasFailure() bool { + if s.Status == RunStatusFailed || s.Status == RunStatusIncomplete { + return true + } + for _, aggregate := range s.Aggregates { + if aggregate.Failed > 0 { + return true + } + } + return false +} + +// Runner is the common reporting and lifecycle surface for every operation mode. +type Runner interface { + Run(ctx context.Context) + Snapshot() RunSnapshot + Done() <-chan struct{} +} + +type runnerState struct { + mu sync.RWMutex + snapshot RunSnapshot + done chan struct{} + doneOnce sync.Once +} + +func newRunnerState(snapshot RunSnapshot) runnerState { + return runnerState{snapshot: snapshot, done: make(chan struct{})} +} + +func (s *runnerState) Snapshot() RunSnapshot { + s.mu.RLock() + defer s.mu.RUnlock() + snapshot := s.snapshot + snapshot.Results = append([]OperationResult(nil), s.snapshot.Results...) + snapshot.Aggregates = append([]OperationAggregate(nil), s.snapshot.Aggregates...) + return snapshot +} + +func (s *runnerState) Done() <-chan struct{} { + return s.done +} + +func (s *runnerState) closeDone() { + s.doneOnce.Do(func() { close(s.done) }) +} + +// DisabledRunner performs no operations and has no completion requirement. +type DisabledRunner struct { + state runnerState +} + +// NewDisabledRunner creates a runner for disabled operation mode. +func NewDisabledRunner() *DisabledRunner { + return &DisabledRunner{state: newRunnerState(RunSnapshot{ + Mode: config.OperationModeDisabled, + Status: RunStatusDisabled, + })} +} + +// Run waits for shutdown without scheduling operations. +func (r *DisabledRunner) Run(ctx context.Context) { + <-ctx.Done() + r.state.closeDone() +} + +// Snapshot returns the disabled runner state. +func (r *DisabledRunner) Snapshot() RunSnapshot { + return r.state.Snapshot() +} + +// Done closes when Run returns after cancellation. +func (r *DisabledRunner) Done() <-chan struct{} { + return r.state.Done() +} diff --git a/test/longhaul/operations/scale.go b/test/longhaul/operations/scale.go index 9a1672959..f002fdc61 100644 --- a/test/longhaul/operations/scale.go +++ b/test/longhaul/operations/scale.go @@ -77,6 +77,9 @@ type ScaleUp struct{ scaleOp } // NewScaleUp creates a ScaleUp operation. maxInstances is clamped to the // CRD upper bound (3) to avoid admission rejections. +// +// Scaling up only adds a standby replica (the primary and thus the write path +// is untouched), so it uses the near-zero NoOutagePolicy budget. func NewScaleUp(client monitor.ClusterClient, health *monitor.HealthMonitor, maxInstances int, recovery time.Duration) *ScaleUp { if maxInstances > 3 { maxInstances = 3 @@ -90,10 +93,7 @@ func NewScaleUp(client monitor.ClusterClient, health *monitor.HealthMonitor, max bound: maxInstances, boundKind: "max", recovery: recovery, - policy: journal.OutagePolicy{ - AllowedWriteFailures: 20, - MustRecoverWithin: recovery, - }, + policy: journal.NoOutagePolicy(recovery), }} } @@ -105,6 +105,10 @@ type ScaleDown struct{ scaleOp } // NewScaleDown creates a ScaleDown operation. minInstances is clamped to the // CRD lower bound (1) to avoid admission rejections. +// +// Scaling down removes the highest-ordinal standby (CNPG never removes the +// primary), so the write path stays up and it uses the same near-zero +// NoOutagePolicy budget as scale-up. func NewScaleDown(client monitor.ClusterClient, health *monitor.HealthMonitor, minInstances int, recovery time.Duration) *ScaleDown { if minInstances < 1 { minInstances = 1 @@ -118,10 +122,7 @@ func NewScaleDown(client monitor.ClusterClient, health *monitor.HealthMonitor, m bound: minInstances, boundKind: "min", recovery: recovery, - policy: journal.OutagePolicy{ - AllowedWriteFailures: 50, - MustRecoverWithin: recovery, - }, + policy: journal.NoOutagePolicy(recovery), }} } diff --git a/test/longhaul/operations/scale_test.go b/test/longhaul/operations/scale_test.go index 3df032c05..e929033d6 100644 --- a/test/longhaul/operations/scale_test.go +++ b/test/longhaul/operations/scale_test.go @@ -12,17 +12,23 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" "github.com/documentdb/documentdb-operator/test/longhaul/monitor" ) // fakeClient is a minimal monitor.ClusterClient stub for unit tests. type fakeClient struct { - mu sync.Mutex - instancesPerNode int - ipnErr error - imageTag string - scaleCalls []int - upgradeCalls []string + mu sync.Mutex + instancesPerNode int + ipnErr error + imageTag string + scaleCalls []int + upgradeCalls []string + primary string + primaryErr error + replacementPrimary string + deleteErr error + deletedPods []string } func (f *fakeClient) GetClusterHealth(_ context.Context) (monitor.ClusterHealth, error) { @@ -51,6 +57,23 @@ func (f *fakeClient) UpgradeDocumentDB(_ context.Context, v string) error { f.upgradeCalls = append(f.upgradeCalls, v) return nil } +func (f *fakeClient) GetPrimaryInstance(_ context.Context) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.primary, f.primaryErr +} +func (f *fakeClient) DeletePod(_ context.Context, name string) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.deleteErr != nil { + return f.deleteErr + } + f.deletedPods = append(f.deletedPods, name) + if f.replacementPrimary != "" { + f.primary = f.replacementPrimary + } + return nil +} var _ = Describe("ScaleUp", func() { DescribeTable("clamps maxInstances to the CRD upper bound", @@ -87,10 +110,10 @@ var _ = Describe("ScaleUp", func() { Entry("blocked: ipn read error", 0, errors.New("apiserver down"), 3, false, "cannot get instancesPerNode"), ) - It("OutagePolicy uses tighter budgets and echoes MustRecoverWithin", func() { + It("OutagePolicy uses the near-zero NoOutagePolicy budget and echoes MustRecoverWithin", func() { s := NewScaleUp(&fakeClient{}, nil, 3, 5*time.Minute) p := s.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(int64(20))) + Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion)) Expect(p.MustRecoverWithin).To(Equal(5 * time.Minute)) }) }) @@ -130,9 +153,9 @@ var _ = Describe("ScaleDown", func() { Entry("blocked: ipn read error", 0, errors.New("apiserver down"), 1, false, "cannot get instancesPerNode"), ) - It("OutagePolicy is more lenient than scale-up", func() { + It("OutagePolicy shares the near-zero NoOutagePolicy budget with scale-up", func() { s := NewScaleDown(&fakeClient{}, nil, 1, 5*time.Minute) p := s.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(int64(50))) + Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion)) }) }) diff --git a/test/longhaul/operations/scheduler.go b/test/longhaul/operations/scheduler.go index a9351d18d..a486febaa 100644 --- a/test/longhaul/operations/scheduler.go +++ b/test/longhaul/operations/scheduler.go @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// Package operations implements the operation scheduler and individual -// disruptive operations for long haul tests. +// Package operations implements operation runners and individual disruptive +// operations for long haul tests. package operations import ( @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/documentdb/documentdb-operator/test/longhaul/config" "github.com/documentdb/documentdb-operator/test/longhaul/journal" "github.com/documentdb/documentdb-operator/test/longhaul/monitor" ) @@ -42,10 +43,38 @@ type Scheduler struct { journal *journal.Journal cooldown time.Duration + // rng, when non-nil, pins weighted-random selection for reproducibility. + // When nil the process-global generator is used (production behavior). + rng *rand.Rand + // coverage draws each operation without replacement and completes the run + // once every operation has run at least once. + coverage bool + mu sync.Mutex lastOpTime time.Time opsExecuted int inProgress bool + + state runnerState + aggregateIndex map[string]int +} + +// SchedulerOption configures optional Scheduler behavior. +type SchedulerOption func(*Scheduler) + +// WithSeed pins weighted-random selection to a fixed seed so the run is +// reproducible. Without it the scheduler uses the process-global generator. +func WithSeed(seed int64) SchedulerOption { + return func(s *Scheduler) { + s.rng = rand.New(rand.NewPCG(uint64(seed), uint64(seed))) + } +} + +// WithCoverage enables coverage mode: the scheduler draws each operation +// without replacement and completes once every operation has run at least once, +// rather than running until context cancellation. +func WithCoverage() SchedulerOption { + return func(s *Scheduler) { s.coverage = true } } // NewScheduler creates an operation scheduler. @@ -54,19 +83,68 @@ func NewScheduler( health *monitor.HealthMonitor, j *journal.Journal, cooldown time.Duration, + opts ...SchedulerOption, ) *Scheduler { - return &Scheduler{ + aggregates := make([]OperationAggregate, 0, len(ops)) + aggregateIndex := make(map[string]int, len(ops)) + for _, op := range ops { + if _, exists := aggregateIndex[op.Name()]; exists { + continue + } + aggregateIndex[op.Name()] = len(aggregates) + aggregates = append(aggregates, OperationAggregate{Name: op.Name()}) + } + s := &Scheduler{ operations: ops, healthMonitor: health, journal: j, cooldown: cooldown, + state: newRunnerState(RunSnapshot{ + Mode: config.OperationModeRandom, + Status: RunStatusPending, + Aggregates: aggregates, + }), + aggregateIndex: aggregateIndex, + } + for _, opt := range opts { + opt(s) + } + return s +} + +// intn returns a non-negative pseudo-random int in [0,n) from the scheduler's +// seeded generator when present, otherwise the process-global generator. +func (s *Scheduler) intn(n int) int { + if s.rng != nil { + return s.rng.IntN(n) } + return rand.IntN(n) } // Run starts the scheduler loop. It blocks until context is cancelled. func (s *Scheduler) Run(ctx context.Context) { s.journal.Info("scheduler", "operation scheduler started") - defer s.journal.Info("scheduler", "operation scheduler stopped") + s.state.mu.Lock() + s.state.snapshot.Status = RunStatusRunning + s.state.mu.Unlock() + defer func() { + s.state.mu.Lock() + if s.state.snapshot.Status == RunStatusRunning { + // Coverage runs that stop before covering every operation (watchdog + // or shutdown) are terminally incomplete, not complete. + if s.coverage && !s.allCoveredLocked() { + s.state.snapshot.Status = RunStatusIncomplete + if s.state.snapshot.FailureReason == "" { + s.state.snapshot.FailureReason = "operation coverage incomplete: run stopped before every operation ran" + } + } else { + s.state.snapshot.Status = RunStatusComplete + } + } + s.state.mu.Unlock() + s.state.closeDone() + s.journal.Info("scheduler", "operation scheduler stopped") + }() ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() @@ -77,8 +155,49 @@ func (s *Scheduler) Run(ctx context.Context) { return case <-ticker.C: s.tryExecute(ctx) + // Coverage mode is completion-driven: stop as soon as the run + // reaches a terminal state (all operations covered, or a failure). + if s.coverage && s.coverageTerminalReached() { + return + } + } + } +} + +// coverageTerminalReached reports whether a coverage run has reached a terminal +// state and its loop should return. +func (s *Scheduler) coverageTerminalReached() bool { + s.state.mu.RLock() + defer s.state.mu.RUnlock() + return s.state.snapshot.Status == RunStatusComplete || + s.state.snapshot.Status == RunStatusFailed +} + +// allCoveredLocked reports whether every registered operation has run at least +// once. Callers must hold s.state.mu. +func (s *Scheduler) allCoveredLocked() bool { + if len(s.state.snapshot.Aggregates) == 0 { + return false + } + for _, a := range s.state.snapshot.Aggregates { + if a.Passed+a.Failed == 0 { + return false } } + return true +} + +// coveredSet returns the set of operation names that have run at least once. +func (s *Scheduler) coveredSet() map[string]bool { + s.state.mu.RLock() + defer s.state.mu.RUnlock() + covered := make(map[string]bool, len(s.state.snapshot.Aggregates)) + for _, a := range s.state.snapshot.Aggregates { + if a.Passed+a.Failed > 0 { + covered[a.Name] = true + } + } + return covered } func (s *Scheduler) tryExecute(ctx context.Context) { @@ -111,16 +230,25 @@ func (s *Scheduler) tryExecute(ctx context.Context) { s.inProgress = true s.mu.Unlock() - s.executeOp(ctx, op) + err := s.executeOp(ctx, op) s.mu.Lock() s.inProgress = false s.lastOpTime = time.Now() s.opsExecuted++ s.mu.Unlock() + + s.recordExecution(op.Name(), err) } func (s *Scheduler) selectOperation(ctx context.Context) Operation { + // In coverage mode, exclude operations that have already run so each is + // drawn without replacement until every operation has been covered. + var covered map[string]bool + if s.coverage { + covered = s.coveredSet() + } + // Filter by preconditions and build weighted list. type candidate struct { op Operation @@ -130,6 +258,9 @@ func (s *Scheduler) selectOperation(ctx context.Context) Operation { totalWeight := 0 for _, op := range s.operations { + if s.coverage && covered[op.Name()] { + continue + } ok, _ := op.Precondition(ctx) if ok { w := op.Weight() @@ -143,7 +274,7 @@ func (s *Scheduler) selectOperation(ctx context.Context) Operation { } // Weighted random selection. - r := rand.IntN(totalWeight) + r := s.intn(totalWeight) for _, c := range candidates { r -= c.weight if r < 0 { @@ -153,18 +284,51 @@ func (s *Scheduler) selectOperation(ctx context.Context) Operation { return candidates[len(candidates)-1].op } -func (s *Scheduler) executeOp(ctx context.Context, op Operation) { +func (s *Scheduler) executeOp(ctx context.Context, op Operation) error { s.journal.Info("scheduler", fmt.Sprintf("executing operation: %s", op.Name())) s.journal.OpenDisruptionWindow(op.Name(), op.OutagePolicy()) err := op.Execute(ctx) - - s.journal.CloseDisruptionWindow() + window := s.journal.CloseDisruptionWindow() if err != nil { s.journal.Error("scheduler", fmt.Sprintf("operation %s failed: %v", op.Name(), err)) - } else { - s.journal.Info("scheduler", fmt.Sprintf("operation %s completed successfully", op.Name())) + return fmt.Errorf("operation %s execute failed: %w", op.Name(), err) + } + if window == nil { + err = fmt.Errorf("operation %s closed without a disruption window", op.Name()) + s.journal.Error("scheduler", err.Error()) + return err + } + if window.ExceededPolicy() { + err = fmt.Errorf("operation %s exceeded its outage policy", op.Name()) + s.journal.Error("scheduler", err.Error()) + return err + } + + s.journal.Info("scheduler", fmt.Sprintf("operation %s completed successfully", op.Name())) + return nil +} + +func (s *Scheduler) recordExecution(name string, err error) { + s.state.mu.Lock() + defer s.state.mu.Unlock() + index, ok := s.aggregateIndex[name] + if !ok { + return + } + if err != nil { + s.state.snapshot.Aggregates[index].Failed++ + s.state.snapshot.Status = RunStatusFailed + if s.state.snapshot.FailureReason == "" { + s.state.snapshot.FailureReason = err.Error() + } + return + } + s.state.snapshot.Aggregates[index].Passed++ + // Coverage mode completes once every operation has run at least once. + if s.coverage && s.state.snapshot.Status == RunStatusRunning && s.allCoveredLocked() { + s.state.snapshot.Status = RunStatusComplete } } @@ -174,3 +338,13 @@ func (s *Scheduler) OpsExecuted() int { defer s.mu.Unlock() return s.opsExecuted } + +// Snapshot returns bounded aggregate counters in registration order. +func (s *Scheduler) Snapshot() RunSnapshot { + return s.state.Snapshot() +} + +// Done closes when the scheduler stops after context cancellation. +func (s *Scheduler) Done() <-chan struct{} { + return s.state.Done() +} diff --git a/test/longhaul/operations/scheduler_test.go b/test/longhaul/operations/scheduler_test.go index cd351eb9c..a12baf7a6 100644 --- a/test/longhaul/operations/scheduler_test.go +++ b/test/longhaul/operations/scheduler_test.go @@ -120,4 +120,92 @@ var _ = Describe("Scheduler", func() { s.opsExecuted = 7 Expect(s.OpsExecuted()).To(Equal(7)) }) + + It("keeps bounded aggregate counters and exposes failures", func() { + a := &fakeOp{name: "a"} + b := &fakeOp{name: "b"} + s := NewScheduler([]Operation{a, b}, nil, journal.New(), time.Hour) + + for i := 0; i < 1000; i++ { + s.recordExecution("a", nil) + } + s.recordExecution("b", errors.New("first failure")) + s.recordExecution("b", errors.New("second failure")) + + snapshot := s.Snapshot() + Expect(snapshot.Aggregates).To(Equal([]OperationAggregate{ + {Name: "a", Passed: 1000}, + {Name: "b", Failed: 2}, + })) + Expect(snapshot.Status).To(Equal(RunStatusFailed)) + Expect(snapshot.HasFailure()).To(BeTrue()) + Expect(snapshot.FailureReason).To(ContainSubstring("first failure")) + }) + + Describe("WithSeed", func() { + It("makes weighted selection reproducible across schedulers", func() { + mk := func() *Scheduler { + return NewScheduler([]Operation{ + &fakeOp{name: "a", weight: 1, available: true}, + &fakeOp{name: "b", weight: 1, available: true}, + &fakeOp{name: "c", weight: 1, available: true}, + }, nil, journal.New(), time.Hour, WithSeed(42)) + } + s1, s2 := mk(), mk() + var seq1, seq2 []string + for i := 0; i < 25; i++ { + seq1 = append(seq1, s1.selectOperation(context.Background()).Name()) + seq2 = append(seq2, s2.selectOperation(context.Background()).Name()) + } + Expect(seq1).To(Equal(seq2)) + }) + }) + + Describe("WithCoverage", func() { + It("draws each operation without replacement", func() { + a := &fakeOp{name: "a", weight: 1, available: true} + b := &fakeOp{name: "b", weight: 1, available: true} + s := NewScheduler([]Operation{a, b}, nil, journal.New(), time.Hour, WithCoverage()) + + // Once "a" is covered, selection must never return it again. + s.recordExecution("a", nil) + for i := 0; i < 50; i++ { + got := s.selectOperation(context.Background()) + Expect(got).NotTo(BeNil(), "iter %d", i) + Expect(got.Name()).To(Equal("b")) + } + + // Once every operation is covered there are no candidates left. + s.recordExecution("b", nil) + Expect(s.selectOperation(context.Background())).To(BeNil()) + }) + + It("completes the run once every operation has run at least once", func() { + a := &fakeOp{name: "a"} + b := &fakeOp{name: "b"} + s := NewScheduler([]Operation{a, b}, nil, journal.New(), time.Hour, WithCoverage()) + s.state.snapshot.Status = RunStatusRunning + + s.recordExecution("a", nil) + Expect(s.Snapshot().Status).To(Equal(RunStatusRunning)) + + s.recordExecution("b", nil) + Expect(s.Snapshot().Status).To(Equal(RunStatusComplete)) + }) + + It("does not complete a partially covered run", func() { + a := &fakeOp{name: "a"} + b := &fakeOp{name: "b"} + s := NewScheduler([]Operation{a, b}, nil, journal.New(), time.Hour, WithCoverage()) + s.state.snapshot.Status = RunStatusRunning + + s.recordExecution("a", nil) + + s.state.mu.RLock() + covered := s.allCoveredLocked() + s.state.mu.RUnlock() + Expect(covered).To(BeFalse()) + Expect(s.Snapshot().Status).To(Equal(RunStatusRunning)) + }) + }) }) diff --git a/test/longhaul/operations/sequence.go b/test/longhaul/operations/sequence.go new file mode 100644 index 000000000..2426f1e64 --- /dev/null +++ b/test/longhaul/operations/sequence.go @@ -0,0 +1,247 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "fmt" + "time" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" +) + +const defaultPreconditionPollInterval = time.Second + +// SteadyStateGate is the health-monitor surface needed by sequence mode. +type SteadyStateGate interface { + WaitForSteadyState(ctx context.Context) error +} + +type preconditionWaitFunc func(context.Context, Operation) error + +// SequenceRunner executes each requested operation exactly once and in order. +type SequenceRunner struct { + operations []Operation + steadyStateGate SteadyStateGate + journal *journal.Journal + recoveryTimeout time.Duration + state runnerState + + waitForPrecondition preconditionWaitFunc + terminal bool +} + +// NewSequenceRunner creates a deterministic sequential operation runner. +func NewSequenceRunner( + ops []Operation, + gate SteadyStateGate, + j *journal.Journal, + recoveryTimeout time.Duration, +) *SequenceRunner { + results := make([]OperationResult, len(ops)) + for i, op := range ops { + results[i] = OperationResult{Name: op.Name(), Status: OperationPending} + } + runner := &SequenceRunner{ + operations: append([]Operation(nil), ops...), + steadyStateGate: gate, + journal: j, + recoveryTimeout: recoveryTimeout, + state: newRunnerState(RunSnapshot{ + Mode: config.OperationModeSequence, + Status: RunStatusPending, + Results: results, + }), + } + runner.waitForPrecondition = runner.pollPrecondition + return runner +} + +// Run executes the configured sequence and stops on the first failure. +func (r *SequenceRunner) Run(ctx context.Context) { + defer r.state.closeDone() + + r.state.mu.Lock() + if r.terminal { + r.state.mu.Unlock() + return + } + r.state.snapshot.Status = RunStatusRunning + r.state.mu.Unlock() + + for i, op := range r.operations { + if !r.setResult(i, OperationRunning, "") { + return + } + if err := r.runOne(ctx, op); err != nil { + status := RunStatusFailed + reason := err.Error() + if ctx.Err() != nil { + status = RunStatusIncomplete + reason = fmt.Sprintf("operation sequence incomplete during %s: %v", op.Name(), ctx.Err()) + } + r.setFailure(i, status, reason) + return + } + if !r.setResult(i, OperationPassed, "") { + return + } + } + + r.state.mu.Lock() + if !r.terminal { + r.terminal = true + r.state.snapshot.Status = RunStatusComplete + } + r.state.mu.Unlock() +} + +func (r *SequenceRunner) runOne(ctx context.Context, op Operation) error { + if r.steadyStateGate == nil { + return fmt.Errorf("operation %s steady-state gate is nil", op.Name()) + } + + steadyCtx, cancelSteady := context.WithTimeout(ctx, r.recoveryTimeout) + err := r.steadyStateGate.WaitForSteadyState(steadyCtx) + cancelSteady() + if err != nil { + return fmt.Errorf("operation %s initial steady-state gate failed: %w", op.Name(), err) + } + + preconditionCtx, cancelPrecondition := context.WithTimeout(ctx, r.recoveryTimeout) + err = r.waitForPrecondition(preconditionCtx, op) + cancelPrecondition() + if err != nil { + return fmt.Errorf("operation %s precondition timeout: %w", op.Name(), err) + } + + r.journal.Info("sequence", fmt.Sprintf("executing operation: %s", op.Name())) + r.journal.OpenDisruptionWindow(op.Name(), op.OutagePolicy()) + + executeCtx, cancelExecute := context.WithTimeout(ctx, r.recoveryTimeout) + executeErr := op.Execute(executeCtx) + cancelExecute() + window := r.journal.CloseDisruptionWindow() + + if executeErr != nil { + r.journal.Error("sequence", fmt.Sprintf("operation %s failed: %v", op.Name(), executeErr)) + return fmt.Errorf("operation %s execute failed: %w", op.Name(), executeErr) + } + if window == nil { + return fmt.Errorf("operation %s closed without a disruption window", op.Name()) + } + if window.ExceededPolicy() { + err := fmt.Errorf("operation %s exceeded its outage policy", op.Name()) + r.journal.Error("sequence", err.Error()) + return err + } + + recoveryCtx, cancelRecovery := context.WithTimeout(ctx, r.recoveryTimeout) + err = r.steadyStateGate.WaitForSteadyState(recoveryCtx) + cancelRecovery() + if err != nil { + return fmt.Errorf("operation %s post-recovery steady-state gate failed: %w", op.Name(), err) + } + + r.journal.Info("sequence", fmt.Sprintf("operation %s completed successfully", op.Name())) + return nil +} + +func (r *SequenceRunner) pollPrecondition(ctx context.Context, op Operation) error { + ticker := time.NewTicker(defaultPreconditionPollInterval) + defer ticker.Stop() + + lastReason := "precondition not met" + for { + ok, reason := op.Precondition(ctx) + if ok { + return nil + } + if reason != "" { + lastReason = reason + } + + select { + case <-ctx.Done(): + return fmt.Errorf("%s: %w", lastReason, ctx.Err()) + case <-ticker.C: + } + } +} + +func (r *SequenceRunner) setResult(index int, status OperationResultStatus, reason string) bool { + r.state.mu.Lock() + defer r.state.mu.Unlock() + if r.terminal { + return false + } + r.state.snapshot.Results[index].Status = status + r.state.snapshot.Results[index].Error = reason + return true +} + +func (r *SequenceRunner) setFailure(index int, status RunStatus, reason string) { + r.state.mu.Lock() + defer r.state.mu.Unlock() + if r.terminal { + return + } + r.terminal = true + r.state.snapshot.Status = status + r.state.snapshot.FailureReason = reason + r.state.snapshot.Results[index].Status = OperationFailed + r.state.snapshot.Results[index].Error = reason +} + +// MarkIncomplete terminally fails a sequence whose watchdog or shutdown +// cancellation fired before Run could publish its own terminal snapshot. +func (r *SequenceRunner) MarkIncomplete(reason string) { + r.state.mu.Lock() + if r.terminal { + r.state.mu.Unlock() + return + } + allPassed := len(r.state.snapshot.Results) > 0 + for _, result := range r.state.snapshot.Results { + if result.Status != OperationPassed { + allPassed = false + break + } + } + if allPassed { + r.terminal = true + r.state.snapshot.Status = RunStatusComplete + r.state.mu.Unlock() + r.state.closeDone() + return + } + r.terminal = true + r.state.snapshot.Status = RunStatusIncomplete + r.state.snapshot.FailureReason = reason + for i := range r.state.snapshot.Results { + if r.state.snapshot.Results[i].Status == OperationRunning { + r.state.snapshot.Results[i].Status = OperationFailed + r.state.snapshot.Results[i].Error = reason + break + } + if r.state.snapshot.Results[i].Status == OperationPending { + r.state.snapshot.Results[i].Status = OperationFailed + r.state.snapshot.Results[i].Error = reason + break + } + } + r.state.mu.Unlock() + r.state.closeDone() +} + +// Snapshot returns a deterministic copy ordered by the configured sequence. +func (r *SequenceRunner) Snapshot() RunSnapshot { + return r.state.Snapshot() +} + +// Done closes when the sequence completes or stops on failure/cancellation. +func (r *SequenceRunner) Done() <-chan struct{} { + return r.state.Done() +} diff --git a/test/longhaul/operations/sequence_test.go b/test/longhaul/operations/sequence_test.go new file mode 100644 index 000000000..f2e8541da --- /dev/null +++ b/test/longhaul/operations/sequence_test.go @@ -0,0 +1,217 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "errors" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" +) + +type sequenceTestOp struct { + name string + execute func(context.Context) error + precondition func(context.Context) (bool, string) + policy journal.OutagePolicy +} + +func (o *sequenceTestOp) Name() string { return o.name } +func (o *sequenceTestOp) Weight() int { return 1 } +func (o *sequenceTestOp) Precondition(ctx context.Context) (bool, string) { + if o.precondition != nil { + return o.precondition(ctx) + } + return true, "" +} +func (o *sequenceTestOp) Execute(ctx context.Context) error { + if o.execute != nil { + return o.execute(ctx) + } + return nil +} +func (o *sequenceTestOp) OutagePolicy() journal.OutagePolicy { + if o.policy.MustRecoverWithin != 0 || o.policy.MaxWriteOutage != 0 { + return o.policy + } + return journal.DefaultOutagePolicy() +} + +type sequenceTestGate struct { + mu sync.Mutex + calls int + err error +} + +func (g *sequenceTestGate) WaitForSteadyState(context.Context) error { + g.mu.Lock() + defer g.mu.Unlock() + g.calls++ + return g.err +} + +func runSequence(runner *SequenceRunner, ctx context.Context) RunSnapshot { + go runner.Run(ctx) + Eventually(runner.Done()).Should(BeClosed()) + return runner.Snapshot() +} + +var _ = Describe("SequenceRunner", func() { + It("executes each operation exactly once in exact order", func() { + var order []string + ops := []Operation{ + &sequenceTestOp{name: "first", execute: func(context.Context) error { + order = append(order, "first") + return nil + }}, + &sequenceTestOp{name: "second", execute: func(context.Context) error { + order = append(order, "second") + return nil + }}, + } + gate := &sequenceTestGate{} + snapshot := runSequence(NewSequenceRunner(ops, gate, journal.New(), time.Second), context.Background()) + + Expect(order).To(Equal([]string{"first", "second"})) + Expect(snapshot.Status).To(Equal(RunStatusComplete)) + Expect(snapshot.Results).To(Equal([]OperationResult{ + {Name: "first", Status: OperationPassed}, + {Name: "second", Status: OperationPassed}, + })) + Expect(gate.calls).To(Equal(4), "initial and post-recovery gate for each operation") + }) + + It("records an execute error and stops before later operations", func() { + var order []string + ops := []Operation{ + &sequenceTestOp{name: "first", execute: func(context.Context) error { + order = append(order, "first") + return nil + }}, + &sequenceTestOp{name: "broken", execute: func(context.Context) error { + order = append(order, "broken") + return errors.New("kaboom") + }}, + &sequenceTestOp{name: "never", execute: func(context.Context) error { + order = append(order, "never") + return nil + }}, + } + snapshot := runSequence( + NewSequenceRunner(ops, &sequenceTestGate{}, journal.New(), time.Second), + context.Background(), + ) + + Expect(order).To(Equal([]string{"first", "broken"})) + Expect(snapshot.Status).To(Equal(RunStatusFailed)) + Expect(snapshot.FailureReason).To(ContainSubstring("kaboom")) + Expect(snapshot.Results).To(Equal([]OperationResult{ + {Name: "first", Status: OperationPassed}, + {Name: "broken", Status: OperationFailed, Error: snapshot.FailureReason}, + {Name: "never", Status: OperationPending}, + })) + }) + + It("fails deterministically when a precondition times out", func() { + op := &sequenceTestOp{name: "blocked"} + runner := NewSequenceRunner([]Operation{op}, &sequenceTestGate{}, journal.New(), time.Second) + runner.waitForPrecondition = func(context.Context, Operation) error { + return context.DeadlineExceeded + } + + snapshot := runSequence(runner, context.Background()) + Expect(snapshot.Status).To(Equal(RunStatusFailed)) + Expect(snapshot.Results[0].Status).To(Equal(OperationFailed)) + Expect(snapshot.FailureReason).To(ContainSubstring("precondition timeout")) + }) + + It("marks cancellation as incomplete and leaves later operations pending", func() { + started := make(chan struct{}) + op := &sequenceTestOp{name: "cancelled", execute: func(ctx context.Context) error { + close(started) + <-ctx.Done() + return ctx.Err() + }} + runner := NewSequenceRunner( + []Operation{op, &sequenceTestOp{name: "never"}}, + &sequenceTestGate{}, + journal.New(), + time.Minute, + ) + ctx, cancel := context.WithCancel(context.Background()) + go runner.Run(ctx) + Eventually(started).Should(BeClosed()) + cancel() + Eventually(runner.Done()).Should(BeClosed()) + + snapshot := runner.Snapshot() + Expect(snapshot.Status).To(Equal(RunStatusIncomplete)) + Expect(snapshot.Results[0].Status).To(Equal(OperationFailed)) + Expect(snapshot.Results[1].Status).To(Equal(OperationPending)) + Expect(snapshot.FailureReason).To(ContainSubstring("incomplete")) + }) + + It("publishes a terminal incomplete snapshot when the watchdog wins", func() { + runner := NewSequenceRunner( + []Operation{ + &sequenceTestOp{name: "first"}, + &sequenceTestOp{name: "second"}, + }, + &sequenceTestGate{}, + journal.New(), + time.Minute, + ) + + runner.MarkIncomplete("watchdog fired") + Expect(runner.Done()).To(BeClosed()) + snapshot := runner.Snapshot() + Expect(snapshot.Status).To(Equal(RunStatusIncomplete)) + Expect(snapshot.FailureReason).To(Equal("watchdog fired")) + Expect(snapshot.Results).To(Equal([]OperationResult{ + {Name: "first", Status: OperationFailed, Error: "watchdog fired"}, + {Name: "second", Status: OperationPending}, + })) + }) + + It("preserves completion when the watchdog races after every operation passed", func() { + runner := NewSequenceRunner( + []Operation{&sequenceTestOp{name: "done"}}, + &sequenceTestGate{}, + journal.New(), + time.Second, + ) + runner.state.snapshot.Results[0].Status = OperationPassed + + runner.MarkIncomplete("watchdog fired") + + snapshot := runner.Snapshot() + Expect(snapshot.Status).To(Equal(RunStatusComplete)) + Expect(snapshot.Results).To(Equal([]OperationResult{{ + Name: "done", + Status: OperationPassed, + }})) + }) + + It("fails when the closed disruption window exceeds policy", func() { + op := &sequenceTestOp{ + name: "policy", + policy: journal.OutagePolicy{ + MaxWriteOutage: time.Hour, + MustRecoverWithin: -time.Nanosecond, + }, + } + snapshot := runSequence( + NewSequenceRunner([]Operation{op}, &sequenceTestGate{}, journal.New(), time.Second), + context.Background(), + ) + + Expect(snapshot.Status).To(Equal(RunStatusFailed)) + Expect(snapshot.FailureReason).To(ContainSubstring("exceeded its outage policy")) + }) +}) diff --git a/test/longhaul/operations/upgrade.go b/test/longhaul/operations/upgrade.go index f9a586b6e..554356997 100644 --- a/test/longhaul/operations/upgrade.go +++ b/test/longhaul/operations/upgrade.go @@ -170,11 +170,11 @@ func (u *UpgradeDocumentDB) readDesiredVersion(ctx context.Context) (string, err return cm.Data[VersionConfigMapKey], nil } -// OutagePolicy allows for a longer disruption window during an upgrade -// because rolling restarts touch every pod sequentially. +// OutagePolicy bounds the write outage of a rolling upgrade. Standby restarts +// do not block writes; the write path is only interrupted during the single +// graceful primary switchover, so it shares the primary-handover budget with +// kill-primary-pod (see journal.PrimaryHandoverPolicy). The upgrade's longer +// whole-topology restart is bounded separately by MustRecoverWithin. func (u *UpgradeDocumentDB) OutagePolicy() journal.OutagePolicy { - return journal.OutagePolicy{ - AllowedWriteFailures: 200, - MustRecoverWithin: u.recovery, - } + return journal.PrimaryHandoverPolicy(u.recovery) } diff --git a/test/longhaul/operations/upgrade_test.go b/test/longhaul/operations/upgrade_test.go index f442e5b58..ae5b7450e 100644 --- a/test/longhaul/operations/upgrade_test.go +++ b/test/longhaul/operations/upgrade_test.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -22,10 +23,10 @@ var _ = Describe("UpgradeDocumentDB", func() { Expect(u.Weight()).To(Equal(1)) }) - It("OutagePolicy gives upgrades a more lenient failure budget", func() { + It("OutagePolicy shares the single-primary-handover budget with kill-primary", func() { u := NewUpgradeDocumentDB(&fakeClient{}, fake.NewSimpleClientset(), nil, nil, "ns", 10*time.Minute) p := u.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(int64(200))) + Expect(p.MaxWriteOutage).To(Equal(journal.PrimaryHandoverWriteOutage)) Expect(p.MustRecoverWithin).To(Equal(10 * time.Minute)) }) diff --git a/test/longhaul/report/checkpoint.go b/test/longhaul/report/checkpoint.go index 5875412c7..d6d28130f 100644 --- a/test/longhaul/report/checkpoint.go +++ b/test/longhaul/report/checkpoint.go @@ -8,8 +8,11 @@ import ( "encoding/json" "fmt" "log" + "sync" "time" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,8 +24,9 @@ const ( ConfigMapName = "longhaul-report" ) -// SummaryFunc is called to generate the current test summary. -type SummaryFunc func() Summary +// SummaryFunc is called to generate the current test summary. final is true +// only for the terminal emit, when incomplete sequence execution must fail. +type SummaryFunc func(final bool) Summary // CheckpointReporter periodically generates and persists reports. type CheckpointReporter struct { @@ -30,6 +34,10 @@ type CheckpointReporter struct { namespace string interval time.Duration summaryFunc SummaryFunc + + emitMu sync.Mutex + finalEmitted bool + finalSummary Summary } // NewCheckpointReporter creates a periodic reporter that writes to stdout and ConfigMap. @@ -67,18 +75,31 @@ func (r *CheckpointReporter) Run(ctx context.Context) { // not as RUNNING) using a bounded context. Safe to call after the main // context has been cancelled. Intended to be called synchronously from main // just before exit so the verdict is durable in the ConfigMap. -func (r *CheckpointReporter) EmitFinal() { +func (r *CheckpointReporter) EmitFinal() Summary { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - r.emit(ctx, true) + return r.emit(ctx, true) } // emit writes the current summary to stdout, GH Actions annotations, and the // status ConfigMap. final=true means this is the shutdown emit, in which case // PASS is persisted as "PASS" (not "RUNNING") so consumers can distinguish a // finished clean run from an in-flight checkpoint. -func (r *CheckpointReporter) emit(ctx context.Context, final bool) { - summary := r.summaryFunc() +func (r *CheckpointReporter) emit(ctx context.Context, final bool) Summary { + r.emitMu.Lock() + defer r.emitMu.Unlock() + if final && r.finalEmitted { + return r.finalSummary + } + if !final && r.finalEmitted { + return Summary{} + } + + summary := r.summaryFunc(final) + if final { + r.finalEmitted = true + r.finalSummary = summary + } // Intermediate PASS checkpoints surface as RUNNING; the final emit // preserves the true PASS/FAIL outcome. @@ -99,13 +120,18 @@ func (r *CheckpointReporter) emit(ctx context.Context, final bool) { // Persist to ConfigMap. if r.clientset == nil { - return + return summary } data := map[string]string{ - "latest-report": markdown, - "last-updated": time.Now().UTC().Format(time.RFC3339), - "result": resultStr, + "latest-report": markdown, + "last-updated": time.Now().UTC().Format(time.RFC3339), + "result": resultStr, + "operation-status": string(summary.OperationRun.Status), + "operation-results": marshalOperationResults(summary.OperationRun.Results), + } + if len(summary.OperationRun.Aggregates) > 0 { + data["operation-aggregates"] = marshalOperationAggregates(summary.OperationRun.Aggregates) } cm := &corev1.ConfigMap{ @@ -142,14 +168,32 @@ func (r *CheckpointReporter) emit(ctx context.Context, final bool) { // Also log the summary as JSON for structured log consumers. summaryJSON, _ := json.Marshal(map[string]any{ - "result": resultStr, - "elapsed": summary.Duration.String(), - "writes": summary.Metrics.WriteAttempted, - "gaps": summary.Metrics.GapsDetected, - "ops": summary.OpsExecuted, - "memory_leak": summary.LeakAnalysis.HasLeak, - "memory_slope": fmt.Sprintf("%.2f MB/h", summary.LeakAnalysis.MemorySlopeMB), - "checkpoint_time": time.Now().UTC().Format(time.RFC3339), + "result": resultStr, + "elapsed": summary.Duration.String(), + "writes": summary.Metrics.WriteAttempted, + "gaps": summary.Metrics.GapsDetected, + "ops": summary.OpsExecuted, + "memory_leak": summary.LeakAnalysis.HasLeak, + "memory_slope": fmt.Sprintf("%.2f MB/h", summary.LeakAnalysis.MemorySlopeMB), + "operation_status": summary.OperationRun.Status, + "checkpoint_time": time.Now().UTC().Format(time.RFC3339), }) log.Printf("[checkpoint] %s", string(summaryJSON)) + return summary +} + +func marshalOperationResults(results []operations.OperationResult) string { + if results == nil { + results = []operations.OperationResult{} + } + data, _ := json.Marshal(results) + return string(data) +} + +func marshalOperationAggregates(aggregates []operations.OperationAggregate) string { + if aggregates == nil { + aggregates = []operations.OperationAggregate{} + } + data, _ := json.Marshal(aggregates) + return string(data) } diff --git a/test/longhaul/report/checkpoint_test.go b/test/longhaul/report/checkpoint_test.go index 891e90b4d..d9c80f7ef 100644 --- a/test/longhaul/report/checkpoint_test.go +++ b/test/longhaul/report/checkpoint_test.go @@ -5,6 +5,7 @@ package report import ( "context" + "encoding/json" "time" . "github.com/onsi/ginkgo/v2" @@ -12,11 +13,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" ) var _ = Describe("CheckpointReporter", func() { It("emit() is safe with a nil clientset (logs to stdout, does not panic)", func() { - r := NewCheckpointReporter(nil, "ns", time.Second, func() Summary { + r := NewCheckpointReporter(nil, "ns", time.Second, func(bool) Summary { return Summary{Result: ResultPass, Duration: time.Minute} }) Expect(func() { r.emit(context.Background(), false) }).NotTo(Panic()) @@ -24,7 +28,7 @@ var _ = Describe("CheckpointReporter", func() { It("creates the ConfigMap on first emit and labels it identifiably", func() { cs := fake.NewSimpleClientset() - r := NewCheckpointReporter(cs, "ns", time.Second, func() Summary { + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { return Summary{Result: ResultPass, Duration: 2 * time.Hour, OpsExecuted: 5} }) @@ -35,6 +39,8 @@ var _ = Describe("CheckpointReporter", func() { Expect(cm.Data).To(HaveKey("latest-report")) Expect(cm.Data).To(HaveKey("last-updated")) Expect(cm.Data).To(HaveKey("result")) + Expect(cm.Data).To(HaveKeyWithValue("operation-status", "")) + Expect(cm.Data).To(HaveKeyWithValue("operation-results", "[]")) // PASS at intermediate checkpoint is persisted as RUNNING so consumers // can distinguish in-flight from final state. Expect(cm.Data["result"]).To(Equal("RUNNING")) @@ -43,7 +49,7 @@ var _ = Describe("CheckpointReporter", func() { It("persists FAIL results as FAIL", func() { cs := fake.NewSimpleClientset() - r := NewCheckpointReporter(cs, "ns", time.Second, func() Summary { + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { return Summary{Result: ResultFail, FailReason: "data loss"} }) @@ -58,7 +64,7 @@ var _ = Describe("CheckpointReporter", func() { cs := fake.NewSimpleClientset() calls := 0 - r := NewCheckpointReporter(cs, "ns", time.Second, func() Summary { + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { calls++ return Summary{Result: ResultPass, Duration: time.Duration(calls) * time.Hour, OpsExecuted: calls * 10} }) @@ -76,4 +82,87 @@ var _ = Describe("CheckpointReporter", func() { Expect(cm2.Data["latest-report"]).NotTo(Equal(report1)) Expect(calls).To(Equal(2)) }) + + It("persists ordered sequence results as bounded JSON", func() { + cs := fake.NewSimpleClientset() + results := []operations.OperationResult{ + {Name: "kill-operator-pod", Status: operations.OperationPassed}, + {Name: "kill-primary-pod", Status: operations.OperationPassed}, + } + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { + return Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusComplete, + Results: results, + }, + } + }) + + r.emit(context.Background(), true) + cm, err := cs.CoreV1().ConfigMaps("ns").Get(context.Background(), ConfigMapName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data["operation-status"]).To(Equal("COMPLETE")) + + var persisted []operations.OperationResult + Expect(json.Unmarshal([]byte(cm.Data["operation-results"]), &persisted)).To(Succeed()) + Expect(persisted).To(Equal(results)) + Expect(cm.Data).NotTo(HaveKey("operation-aggregates")) + }) + + It("overwrites mode-specific fields instead of retaining stale aggregates", func() { + cs := fake.NewSimpleClientset() + random := true + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { + if random { + return Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeRandom, + Status: operations.RunStatusRunning, + Aggregates: []operations.OperationAggregate{{Name: "scale-up", Passed: 3}}, + }, + } + } + return Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusComplete, + Results: []operations.OperationResult{{Name: "scale-up", Status: operations.OperationPassed}}, + }, + } + }) + + r.emit(context.Background(), false) + random = false + r.emit(context.Background(), true) + + cm, err := cs.CoreV1().ConfigMaps("ns").Get(context.Background(), ConfigMapName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).NotTo(HaveKey("operation-aggregates")) + Expect(cm.Data["operation-results"]).To(MatchJSON(`[{"name":"scale-up","status":"PASSED"}]`)) + }) + + It("emits the final report exactly once and rejects later checkpoints", func() { + cs := fake.NewSimpleClientset() + calls := 0 + r := NewCheckpointReporter(cs, "ns", time.Second, func(final bool) Summary { + calls++ + Expect(final).To(BeTrue()) + return Summary{Result: ResultPass, Duration: time.Duration(calls) * time.Minute} + }) + + first := r.EmitFinal() + second := r.EmitFinal() + r.emit(context.Background(), false) + + Expect(calls).To(Equal(1)) + Expect(second).To(Equal(first)) + cm, err := cs.CoreV1().ConfigMaps("ns").Get(context.Background(), ConfigMapName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data["result"]).To(Equal("PASS")) + Expect(cm.Data["latest-report"]).To(ContainSubstring("**Duration:** 1m0s")) + }) }) diff --git a/test/longhaul/report/report.go b/test/longhaul/report/report.go index 37cd17142..f31abf87a 100644 --- a/test/longhaul/report/report.go +++ b/test/longhaul/report/report.go @@ -9,8 +9,10 @@ import ( "time" "github.com/documentdb/documentdb-operator/test/longhaul/backup" + "github.com/documentdb/documentdb-operator/test/longhaul/config" "github.com/documentdb/documentdb-operator/test/longhaul/journal" "github.com/documentdb/documentdb-operator/test/longhaul/monitor" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" "github.com/documentdb/documentdb-operator/test/longhaul/workload" ) @@ -26,9 +28,8 @@ const ( // It is a pure value snapshot — no live counters, no channels — so it can be // passed across goroutines and re-rendered offline. type Summary struct { - // Result is the current verdict. PASS while data-loss counters stay zero, - // flipped to FAIL when the durability oracle detects gaps/checksum errors - // or a disruption window blows its policy budget. + // Result is the current verdict. It flips to FAIL for durability errors, + // operation failures/incomplete sequences, or outage-policy violations. Result Result // Duration is wall-clock time since the run started (process StartTime), @@ -49,13 +50,14 @@ type Summary struct { // only emits a warning annotation. LeakAnalysis monitor.LeakAnalysis - // OpsExecuted is the count of operations (scale up/down, restart, etc.) - // the operations scheduler has run since startup. + // OpsExecuted is the count of terminal operation attempts since startup. OpsExecuted int - // Windows is every disruption window opened during the run, in start - // order. Each window records its op, duration, write-failure count, and - // whether it exceeded its policy budget. + // OperationRun is the bounded sequence result or random aggregate snapshot. + OperationRun operations.RunSnapshot + + // Windows is the journal's bounded set of recent closed disruption windows, + // in start order. Windows []journal.DisruptionWindow // Events is the journal's full event ring (info/warn/error log lines). @@ -83,6 +85,27 @@ func GenerateMarkdown(s Summary) string { } b.WriteString("\n") + switch s.OperationRun.Mode { + case config.OperationModeSequence: + b.WriteString("## Operation Results\n\n") + b.WriteString("| # | Operation | Status | Error |\n") + b.WriteString("|---|-----------|--------|-------|\n") + for i, result := range s.OperationRun.Results { + fmt.Fprintf(&b, "| %d | %s | %s | %s |\n", + i+1, result.Name, result.Status, markdownCell(result.Error)) + } + b.WriteString("\n") + case config.OperationModeRandom: + b.WriteString("## Operation Summary\n\n") + b.WriteString("| Operation | Passed | Failed |\n") + b.WriteString("|-----------|--------|--------|\n") + for _, aggregate := range s.OperationRun.Aggregates { + fmt.Fprintf(&b, "| %s | %d | %d |\n", + aggregate.Name, aggregate.Passed, aggregate.Failed) + } + b.WriteString("\n") + } + // Data Plane Metrics b.WriteString("## Data Plane Metrics\n\n") b.WriteString("| Metric | Value |\n") @@ -115,15 +138,16 @@ func GenerateMarkdown(s Summary) string { // Disruption Windows if len(s.Windows) > 0 { b.WriteString("## Disruption Windows\n\n") - b.WriteString("| Operation | Duration | Write Failures | Policy Exceeded |\n") - b.WriteString("|-----------|----------|----------------|------------------|\n") + b.WriteString("| Operation | Duration | Write Failures | Est. Write Outage | Policy Exceeded |\n") + b.WriteString("|-----------|----------|----------------|-------------------|------------------|\n") for _, w := range s.Windows { exceeded := "No" if w.ExceededPolicy() { exceeded = "**YES**" } - fmt.Fprintf(&b, "| %s | %s | %d | %s |\n", - w.OperationName, w.Duration().Round(time.Second), w.WriteFailures, exceeded) + fmt.Fprintf(&b, "| %s | %s | %d | %s | %s |\n", + w.OperationName, w.Duration().Round(time.Second), w.WriteFailures, + w.EstimatedWriteOutage().Round(time.Millisecond), exceeded) } b.WriteString("\n") } @@ -156,3 +180,11 @@ func GenerateMarkdown(s Summary) string { return b.String() } + +func markdownCell(value string) string { + if value == "" { + return "—" + } + value = strings.ReplaceAll(value, "|", "\\|") + return strings.ReplaceAll(value, "\n", " ") +} diff --git a/test/longhaul/report/report_test.go b/test/longhaul/report/report_test.go index f8b77297e..bf3e3dc62 100644 --- a/test/longhaul/report/report_test.go +++ b/test/longhaul/report/report_test.go @@ -10,8 +10,10 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/documentdb/documentdb-operator/test/longhaul/config" "github.com/documentdb/documentdb-operator/test/longhaul/journal" "github.com/documentdb/documentdb-operator/test/longhaul/monitor" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" "github.com/documentdb/documentdb-operator/test/longhaul/workload" ) @@ -69,16 +71,52 @@ var _ = Describe("GenerateMarkdown", func() { Expect(md).NotTo(ContainSubstring("Disruption Windows")) }) + It("renders ordered sequence operation results", func() { + md := GenerateMarkdown(Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusComplete, + Results: []operations.OperationResult{ + {Name: "kill-operator-pod", Status: operations.OperationPassed}, + {Name: "kill-primary-pod", Status: operations.OperationFailed, Error: "primary unchanged"}, + }, + }, + }) + Expect(md).To(ContainSubstring("## Operation Results")) + Expect(md).To(ContainSubstring("| 1 | kill-operator-pod | PASSED |")) + Expect(md).To(ContainSubstring("| 2 | kill-primary-pod | FAILED | primary unchanged |")) + Expect(strings.Index(md, "kill-operator-pod")).To(BeNumerically("<", strings.Index(md, "kill-primary-pod"))) + }) + + It("renders bounded random aggregate counters", func() { + md := GenerateMarkdown(Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeRandom, + Status: operations.RunStatusRunning, + Aggregates: []operations.OperationAggregate{ + {Name: "scale-up", Passed: 12, Failed: 1}, + {Name: "scale-down", Passed: 9}, + }, + }, + }) + Expect(md).To(ContainSubstring("## Operation Summary")) + Expect(md).To(ContainSubstring("| scale-up | 12 | 1 |")) + Expect(md).To(ContainSubstring("| scale-down | 9 | 0 |")) + }) + It("appears with the operation name when at least one window exists", func() { now := time.Now() md := GenerateMarkdown(Summary{ Result: ResultPass, Windows: []journal.DisruptionWindow{{ - OperationName: "scale-up", - StartTime: now.Add(-30 * time.Second), - EndTime: now, - WriteFailures: 3, - Policy: journal.OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + OperationName: "scale-up", + StartTime: now.Add(-30 * time.Second), + EndTime: now, + WriteFailures: 3, + WritesPerSecond: 50, + Policy: journal.OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }}, }) Expect(md).To(ContainSubstring("Disruption Windows")) diff --git a/test/longhaul/workload/writer.go b/test/longhaul/workload/writer.go index c2fdcc3e2..dddfd0e54 100644 --- a/test/longhaul/workload/writer.go +++ b/test/longhaul/workload/writer.go @@ -28,6 +28,18 @@ const ( writeInterval = 100 * time.Millisecond ) +// AggregateWriteRate returns the workload's aggregate write rate in writes per +// second across all writer goroutines, given the configured writer count. It is +// the reciprocal of the per-writer writeInterval scaled by numWriters, and is +// used to convert observed write-failure counts into an estimated outage +// duration (see journal.DisruptionWindow). Returns 0 for a non-positive count. +func AggregateWriteRate(numWriters int) float64 { + if numWriters <= 0 { + return 0 + } + return float64(numWriters) / writeInterval.Seconds() +} + // WriteDocument is the schema for data-plane durability tracking. type WriteDocument struct { WriterID string `bson:"writer_id"`