documentdb · WentingWu666666 · Jul 10, 2026 · Jul 10, 2026 · Jul 10, 2026 · Jul 10, 2026
@@ -42,23 +42,33 @@
 #   longhaul-report ConfigMap's result field.
 #
 # CI budget
-#   Scale/upgrade disruption ops are disabled for the smoke run
-#   (LONGHAUL_MIN_INSTANCES == LONGHAUL_MAX_INSTANCES) so the gate is a fast,
-#   deterministic data-durability check (writers + verifier) that fits a
-#   GitHub-hosted runner and finishes in a few minutes.
+#   Scale ops run for real (MIN=2, MAX=3). The gate exercises every registered
+#   operation once and finishes within a GitHub-hosted runner's budget.
+#
+# Operation coverage (random mode)
+#   The smoke runs the real random scheduler — the exact path the multi-day
+#   long-haul run uses — but in coverage mode (LONGHAUL_OPERATION_COVERAGE) with
+#   a pinned seed (LONGHAUL_OPERATION_SEED). Coverage mode draws each operation
+#   without replacement and completes once every operation has run at least once,
+#   so the gate exercises scheduler.go's weighted selection, cooldown, and
+#   steady-state gates while still guaranteeing per-op coverage and a
+#   deterministic PASS/FAIL verdict. The upgrade uses a second tag for the same
+#   database image payload: this gates the rolling-update mechanics without
+#   turning this smoke test into a cross-version compatibility suite. MAX_DURATION
+#   is only the completion watchdog.
 #
 # Data-protection gate
 #   The backup verifier is exercised for real, not just compiled in. The kind
 #   cluster already has CSI VolumeSnapshot support (setup-test-environment runs
 #   deploy-csi-driver.sh: external-snapshotter + a default csi-hostpath
-#   VolumeSnapshotClass), so a single-instance cluster can complete snapshot
-#   backups — exactly as the e2e scheduled-backup test proves. The smoke run
-#   sets a per-minute backup schedule and a 30s verify interval (vs the 5m
-#   default, via LONGHAUL_BACKUP_VERIFY_INTERVAL) so the verifier's periodic
-#   loop fires many times within the window and reliably observes a
-#   scheduled+completed backup. It then asserts scheduled AND completed >= 1
-#   (with no retention leak or completion stall), so a broken backup path fails
-#   the PR rather than passing silently as a no-op.
+#   VolumeSnapshotClass), so the cluster can complete snapshot backups —
+#   exactly as the e2e scheduled-backup test proves. The smoke run sets a
+#   per-minute backup schedule and a 30s verify interval (vs the 5m default,
+#   via LONGHAUL_BACKUP_VERIFY_INTERVAL) so the verifier's periodic loop fires
+#   many times within the window and reliably observes a scheduled+completed
+#   backup. It then asserts scheduled AND completed >= 1 (with no retention leak
+#   or completion stall), so a broken backup path fails the PR rather than
+#   passing silently as a no-op.
 
 name: Long-Haul Smoke Gate
 
@@ -74,9 +84,9 @@ on:
   workflow_dispatch:
     inputs:
       max_duration:
-        description: "Bounded driver run length (Go duration). Keep >= 6m so at least one per-minute backup is scheduled and completed within the window."
+        description: "Coverage watchdog duration (Go duration, e.g. 20m). Keep >= 6m so at least one per-minute backup is scheduled and completed within the window."
         required: false
-        default: "6m"
+        default: "20m"
 
 permissions:
   contents: read
@@ -119,7 +129,7 @@ jobs:
     needs: build
     if: always() && needs.build.result == 'success'
     runs-on: ubuntu-22.04
-    timeout-minutes: 40
+    timeout-minutes: 50
     env:
       IMAGE_TAG: ${{ needs.build.outputs.image_tag }}
       EXT_IMAGE_TAG: ${{ needs.build.outputs.ext_image_tag }}
@@ -128,8 +138,10 @@ jobs:
       # Must match the cluster name the composite action derives:
       #   documentdb-<test-type>-<architecture>-<test-scenario-name>
       KIND_CLUSTER: documentdb-longhaul-amd64-smoke
-      # The pruner's first tick is at 5m; the default must run beyond it.
-      MAX_DURATION: ${{ github.event.inputs.max_duration || '6m' }}
+      # Coverage watchdog; also kept beyond the 5m pruner tick and long enough
+      # for at least one per-minute backup to schedule and complete.
+      MAX_DURATION: ${{ github.event.inputs.max_duration || '20m' }}
+      UPGRADE_IMAGE_TAG: ${{ needs.build.outputs.ext_image_tag }}-smoke-upgrade
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -167,7 +179,7 @@ jobs:
           runner: "ubuntu-22.04"
           test-scenario-name: "smoke"
           node-count: "1"
-          instances-per-node: "1"
+          instances-per-node: "2"
           cert-manager-namespace: ${{ env.CERT_MANAGER_NS }}
           operator-namespace: ${{ env.OPERATOR_NS }}
           db-namespace: ${{ env.DB_NS }}
@@ -198,6 +210,48 @@ jobs:
             --from-literal=uri="${URI}" \
             --dry-run=client -o yaml | kubectl apply -f -
 
+      - name: Prepare deterministic upgrade target
+        run: |
+          set -euo pipefail
+          owner=$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')
+          source_tag="${EXT_IMAGE_TAG}-amd64"
+
+          # The operator resolves documentDBVersion against the canonical image
+          # repositories. Give the existing payload a second local tag so the
+          # upgrade operation performs a real rolling image-reference update
+          # without depending on a second database release.
+          for component in documentdb gateway; do
+            source="ghcr.io/${owner}/documentdb-kubernetes-operator/${component}:${source_tag}"
+            base_target="ghcr.io/documentdb/documentdb-kubernetes-operator/${component}:${source_tag}"
+            upgrade_target="ghcr.io/documentdb/documentdb-kubernetes-operator/${component}:${UPGRADE_IMAGE_TAG}"
+            docker image inspect "${source}" >/dev/null
+            docker tag "${source}" "${base_target}"
+            docker tag "${source}" "${upgrade_target}"
+            kind load docker-image "${base_target}" --name "${KIND_CLUSTER}"
+            kind load docker-image "${upgrade_target}" --name "${KIND_CLUSTER}"
+          done
+
+          # setup-test-environment pins explicit image fields, which take
+          # precedence over documentDBVersion. Move to the equivalent
+          # version-based reference before starting the driver so its upgrade
+          # operation can change both database component image tags.
+          version_patch=$(BASE_VERSION="${source_tag}" jq -nc '{
+              spec: {
+                documentDBVersion: env.BASE_VERSION,
+                image: {
+                  documentDB: null,
+                  gateway: null
+                }
+              }
+            }')
+          kubectl patch documentdb "${DB_NAME}" -n "${DB_NS}" \
+            --type merge -p "${version_patch}"
+
+          kubectl create configmap longhaul-versions \
+            -n "${DB_NS}" \
+            --from-literal="desired-documentdb-version=${UPGRADE_IMAGE_TAG}" \
+            --dry-run=client -o yaml | kubectl apply -f -
+
       - name: Deploy long-haul driver (real manifests, bounded override)
         run: |
           # RBAC applies unmodified (namespace matches DB_NS).
@@ -215,11 +269,17 @@ jobs:
 
           # Bounded, deterministic smoke override — patch ONLY runtime knobs on
           # the shipped ConfigMap; the manifest structure is unchanged.
-          #  - MAX_DURATION: finite run
+          #  - MAX_DURATION: finite run (coverage-completion watchdog)
           #  - RESET_DATA: fresh collection each CI run
           #  - RETAIN_PER_WRITER: low enough to force a real prune at 5m
-          #  - MIN==MAX instances: disable disruptive scale ops (fast + stable)
+          #  - OPERATION_MODE=random + COVERAGE: run the real scheduler but draw
+          #    each operation without replacement and finish once every operation
+          #    (scale-up/-down, kill-operator-pod, kill-primary-pod,
+          #    upgrade-documentdb) has run once — deterministic coverage of the
+          #    production path.
+          #  - OPERATION_SEED: pin selection so the run is reproducible.
           #  - short cadences so the verifier gets several cycles in the window
+          #  - short steady-state gate with a bounded recovery budget
           #  - BACKUP_*: exercise the data-protection verifier for real — a
           #    per-minute schedule so at least one backup is scheduled and
           #    completed within the bounded window, plus a 30s verify interval
@@ -233,12 +293,15 @@ jobs:
               LONGHAUL_RESET_DATA: "true",
               LONGHAUL_NUM_WRITERS: "2",
               LONGHAUL_RETAIN_PER_WRITER: "100",
+              LONGHAUL_OPERATION_MODE: "random",
+              LONGHAUL_OPERATION_COVERAGE: "true",
+              LONGHAUL_OPERATION_SEED: "1",
               LONGHAUL_OP_COOLDOWN: "30s",
-              LONGHAUL_STEADY_STATE_WAIT: "10s",
-              LONGHAUL_RECOVERY_TIMEOUT: "2m",
+              LONGHAUL_STEADY_STATE_WAIT: "5s",
+              LONGHAUL_RECOVERY_TIMEOUT: "5m",
               LONGHAUL_REPORT_INTERVAL: "30s",
-              LONGHAUL_MIN_INSTANCES: "1",
-              LONGHAUL_MAX_INSTANCES: "1",
+              LONGHAUL_MIN_INSTANCES: "2",
+              LONGHAUL_MAX_INSTANCES: "3",
               LONGHAUL_BACKUP_ENABLED: "true",
               LONGHAUL_BACKUP_SCHEDULE: "*/1 * * * *",
               LONGHAUL_BACKUP_RETENTION_DAYS: "1",
@@ -255,7 +318,7 @@ jobs:
         id: wait
         run: |
           set -euo pipefail
-          deadline=$(( $(date +%s) + 900 ))  # 15 min hard cap
+          deadline=$(( $(date +%s) + 1800 ))  # 30 min hard cap
           exit_code=""
           while [[ $(date +%s) -lt ${deadline} ]]; do
             pod=$(kubectl get pods -n "${DB_NS}" \
@@ -301,8 +364,14 @@ jobs:
             -o jsonpath='{.data.result}' 2>/dev/null || echo "MISSING")
           report=$(kubectl get configmap longhaul-report -n "${DB_NS}" \
             -o jsonpath='{.data.latest-report}' 2>/dev/null || echo "MISSING")
-          echo "Driver exit code : ${exit_code}"
-          echo "Report result    : ${result}"
+          operation_status=$(kubectl get configmap longhaul-report -n "${DB_NS}" \
+            -o json | jq -r '.data["operation-status"] // "MISSING"')
+          operation_aggregates=$(kubectl get configmap longhaul-report -n "${DB_NS}" \
+            -o json | jq -r '.data["operation-aggregates"] // "MISSING"')
+          echo "Driver exit code    : ${exit_code}"
+          echo "Report result       : ${result}"
+          echo "Operation status    : ${operation_status}"
+          echo "Operation aggregates: ${operation_aggregates}"
 
           if [[ "${exit_code}" != "0" ]]; then
             echo "::error::Driver exited non-zero (${exit_code})."
@@ -312,11 +381,25 @@ jobs:
             echo "::error::longhaul-report result is '${result}', expected PASS."
             exit 1
           fi
+          if [[ "${operation_status}" != "COMPLETE" ]]; then
+            echo "::error::operation-status is '${operation_status}', expected COMPLETE."
+            exit 1
+          fi
+          # Coverage mode: assert every registered operation ran at least once
+          # (passed >= 1) and none failed. Order is not asserted — coverage,
+          # not sequence, is the guarantee.
+          if ! jq -e '
+            (map(.name) | sort) == (["scale-up","scale-down","kill-operator-pod","kill-primary-pod","upgrade-documentdb"] | sort) and
+            all(.[]; .passed >= 1 and .failed == 0)
+          ' <<<"${operation_aggregates}" >/dev/null; then
+            echo "::error::operation-aggregates did not show every operation covered (passed>=1, failed==0)."
+            exit 1
+          fi
           if ! grep -Eq 'pruner: pruned [1-9][0-9]* docs' <<<"${report}"; then
             echo "::error::Retention pruner did not report deleting any documents."
             exit 1
           fi
-          echo "✅ Long-haul smoke gate passed (exit 0, report PASS, retention pruned documents)."
+          echo "✅ Long-haul smoke gate passed (random coverage COMPLETE, all operations covered, report PASS, retention pruned documents)."
 
       - name: Assert data-protection verifier ran
         run: |

@@ -44,7 +44,7 @@ flowchart LR
 | Component | Role | Output |
 |---|---|---|
 | **Writer/Verifier** | Data-plane workload. Connects via `mongodb://` only — no k8s imports. Writers insert monotonic sequences with checksums under majority write concern; verifiers scan for gaps and bad checksums. | Counters (acked, failed, verify passes, gaps, checksum errors); errors to journal. |
-| **Operation Scheduler** | Control plane. Applies weighted-random ops (scale, kill, failover, backup, upgrade) with preconditions and cooldowns. | Operation start/end events to journal. |
+| **Operation Runner** | Control plane. Applies weighted-random ops for production long-haul runs, a deterministic named sequence for smoke/reproduction, or no ops when disabled. | Bounded per-operation results/aggregates plus operation events to journal. |
 | **Monitor** | Polls pod RSS/CPU and checks readiness of operator + DB pods. | Periodic samples + readiness events to journal. |
 | **Journal** | In-process append-only event log shared by all components. | Reproducible event stream for the report. |
 | **Report** | Aggregates the journal into a markdown summary at a configurable interval; raises alerts on threshold breaches. | Markdown report; alert lines. |
@@ -77,7 +77,11 @@ The test runs **continuously** — no cycles, no scheduled resets. Workload, met
 
 ## Operations
 
-The scheduler picks operations from these categories with weighted randomization:
+Production runs use weighted randomization. Deterministic smoke and reproduction
+runs can instead request a comma-separated sequence of stable operation names;
+each operation runs exactly once in order and the driver exits as soon as the
+sequence completes or fails. A disabled mode leaves the workload running without
+management operations.
 
 | Category | Examples |
 |---|---|
@@ -87,16 +91,34 @@ The scheduler picks operations from these categories with weighted randomization
 | **Chaos** | kill primary pod, drain node, kill operator pod |
 | **Data protection** | trigger backup, verify backup |
 
-**Sequencing invariants** (enforced by the scheduler — exact values live in code):
+**Operation invariants** (exact values live in code):
 
-- One disruptive op at a time. Overlapping disruptions are non-diagnosable.
-- Per-category cooldown between ops. Lets the cluster stabilize.
-- Steady-state gate — health check must pass before the next op fires.
+- One disruptive op at a time in every mode. Overlapping disruptions are
+  non-diagnosable.
+- Random mode applies the global cooldown between attempts.
+- The steady-state gate must pass before each operation. Sequence mode also
+  requires each named precondition to become true within the recovery timeout.
 
 **Backup is not isolated.** It runs concurrently with topology changes and chaos so that backup-vs-topology serialization bugs surface here rather than in production — that serialization is the backup feature's job, not the harness's.
 
 Each operation declares an **outage policy**: tolerated write failures during its disruption window and a max recovery time. Breaching the policy is recorded as a Tier-1 failure (see Failure Tiers).
 
+Operation state is intentionally bounded for multi-day runs. Random mode keeps
+only passed/failed counters per registered operation type; sequence mode keeps
+one mutable `PENDING`/`RUNNING`/`PASSED`/`FAILED` result per requested item.
+Execution errors, precondition timeouts, outage-policy violations, and an
+incomplete sequence at shutdown all produce a failing final verdict.
+
+**Random coverage mode** (used by the PR smoke gate) is a variant of random
+mode: the scheduler draws each operation *without replacement* and completes
+once every registered operation has run at least once, rather than running for
+the full duration. A fixed seed (`LONGHAUL_OPERATION_SEED`) makes selection
+reproducible. This lets the smoke gate exercise the production scheduler path —
+weighted selection, cooldown, and steady-state gates — while guaranteeing per-op
+coverage and a deterministic PASS/FAIL verdict; `MAX_DURATION` becomes the
+completion watchdog, and a run that stops before covering every operation is a
+failing `INCOMPLETE` verdict.
+
 ---
 
 ## Data Plane Workload