feat(storage): replace the LMDB chunk store with one file per chunk, and migrate onto it

.github/workflows/ci.yml

@@ -50,6 +50,57 @@ jobs:
           version: ${{ env.FOUNDRY_VERSION }}
       - name: Run unit tests
         run: cargo test --lib --features test-utils
+      # Before the e2e suite, deliberately. These are fast and deterministic, and the e2e
+      # suite flakes on hosted runners for transport reasons that have nothing to do with
+      # storage. A failing step aborts the job, so anything sequenced after a flaky one
+      # never reports, which is how these ran on no platform at all for a whole run.
+      - name: Prove the migration returns disk to the filesystem
+        shell: bash
+        run: |
+          set -euo pipefail
+          cargo test --test migration_reclaims_disk --features test-utils -- --nocapture --test-threads=1 2>&1 | tee /tmp/reclaims_disk.log
+          # A target whose required features are not passed is skipped with a
+          # warning and a zero exit, so a harness can stop running without anyone
+          # noticing. This is what makes that loud.
+          grep -qE 'test result: ok\. [1-9]' /tmp/reclaims_disk.log \
+            || { echo 'reclaims_disk ran no tests'; exit 1; }
+      - name: Kill a node mid-migration and check what survived
+        shell: bash
+        run: |
+          set -euo pipefail
+          cargo test --test migration_crash_safety --features test-utils -- --test-threads=1 2>&1 | tee /tmp/crash_safety.log
+          # A target whose required features are not passed is skipped with a
+          # warning and a zero exit, so a harness can stop running without anyone
+          # noticing. This is what makes that loud.
+          grep -qE 'test result: ok\. [1-9]' /tmp/crash_safety.log \
+            || { echo 'crash_safety ran no tests'; exit 1; }
+      - name: Several nodes migrating on one disk
+        shell: bash
+        run: |
+          set -euo pipefail
+          cargo test --test migration_shared_volume --features test-utils 2>&1 | tee /tmp/shared_volume.log
+          # A target whose required features are not passed is skipped with a
+          # warning and a zero exit, so a harness can stop running without anyone
+          # noticing. This is what makes that loud.
+          grep -qE 'test result: ok\. [1-9]' /tmp/shared_volume.log \
+            || { echo 'shared_volume ran no tests'; exit 1; }
+      # Linux only. This one plants a hundred thousand files to measure what a restart
+      # costs, and the answer it is after is a fleet answer, where every node is Linux.
+      # The scan itself reads names and nothing else, which is not a platform-specific
+      # path, and opening a store is covered on all three by the unit tests. Planting that
+      # many files on the Windows runner would cost minutes of every run to re-measure
+      # something no node will ever do there.
+      - name: Startup scan, index memory and inode cost at scale
+        if: runner.os == 'Linux'
+        shell: bash
+        run: |
+          set -euo pipefail
+          cargo test --test storage_scale --features test-utils -- --nocapture --test-threads=1 2>&1 | tee /tmp/scale.log
+          # A target whose required features are not passed is skipped with a
+          # warning and a zero exit, so a harness can stop running without anyone
+          # noticing. This is what makes that loud.
+          grep -qE 'test result: ok\. [1-9]' /tmp/scale.log \
+            || { echo 'scale ran no tests'; exit 1; }
       - name: Run e2e tests
         run: cargo test --test e2e --features test-utils -- --test-threads=1
       - name: Run v12 storage-bound audit attack PoCs
@@ -58,6 +109,82 @@ jobs:
         run: cargo test --test poc_audit_handler_live --features test-utils
       - name: Run bootstrap-stall PoC regression marker
         run: cargo test --test poc_bootstrap_stall --features test-utils
+      - name: Shutdown waits for writes whose caller has gone
+        run: cargo test --test poc_shutdown_lmdb_drain --features test-utils
+
+  # Runs the storage tests against real ext4, XFS and btrfs rather than whatever the
+  # runner provides. Deliberately NOT named durability: killing a process and reopening
+  # the same mounted filesystem keeps the page cache, so this exercises each filesystem's
+  # syscall, locking, rename and delete behaviour, not its behaviour under power loss.
+  # That still needs block-device fault injection or a real machine, and remains a fleet
+  # gate.
+  filesystems:
+    name: Storage on ${{ matrix.fs }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        fs: [ext4, xfs, btrfs]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - name: Install the filesystem tools
+        run: sudo apt-get update && sudo apt-get install -y xfsprogs btrfs-progs
+      - name: Make a ${{ matrix.fs }} volume and mount it
+        shell: bash
+        run: |
+          set -euo pipefail
+          # A loopback image, so these run on a filesystem of the right kind rather than
+          # on whatever the runner happens to give us. ext4 is what most of the fleet is
+          # on; XFS and btrfs are the two the design reasons about separately, btrfs
+          # because it has been observed reordering writes around a rename.
+          # 3 GiB is ample: these tests use tens of MiB. The scale harness, which is
+          # the one that needs room, is not in this job.
+          truncate -s 3G /tmp/${{ matrix.fs }}.img
+          mkfs.${{ matrix.fs }} -q /tmp/${{ matrix.fs }}.img
+          sudo mkdir -p /mnt/antfs
+          sudo mount -o loop /tmp/${{ matrix.fs }}.img /mnt/antfs
+          sudo chown "$USER" /mnt/antfs
+          df -hT /mnt/antfs
+      # TMPDIR is what `TempDir::new` uses, so this is what puts the test data on the
+      # mounted filesystem rather than on the runner's root.
+      - name: The migration returns disk on ${{ matrix.fs }}
+        env:
+          TMPDIR: /mnt/antfs
+        shell: bash
+        run: |
+          set -euo pipefail
+          cargo test --test migration_reclaims_disk --features test-utils -- --nocapture --test-threads=1 2>&1 | tee /tmp/reclaims_disk.log
+          # A target whose required features are not passed is skipped with a
+          # warning and a zero exit, so a harness can stop running without anyone
+          # noticing. This is what makes that loud.
+          grep -qE 'test result: ok\. [1-9]' /tmp/reclaims_disk.log \
+            || { echo 'reclaims_disk ran no tests'; exit 1; }
+      - name: A node killed mid-write on ${{ matrix.fs }} loses nothing
+        env:
+          TMPDIR: /mnt/antfs
+        shell: bash
+        run: |
+          set -euo pipefail
+          cargo test --test migration_crash_safety --features test-utils -- --test-threads=1 2>&1 | tee /tmp/crash_safety.log
+          # A target whose required features are not passed is skipped with a
+          # warning and a zero exit, so a harness can stop running without anyone
+          # noticing. This is what makes that loud.
+          grep -qE 'test result: ok\. [1-9]' /tmp/crash_safety.log \
+            || { echo 'crash_safety ran no tests'; exit 1; }
+      - name: Several nodes on one ${{ matrix.fs }} volume
+        env:
+          TMPDIR: /mnt/antfs
+        shell: bash
+        run: |
+          set -euo pipefail
+          cargo test --test migration_shared_volume --features test-utils 2>&1 | tee /tmp/shared_volume.log
+          # A target whose required features are not passed is skipped with a
+          # warning and a zero exit, so a harness can stop running without anyone
+          # noticing. This is what makes that loud.
+          grep -qE 'test result: ok\. [1-9]' /tmp/shared_volume.log \
+            || { echo 'shared_volume ran no tests'; exit 1; }
 
   doc:
     name: Documentation
@@ -84,6 +211,28 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       - name: Build release (no logging)
         run: cargo build --release --no-default-features
+      # The crash harness drives the store through a failpoint that parks the process
+      # forever on an environment variable. It is compiled only under `test-utils`, which
+      # is not a default feature and is not passed by the release workflow, so a shipped
+      # binary does not contain it. This proves that rather than trusting it: the variable
+      # name is a string literal, so it survives into the binary whenever the code that
+      # reads it is compiled, and its absence is the absence of the failpoint.
+      - name: A shipped binary carries no failpoint
+        if: runner.os == 'Linux'
+        shell: bash
+        run: |
+          set -euo pipefail
+          cargo build --bin ant-node
+          found=$(strings -a target/debug/ant-node | grep -c 'ANT_HALT_' || true)
+          # With --features test-utils this count is not zero, which is what makes a zero
+          # here evidence rather than an accident of how the binary was stripped. The exact
+          # number is one per failpoint and is deliberately not asserted, so that adding a
+          # failpoint does not fail this check.
+          if [ "$found" != "0" ]; then
+            echo "the publish failpoint is compiled into a default-feature build"
+            exit 1
+          fi
+          echo "no failpoint in a default-feature build"
 
   test-no-logging:
     name: Test (no logging)

Cargo.toml

@@ -129,6 +129,34 @@ proptest = "1"
 alloy = { version = "1", features = ["node-bindings"] }
 serial_test = "3"
 
+# Proves the migration returns disk to the filesystem, which is the claim the whole
+# change exists to make good. Needs the test-only migration-state accessor.
+[[test]]
+name = "migration_reclaims_disk"
+path = "tests/migration_reclaims_disk.rs"
+required-features = ["test-utils"]
+
+# Kills a real child process part-way through writing and migrating, then checks what
+# survived. The automatable half of the power-loss gate.
+[[test]]
+name = "migration_crash_safety"
+path = "tests/migration_crash_safety.rs"
+required-features = ["test-utils"]
+
+# Startup scan time, index memory and inode cost at scale. Regression gates, not
+# benchmarks; ANT_SCALE_KEYS raises the count for a deliberate larger run.
+[[test]]
+name = "storage_scale"
+path = "tests/storage_scale.rs"
+required-features = ["test-utils"]
+
+# Several nodes migrating on one disk: the volume lock, and that each finishes with its
+# own chunks and only its own.
+[[test]]
+name = "migration_shared_volume"
+path = "tests/migration_shared_volume.rs"
+required-features = ["test-utils"]
+
 # E2E test infrastructure (run with --features test-utils)
 [[test]]
 name = "e2e"

config/production.toml

@@ -46,9 +46,59 @@ enabled = true
 # Verify content hash on read
 verify_on_read = true
 
-# Maximum LMDB database size in GiB (0 = default 32 GiB)
+# Maximum size in GiB of the legacy LMDB store, while one still exists
+# (0 = derive it from available disk). Retired along with LMDB itself.
 db_size_gb = 0
 
+# --- Moving off the legacy LMDB chunk store ---
+#
+# Chunks are now one file each, under {root_dir}/chunks/. A node that still has a
+# chunks.mdb copies it into files in the background, then deletes it whole, which is the
+# only moment LMDB's disk comes back.
+#
+# The two release-level switches (whether to delete the old store, and whether audits
+# still penalise) belong to the build, not to this file, so they are deliberately absent.
+[storage.migration]
+# Run the copier. Turning this off leaves both stores in place forever and never
+# returns the old store's disk.
+enabled = true
+
+# Also write new chunks to the legacy store while it exists, so a fleet rollback to an
+# older build cannot lose a chunk uploaded during the migration.
+dual_write_legacy = true
+
+# Allow a node that cannot fit its chunks to give up the ones it is furthest from.
+#
+# Whatever this is set to, a chunk is only ever given up when the node is near the back of
+# its group for it, its close group has received the node's reduced commitment, AND all but
+# one of that group has cryptographically proven it holds a copy. A node that cannot show
+# all three keeps both stores and asks for more disk. Turn this off if you would rather add
+# disk than have the node give anything up at all.
+allow_shed = true
+
+# Hours after this build first starts before a node may give anything up, so peers on
+# older builds have upgraded and stopped penalising it for doing so.
+shed_hold_hours = 72
+
+# Hours between one migration wave opening and the next.
+#
+# A close group is split into waves so only two of its members give chunks up at a time.
+# If all seven went together none could prove to the others that a copy survived, and the
+# group would deadlock waiting on each other. A node with room to copy everything does not
+# wait for a wave: it is never unable to serve, so it is not part of that problem.
+wave_hours = 24
+
+# Hours between a node committing to what it will keep and deleting the old store.
+# Never shorter than 4: that is what the answerability window needs.
+retire_delay_hours = 4
+
+# Free space, in MiB, the copier leaves untouched on top of disk_reserve_mb.
+copier_slack_mb = 2048
+
+# Copy rate ceiling, in MiB/s. Keep it modest: an unthrottled copier competing with the
+# audit responder for disk turns a storage migration into an audit incident.
+copier_throttle_mib_per_sec = 32
+
 # --- Upgrade ---
 [upgrade]
 enabled = false

deploy/scripts/spawn-nodes.sh

@@ -68,6 +68,15 @@ fi
 # Create directories
 mkdir -p "$BASE_DIR" "$LOG_DIR"
 
+# The per-volume migration lock. Every node on this host shares it and nothing else, so
+# they can serialise their copies off LMDB without being able to reach each other's data.
+# It needs its own directory because PrivateTmp=true below gives each unit a /tmp of its
+# own, and the node's default lock location is in there: without this every node takes a
+# lock nobody else can see, all of them start copying at once, and the host runs out of
+# space with several half-finished migrations on it.
+LOCK_DIR="${BASE_DIR%/*}/migration"
+mkdir -p "$LOCK_DIR"
+
 # Create ant user if not exists
 if ! id -u ant &>/dev/null; then
     useradd -r -s /bin/false ant || true
@@ -90,6 +99,8 @@ for i in $(seq 0 $((NODE_COUNT - 1))); do
     # Create node directory
     mkdir -p "$NODE_DIR"
     chown ant:ant "$NODE_DIR"
+    chown ant:ant "$LOCK_DIR"
+    chmod 0750 "$LOCK_DIR"
 
     # Create systemd service
     cat > "/etc/systemd/system/$SERVICE_NAME.service" <<EOF
@@ -120,6 +131,10 @@ NoNewPrivileges=true
 ProtectSystem=strict
 ProtectHome=true
 ReadWritePaths=$NODE_DIR
+# Only the lock lives here. Granting write access to the shared node directory instead
+# would let every node write into every other node's data.
+ReadWritePaths=$LOCK_DIR
+Environment=ANT_MIGRATION_LOCK_DIR=$LOCK_DIR
 PrivateTmp=true
 ProtectKernelTunables=true
 ProtectKernelModules=true

deploy/terraform/cloud-init/worker.yml

@@ -85,11 +85,25 @@ write_files:
       MemoryMax=350M
       CPUQuota=15%
 
+      # Every node on this host shares one disk, and the migration off LMDB copies every
+      # chunk before it deletes anything, so two nodes copying at once need two full
+      # copies of the disk at the same time. The node takes a per-volume lock to stop
+      # that, and the lock only works if all the nodes on the host can see the same file.
+      # PrivateTmp=true gives each unit a tmpfs of its own, so the default location in
+      # /tmp is private to each node and every one of them takes its own lock. This names
+      # a directory on the shared volume instead, holding nothing but the lock, and grants
+      # write access to that one directory.
+      Environment=ANT_MIGRATION_LOCK_DIR=/var/lib/ant/migration
+
       # Security hardening
       NoNewPrivileges=true
       ProtectSystem=strict
       ProtectHome=true
       ReadWritePaths=$${NODE_DIR}
+      # Only the lock lives here. Granting write access to the shared nodes/ directory
+      # instead would let every node write into every other node's data directory, which
+      # is what ReadWritePaths=$${NODE_DIR} exists to prevent.
+      ReadWritePaths=/var/lib/ant/migration
       PrivateTmp=true
 
       [Install]
@@ -174,7 +188,11 @@ runcmd:
 
   # Create data directories
   - mkdir -p /var/lib/ant/nodes
+  # The per-volume migration lock. Shared by every node on this host and nothing else, so
+  # they can serialise their copies without being able to reach each other's data.
+  - mkdir -p /var/lib/ant/migration
   - chown -R ant:ant /var/lib/ant
+  - chmod 0750 /var/lib/ant/migration
 
   # Increase file limits
   - echo "* soft nofile 65535" >> /etc/security/limits.conf