diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab7f31f5..323ec55b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,57 @@ jobs: version: ${{ env.FOUNDRY_VERSION }} - name: Run unit tests run: cargo test --lib --features test-utils + # Before the e2e suite, deliberately. These are fast and deterministic, and the e2e + # suite flakes on hosted runners for transport reasons that have nothing to do with + # storage. A failing step aborts the job, so anything sequenced after a flaky one + # never reports, which is how these ran on no platform at all for a whole run. + - name: Prove the migration returns disk to the filesystem + shell: bash + run: | + set -euo pipefail + cargo test --test migration_reclaims_disk --features test-utils -- --nocapture --test-threads=1 2>&1 | tee /tmp/reclaims_disk.log + # A target whose required features are not passed is skipped with a + # warning and a zero exit, so a harness can stop running without anyone + # noticing. This is what makes that loud. + grep -qE 'test result: ok\. [1-9]' /tmp/reclaims_disk.log \ + || { echo 'reclaims_disk ran no tests'; exit 1; } + - name: Kill a node mid-migration and check what survived + shell: bash + run: | + set -euo pipefail + cargo test --test migration_crash_safety --features test-utils -- --test-threads=1 2>&1 | tee /tmp/crash_safety.log + # A target whose required features are not passed is skipped with a + # warning and a zero exit, so a harness can stop running without anyone + # noticing. This is what makes that loud. + grep -qE 'test result: ok\. [1-9]' /tmp/crash_safety.log \ + || { echo 'crash_safety ran no tests'; exit 1; } + - name: Several nodes migrating on one disk + shell: bash + run: | + set -euo pipefail + cargo test --test migration_shared_volume --features test-utils 2>&1 | tee /tmp/shared_volume.log + # A target whose required features are not passed is skipped with a + # warning and a zero exit, so a harness can stop running without anyone + # noticing. This is what makes that loud. + grep -qE 'test result: ok\. [1-9]' /tmp/shared_volume.log \ + || { echo 'shared_volume ran no tests'; exit 1; } + # Linux only. This one plants a hundred thousand files to measure what a restart + # costs, and the answer it is after is a fleet answer, where every node is Linux. + # The scan itself reads names and nothing else, which is not a platform-specific + # path, and opening a store is covered on all three by the unit tests. Planting that + # many files on the Windows runner would cost minutes of every run to re-measure + # something no node will ever do there. + - name: Startup scan, index memory and inode cost at scale + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + cargo test --test storage_scale --features test-utils -- --nocapture --test-threads=1 2>&1 | tee /tmp/scale.log + # A target whose required features are not passed is skipped with a + # warning and a zero exit, so a harness can stop running without anyone + # noticing. This is what makes that loud. + grep -qE 'test result: ok\. [1-9]' /tmp/scale.log \ + || { echo 'scale ran no tests'; exit 1; } - name: Run e2e tests run: cargo test --test e2e --features test-utils -- --test-threads=1 - name: Run v12 storage-bound audit attack PoCs @@ -58,6 +109,82 @@ jobs: run: cargo test --test poc_audit_handler_live --features test-utils - name: Run bootstrap-stall PoC regression marker run: cargo test --test poc_bootstrap_stall --features test-utils + - name: Shutdown waits for writes whose caller has gone + run: cargo test --test poc_shutdown_lmdb_drain --features test-utils + + # Runs the storage tests against real ext4, XFS and btrfs rather than whatever the + # runner provides. Deliberately NOT named durability: killing a process and reopening + # the same mounted filesystem keeps the page cache, so this exercises each filesystem's + # syscall, locking, rename and delete behaviour, not its behaviour under power loss. + # That still needs block-device fault injection or a real machine, and remains a fleet + # gate. + filesystems: + name: Storage on ${{ matrix.fs }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + fs: [ext4, xfs, btrfs] + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Install the filesystem tools + run: sudo apt-get update && sudo apt-get install -y xfsprogs btrfs-progs + - name: Make a ${{ matrix.fs }} volume and mount it + shell: bash + run: | + set -euo pipefail + # A loopback image, so these run on a filesystem of the right kind rather than + # on whatever the runner happens to give us. ext4 is what most of the fleet is + # on; XFS and btrfs are the two the design reasons about separately, btrfs + # because it has been observed reordering writes around a rename. + # 3 GiB is ample: these tests use tens of MiB. The scale harness, which is + # the one that needs room, is not in this job. + truncate -s 3G /tmp/${{ matrix.fs }}.img + mkfs.${{ matrix.fs }} -q /tmp/${{ matrix.fs }}.img + sudo mkdir -p /mnt/antfs + sudo mount -o loop /tmp/${{ matrix.fs }}.img /mnt/antfs + sudo chown "$USER" /mnt/antfs + df -hT /mnt/antfs + # TMPDIR is what `TempDir::new` uses, so this is what puts the test data on the + # mounted filesystem rather than on the runner's root. + - name: The migration returns disk on ${{ matrix.fs }} + env: + TMPDIR: /mnt/antfs + shell: bash + run: | + set -euo pipefail + cargo test --test migration_reclaims_disk --features test-utils -- --nocapture --test-threads=1 2>&1 | tee /tmp/reclaims_disk.log + # A target whose required features are not passed is skipped with a + # warning and a zero exit, so a harness can stop running without anyone + # noticing. This is what makes that loud. + grep -qE 'test result: ok\. [1-9]' /tmp/reclaims_disk.log \ + || { echo 'reclaims_disk ran no tests'; exit 1; } + - name: A node killed mid-write on ${{ matrix.fs }} loses nothing + env: + TMPDIR: /mnt/antfs + shell: bash + run: | + set -euo pipefail + cargo test --test migration_crash_safety --features test-utils -- --test-threads=1 2>&1 | tee /tmp/crash_safety.log + # A target whose required features are not passed is skipped with a + # warning and a zero exit, so a harness can stop running without anyone + # noticing. This is what makes that loud. + grep -qE 'test result: ok\. [1-9]' /tmp/crash_safety.log \ + || { echo 'crash_safety ran no tests'; exit 1; } + - name: Several nodes on one ${{ matrix.fs }} volume + env: + TMPDIR: /mnt/antfs + shell: bash + run: | + set -euo pipefail + cargo test --test migration_shared_volume --features test-utils 2>&1 | tee /tmp/shared_volume.log + # A target whose required features are not passed is skipped with a + # warning and a zero exit, so a harness can stop running without anyone + # noticing. This is what makes that loud. + grep -qE 'test result: ok\. [1-9]' /tmp/shared_volume.log \ + || { echo 'shared_volume ran no tests'; exit 1; } doc: name: Documentation @@ -84,6 +211,28 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Build release (no logging) run: cargo build --release --no-default-features + # The crash harness drives the store through a failpoint that parks the process + # forever on an environment variable. It is compiled only under `test-utils`, which + # is not a default feature and is not passed by the release workflow, so a shipped + # binary does not contain it. This proves that rather than trusting it: the variable + # name is a string literal, so it survives into the binary whenever the code that + # reads it is compiled, and its absence is the absence of the failpoint. + - name: A shipped binary carries no failpoint + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + cargo build --bin ant-node + found=$(strings -a target/debug/ant-node | grep -c 'ANT_HALT_' || true) + # With --features test-utils this count is not zero, which is what makes a zero + # here evidence rather than an accident of how the binary was stripped. The exact + # number is one per failpoint and is deliberately not asserted, so that adding a + # failpoint does not fail this check. + if [ "$found" != "0" ]; then + echo "the publish failpoint is compiled into a default-feature build" + exit 1 + fi + echo "no failpoint in a default-feature build" test-no-logging: name: Test (no logging) diff --git a/Cargo.toml b/Cargo.toml index 2b99f0f1..4c1e7029 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -129,6 +129,34 @@ proptest = "1" alloy = { version = "1", features = ["node-bindings"] } serial_test = "3" +# Proves the migration returns disk to the filesystem, which is the claim the whole +# change exists to make good. Needs the test-only migration-state accessor. +[[test]] +name = "migration_reclaims_disk" +path = "tests/migration_reclaims_disk.rs" +required-features = ["test-utils"] + +# Kills a real child process part-way through writing and migrating, then checks what +# survived. The automatable half of the power-loss gate. +[[test]] +name = "migration_crash_safety" +path = "tests/migration_crash_safety.rs" +required-features = ["test-utils"] + +# Startup scan time, index memory and inode cost at scale. Regression gates, not +# benchmarks; ANT_SCALE_KEYS raises the count for a deliberate larger run. +[[test]] +name = "storage_scale" +path = "tests/storage_scale.rs" +required-features = ["test-utils"] + +# Several nodes migrating on one disk: the volume lock, and that each finishes with its +# own chunks and only its own. +[[test]] +name = "migration_shared_volume" +path = "tests/migration_shared_volume.rs" +required-features = ["test-utils"] + # E2E test infrastructure (run with --features test-utils) [[test]] name = "e2e" diff --git a/config/production.toml b/config/production.toml index ce44e017..72e82112 100644 --- a/config/production.toml +++ b/config/production.toml @@ -46,9 +46,59 @@ enabled = true # Verify content hash on read verify_on_read = true -# Maximum LMDB database size in GiB (0 = default 32 GiB) +# Maximum size in GiB of the legacy LMDB store, while one still exists +# (0 = derive it from available disk). Retired along with LMDB itself. db_size_gb = 0 +# --- Moving off the legacy LMDB chunk store --- +# +# Chunks are now one file each, under {root_dir}/chunks/. A node that still has a +# chunks.mdb copies it into files in the background, then deletes it whole, which is the +# only moment LMDB's disk comes back. +# +# The two release-level switches (whether to delete the old store, and whether audits +# still penalise) belong to the build, not to this file, so they are deliberately absent. +[storage.migration] +# Run the copier. Turning this off leaves both stores in place forever and never +# returns the old store's disk. +enabled = true + +# Also write new chunks to the legacy store while it exists, so a fleet rollback to an +# older build cannot lose a chunk uploaded during the migration. +dual_write_legacy = true + +# Allow a node that cannot fit its chunks to give up the ones it is furthest from. +# +# Whatever this is set to, a chunk is only ever given up when the node is near the back of +# its group for it, its close group has received the node's reduced commitment, AND all but +# one of that group has cryptographically proven it holds a copy. A node that cannot show +# all three keeps both stores and asks for more disk. Turn this off if you would rather add +# disk than have the node give anything up at all. +allow_shed = true + +# Hours after this build first starts before a node may give anything up, so peers on +# older builds have upgraded and stopped penalising it for doing so. +shed_hold_hours = 72 + +# Hours between one migration wave opening and the next. +# +# A close group is split into waves so only two of its members give chunks up at a time. +# If all seven went together none could prove to the others that a copy survived, and the +# group would deadlock waiting on each other. A node with room to copy everything does not +# wait for a wave: it is never unable to serve, so it is not part of that problem. +wave_hours = 24 + +# Hours between a node committing to what it will keep and deleting the old store. +# Never shorter than 4: that is what the answerability window needs. +retire_delay_hours = 4 + +# Free space, in MiB, the copier leaves untouched on top of disk_reserve_mb. +copier_slack_mb = 2048 + +# Copy rate ceiling, in MiB/s. Keep it modest: an unthrottled copier competing with the +# audit responder for disk turns a storage migration into an audit incident. +copier_throttle_mib_per_sec = 32 + # --- Upgrade --- [upgrade] enabled = false diff --git a/deploy/scripts/spawn-nodes.sh b/deploy/scripts/spawn-nodes.sh index 49f741c4..445bca62 100755 --- a/deploy/scripts/spawn-nodes.sh +++ b/deploy/scripts/spawn-nodes.sh @@ -68,6 +68,15 @@ fi # Create directories mkdir -p "$BASE_DIR" "$LOG_DIR" +# The per-volume migration lock. Every node on this host shares it and nothing else, so +# they can serialise their copies off LMDB without being able to reach each other's data. +# It needs its own directory because PrivateTmp=true below gives each unit a /tmp of its +# own, and the node's default lock location is in there: without this every node takes a +# lock nobody else can see, all of them start copying at once, and the host runs out of +# space with several half-finished migrations on it. +LOCK_DIR="${BASE_DIR%/*}/migration" +mkdir -p "$LOCK_DIR" + # Create ant user if not exists if ! id -u ant &>/dev/null; then useradd -r -s /bin/false ant || true @@ -90,6 +99,8 @@ for i in $(seq 0 $((NODE_COUNT - 1))); do # Create node directory mkdir -p "$NODE_DIR" chown ant:ant "$NODE_DIR" + chown ant:ant "$LOCK_DIR" + chmod 0750 "$LOCK_DIR" # Create systemd service cat > "/etc/systemd/system/$SERVICE_NAME.service" <> /etc/security/limits.conf diff --git a/docs/adr/ADR-0014-file-based-chunk-store-and-lmdb-retirement.md b/docs/adr/ADR-0014-file-based-chunk-store-and-lmdb-retirement.md new file mode 100644 index 00000000..c0631a2f --- /dev/null +++ b/docs/adr/ADR-0014-file-based-chunk-store-and-lmdb-retirement.md @@ -0,0 +1,558 @@ +# ADR-0014: One File Per Chunk, and Retiring LMDB Without Losing Data + +- **Status:** Proposed +- **Date:** 2026-08-25 +- **Decision owners:** Anselme Gaeremynck +- **Reviewers:** David Irvine, Chris O'Neil, Mick van der Most van Spijk +- **Supersedes:** none +- **Superseded by:** none +- **Related:** ADR-0002 (gossip-triggered subtree audit), ADR-0003 (possession checks), + ADR-0004 (commitment-bound quote pricing), ADR-0007 (Windows LMDB map headroom cap, + retired by this decision) + +## Context + +The node stores chunks in LMDB. LMDB returns a deleted page to its own free list and never +to the filesystem, so **deleting chunks does not free disk**. In one week the fleet deleted +2.29 million chunks and got back zero bytes. Operators read that as a bug and are tempted +to wipe node directories to reclaim space, which costs the network real replicas. + +There is no partial way out. Compaction needs free space equal to the live data, which is +exactly the condition a full node does not meet, and it does not get us off LMDB anyway. +Punching holes in `data.mdb` is Linux-only, needs LMDB internals to identify free pages, +and reads back as zeros. Disk comes back exactly once: when `chunks.mdb` is removed whole. + + peak disk during migration = allocated chunks.mdb (unchanged) + files written so far + +So a local migration is possible if and only if `free >= live payload`. On production +volumes today (55 volumes at 492 GiB, free median 25.8 GiB, p10 9.8 GiB, about 12 nodes +per volume, about 38 GiB of LMDB per node of which about 24 GiB is live) migrating one +node costs 24 GiB and returns 38 GiB. One at a time the host gains about 14 GiB per node +and the queue accelerates. All twelve at once need 288 GiB and all twelve stall. + +The chunk workload is the easiest possible case for a filesystem: content-addressed, +immutable, write once, read many, delete whole, and **4 MiB**, confirmed by the team +rather than assumed. That size is what makes one file per chunk the right shape; see the +Storj and borgbackup note under Validation for what would change the answer. + +## Decision Drivers + +- Deleting a chunk must return its blocks to the filesystem, on a full disk, with no free + space required and no compaction to schedule. +- No chunk may lose its last replica, including during a fleet rollback, a skipped + upgrade, or a crash halfway through the migration. +- Mass audit failures are as damaging as data loss. Nothing here may cause them. +- It has to work for every operator, not for our fleet. Most node operators are not us and + cannot be told to attach a second volume. +- No opt-in. Whatever we ship is what every node does by default. + +## Considered Options + +1. **Stay on LMDB and compact.** Needs free space equal to the live data, which is the + condition we are trying to escape, and leaves us on LMDB. +2. **Append-only packs** (borg segments, Storj hashstore). Reintroduces compaction, a free + list, and a cross-file index. That is LMDB's disease with a different allocator. +3. **Fixed-size slots** (Sia `hostd`, Swarm sharky). Cheaper than log packing, and Sia's + sector size is exactly our 4 MiB. But a freed slot returns space to the *store*, never + to the *filesystem*: the volume file never shrinks. It is the right design once a node + has a declared capacity, and the wrong one while our whole complaint is that disks stay + full as chunk counts drop. +4. **One file per chunk, sharded on the address prefix.** Broken for us, see below. +5. **One file per chunk, sharded on the address suffix.** Chosen. + +## Decision + +### The store + +One immutable file per chunk: + +```text +{root}/chunks/layout.json versioned layout marker +{root}/chunks//<64-hex> xy = the LAST two hex characters of the address +``` + +**Suffix, never prefix.** A node holds keys it is among the `CLOSE_GROUP_SIZE` closest to, +so its holdings share roughly `log2(N / 7)` leading bits with its own node ID, and that +shared prefix grows as the network grows. Distinct directories a single node would actually +use, sharding on the first hex characters: + +| nodes | shared bits | 2 hex | 3 hex | 4 hex | +|---:|---:|---:|---:|---:| +| 1,000 | 7.2 | 1.8 | 29 | 459 | +| 10,000 | 10.5 | 1 | 2.9 | 46 | +| 100,000 | 13.8 | 1 | 1 | 4.6 | +| 1,000,000 | 17.1 | 1 | 1 | 1 | + +At today's ~800 nodes a two-hex prefix is already down to about two directories. Prefix +sharding does not degrade, it fails, and it fails later for the nodes that grow into it. +Close-group membership constrains the leading bits and places no constraint at all on the +trailing ones, and the address is a BLAKE3 output, so the last byte is uniform by +construction at every network size. IPFS shipped the same fix for a different reason: its +prefixes were constant because of the CID encoding, not because of clustering, and the +flatfs `_README` still says *"Previously, we used prefixes, we now use the next-to-last two +characters."* The generalisation is the part worth keeping: **shard on bits you can prove +are uniform, not on bits that happen to be uniform today.** + +**256 shards, one level.** 23 files per directory at today's ~6,000 chunks per node, 977 at +a 1 TiB node, 39,000 at 10 TiB, for 1 MiB of directory inodes. 4,096 shards only starts to +pay past several million chunks and costs sixteen times the directory overhead for every +node that is not that large. + +**Lowercase hex filenames, full 64 characters.** NTFS and default APFS fold case, so under +base64url or base58 two distinct keys can share one case-folded filename, which is a silent +overwrite. No hex string can spell `CON`, `NUL`, `AUX`, `COM1` or `LPT1`, because none of +those letters is in `0-9a-f`. Keeping the whole key in the name means a `find` over the tree +recovers the store even if the directory layer is lost. + +**The scheme is recorded in `layout.json` at creation.** Nobody in this survey shipped an +in-place re-sharder and all of them paid for it: IPFS says export and re-import, Storj ran a +multi-year satellite-controlled backend migration, borg rewrites only on the next +compaction. One small file is the difference between changing the default later and never +being able to. + +### The index + +**The filesystem is the sole authority.** The key set is a `BTreeSet` rebuilt at +every open by reading directory entries, names only: no `stat`, no content read. A `stat` +per entry costs about ten times the enumeration on Linux and macOS and fifty to sixty times +on Windows, and buys nothing, because the filename is the key. + +No sidecar database, because a persistent index **cannot remove reconciliation**. Commit +the index first and a crash leaves a phantom key; rename the file first and a crash leaves +an unindexed file. Repairing either means looking at the filesystem anyway, so the +filesystem may as well be the authority, and then nothing can drift. Ceph FileStore's +tracker #17177 is the cautionary tale: a crash between `unlink` and the LevelDB flush +orphaned omap keys that were silently reattached to a different object later. + +`BTreeSet` rather than a hash set for three reasons: `all_keys()` must be sorted (the +commitment builder truncates the responsible subset with `take(cap)` *before* the Merkle +tree sorts it, so an unstable order would make the published commitment depend on iteration +luck), it never spikes memory while growing, and bulk-building it from a sorted vector packs +every node to capacity where repeated insertion converges on 68% fill for the same keys. + +**One process per data directory, enforced.** LMDB was genuinely multi-process safe. This +store is not: two of them keep independent in-memory indices, so both would report the same +write as newly stored and each would keep serving keys the other had deleted. A node whose +store is already held by another process refuses to start and says so. + +**Every in-memory mutation mirrors a filesystem operation that has already completed**, and +never anticipates one. Bitcask's issue #114 is what the opposite order looks like: an index +rebuilt at startup and then mutated in place drifted to 2,400 keys pointing at fewer than +100 files. + +### Durability + +Write, on Unix: reserve capacity, create a temp in the **destination** directory, write, +flush the file, rename, flush the shard directory, then admit the key. The publish is an +intra-directory rename, so it is atomic on every Unix filesystem we support and only that +one directory needs flushing. Off Unix there is no rename at all, for the reason the table +below gives; the file is created under its final name and flushed. Either way the final +name can never appear on partial content, because the name is the hash and a name that does +appear over the wrong bytes is caught on read. Delete: unlink, flush the shard directory, +then drop the key. + +Per platform, honestly: + +| | rename atomic | fsync(temp) + rename durable | directory fsync | +|---|---|---|---| +| ext4 | yes | **no**, `auto_da_alloc` only orders data before the rename's commit | yes, required | +| XFS | yes | not by that sequence | yes | +| btrfs | yes | **uncertain**, ALICE found reordering | yes | +| APFS | yes | `sync_all` already uses `F_FULLFSYNC` on Apple targets | returns 0, effect undocumented | +| NTFS | **not documented as atomic** | see below | **no documented way** | + +On Windows a node cannot make the rename durable through the standard library at all. Two +places in this design leaned on one, and neither leans on it now. + +Publishing a chunk off Unix does not rename: it creates the file under its final name and +flushes it, which Microsoft documents as flushing the creation metadata with it. The +content is content-addressed and re-replicable either way, so a file that does not survive +is detected on read and repaired from the network. + +Retirement still renames the environment aside before deleting it, and that rename is not +durable off Unix. What makes it safe is that **the mark goes inside the directory, not +beside it**. A power loss that reverts the rename brings the directory back under its live +name still carrying its mark, and a marked directory under the live name is never opened or +served from: its chunks are in the file store, which is what the mark records. A loss +before the mark leaves the directory unmarked under either name, and an unmarked directory +is always restored and reopened. Every one of those four states has a test. + +So the earlier position, that Windows should refuse to delete the legacy environment until +an operator overrode it, is not what ships. It has been replaced by a mechanism rather than +by a policy, which is the better answer: a switch nobody turns on is a migration that never +finishes. `ANT_MIGRATION_RETIRE_LEGACY=0` remains, per node, for an operator who wants to +hold retirement off one machine, and the forced power-loss run below is still an open fleet +gate on every platform including this one. What that run is now checking is directory +creation, which has no portable flush. + +### Retiring LMDB + +Three releases, because slashing is the *auditor's* decision. A node that has to give up +chunks cannot stop its auditors from penalising it, so the auditors have to stop first. + +| Release | Penalise a peer for not holding a close-group chunk? | Delete `chunks.mdb`? | +|---|---|---| +| **First**: stop one penalty | no | no | +| **Second**: migrate | no | yes | +| **Third**: restore it | yes | yes | + +What the first release withholds is deliberately narrow: only the penalty for **not holding a close-group +chunk you were supposed to be holding**. The commitment-bound subtree audit still +penalises, in every release. So does a responder whose own storage fails: a fetch answered +with an error means the read faulted or the bytes no longer hash to their address, which is +never what a node giving chunks up looks like, and a node that does not hold the chunk says +so with `NotFound` instead. That is not a compromise, it is what makes the rest work: a +node reduces its commitment precisely so its peers hold it to the smaller claim, and +suspending that enforcement would make the reduction meaningless. Audits of both kinds run +and record throughout. + +Both are **build constants with environment overrides, never serialised config**. A node +writes its effective configuration back to disk, so shipping them as ordinary fields would +bake the first release's values into every operator's file and the next would change nothing. + +Per node, in order: + +1. **Open both stores.** Reads are the union, writes go to files. New chunks are also + written to LMDB **first** while it exists: a chunk uploaded during the bridge to holders + that all revert to a pre-migration build would otherwise be gone from every one of them, + and that is client data, not a replica. +2. **Copy closest first**, throttled, stopping at a slack floor above the disk reserve. +3. **Settle.** The node commits only to its file-backed keys from here, while still serving + everything it ever committed to. Serving reads the union; the commitment reads the + file-backed set. A node is at worst over-honest. Nothing is deleted at this step: it + only narrows the claim, so the close group can learn the new one before anything goes. +4. **Verify.** Every chunk both stores hold is re-hashed and recopied from LMDB on + mismatch. A filename is not proof the bytes behind it are good, and the startup scan + reads names only. +5. **Retire.** Once the retirement delay has elapsed, at least two commitment rebuilds have + been published, and no key the node is giving up is still answerable under a retained + commitment slot: rename `chunks.mdb` aside, flush the parent, record the node as + file-only, and only then delete it. The rename is what makes the state change atomic, + because `remove_dir_all` is not: a failure partway through leaves a directory that can + no longer be opened as an environment, and recording completion on top of that would + have the node claim it had finished over a half-deleted store. **This is where the disk + comes back.** The gates that can change while nobody is looking are rechecked inside + the destructive step itself, in the same critical section that proves no other task + holds the store: the proof's health generation, the answerability veto, the announced + writes, and that every legacy-only key is in the approved set. The network gates, rank + and commitment delivery and possession, are rechecked immediately before that call and + outside the guard, so the window on those is the seconds it takes to take the guard + rather than the hours the verification pass can run for. Both matter, and they are not + the same claim. +6. **Refetch** the shortfall through ordinary replication, with the freed space to do it in. + +The delete gate is the pruner's existing retention contract +(`ResponderCommitmentState::is_held`, `GOSSIP_ANSWERABILITY_TTL` three hours). No new +protocol. + +**Nothing is given up without proof it exists elsewhere.** Only nodes that cannot fit +their payload give up anything at all, and such a node must clear three gates, in this +order, before a byte is deleted: + +1. **It is not near the front of the group for the chunk.** Only the last two positions of + the *admission group* (`storage_admission_width`, the close group plus its margin) are + eligible, which is the width the pruner treats as strictly in-range and refuses to + delete inside. A one-off migration must not be more willing to drop a chunk than the + thing that runs every day. +2. **Its close group has received the reduced commitment.** The node narrows what it claims + first, and only once peers have demonstrably received that narrower claim, proven by + them answering a neighbour sync that carried it, may anything be deleted. Until then + they audit it against the set it used to hold, and a wave of audit failures is as + damaging as losing the chunks. +3. **Other nodes have proven they hold the chunk, and are currently publishing a claim.** + All but one of its current close group must answer a cryptographic possession challenge + over a nonce they have never seen. This is the pruner's own evidence, reused + deliberately, and it is deliberately not the cheap `VerificationRequest`: that carries a + self-reported `present: bool`, and a node that has silently lost a chunk still answers + yes. A peer only counts if this node has also heard a commitment from it recently, which + excludes a peer sitting between a retired commitment and its next rotation. That gap is + exactly what a node in the middle of its own migration looks like, and counting it would + let two migrating nodes each conclude the other was covering the chunk. + +Rank alone would not do. Being far from a chunk says something about who *should* hold it, +not about who *does*, and in a fleet-wide migration the nodes that should hold it are +exactly the ones that may also be short of space. Without gate 3 the safety property is +merely statistical: every holder could be short at once and each drop the same chunk, and a +per-volume lock cannot see that, because it serialises one volume and this is a +network-wide question. + +A node that cannot clear these gates keeps both stores, does not free its disk, and tells +the operator to add storage. That is the correct answer, not a smaller replica count. + +Gates 2 and 3 are re-checked immediately before the environment is removed, not once when +the node settled hours earlier. The group moves, and two paths can put a key back into the +legacy-only set in between: a file that failed verification and is now being served from +the legacy copy, and a write whose file half failed. + +**Two of a close group at a time, not seven.** The gates above are per chunk, and they are +safe, but on their own they deadlock: if every holder migrates at once, none can prove to +the others that a copy survives and the whole group sits waiting. So each node derives a +migration wave from a hash of its own ID, and a group of seven is split into four waves. +Wave `w` opens `w * wave_hours` after the build first starts. It needs no coordination and +no protocol change, which matters because a node cannot usefully ask its close group "are +you migrating?" and would not trust the answer by the time it arrived. + +It is a stagger, not a guarantee: seven IDs hashed into four waves will not always land two, +two, two, one. What makes it safe rather than merely tidy is that it composes with the +possession gate. A node whose turn has come still cannot give a chunk up until its +neighbours prove they hold it, so an unlucky wave waits instead of over-shedding. Only nodes +that have to give something up wait for a wave; a node with room copies and retires +immediately, because it is never unable to serve. + +Separately, a host-wide advisory lock serialises migrations sharing a volume, held from the +first copy through retirement, so a node cannot release it and let eleven others start +before it has finished copying. It is released when the environment is unlinked and its +directory renamed aside, not when the last byte comes back: the deletion itself runs +detached so the node can serve while it happens, and it can take minutes on a large store. +So the next node in the queue can begin its copy while the previous one's tombstone is +still on the disk. That is deliberate, and it is worth stating rather than claiming a +tighter guarantee than there is. The two limits answer different questions: the lock is +about one machine's disk, the wave is about one chunk's replicas. + +Where the lock file lives is a deployment fact, and the wrong answer is silent: nodes that +cannot see each other's lock each take one and report success. A host whose nodes do not +share a `/tmp`, which is any host using `PrivateTmp=true`, has to be told where the lock +lives through `ANT_MIGRATION_LOCK_DIR`. The node logs the path it locked at so this can be +answered from a log rather than inferred from a unit file. + +## What the review added + +Five mechanisms are in the implementation that are not in the design above. Each exists +because adversarial review found a way for the destructive step to run on a belief that +was no longer true. They are recorded here because they are load-bearing, not incidental. + +**A directory that has been retired says so from the inside.** The rename that moves the +environment aside cannot be shown to be durable off Unix, so a power loss can bring it back +under its old name with its contents already deleted, and a node that opened that would +fail to start. A file written inside it after the rename and before any deletion travels +with the directory, so what it is never has to be inferred. A mark beside the environment +was tried first and was wrong: it would have to be cancelled when a retirement is abandoned, +cancellation can fail or be lost, and a stale one authorises deleting an environment that +has since taken a chunk. Deletion removes the mark last, so a failed deletion never leaves a +retired directory looking intact. + +**A chunk the node cannot serve is kept but not claimed.** Deleting it, or dropping it from +the index, puts the key in neither the file store's view nor the legacy one, and what +neither view protects is what retirement destroys. Claiming it puts the key in signed +commitments and answers presence probes with a yes for a chunk that cannot be served, which +the commitment-bound audit penalises. So the file stays and the answers stop. Two states, +not one: a chunk that could not be *read* is settled by a later read, and one whose bytes +were *proven wrong* is not, because reading them again says the same thing. + +**A verification proof expires.** The pre-retirement pass reads every chunk, and its result +is reused rather than re-read on every tick, because retirement is usually deferred by a +gate that has nothing to do with the files. A kept chunk that stops being servable in that +window is invisible: ordinary requests are still served from the legacy copy. The store +counts the times a chunk stops being servable, a proof records that count, and retirement +refuses a proof the store has outrun. + +**A write announces itself before it starts.** The work runs on a blocking thread that +outlives the future waiting for it, so a cancelled write can leave the environment holding a +chunk that nothing recorded. The announcement is deliberately not part of what the node +claims to hold: it vetoes retirement and is reconciled against the disk, but no commitment, +quote or presence answer sees it. A delete drains both halves of any announced write for its +key, so a publish cannot land afterwards and undo it. + +**The legacy environment never grows again.** Both stores sit on one disk, each measures +the same free space, and neither knows what the other is about to spend, so a chunk written +to both can be admitted twice against one lot of headroom and enough of them can cross the +reserve together and fill the volume this whole exercise exists to free. From the moment it +is adopted the environment is pinned to what it already occupies: it writes only from pages +it already has, and the file store's accounting becomes the only claim on free disk. The +cost is that the rollback copy is made only when the environment has room of its own, which +on a real node it usually does, because this migration exists precisely because deleting +millions of chunks filled the free list and returned nothing to the filesystem. + +The category underneath all five is the same: **a fact established at one moment being acted +on at another.** Copying, verifying and retiring are separated by hours by design, and every +gap between them is somewhere the store can move. The pattern that works is to make the +belief carry its own expiry — the directory carries its mark, the proof carries the count it +saw, the write carries its note — rather than to check again and hope the check is close +enough to the act. + +## Consequences + +### Positive + +- `unlink` returns blocks immediately. No free list, no compaction, no free space required + to reclaim space. This is the entire point. +- `exists()` and `current_chunks()` become in-memory lookups with no syscall, cheaper than + the LMDB reads they replace. +- `all_keys()` gains a stable ascending order, which the commitment builder needs and the + pruning cursor wants. +- A fresh node never opens a memory map at all. `storage.db_size_gb` and ADR-0007's Windows + map headroom cap die with LMDB. +- The store is self-describing: the filename is the hash, so an operator can verify a chunk + with `b3sum`, and a scrambled directory layer is recoverable with `find`. + +### Negative / Trade-offs + +- **There is no rollback once a node has deleted its LMDB.** The staged rollout is the only + control: a small leading batch, ours, and a wide window. +- **The window between the first and third releases is publicly known, and in it nobody is + penalised for failing to hold a close-group chunk.** The cheapest way to exploit it is + precise and worth writing down: a modified peer that never gossips a commitment at all is + credited as a legacy node, can answer `Present`, and can then return `NotFound` or fail a + possession check with no trust cost. It pays only for an identity and the traffic. One + such identity removes one of seven replicas; control of all seven positions removes the + chunk's availability. The commitment-bound audit is untouched, so this only works for a + peer that publishes no commitment at all, which is itself visible. The mitigation is not + a code change, it is not letting the third release slip. + It is bounded, because the third release evicts afterwards, and audits keep recording so we + can see it happening, but it is a real invitation for the duration. +- `exists()` is now an index lookup rather than a read of the backing store, so something + outside the node deleting files is not noticed until the next read of that key. The read + path self-heals, and a `stat` per call on the node's hottest path is not worth it. +- One inode and one directory entry per chunk. At 4 MiB per object that is 0.05% overhead + and block rounding for a full chunk is exactly zero, but it is real. +- Windows publishes chunks under their final name rather than by rename, so a crash + mid-write leaves a partial file that the write, read and pre-retirement paths each have + to detect rather than trust. +- The paid list is still LMDB. It is a fixed 256 MiB map that contributes nothing to the + disk problem, but it is why `heed` cannot be dropped yet. +- **Narrowing the commitment cuts the quoted price.** Price is quadratic in the committed + key count, so a node that has just proved it is short of disk advertises a cheaper quote + than its close-group peers and then refuses the store on capacity. A wasted round trip + rather than a mispayment. The fix belongs to the quote path and is a separate decision. +- **A cancelled awaiter drops the per-key lock while its blocking write runs on.** This was + accepted as bounded and is no longer accepted: review showed both consequences were worse + than they look. The file store now records what it is writing, per key, cleared by the + worker rather than the caller, and a delete waits out whatever is already writing its + key, so a publish cannot land afterwards and undo a prune. A write into the legacy + environment announces itself before it starts and is reconciled against the disk, so a + cancelled one cannot leave a chunk that neither view protects. + +### Neutral / Operational + +- Startup cost is the directory scan: 122 ms warm and 1.55 s cold at 250,000 files across + 256 shards on APFS, of which the index build is 2 to 11 ms. No fast-start snapshot in v1. + If one is ever added, validate it with the Merkle root of the sorted key set (which + ADR-0004 already computes) rather than a checksum, because a checksum passes for an + operator who restores yesterday's data directory and leaves yesterday's snapshot. +- APFS enumeration degrades with churn, not just size: a million files went from about 72 + to about 306 microseconds per entry over twenty cycles of 5% replacement. A long-lived + macOS node will get slower to start in a way a fresh benchmark never shows. +- NTFS 8.3 short-name generation is worse for us than for most, because a node's filenames + genuinely share a long prefix. Microsoft advises disabling it above 300,000 files per + directory. + +## Validation + +**Already proved, locally:** publish is exactly-once under sixteen concurrent writers of one +address; the index rebuilds from the filesystem across restarts with a stable order; a file +in the wrong shard, an uppercase name, and a non-hex name are all refused; an interrupted +write is swept; a corrupt file is removed and repaired from the legacy copy; a missing file +drops out of the index so replication repairs it; the copier is resumable and cannot +resurrect a pruned chunk; retirement is refused while any gate is unmet and removes the +environment when they are all met; the release switches never round-trip through a config +file. + +For the four mechanisms above: an environment carrying no mark is kept however badly it +reads, one carrying its own mark is removed whatever it is named, and the mark survives the +rename it exists to outlive; a chunk that cannot be read is kept on disk, not acknowledged, +not advertised, and answered for again once it can be read; a verification overtaken by a +file that stopped being readable does not authorise a deletion; a write in flight is not +claimed but does stop retirement; a delete outlasts a write nobody waited for; and a key the +environment holds that is in neither view refuses the proof and is put back where the gates +can see it. Each was verified by removing the fix and confirming the test fails. + +**Proved in CI, on every commit.** The three harnesses that touch durability run on Linux, +macOS and Windows, and again on ext4, XFS and btrfs loopback volumes. The fourth measures +what one file per chunk costs at scale, which is a fleet question on a fleet that is Linux, +so it runs there: + +- *The disk comes back.* Free space is sampled from the filesystem three times: before + anything is written, at the peak where both stores hold everything, and after the + environment is gone. Unlinking the environment while holding it open, which makes the + paths disappear and keeps every block, fails it. This is the claim the whole decision + rests on and the one the old store could not meet. +- *A crash loses nothing.* A child process is killed at a failpoint inside a publish, not + after a sleep, so the kill lands where a half-finished chunk exists. What the parent then + checks is that nothing is claimed that cannot be served, that a leftover is swept, and + that a chunk caught between the environment write and the file write is named on the + copier's list rather than lost between them. The same is done to a retirement: a child is + killed with the environment renamed aside and marked, nothing yet deleted, which is the + most destructive moment in the migration. The next start must finish that deletion and + never reopen the directory, because the node has already told the network it serves those + chunks from the file store. Refusing to believe the mark fails it. +- *Nodes sharing a disk take turns.* Two drivers on one volume, driving `migration::run` + rather than the copier, with the lock held first by an outsider so neither can be observed + making progress. Held through retirement as well as through copying, which is the heavier + half. Removing the lock from either branch of the driver fails these. +- *One file per chunk costs what was claimed.* 100,000 chunks, measured rather than + asserted: the startup scan takes about 100 ms, the index costs 52 bytes per chunk, opening + the store reads 125 bytes whatever the chunks contain, and `put` writes exactly one + directory entry per chunk. The claim underneath the scan's cost, that it reads names and + does not `stat` behind each one, is checked against the machine rather than against a + number: the same directory is walked twice in the same process, once reading names and + once calling `metadata` on every entry, and the scan has to land on the names-only side of + the two. A flat ceiling cannot settle that, because one `stat` per entry costs about three + times a bare walk and stays well inside any ceiling loose enough not to flake. Adding that + `stat` to the scan fails it. + +Each of these was checked by mutation: the fix removed, the test confirmed red, the fix +restored. + +**Fleet gates, which cannot be closed from a workstation:** + +- Forced power loss on ext4, XFS, btrfs, APFS and NTFS showing old-or-new, with antivirus + and 8.3 generation enabled on the NTFS run. The publish path off Unix does not rename at + all, precisely because a rename cannot be shown to be durable there: it creates the chunk + under its final name and flushes the file, which is documented to flush the creation + metadata with it. What that leaves unproven is directory creation, which has no portable + flush, so this run is what closes it. `ANT_MIGRATION_RETIRE_LEGACY=0` holds retirement off + a node until then, per node, without a separate build. + + The loopback jobs above do **not** close this and are not offered as doing so. Killing a + process and reopening the same mounted filesystem keeps the kernel page cache, so the + bytes written before the kill are still there to be read; removing every flush from the + publish path would leave those jobs green. What they do cover is the rest of what a + filesystem decides: rename behaviour, locking, deletion, and whether the space is actually + returned, which btrfs in particular accounts for differently from ext4. +- Startup scan, RSS and inode use at 1M and 10M keys, and on each filesystem. CI answers + 100,000 keys on ext4 and prints every number it measures, so drift is visible in the log + before it trips a gate; `ANT_SCALE_KEYS` raises the count for a deliberate larger run on a + machine with the disk for it. What CI cannot answer is where the curve stops being linear, + which is a question about a machine holding ten million files, not about the code. +- The first release gates on no audit-timeout regression on the quiet responsible lane and on + disk growth + matching prediction. +- The second gates on a soak of the first, plus a verified retirement returning the + predicted space. +- The third gates on migration-complete lines across the fleet, refetch backlogs drained, and the + recorded audit failure rate back to its pre-migration baseline. The first release's + observability is + what makes that decidable. +- **How often a short-of-disk node can actually clear the possession gate.** A node whose + close group is also short of space will not clear it, will not free its disk, and will + tell its operator to add storage. That is the intended answer, but the fleet needs to + show how large that population is before the second release, because it decides whether the + migration + completes on its own or needs operator action at scale. +- **Chunk size is 4 MiB, confirmed.** This was the open question that gated the whole + design and it is now answered. Storj and borgbackup both ran one file per object at scale + and reversed to packing, and both did so for *small* objects: Storj's pieces are *"often + smaller than a hard drive sector"* and over 60% of borg's chunks are under 8 KiB. Nobody + has reversed this decision for large objects. The tripwire remains: if the network ever + starts storing a large share of small records, this ADR should be revisited, and the + inode exposure below comes with it. + +**Review trigger:** if the network ever adopts a declared node capacity, fixed-slot packing +becomes the better store design and this decision should be reopened. + +## Implementation slices + +This ADR is landed by two pull requests, in this order: + +1. **Stop penalising a node for not holding a close-group chunk.** One switch, one helper, + six call sites. It must ship a release ahead of the migration, because the penalty is + the auditor's decision and a node cannot stop its peers applying it. The commitment-bound + subtree audit keeps penalising throughout. +2. **The file store and the migration.** Everything else in this document. + +A third release flips the switch from (1) back, gated on fleet evidence rather than a date, +which is why it is a release and not an expiry constant compiled into the first one. + +## Notes for AI-assisted work + +Drafted with AI assistance. Not to be marked Accepted without human review. diff --git a/scripts/adr-governance.py b/scripts/adr-governance.py index 7f56bde9..c8e6da44 100755 --- a/scripts/adr-governance.py +++ b/scripts/adr-governance.py @@ -53,6 +53,12 @@ def changed_files_against_base(base: str) -> list[str]: return [] +def base_adr_names(ref: str) -> list[str]: + """ADR filenames present on `ref`.""" + listing = run(["git", "ls-tree", "--name-only", ref, "docs/adr/"]) + return [Path(line).name for line in listing.splitlines() if line.startswith("docs/adr/ADR-")] + + def file_at(ref: str, path: str) -> str | None: try: return run(["git", "show", f"{ref}:{path}"]) @@ -83,6 +89,29 @@ def main() -> int: errors.append(f"{path}: duplicate ADR number also used by {seen_numbers[number]}") seen_numbers[number] = path + # And against the base branch, which is the check that actually catches this. A branch + # cut before another ADR merged does not contain it, so the loop above sees one file + # per number and passes, and the duplicate only exists once the two are merged + # together. That has happened here: a branch claimed a number main had already used and + # its governance run was green the whole time. + if base: + for path in sorted(changed_adr_paths): + if not path.exists() or file_at(base, str(path)) is not None: + # Not added by this PR: either gone, or already on the base under this + # exact name, in which case it is the same ADR rather than a clash. + continue + number = path.name.split("-", 2)[1] if "-" in path.name else path.name + for taken in base_adr_names(base): + if taken == path.name: + continue + taken_number = taken.split("-", 2)[1] if "-" in taken else taken + if taken_number == number: + errors.append( + f"{path}: ADR number {number} is already used on {base} by " + f"docs/adr/{taken}. Pick the next free number; merging both would " + f"leave two different ADRs wearing one number." + ) + for path in files_to_validate: if not FILENAME_RE.match(path.name): errors.append(f"{path}: filename must match ADR-NNNN-short-title.md") diff --git a/src/config.rs b/src/config.rs index 2319f96b..5f11a49d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,6 @@ //! Configuration for ant-node. +use crate::storage::MigrationConfig; use evmlib::Network as EvmNetwork; use serde::{Deserialize, Serialize}; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; @@ -436,6 +437,13 @@ pub struct StorageConfig { /// preventing the node from filling the disk completely. Default: 500 MiB. #[serde(default = "default_disk_reserve_mb")] pub disk_reserve_mb: u64, + + /// Controls for moving this node off the legacy LMDB chunk store. + /// + /// The two release switches inside it are deliberately not serialised: see + /// [`MigrationConfig`]. + #[serde(default)] + pub migration: MigrationConfig, } impl Default for StorageConfig { @@ -445,6 +453,7 @@ impl Default for StorageConfig { verify_on_read: default_storage_verify_on_read(), db_size_gb: 0, disk_reserve_mb: default_disk_reserve_mb(), + migration: MigrationConfig::default(), } } } @@ -598,9 +607,33 @@ fn default_testnet_bootstrap() -> Vec { #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used)] mod tests { + use super::*; use serial_test::serial; + #[test] + fn the_shipped_storage_config_parses() { + // The migration section is operator-facing, so a typo in it would only surface on + // a node that had already shipped. Only `[storage]` is checked: the rest of + // `production.toml` does not currently deserialize as a `NodeConfig` (its + // `evm_network` is a bare string where an internally tagged enum is expected), + // which is a separate, pre-existing problem. + let raw = include_str!("../config/production.toml"); + let doc: toml::Value = toml::from_str(raw).expect("production.toml must be valid TOML"); + let storage = doc.get("storage").expect("a [storage] section").clone(); + let config: StorageConfig = storage.try_into().expect("[storage] must deserialize"); + + assert!(config.migration.enabled); + assert!(config.migration.dual_write_legacy); + assert_eq!(config.migration.shed_hold_hours, 72); + assert_eq!(config.migration.copier_throttle_mib_per_sec, 32); + assert_eq!(config.migration.copier_slack_mb, 2048); + // The release switches are absent from the file on purpose, so they come from the + // build rather than from whatever an operator's config last recorded. + let build = MigrationConfig::default(); + assert_eq!(config.migration.retire_legacy, build.retire_legacy); + } + #[test] fn test_default_config_has_cache_capacity() { let config = PaymentConfig::default(); diff --git a/src/devnet.rs b/src/devnet.rs index d9e9de09..5cf16b06 100644 --- a/src/devnet.rs +++ b/src/devnet.rs @@ -11,7 +11,7 @@ use crate::payment::{ QuotingMetricsTracker, }; use crate::replication::config::ReplicationConfig; -use crate::storage::{AntProtocol, ChunkRequestContext, LmdbStorage, LmdbStorageConfig}; +use crate::storage::{AntProtocol, ChunkRequestContext, ChunkStore, ChunkStoreConfig}; use evmlib::Network as EvmNetwork; use evmlib::RewardsAddress; use rand::Rng; @@ -595,12 +595,12 @@ impl Devnet { identity: &NodeIdentity, config: &DevnetConfig, ) -> Result { - let storage_config = LmdbStorageConfig { + let storage_config = ChunkStoreConfig { root_dir: data_dir.to_path_buf(), verify_on_read: true, - ..LmdbStorageConfig::default() + ..ChunkStoreConfig::default() }; - let storage = LmdbStorage::new(storage_config) + let storage = ChunkStore::new(storage_config) .await .map_err(|e| DevnetError::Core(format!("Failed to create LMDB storage: {e}")))?; diff --git a/src/lib.rs b/src/lib.rs index 38cc9096..83d19fec 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -71,7 +71,7 @@ pub use event::{NodeEvent, NodeEventsChannel}; pub use node::{NodeBuilder, RunningNode}; pub use payment::{PaymentStatus, PaymentVerifier, PaymentVerifierConfig}; pub use replication::{config::ReplicationConfig, ReplicationEngine}; -pub use storage::{AntProtocol, LmdbStorage, LmdbStorageConfig}; +pub use storage::{AntProtocol, ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; /// Re-exports from `saorsa-core` so downstream crates (e.g. `ant-client`) /// can depend on `ant-node` alone without a direct `saorsa-core` dependency. diff --git a/src/node.rs b/src/node.rs index 65b66b4f..aa5d1fe7 100644 --- a/src/node.rs +++ b/src/node.rs @@ -13,9 +13,10 @@ use crate::payment::{ EvmVerifierConfig, PaymentVerifier, PaymentVerifierConfig, PriceFloorConfig, QuoteGenerator, }; use crate::replication::config::ReplicationConfig; +use crate::replication::fresh::FreshWriteEvent; use crate::replication::ReplicationEngine; -use crate::storage::lmdb::MIB; -use crate::storage::{AntProtocol, ChunkRequestContext, LmdbStorage, LmdbStorageConfig}; +use crate::storage::MIB; +use crate::storage::{AntProtocol, ChunkRequestContext, ChunkStore, ChunkStoreConfig}; use crate::upgrade::{ upgrade_cache_dir, AutoApplyUpgrader, BinaryCache, ReleaseCache, UpgradeMonitor, UpgradeResult, }; @@ -25,17 +26,33 @@ use saorsa_core::{ IPDiversityConfig as CoreDiversityConfig, MultiAddr, NodeConfig as CoreNodeConfig, P2PEvent, P2PNode, }; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicI32, Ordering}; use std::sync::Arc; use std::time::Instant; +use tokio::sync::mpsc::UnboundedReceiver; use tokio::sync::Semaphore; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; #[cfg(unix)] use tokio::signal::unix::{signal, SignalKind}; +/// How long shutdown waits for the storage migration to reach a stopping point. +/// +/// Generous, because interrupting a copy mid-chunk costs nothing (every step is +/// idempotent and re-derived at the next start) but interrupting the drain that precedes +/// removing the legacy store is worth avoiding. Bounded, because a step that will not +/// finish must not hold the process open. +const MIGRATION_SHUTDOWN_GRACE: std::time::Duration = std::time::Duration::from_secs(30); + +/// How long shutdown waits for in-flight request handlers to finish. +/// +/// Short, because these are single request/response exchanges and the peer will retry. +/// The point is to stop new legacy reads starting, not to see every last one through. +const PROTOCOL_DRAIN_GRACE: std::time::Duration = std::time::Duration::from_secs(5); + /// Builder for constructing an Ant node. pub struct NodeBuilder { config: NodeConfig, @@ -97,6 +114,12 @@ impl NodeBuilder { // Ensure root directory exists std::fs::create_dir_all(&self.config.root_dir)?; + // One release-level decision, applied before anything can audit: while the fleet + // moves off the legacy chunk store, a peer is not penalised for failing to hold a + // chunk it was supposed to be holding. It is still penalised for failing a + // commitment-bound audit. Audits of both kinds run and record throughout. + crate::replication::config::apply_close_group_storage_penalty_policy(); + // Create shutdown token let shutdown = CancellationToken::new(); @@ -145,56 +168,20 @@ impl NodeBuilder { protocol.attach_p2p_node(Arc::clone(&p2p_arc)); } - // Initialize replication engine (if storage is enabled) - let replication_engine = if let (Some(ref protocol), Some(fresh_rx)) = - (&ant_protocol, fresh_write_rx) - { - let storage_arc = protocol.storage(); - let payment_verifier_arc = protocol.payment_verifier_arc(); - match ReplicationEngine::new( - repl_config, - Arc::clone(&p2p_arc), - storage_arc, - payment_verifier_arc, - Arc::clone(&identity), - &self.config.root_dir, - fresh_rx, - shutdown.clone(), - ) - .await - { - Ok(engine) => { - // ADR-0004: wire the engine's commitment state as the - // quote generator's commitment source so quotes force - // their price from the live storage commitment. Done - // here because the engine owns the commitment state and - // is built after the protocol. - if let Some(ref protocol) = ant_protocol { - let concrete = Arc::clone(engine.commitment_state()); - let source: Arc = concrete; - protocol.attach_commitment_source(source); - // ADR-0004: share the engine's gossip commitment - // cache with the verifier so the cross-check can - // resolve quote pins against neighbours' commitments. - protocol - .payment_verifier_arc() - .attach_commitment_cache(Arc::clone(engine.last_commitment_by_peer())); - // ADR-0004: give the verifier the monetized-pin sender so - // commitments that back a payment get a deterministic - // first audit from the engine's drainer. - protocol - .payment_verifier_arc() - .attach_monetized_pin_sender(engine.monetized_pin_sender()); - } - Some(engine) - } - Err(e) => { - warn!("Failed to initialize replication engine: {e}"); - None - } + let (replication_engine, migration_task) = match (&ant_protocol, fresh_write_rx) { + (Some(protocol), Some(fresh_rx)) => { + Self::build_replication_engine( + protocol, + repl_config, + &p2p_arc, + &identity, + &self.config.root_dir, + fresh_rx, + &shutdown, + ) + .await? } - } else { - None + _ => (None, None), }; let node = RunningNode { @@ -207,12 +194,92 @@ impl NodeBuilder { ant_protocol, replication_engine, protocol_task: None, + migration_task, + protocol_children: TaskTracker::new(), upgrade_exit_code: Arc::new(AtomicI32::new(-1)), }; Ok(node) } + /// Start the replication engine and, if this node still has one, the migration off + /// the legacy chunk store. + /// + /// The two are built together because the migration cannot run without the engine: + /// it needs the commitment state, which holds the veto on deleting the old store, + /// and the live routing view that says which chunks this node must never give up. + /// + /// # Errors + /// + /// Returns an error only when the engine fails to start on a node that has a legacy + /// store to migrate. On a node with nothing to migrate an engine failure is logged + /// and the node runs without one, as it always has. + async fn build_replication_engine( + protocol: &Arc, + repl_config: ReplicationConfig, + p2p: &Arc, + identity: &Arc, + root_dir: &Path, + fresh_rx: UnboundedReceiver, + shutdown: &CancellationToken, + ) -> Result<(Option, Option>)> { + let engine = match ReplicationEngine::new( + repl_config, + Arc::clone(p2p), + protocol.storage(), + protocol.payment_verifier_arc(), + Arc::clone(identity), + root_dir, + fresh_rx, + shutdown.clone(), + ) + .await + { + Ok(engine) => engine, + Err(e) => { + // A node that still has a legacy chunk store depends on this engine for + // the commitment state, the routing view and the possession challenges + // the migration cannot proceed without. Carrying on would leave it + // serving from both stores forever, never reclaiming its disk, which is + // the condition this release exists to end. Refuse to start instead of + // running in it indefinitely. + if protocol.storage().has_legacy() { + return Err(Error::Startup(format!( + "This node has a legacy chunk store to migrate but the \ + replication engine did not start: {e}. Without it the \ + migration cannot run and the disk is never reclaimed. \ + Fix the cause rather than running on." + ))); + } + warn!("Failed to initialize replication engine: {e}"); + return Ok((None, None)); + } + }; + + // ADR-0004: wire the engine's commitment state as the quote generator's + // commitment source so quotes force their price from the live storage + // commitment. Done here because the engine owns the commitment state and is + // built after the protocol. + let concrete = Arc::clone(engine.commitment_state()); + let source: Arc = concrete; + protocol.attach_commitment_source(source); + // ADR-0004: share the engine's gossip commitment cache with the verifier so the + // cross-check can resolve quote pins against neighbours' commitments. + protocol + .payment_verifier_arc() + .attach_commitment_cache(Arc::clone(engine.last_commitment_by_peer())); + // ADR-0004: give the verifier the monetized-pin sender so commitments that back + // a payment get a deterministic first audit from the engine's drainer. + protocol + .payment_verifier_arc() + .attach_monetized_pin_sender(engine.monetized_pin_sender()); + + let migration_task = + Self::spawn_storage_migration(protocol.storage(), p2p, &engine, shutdown.clone()); + + Ok((Some(engine), migration_task)) + } + /// Build the saorsa-core `NodeConfig` from our config. fn build_core_config(config: &NodeConfig) -> Result { let local = matches!(config.network_mode, NetworkMode::Development); @@ -382,6 +449,37 @@ impl NodeBuilder { monitor } + /// Start moving this node off the legacy LMDB chunk store, if it still has one. + /// + /// Started after the replication engine rather than with the store, because the + /// copier needs two things only the engine has: the commitment state, which owns the + /// retention veto on deleting the old store, and live routing, which is how the node + /// knows which chunks it is among the closest to and therefore must never give up. + fn spawn_storage_migration( + store: Arc, + p2p: &Arc, + engine: &ReplicationEngine, + shutdown: CancellationToken, + ) -> Option> { + if !crate::storage::migration::should_migrate(&store) { + return None; + } + let context = crate::storage::migration::MigrationContext { + p2p: Some(Arc::clone(p2p)), + self_id: Some(*p2p.peer_id()), + self_xor: crate::client::peer_id_to_xor_name(&p2p.peer_id().to_string()), + commitment: Some(Arc::clone(engine.commitment_state())), + replication: Some(Arc::clone(engine.config())), + sync_state: Some(Arc::clone(engine.sync_state())), + audit_challenge_coordinator: Some(Arc::clone(engine.audit_challenge_coordinator())), + peer_commitments: Some(Arc::clone(engine.last_commitment_by_peer())), + close_group_size: engine.config().close_group_size, + }; + Some(tokio::spawn(async move { + crate::storage::migration::run(store, context, shutdown).await; + })) + } + /// Build the ANT protocol handler from config. /// /// Initializes LMDB storage, payment verifier, and quote generator. @@ -392,13 +490,14 @@ impl NodeBuilder { close_group_size: usize, ) -> Result { // Create LMDB storage - let storage_config = LmdbStorageConfig { + let storage_config = ChunkStoreConfig { root_dir: config.root_dir.clone(), verify_on_read: config.storage.verify_on_read, max_map_size: config.storage.db_size_gb.saturating_mul(1024 * 1024 * 1024), disk_reserve: config.storage.disk_reserve_mb.saturating_mul(MIB), + migration: config.storage.migration.clone(), }; - let storage = LmdbStorage::new(storage_config) + let storage = ChunkStore::new(storage_config) .await .map_err(|e| Error::Startup(format!("Failed to create LMDB storage: {e}")))?; @@ -466,6 +565,18 @@ pub struct RunningNode { replication_engine: Option, /// Protocol message routing background task. protocol_task: Option>, + /// The task moving this node off the legacy chunk store, if it has one. + /// + /// Awaited before the replication engine and the P2P layer are torn down, because it + /// holds handles to both and is in the middle of reading and writing the chunk store. + migration_task: Option>, + /// The per-message handler tasks the protocol loop spawns. + /// + /// Tracked rather than detached so shutdown can stop accepting work and then wait for + /// what is already in flight. Aborting only the loop leaves its children running, and + /// a chunk read that outlives the loop keeps the legacy store busy exactly while the + /// migration is trying to drain it. + protocol_children: TaskTracker, /// Exit code requested by a successful upgrade (-1 = no upgrade exit pending). upgrade_exit_code: Arc, } @@ -689,22 +800,82 @@ impl RunningNode { }); } + // A node that still has a legacy chunk store and no task moving it off one is the + // failure this cannot be allowed to have silently: the store opens, serves the + // union of both, and never frees a byte. It happened once, during a rebase that + // dropped the spawn, and nothing noticed because a node without a legacy store + // starts no migration and every test built the store directly. Say so loudly. + if let Some(ref protocol) = self.ant_protocol { + if protocol.storage().has_legacy() && self.migration_task.is_none() { + error!( + migration_event = "not_started", + "This node still has a legacy chunk store but nothing is migrating it. \ + Its disk will never be reclaimed. This is a wiring fault, not a \ + configuration one: report it rather than working around it." + ); + } + } + info!("Node running, waiting for shutdown signal"); // Run the main event loop with signal handling self.run_event_loop().await?; + // Protocol routing stops FIRST, loop and children both. The migration's last step + // drains the legacy store's in-flight reads, and inbound protocol traffic keeps + // starting new ones, so waiting on the migration while still serving requests can + // keep that drain from ever completing and hang shutdown. Aborting the accept loop + // alone would not do it: the requests already in flight run in their own tasks. + if let Some(handle) = self.protocol_task.take() { + handle.abort(); + } + // Cancelled first, so anything still queued behind the concurrency permits gives + // up rather than starting fresh storage work, then given a moment to finish what + // is genuinely in flight. + self.shutdown.cancel(); + self.protocol_children.close(); + if tokio::time::timeout(PROTOCOL_DRAIN_GRACE, self.protocol_children.wait()) + .await + .is_err() + { + warn!( + "{} request handler(s) had not finished after {}s; continuing shutdown \ + without them.", + self.protocol_children.len(), + PROTOCOL_DRAIN_GRACE.as_secs() + ); + } + + // Then the migration, awaited rather than aborted: it is mid-way through reading + // and writing the chunk store, and it holds the commitment state and the routing + // handle that the shutdown below is about to invalidate. It watches the same + // cancellation token, so this returns as soon as its current step does. Bounded, + // because a step that will not finish must not hold the process open. + if let Some(mut handle) = self.migration_task.take() { + // Awaited by reference, so a timeout leaves the handle here to abort rather + // than dropping it and letting the task run on detached through the engine and + // P2P teardown it depends on. + match tokio::time::timeout(MIGRATION_SHUTDOWN_GRACE, &mut handle).await { + Ok(Ok(())) => {} + Ok(Err(e)) => warn!("Storage migration task did not stop cleanly: {e}"), + Err(_) => { + warn!( + "Storage migration did not stop within {}s; stopping it. \ + Everything it does is idempotent and re-derived at the next start.", + MIGRATION_SHUTDOWN_GRACE.as_secs() + ); + handle.abort(); + let _ = handle.await; + } + } + } + // Shutdown replication engine before P2P so background tasks don't - // use a dead P2P layer, and Arc references are released. + // use a dead P2P layer, and Arc references are released. if let Some(ref mut engine) = self.replication_engine { engine.shutdown().await; } - // Stop protocol routing task - if let Some(handle) = self.protocol_task.take() { - handle.abort(); - } - // Shutdown P2P node info!("Shutting down P2P node..."); if let Err(e) = self.p2p_node.shutdown().await { @@ -777,6 +948,58 @@ impl RunningNode { Ok(()) } + /// Handle one inbound protocol message and send whatever it produced. + async fn answer_one_request( + protocol: &Arc, + p2p: &Arc, + source: &saorsa_core::identity::PeerId, + data: &[u8], + data_type: &str, + response_topic: &str, + received_at: Instant, + ) { + if data_type != "chunk" { + return; + } + let queue_wait = received_at.elapsed(); + let handled = protocol + .try_handle_request_with_context( + data, + Some(ChunkRequestContext::new( + source.to_string(), + received_at, + queue_wait, + )), + ) + .await; + let telemetry = handled.get_telemetry; + match handled.response { + Ok(Some(response)) => { + let send_started = Instant::now(); + let send_result = p2p + .send_message(source, response_topic, response.to_vec(), &[]) + .await; + if let Some(telemetry) = telemetry { + telemetry.finish_send(send_started.elapsed(), send_result.is_ok()); + } + if let Err(e) = send_result { + warn!("Failed to send {data_type} protocol response to {source}: {e}"); + } + } + Ok(None) => { + if let Some(telemetry) = telemetry { + telemetry.finish_without_send("no_response"); + } + } + Err(e) => { + if let Some(telemetry) = telemetry { + telemetry.finish_without_send("encode_error"); + } + warn!("{data_type} protocol handler error: {e}"); + } + } + } + /// Start the protocol message routing background task. /// /// Subscribes to P2P events and routes incoming chunk protocol messages @@ -790,6 +1013,8 @@ impl RunningNode { let mut events = self.p2p_node.subscribe_events(); let p2p = Arc::clone(&self.p2p_node); let semaphore = Arc::new(Semaphore::new(64)); + let children = self.protocol_children.clone(); + let stopping = self.shutdown.clone(); self.protocol_task = Some(tokio::spawn(async move { while let Ok(event) = events.recv().await { @@ -812,60 +1037,38 @@ impl RunningNode { let protocol = Arc::clone(&protocol); let p2p = Arc::clone(&p2p); let sem = semaphore.clone(); - tokio::spawn(async move { - let Ok(_permit) = sem.acquire().await else { - return; - }; - let queue_wait = received_at.elapsed(); - let handled = match data_type { - "chunk" => { - protocol - .try_handle_request_with_context( - &data, - Some(ChunkRequestContext::new( - source.to_string(), - received_at, - queue_wait, - )), - ) - .await + let stopping = stopping.clone(); + children.spawn(async move { + // A queued handler must not start work once shutdown has + // begun. With 64 permits and a busy node the queue behind them + // can be long, and every one of those would otherwise start + // fresh storage reads while the store beneath is being torn + // down. + let _permit = { + let acquired = tokio::select! { + biased; + () = stopping.cancelled() => return, + p = sem.acquire() => p, + }; + match acquired { + Ok(permit) => permit, + Err(_) => return, } - _ => return, }; - let telemetry = handled.get_telemetry; - match handled.response { - Ok(Some(response)) => { - let send_started = Instant::now(); - let send_result = p2p - .send_message( - &source, - response_topic, - response.to_vec(), - &[], - ) - .await; - if let Some(telemetry) = telemetry { - telemetry.finish_send( - send_started.elapsed(), - send_result.is_ok(), - ); - } - if let Err(e) = send_result { - warn!("Failed to send {data_type} protocol response to {source}: {e}"); - } - } - Ok(None) => { - if let Some(telemetry) = telemetry { - telemetry.finish_without_send("no_response"); - } - } - Err(e) => { - if let Some(telemetry) = telemetry { - telemetry.finish_without_send("encode_error"); - } - warn!("{data_type} protocol handler error: {e}"); - } + // Checked again: the wait for a permit may have been long. + if stopping.is_cancelled() { + return; } + Self::answer_one_request( + &protocol, + &p2p, + &source, + &data, + data_type, + response_topic, + received_at, + ) + .await; }); } } @@ -895,6 +1098,196 @@ fn jittered_interval(base: std::time::Duration) -> std::time::Duration { #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { + use rand::Rng; + use tempfile::TempDir; + + /// The e2e port range, so a test bind never lands on a production or dev instance. + const TEST_PORT_RANGE: std::ops::Range = 20000..60000; + + /// How many times a bind is retried before the failure is treated as real. + const BIND_ATTEMPTS: u32 = 5; + + /// A well-formed address that receives nothing; no chain is contacted in these tests. + const TEST_REWARDS_ADDRESS: &str = "0x0000000000000000000000000000000000000001"; + + /// A node with a legacy chunk store must get a migration task; one without must not. + /// + /// The spawn helper is tested directly because its *absence* is the failure mode that + /// already happened here: a rebase dropped the call, the store still opened and still + /// served, and no test could tell the difference. + #[tokio::test] + async fn a_legacy_store_gets_a_migration_task_and_a_fresh_node_does_not() { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + // Fresh node: nothing to migrate, so no task. + let fresh = Arc::new( + crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { + root_dir: root.clone(), + ..crate::storage::ChunkStoreConfig::test_default() + }) + .await + .expect("open fresh"), + ); + assert!(!fresh.has_legacy()); + assert!( + !crate::storage::migration::should_migrate(&fresh), + "a node with no legacy store has nothing to migrate" + ); + drop(fresh); + + // Seed a legacy store, then reopen: now there is something to migrate. + { + let lmdb = crate::storage::LmdbStorage::new(crate::storage::LmdbStorageConfig { + root_dir: root.clone(), + verify_on_read: true, + max_map_size: 0, + disk_reserve: 0, + }) + .await + .expect("open legacy"); + let content = b"a chunk from before the migration"; + let addr = crate::client::compute_address(content); + lmdb.put(&addr, content).await.expect("put"); + lmdb.wait_idle().await; + } + let upgrading = Arc::new( + crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { + root_dir: root.clone(), + ..crate::storage::ChunkStoreConfig::test_default() + }) + .await + .expect("open upgrading"), + ); + assert!(upgrading.has_legacy()); + assert!( + crate::storage::migration::should_migrate(&upgrading), + "a node with a legacy store must be migrated, or its disk is never reclaimed" + ); + } + + /// Seed a legacy LMDB store under `root` with one chunk, then close it. + async fn seed_legacy_store(root: &std::path::Path) { + let lmdb = crate::storage::LmdbStorage::new(crate::storage::LmdbStorageConfig { + root_dir: root.to_path_buf(), + verify_on_read: true, + max_map_size: 0, + disk_reserve: 0, + }) + .await + .expect("open legacy"); + let content = b"a chunk written before the migration"; + let addr = crate::client::compute_address(content); + lmdb.put(&addr, content).await.expect("put"); + lmdb.wait_idle().await; + } + + /// A node config that builds without touching a chain or a real network. + fn local_node_config(root: &std::path::Path, port: u16) -> NodeConfig { + NodeConfig { + root_dir: root.to_path_buf(), + port, + ipv4_only: true, + network_mode: NetworkMode::Development, + payment: crate::config::PaymentConfig { + rewards_address: Some(TEST_REWARDS_ADDRESS.to_string()), + ..crate::config::PaymentConfig::default() + }, + ..NodeConfig::default() + } + } + + /// A real, fully built node with a legacy store is actually migrating it. + /// + /// This goes through `build()` rather than calling the spawn helper, because the + /// failure that already happened here was the *call site* going missing, not the + /// helper being wrong. A test of the helper alone stays green through exactly that + /// bug. Deleting the spawn from `build()` must turn this red. + #[tokio::test] + async fn a_built_node_with_a_legacy_store_is_migrating_it() { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + seed_legacy_store(&root).await; + + // Ports are picked at random from the test range and a freshly released one can + // still be held for a moment, so a bind failure is retried rather than reported + // as a wiring fault. + let mut built = None; + let mut last_err = String::new(); + for _ in 0..BIND_ATTEMPTS { + let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); + match NodeBuilder::new(local_node_config(&root, port)) + .build() + .await + { + Ok(node) => { + built = Some(node); + break; + } + Err(e) => { + last_err = e.to_string(); + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + } + } + let Some(node) = built else { + panic!("could not build a node after {BIND_ATTEMPTS} attempts: {last_err}"); + }; + + let storage_has_legacy = node + .ant_protocol + .as_ref() + .is_some_and(|p| p.storage().has_legacy()); + assert!( + storage_has_legacy, + "the node must have opened the legacy store this test seeded" + ); + assert!( + node.migration_task.is_some(), + "a node holding a legacy chunk store came up with nothing migrating it, so \ + its disk would never be reclaimed" + ); + + node.shutdown.cancel(); + if let Some(handle) = node.migration_task { + handle.abort(); + } + } + + /// A node with nothing to migrate does not start a driver for it. + #[tokio::test] + async fn a_built_node_without_a_legacy_store_starts_no_migration() { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + let mut built = None; + let mut last_err = String::new(); + for _ in 0..BIND_ATTEMPTS { + let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); + match NodeBuilder::new(local_node_config(&root, port)) + .build() + .await + { + Ok(node) => { + built = Some(node); + break; + } + Err(e) => { + last_err = e.to_string(); + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + } + } + let Some(node) = built else { + panic!("could not build a node after {BIND_ATTEMPTS} attempts: {last_err}"); + }; + + assert!(node.migration_task.is_none()); + node.shutdown.cancel(); + } use super::*; use crate::config::NODES_SUBDIR; diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index fd550c77..8395950a 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -13,7 +13,7 @@ use crate::payment::proof::{ }; use crate::replication::commitment::MAX_COMMITMENT_KEY_COUNT; use crate::replication::config::K_BUCKET_SIZE; -use crate::storage::lmdb::LmdbStorage; +use crate::storage::ChunkStore; use ant_protocol::payment::verify::{verify_quote_content, verify_quote_signature}; use evmlib::common::{Amount, QuoteHash}; use evmlib::contract::payment_vault; @@ -614,7 +614,7 @@ pub struct PaymentVerifier { /// compared unlike counts and false-rejected honest quotes). `None` in unit /// tests that don't exercise store-backed checks; production wires it via /// [`Self::attach_storage`]. - storage: RwLock>>, + storage: RwLock>>, /// Test-only override for the paid-quote issuer K-closest check. /// /// Production code derives closest peers from the attached [`P2PNode`]. @@ -878,7 +878,7 @@ impl PaymentVerifier { self.config.close_group_size } - /// Attach the node's [`LmdbStorage`] handle for store-backed verifier + /// Attach the node's [`ChunkStore`] handle for store-backed verifier /// checks that read the authoritative on-disk record count. /// /// NOTE: the ADR-0006 price floor does NOT depend on this handle — it is @@ -888,9 +888,9 @@ impl PaymentVerifier { /// attached still admits PUTs; this /// attachment only feeds any current/future store-count-backed checks. /// Idempotent: calling twice replaces the handle. - pub fn attach_storage(&self, storage: Arc) { + pub fn attach_storage(&self, storage: Arc) { *self.storage.write() = Some(storage); - debug!("PaymentVerifier: LmdbStorage attached for paid-quote price-floor checks"); + debug!("PaymentVerifier: ChunkStore attached for paid-quote price-floor checks"); } /// Attach the live commitment source for the price floor: the SAME diff --git a/src/replication/admission.rs b/src/replication/admission.rs index cd881625..445d5644 100644 --- a/src/replication/admission.rs +++ b/src/replication/admission.rs @@ -17,7 +17,7 @@ use saorsa_core::P2PNode; use crate::ant_protocol::XorName; use crate::replication::config::ReplicationConfig; use crate::replication::paid_list::PaidList; -use crate::storage::LmdbStorage; +use crate::storage::ChunkStore; /// Result of admitting a set of hints from a neighbor sync. #[derive(Debug)] @@ -82,7 +82,7 @@ async fn is_relevant( key: &XorName, p2p_node: &Arc, config: &ReplicationConfig, - storage: &Arc, + storage: &Arc, paid_list: &Arc, pending_keys: &HashSet, ) -> bool { @@ -113,7 +113,7 @@ pub async fn admit_hints( paid_hints: &[XorName], p2p_node: &Arc, config: &ReplicationConfig, - storage: &Arc, + storage: &Arc, paid_list: &Arc, pending_keys: &HashSet, ) -> AdmissionResult { diff --git a/src/replication/audit.rs b/src/replication/audit.rs index 90dfd1b5..9b312bb3 100644 --- a/src/replication/audit.rs +++ b/src/replication/audit.rs @@ -21,7 +21,7 @@ use crate::replication::protocol::{ use crate::replication::types::{ AuditFailureReason, AuditFailureSummary, FailureEvidence, PeerSyncRecord, RepairProofs, }; -use crate::storage::LmdbStorage; +use crate::storage::ChunkStore; use saorsa_core::identity::PeerId; use saorsa_core::P2PNode; use tokio::sync::RwLock; @@ -34,7 +34,7 @@ use crate::replication::config::REPAIR_HINT_MIN_AGE; #[cfg(test)] use crate::replication::types::{BootstrapClaimObservation, NeighborSyncState}; #[cfg(test)] -use crate::storage::LmdbStorageConfig; +use crate::storage::ChunkStoreConfig; #[cfg(test)] use tempfile::TempDir; @@ -113,7 +113,7 @@ pub(crate) fn responsible_audit_response_timeout( )] pub async fn audit_tick_with_repair_proofs( p2p_node: &Arc, - storage: &Arc, + storage: &Arc, config: &ReplicationConfig, sync_history: &HashMap, repair_proofs: &Arc>, @@ -543,7 +543,7 @@ async fn verify_digests( nonce: &[u8; 32], keys: &[XorName], digests: &[[u8; 32]], - storage: &Arc, + storage: &Arc, p2p_node: &Arc, config: &ReplicationConfig, ) -> AuditTickResult { @@ -759,7 +759,7 @@ async fn handle_audit_timeout( /// attack where a malicious challenger forges digests for a different peer. pub async fn handle_audit_challenge( challenge: &AuditChallenge, - storage: &LmdbStorage, + storage: &ChunkStore, self_peer_id: &PeerId, is_bootstrapping: bool, stored_chunks: usize, @@ -890,16 +890,17 @@ mod tests { ); } - /// Create a test `LmdbStorage` backed by a temp directory. - async fn create_test_storage() -> (LmdbStorage, TempDir) { + /// Create a test `ChunkStore` backed by a temp directory. + async fn create_test_storage() -> (ChunkStore, TempDir) { let temp_dir = TempDir::new().expect("create temp dir"); - let config = LmdbStorageConfig { + let config = ChunkStoreConfig { root_dir: temp_dir.path().to_path_buf(), verify_on_read: false, max_map_size: 0, disk_reserve: 0, + ..ChunkStoreConfig::test_default() }; - let storage = LmdbStorage::new(config).await.expect("create storage"); + let storage = ChunkStore::new(config).await.expect("create storage"); (storage, temp_dir) } @@ -931,11 +932,11 @@ mod tests { // Store two chunks. let content_a = b"chunk alpha"; - let addr_a = LmdbStorage::compute_address(content_a); + let addr_a = ChunkStore::compute_address(content_a); storage.put(&addr_a, content_a).await.expect("put a"); let content_b = b"chunk beta"; - let addr_b = LmdbStorage::compute_address(content_b); + let addr_b = ChunkStore::compute_address(content_b); storage.put(&addr_b, content_b).await.expect("put b"); let nonce = [0xAA; 32]; @@ -1011,7 +1012,7 @@ mod tests { let (storage, _temp) = create_test_storage().await; let content = b"present chunk"; - let addr_present = LmdbStorage::compute_address(content); + let addr_present = ChunkStore::compute_address(content); storage.put(&addr_present, content).await.expect("put"); let addr_absent = [0xDE; 32]; @@ -1199,7 +1200,7 @@ mod tests { let (storage, _temp) = create_test_storage().await; let content = b"stored but bootstrapping"; - let addr = LmdbStorage::compute_address(content); + let addr = ChunkStore::compute_address(content); storage.put(&addr, content).await.expect("put"); let challenge = make_challenge(200, [0xCC; 32], [0xDD; 32], vec![addr]); @@ -1230,11 +1231,11 @@ mod tests { // Store K1 and K2, but NOT K3 let content_k1 = b"key one data"; - let addr_k1 = LmdbStorage::compute_address(content_k1); + let addr_k1 = ChunkStore::compute_address(content_k1); storage.put(&addr_k1, content_k1).await.unwrap(); let content_k2 = b"key two data"; - let addr_k2 = LmdbStorage::compute_address(content_k2); + let addr_k2 = ChunkStore::compute_address(content_k2); storage.put(&addr_k2, content_k2).await.unwrap(); let addr_k3 = [0xFF; 32]; // Not stored @@ -1283,9 +1284,9 @@ mod tests { let c1 = b"chunk alpha"; let c2 = b"chunk beta"; let c3 = b"chunk gamma"; - let a1 = LmdbStorage::compute_address(c1); - let a2 = LmdbStorage::compute_address(c2); - let a3 = LmdbStorage::compute_address(c3); + let a1 = ChunkStore::compute_address(c1); + let a2 = ChunkStore::compute_address(c2); + let a3 = ChunkStore::compute_address(c3); storage.put(&a1, c1).await.unwrap(); storage.put(&a2, c2).await.unwrap(); storage.put(&a3, c3).await.unwrap(); @@ -1337,8 +1338,8 @@ mod tests { // Store K1 and K2 on the challenger (for expected digest computation). let c1 = b"scenario 55 key one"; let c2 = b"scenario 55 key two"; - let k1 = LmdbStorage::compute_address(c1); - let k2 = LmdbStorage::compute_address(c2); + let k1 = ChunkStore::compute_address(c1); + let k2 = ChunkStore::compute_address(c2); storage.put(&k1, c1).await.expect("put k1"); storage.put(&k2, c2).await.expect("put k2"); @@ -1622,7 +1623,7 @@ mod tests { // Store a single chunk let content = b"single chunk"; - let addr = LmdbStorage::compute_address(content); + let addr = ChunkStore::compute_address(content); storage.put(&addr, content).await.unwrap(); // Challenge with 1 stored + 4 absent = 5 keys total @@ -1682,7 +1683,7 @@ mod tests { // Store data so there *would* be work to audit. let content = b"should not be audited during bootstrap"; - let addr = LmdbStorage::compute_address(content); + let addr = ChunkStore::compute_address(content); storage.put(&addr, content).await.expect("put"); let challenge = make_challenge(2900, [0x29; 32], [0x29; 32], vec![addr]); @@ -1773,7 +1774,7 @@ mod tests { let mut addrs = Vec::new(); for i in 0u8..5 { let content = format!("dynamic challenge key {i}"); - let addr = LmdbStorage::compute_address(content.as_bytes()); + let addr = ChunkStore::compute_address(content.as_bytes()); storage.put(&addr, content.as_bytes()).await.expect("put"); addrs.push(addr); } @@ -1830,7 +1831,7 @@ mod tests { // Store data so there is an auditable key. let content = b"bootstrap grace test"; - let addr = LmdbStorage::compute_address(content); + let addr = ChunkStore::compute_address(content); storage.put(&addr, content).await.expect("put"); let challenge = make_challenge(4700, [0x47; 32], [0x47; 32], vec![addr]); @@ -1894,9 +1895,9 @@ mod tests { let c1 = b"scenario 53 key one"; let c2 = b"scenario 53 key two"; let c3 = b"scenario 53 key three"; - let k1 = LmdbStorage::compute_address(c1); - let k2 = LmdbStorage::compute_address(c2); - let k3 = LmdbStorage::compute_address(c3); + let k1 = ChunkStore::compute_address(c1); + let k2 = ChunkStore::compute_address(c2); + let k3 = ChunkStore::compute_address(c3); storage.put(&k1, c1).await.expect("put k1"); storage.put(&k2, c2).await.expect("put k2"); storage.put(&k3, c3).await.expect("put k3"); diff --git a/src/replication/audit_metrics.rs b/src/replication/audit_metrics.rs index 5646a889..e1f2cf85 100644 --- a/src/replication/audit_metrics.rs +++ b/src/replication/audit_metrics.rs @@ -407,9 +407,12 @@ static DIGEST_DISPATCH_LATENCY_COUNT: AtomicU64 = AtomicU64::new(0); static DIGEST_DISPATCH_LATENCY_TOTAL_MS: AtomicU64 = AtomicU64::new(0); static DIGEST_DISPATCH_LATENCY_MAX_MS: AtomicU64 = AtomicU64::new(0); -#[cfg(feature = "logging")] impl AuditType { /// Stable structured-log label. + /// + /// Not gated on the `logging` feature: it is passed as an ordinary argument to the + /// penalty helper, which evaluates its arguments whether or not the log macro that + /// consumes them compiles to anything. #[must_use] pub const fn as_str(self) -> &'static str { match self { diff --git a/src/replication/commitment_state.rs b/src/replication/commitment_state.rs index 8c7b2840..9daff439 100644 --- a/src/replication/commitment_state.rs +++ b/src/replication/commitment_state.rs @@ -24,9 +24,11 @@ //! its persisted key set — so an honest restarted node can answer every pin that //! is still inside its answerability window, and an unanswerable pin is provable //! misbehaviour rather than an honest crash-restart. Trees are otherwise rebuilt -//! from `LmdbStorage` at the next rotation tick. Memory cost is bounded by +//! from `ChunkStore` at the next rotation tick. Memory cost is bounded by //! `2 × (key_count × ~64 bytes + signature_size)` — for 10k keys, ~1.3 MB. +use saorsa_core::identity::PeerId; +use std::collections::HashSet; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -441,6 +443,18 @@ struct Inner { /// the answerability TTL, not a fixed count). A commitment is retained iff it /// is the live current one or its hash appears here with an unexpired stamp. recently_gossiped: Vec, + /// Peers that have demonstrably received the CURRENT commitment root. + /// + /// Distinct from `recently_gossiped`, which records that a root was put on the wire. + /// This records that a specific peer's node answered afterwards, so the request + /// carrying the root arrived. The storage migration needs that stronger statement: a + /// node must not start giving chunks up until its close group has actually seen the + /// reduced commitment, or those peers keep auditing it against the set it used to + /// hold. + current_recipients: HashSet, + /// The root `current_recipients` refers to. A rotation to a different root empties + /// the set, because nobody has seen the new one yet. + current_recipients_hash: Option<[u8; 32]>, } impl Default for ResponderCommitmentState { @@ -460,6 +474,8 @@ impl ResponderCommitmentState { slots: Vec::with_capacity(RETAINED_GOSSIPED_COMMITMENTS + 1), has_current: false, recently_gossiped: Vec::with_capacity(RETAINED_GOSSIPED_COMMITMENTS), + current_recipients: HashSet::new(), + current_recipients_hash: None, }), } } @@ -470,6 +486,12 @@ impl ResponderCommitmentState { pub fn rotate(&self, new_current: BuiltCommitment) { let new_current = Arc::new(new_current); let mut guard = self.inner.write(); + // Nobody has seen the new root yet, so nobody is a recipient of it. Clearing here + // rather than lazily on the next delivery is what makes the invariant true: the + // lazy version credited whichever peer happened to answer next, for a root that + // peer had never been sent. + guard.current_recipients.clear(); + guard.current_recipients_hash = None; guard.slots.insert(0, new_current); guard.has_current = true; prune_slots(&mut guard, Instant::now()); @@ -503,6 +525,64 @@ impl ResponderCommitmentState { /// `GOSSIP_ANSWERABILITY_TTL` after its last emission, which is what lets /// an out-of-range key age out even when the no-op guard freezes the /// committed key set. + /// Record that `peer` demonstrably received the commitment root `delivered`. + /// + /// Called when a peer answers a neighbour sync that carried that root, which is proof + /// of arrival rather than proof of emission. Ignored if the node has rotated since, + /// because the peer then saw a root that is no longer the one being attested. + pub fn note_commitment_delivered(&self, peer: PeerId, delivered: [u8; 32]) { + let mut guard = self.inner.write(); + if !guard.has_current { + return; + } + let Some(hash) = guard.slots.first().map(|c| c.cached_hash) else { + return; + }; + // The caller names the root it actually put on the wire. A rotation between the + // send and the reply means this peer saw the previous root, and crediting it to + // the current one would attest to something that did not happen. + if delivered != hash { + return; + } + if guard.current_recipients_hash != Some(hash) { + guard.current_recipients.clear(); + guard.current_recipients_hash = Some(hash); + } + guard.current_recipients.insert(peer); + } + + /// How many distinct peers have received the current commitment root. + /// + /// Zero once the root changes, because a rotation is a new claim that nobody has + /// seen yet. + #[must_use] + pub fn current_delivered_peer_count(&self) -> usize { + self.current_delivered_peers().len() + } + + /// Which peers have received the current commitment root. + /// + /// The caller intersects this with whoever is in the close group *now*. A peer that + /// has since left knowing the root is no evidence about the group that will audit + /// this node, and counting it would let a node give chunks up while its actual + /// neighbours still hold it to the larger key set. + #[must_use] + pub fn current_delivered_peers(&self) -> HashSet { + let guard = self.inner.read(); + if !guard.has_current { + return HashSet::new(); + } + let Some(hash) = guard.slots.first().map(|c| c.cached_hash) else { + return HashSet::new(); + }; + if guard.current_recipients_hash == Some(hash) { + guard.current_recipients.clone() + } else { + HashSet::new() + } + } + + /// Stamp `hash` as emitted on the wire, refreshing its answerability window. pub fn mark_gossiped(&self, hash: [u8; 32]) { let now = Instant::now(); let mut guard = self.inner.write(); @@ -875,6 +955,14 @@ mod tests { k } + fn peer(byte: u8) -> PeerId { + let mut bytes = [0u8; 32]; + if let Some(slot) = bytes.first_mut() { + *slot = byte; + } + PeerId::from_bytes(bytes) + } + fn bh(byte: u8) -> [u8; 32] { [byte ^ 0x5A; 32] } @@ -1259,6 +1347,54 @@ mod tests { /// Build a `BuiltCommitment` over the given keys for use in raw `prune_slots` /// tests (each key's `bytes_hash` is `bh(k[0])`). + #[test] + fn commitment_delivery_counts_per_root_and_a_rotation_resets_it() { + let state = ResponderCommitmentState::default(); + + // Nothing advertised, so nobody can have received anything. + state.note_commitment_delivered(peer(1), [0u8; 32]); + assert_eq!(state.current_delivered_peer_count(), 0); + + let first = built(&[1, 2, 3]); + let h_first = first.hash(); + state.rotate(first); + assert_eq!(state.current_delivered_peer_count(), 0); + + state.note_commitment_delivered(peer(1), h_first); + state.note_commitment_delivered(peer(2), h_first); + // The same peer twice is still one peer. + state.note_commitment_delivered(peer(2), h_first); + assert_eq!(state.current_delivered_peer_count(), 2); + + // A different key set is a different claim, and nobody has seen it yet. This is + // what stops a node treating "they knew my old commitment" as "they know my new + // smaller one", which is exactly the confusion the storage migration must avoid. + let second = built(&[1, 2]); + let h_second = second.hash(); + state.rotate(second); + assert_eq!( + state.current_delivered_peer_count(), + 0, + "a rotation must empty the set, not wait to be told" + ); + + // A reply to a sync that carried the OLD root arrives after the rotation. It is + // proof that peer saw the old root, and no evidence at all about the new one. + state.note_commitment_delivered(peer(3), h_first); + assert_eq!( + state.current_delivered_peer_count(), + 0, + "a late reply must not be credited to a root its peer never saw" + ); + + state.note_commitment_delivered(peer(1), h_second); + assert_eq!(state.current_delivered_peer_count(), 1); + + // Retiring the current root means there is nothing being advertised to know. + state.retire_current(); + assert_eq!(state.current_delivered_peer_count(), 0); + } + fn built(keys: &[u8]) -> BuiltCommitment { let (pk, sk) = keypair(); let entries: Vec<_> = keys.iter().map(|&b| (key(b), bh(b))).collect(); @@ -1288,6 +1424,8 @@ mod tests { let base = Instant::now(); let now = base + GOSSIP_ANSWERABILITY_TTL + Duration::from_secs(1); let mut inner = Inner { + current_recipients: HashSet::new(), + current_recipients_hash: None, slots: vec![Arc::clone(&c_current), Arc::clone(&c_stale)], has_current: true, recently_gossiped: vec![ @@ -1337,6 +1475,8 @@ mod tests { let base = Instant::now(); let now = base + GOSSIP_ANSWERABILITY_TTL / 2; let mut inner = Inner { + current_recipients: HashSet::new(), + current_recipients_hash: None, slots: vec![Arc::clone(&c_current), Arc::clone(&c_prev)], has_current: true, recently_gossiped: vec![ @@ -1405,6 +1545,8 @@ mod tests { let base = Instant::now(); let now = base + GOSSIP_ANSWERABILITY_TTL + Duration::from_secs(1); let mut inner = Inner { + current_recipients: HashSet::new(), + current_recipients_hash: None, slots: vec![Arc::clone(&c1)], has_current: false, // already retired recently_gossiped: vec![GossipedAt { @@ -1435,6 +1577,8 @@ mod tests { let base = Instant::now(); let now = base + GOSSIP_ANSWERABILITY_TTL / 2; let mut inner = Inner { + current_recipients: HashSet::new(), + current_recipients_hash: None, slots: vec![Arc::clone(&c1)], has_current: false, // retired recently_gossiped: vec![GossipedAt { diff --git a/src/replication/config.rs b/src/replication/config.rs index 66c8e0bd..55bc7b60 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -15,6 +15,11 @@ use std::time::Duration; use rand::Rng; use crate::ant_protocol::CLOSE_GROUP_SIZE; +use crate::logging::{debug, info, warn}; +use saorsa_core::identity::PeerId; +use saorsa_core::{P2PNode, TrustEvent}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; // --------------------------------------------------------------------------- // Static constants (compile-time reference profile) @@ -668,6 +673,145 @@ pub(crate) const CAPACITY_BLOCKED_RETRY: Duration = /// Trust event weight for confirmed audit failures. pub const AUDIT_FAILURE_TRUST_WEIGHT: f64 = 5.0; +/// Whether this build penalises a peer for not holding a chunk it was supposed to hold. +/// +/// **`true` while the fleet moves off the legacy LMDB chunk store; back to `false` once it +/// has.** Flipping it is a one-line change in one release. +/// +/// Deliberately narrow. It covers exactly one accusation: "you did not have a chunk you +/// were supposed to be holding". It does **not** cover the commitment-bound subtree audit, +/// where a peer published a signed claim to hold specific keys and could not answer for +/// them. That contract stays enforced in every release. +/// +/// The reason it has to exist at all is that the penalty is the *auditor's* decision. A +/// node that has to give up chunks, because it cannot fit them while it moves them out of +/// a store that never returns disk, cannot stop its peers penalising it for that. So the +/// peers stop first, one release ahead, and the node moves in the next one. +/// +/// A build constant rather than a config field on purpose: a node writes its effective +/// configuration back to disk, so shipping this as an ordinary setting would bake this +/// release's value into every operator's file and the next release would change nothing. +pub const RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY: bool = true; + +/// Environment override for [`RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY`], for a canary. +pub const SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV: &str = "ANT_SUSPEND_UNHELD_CHUNK_PENALTY"; + +/// The live switch. +/// +/// Initialised **from the release constant**, not to `false`. That matters: a code path +/// that never applies the policy then behaves like this release rather than the previous +/// one. Defaulting the other way meant any constructor that skipped the startup call would +/// keep penalising nodes for the very thing this release exists to stop penalising, and +/// `ReplicationEngine::new` is public and is constructed directly by test harnesses. +/// +/// Process-wide rather than threaded through a parameter because it is exactly that: one +/// release-level decision that every affected site has to obey identically, and those +/// sites are spread across call graphs that share no configuration object. +static CLOSE_GROUP_STORAGE_PENALTY_SUSPENDED: AtomicBool = + AtomicBool::new(RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY); + +/// Apply this release's decision. Called once, before anything can audit. +pub fn apply_close_group_storage_penalty_policy() { + let Ok(raw) = std::env::var(SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV) else { + apply_and_announce(RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY); + return; + }; + let suspended = match raw.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" => true, + "0" | "false" | "no" | "off" => false, + other => { + warn!( + "{SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV}={other} is not a boolean; \ + using the build default {RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY}" + ); + RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY + } + }; + apply_and_announce(suspended); +} + +/// Set the switch and say so, once, where an operator will see it. +/// +/// Both states are logged. An operator reading "penalties are suspended" and an operator +/// reading nothing at all cannot tell the second from a missing log line, and the state +/// that most needs to be visible is the one that disagrees with what the release intended. +fn apply_and_announce(suspended: bool) { + set_close_group_storage_penalty_suspended(suspended); + if suspended != RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY { + warn!( + close_group_storage_penalty_suspended = suspended, + "{SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV} overrides this build: the penalty \ + for not holding a close-group chunk is {}, where the release intends {}. \ + Clear that variable unless this node is a deliberate canary.", + if suspended { "SUSPENDED" } else { "APPLIED" }, + if RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY { + "SUSPENDED" + } else { + "APPLIED" + } + ); + } + if suspended { + info!( + close_group_storage_penalty_suspended = true, + "This release does NOT penalise a peer for failing to hold a close-group \ + chunk. Commitment-bound audits still penalise. Audits run and record \ + throughout." + ); + } else { + info!( + close_group_storage_penalty_suspended = false, + "This release penalises a peer for failing to hold a close-group chunk." + ); + } +} + +/// Set whether failing to hold a close-group chunk penalises. +/// +/// Startup applies the release policy through this. Tests that mean to exercise the +/// penalty itself set it explicitly, so what they assert is not an accident of whichever +/// release they happen to be compiled against. +pub fn set_close_group_storage_penalty_suspended(suspended: bool) { + CLOSE_GROUP_STORAGE_PENALTY_SUSPENDED.store(suspended, Ordering::Relaxed); +} + +/// Whether failing to hold a close-group chunk currently penalises. +#[must_use] +pub fn close_group_storage_penalty_suspended() -> bool { + CLOSE_GROUP_STORAGE_PENALTY_SUSPENDED.load(Ordering::Relaxed) +} + +/// Penalise `peer` at `weight` for not holding a chunk it was supposed to be holding, +/// unless this release withholds that particular penalty. +/// +/// Covers the responsible-chunk audit, the fresh-replication possession check, the prune +/// audit, and the fetch paths where a peer that answered `Present` could not then serve +/// the bytes. A node short of the disk to hold its chunks produces every one of those, so +/// leaving any of them out would stop some of its accusers and not others. +/// +/// Only the penalty is withheld. The caller has already logged the failure with its type, +/// class and key, and that record is what tells us when it is safe to switch the penalty +/// back on. +pub async fn penalise_unheld_close_group_chunk( + p2p_node: &Arc, + peer: &PeerId, + audit_type: &str, + weight: f64, +) { + if close_group_storage_penalty_suspended() { + debug!( + audit_type, + peer = %peer, + "Recorded but not penalised: this release withholds the penalty for not \ + holding a close-group chunk. Commitment-bound audits still penalise." + ); + return; + } + p2p_node + .report_trust_event(peer, TrustEvent::ApplicationFailure(weight)) + .await; +} + /// Probability of launching a subtree audit when a peer's *changed* commitment /// is ingested via gossip (ADR-0002). Keeps audits occasional surprise exams. pub const AUDIT_ON_GOSSIP_PROBABILITY: f64 = 0.2; @@ -1258,6 +1402,7 @@ fn random_duration_in_range(min: Duration, max: Duration) -> Duration { #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { use super::*; + use serial_test::serial; #[test] fn defaults_pass_validation() { @@ -1290,6 +1435,34 @@ mod tests { assert!((AUDIT_FAILURE_TRUST_WEIGHT - 5.0).abs() <= f64::EPSILON); } + /// One test rather than several, because the switch is process-wide: separate tests + /// would race each other under the default parallel runner. + #[test] + #[serial] + fn the_unheld_chunk_penalty_switch_follows_the_release_it_is_compiled_into() { + // A build that never applies the policy still behaves like THIS release, not the + // previous one. `ReplicationEngine::new` is public and is constructed directly by + // test harnesses, so defaulting the other way would leave those engines penalising + // exactly what the release exists to stop penalising. + assert_eq!( + close_group_storage_penalty_suspended(), + RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY + ); + + set_close_group_storage_penalty_suspended(true); + assert!(close_group_storage_penalty_suspended()); + set_close_group_storage_penalty_suspended(false); + assert!(!close_group_storage_penalty_suspended()); + + // And applying the release policy lands on whatever this build ships, without + // asserting the constant itself, which the follow-up release flips on purpose. + apply_and_announce(RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY); + assert_eq!( + close_group_storage_penalty_suspended(), + RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY + ); + } + #[test] fn core_replication_id_stays_v2_and_subtree_rides_its_own_id() { // Core replication, including all digest audit lanes, stays on v2. diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 0b25e38c..341acc45 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -97,7 +97,7 @@ use crate::replication::types::{ NeighborSyncState, PeerSyncRecord, PresenceEvidence, RepairProofs, VerificationEntry, VerificationState, }; -use crate::storage::{CapacityVerdict, LmdbStorage}; +use crate::storage::{CapacityVerdict, ChunkStore}; use saorsa_core::identity::{NodeIdentity, PeerId}; use saorsa_core::{DhtNetworkEvent, P2PEvent, P2PNode, TrustEvent}; use saorsa_pqc::api::sig::{MlDsaSecretKey, MlDsaVariant}; @@ -1442,7 +1442,7 @@ impl Drop for FreshOfferEntryGuard { struct VerificationCycleContext<'a> { p2p_node: &'a Arc, paid_list: &'a Arc, - storage: &'a Arc, + storage: &'a Arc, queues: &'a Arc>, config: &'a ReplicationConfig, bootstrap_state: &'a Arc>, @@ -1655,7 +1655,7 @@ pub struct ReplicationEngine { /// P2P networking node. p2p_node: Arc, /// Local chunk storage. - storage: Arc, + storage: Arc, /// Persistent paid-for-list. paid_list: Arc, /// Payment verifier for `PoP` validation. @@ -1860,7 +1860,7 @@ impl ReplicationEngine { pub async fn new( config: ReplicationConfig, p2p_node: Arc, - storage: Arc, + storage: Arc, payment_verifier: Arc, identity: Arc, root_dir: &Path, @@ -1988,6 +1988,24 @@ impl ReplicationEngine { &self.commitment_state } + /// Neighbour-sync state, for the storage migration's possession challenges. + #[must_use] + pub fn sync_state(&self) -> &Arc> { + &self.sync_state + } + + /// The audit-challenge coordinator, for the storage migration's possession challenges. + #[must_use] + pub fn audit_challenge_coordinator(&self) -> &Arc { + &self.audit_challenge_coordinator + } + + /// Replication settings, for the storage migration's possession challenges. + #[must_use] + pub fn config(&self) -> &Arc { + &self.config + } + /// Get a reference to the auditor's last-commitment-by-peer table. #[must_use] pub fn last_commitment_by_peer(&self) -> &Arc>> { @@ -2262,11 +2280,11 @@ impl ReplicationEngine { /// Cancel all background tasks and wait for them to terminate. /// /// This must be awaited before dropping the engine when the caller needs - /// the `Arc` references held by background tasks to be + /// the `Arc` references held by background tasks to be /// released (e.g. before reopening the same LMDB environment). /// /// When this returns, no engine-spawned task still holds - /// `Arc` or `Arc`, and no LMDB blocking operation + /// `Arc` or `Arc`, and no LMDB blocking operation /// (read or write, on either the chunk store or the paid-list /// environment) is still running. Engine tasks race their work against /// the shutdown token; a dropped future may leave a `spawn_blocking` @@ -2314,7 +2332,7 @@ impl ReplicationEngine { // while an LMDB transaction still owns the environment. // // Deliberately unbounded: the LMDB contract requires every worker to - // release its `Arc` before the caller may reopen the + // release its `Arc` before the caller may reopen the // environment, and a timeout here could return with one still held. // What makes that safe is that every detached task is now guaranteed to // finish — the pools above are closed, stale work is shed at dequeue, @@ -3603,7 +3621,7 @@ impl ReplicationEngine { in_flight.push(Box::pin(async move { // Tracked so shutdown() still awaits the task if // this awaiter is dropped (e.g. the worker is - // aborted): it holds Arc and must + // aborted): it holds Arc and must // not outlive the engine. let handle = tracker.spawn(async move { // Cancel-aware: abort when the engine shuts down. @@ -4252,7 +4270,7 @@ struct PeerResponderSlot { #[derive(Clone)] struct ReplicationMessageHandlerContext { p2p_node: Arc, - storage: Arc, + storage: Arc, paid_list: Arc, payment_verifier: Arc, queues: Arc>, @@ -5815,7 +5833,10 @@ async fn dispatch_fresh_offer( responder_class = "fresh_offer", source = %source, key = %hex::encode(key), - "Fresh offer refused at admission — this node will be penalised for the resulting absence: {failure}" + penalty_suspended = config::close_group_storage_penalty_suspended(), + "Fresh offer refused at admission; the resulting absence is recorded \ + against this node, and penalised unless the release withholds it: \ + {failure}" ); // Release the key explicitly rather than on drop, so the next offer // opens a fresh entry rather than queueing behind a handler that was @@ -5839,7 +5860,7 @@ async fn dispatch_fresh_offer( let ctx = ctx.clone(); // Track the worker so `ReplicationEngine::shutdown()` can await it: it holds - // an `Arc` while writing, and the shutdown contract requires + // an `Arc` while writing, and the shutdown contract requires // those references be released before the caller reopens the environment. ctx.detached_task_tracker .clone() @@ -6487,7 +6508,7 @@ async fn handle_neighbor_sync_request( source: &PeerId, request: &protocol::NeighborSyncRequest, p2p_node: &Arc, - storage: &Arc, + storage: &Arc, paid_list: &Arc, queues: &Arc>, config: &ReplicationConfig, @@ -6678,7 +6699,7 @@ pub fn verification_requests_for_key_from_for_test(requester: &PeerId, key: &Xor async fn handle_verification_request( source: &PeerId, request: &protocol::VerificationRequest, - storage: &Arc, + storage: &Arc, paid_list: &Arc, p2p_node: &Arc, request_id: u64, @@ -6969,25 +6990,95 @@ fn request_is_stale(received_at: Instant, timeout: Duration) -> bool { received_at.elapsed() >= timeout } +/// How a fetch responder's answer is charged against its reputation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FetchFault { + /// The peer does not hold a chunk it was expected to hold. + /// + /// This is the lane the release withholds, because a node part-way through moving + /// off the legacy store answers exactly this way about chunks it has legitimately + /// given up. + UnheldChunk, + /// The peer's own storage failed, or served bytes that no longer hash to their + /// address. + /// + /// Never withheld. `FetchResponse::Error` has one producer, and it is the responder's + /// storage read returning an error: an I/O fault, an exhausted descriptor table, or a + /// failed integrity check. A peer that merely does not hold the chunk answers + /// `NotFound` instead, so nothing about the migration produces this. + ResponderFault, +} + +/// Classify a fetch response that did not carry the chunk. +/// +/// `Success` yields `None`. Every other answer is a fault of one kind or the other, and +/// which kind decides whether this release charges for it. +fn fetch_fault_for(response: &protocol::FetchResponse) -> Option { + match response { + protocol::FetchResponse::Success { .. } => None, + protocol::FetchResponse::NotFound { .. } => Some(FetchFault::UnheldChunk), + protocol::FetchResponse::Error { .. } => Some(FetchFault::ResponderFault), + } +} + +/// Charge a fetch fault to the responder. +/// +/// The only place the two kinds are treated differently. An unheld chunk goes through the +/// release switch, which is currently withholding it; a responder fault is charged +/// directly and is not affected by the switch at all. +async fn charge_fetch_fault( + p2p_node: &Arc, + source: &PeerId, + fault: FetchFault, + lane: &'static str, +) { + match fault { + FetchFault::UnheldChunk => { + config::penalise_unheld_close_group_chunk( + p2p_node, + source, + lane, + REPLICATION_TRUST_WEIGHT, + ) + .await; + } + FetchFault::ResponderFault => { + p2p_node + .report_trust_event( + source, + TrustEvent::ApplicationFailure(REPLICATION_TRUST_WEIGHT), + ) + .await; + } + } +} + +/// Turn the responder's storage read into the answer it sends back. +/// +/// The whole distinction the fetch lanes rest on is made here. A key this node does not +/// hold reads as `Ok(None)` and is answered `NotFound`. A read that fails, from an I/O +/// fault, an exhausted descriptor table, or a failed integrity check, is answered `Error`. +/// Nothing about a node giving chunks up produces the second. +fn fetch_response_for(key: XorName, read: Result>>) -> protocol::FetchResponse { + match read { + Ok(Some(data)) => protocol::FetchResponse::Success { key, data }, + Ok(None) => protocol::FetchResponse::NotFound { key }, + Err(e) => protocol::FetchResponse::Error { + key, + reason: format!("{e}"), + }, + } +} + async fn handle_fetch_request( source: &PeerId, request: &protocol::FetchRequest, - storage: &Arc, + storage: &Arc, p2p_node: &Arc, request_id: u64, rr_message_id: Option<&str>, ) -> Result<()> { - let response = match storage.get(&request.key).await { - Ok(Some(data)) => protocol::FetchResponse::Success { - key: request.key, - data, - }, - Ok(None) => protocol::FetchResponse::NotFound { key: request.key }, - Err(e) => protocol::FetchResponse::Error { - key: request.key, - reason: format!("{e}"), - }, - }; + let response = fetch_response_for(request.key, storage.get(&request.key).await); send_replication_response( source, @@ -7014,7 +7105,7 @@ struct AuditResponderCompletion { async fn handle_audit_challenge_msg( source: &PeerId, challenge: &protocol::AuditChallenge, - storage: &Arc, + storage: &Arc, p2p_node: &Arc, is_bootstrapping: bool, reply: ReplyRoute<'_>, @@ -7278,7 +7369,7 @@ async fn record_sent_replica_hints( #[allow(clippy::too_many_arguments, clippy::too_many_lines)] async fn run_neighbor_sync_round( p2p_node: &Arc, - storage: &Arc, + storage: &Arc, paid_list: &Arc, queues: &Arc>, config: &ReplicationConfig, @@ -7380,9 +7471,11 @@ async fn run_neighbor_sync_round( // same value across the batch is fine and reduces RwLock churn). Atomically // snapshot + mark-gossiped so we stay answerable for exactly what we emit // (ADR-0002 retention), with no TOCTOU vs a concurrent retire/rotate. - let my_commitment = commitment_state - .current_for_gossip() - .map(|b| b.commitment().clone()); + let gossiped = commitment_state.current_for_gossip(); + // The hash actually put on the wire, captured with the payload. A rotation later in + // the round must not let a reply be credited to a root the peer never saw. + let gossiped_hash = gossiped.as_ref().map(|b| b.hash()); + let my_commitment = gossiped.map(|b| b.commitment().clone()); let mut hints_by_peer = neighbor_sync::build_sync_hints_for_peers( &batch, @@ -7408,6 +7501,13 @@ async fn run_neighbor_sync_round( .await; if let Some(outcome) = outcome { + // The peer answered, so the request that carried our commitment root arrived. + // That is proof of delivery rather than proof of emission, and the storage + // migration will not let a node give anything up until its close group has + // actually seen the reduced root. + if let Some(hash) = gossiped_hash { + commitment_state.note_commitment_delivered(*peer, hash); + } handle_sync_response( &self_id, peer, @@ -7462,6 +7562,14 @@ async fn run_neighbor_sync_round( .await; if let Some(outcome) = replacement_outcome { + // Same payload, same round trip, same proof: a reply can only come + // back if the request carrying the root reached this peer. Omitting it + // here made the counter under-report on any node whose primary syncs + // often fall through to a replacement, which is exactly the node most + // likely to be short of disk, and stalled its migration indefinitely. + if let Some(hash) = gossiped_hash { + commitment_state.note_commitment_delivered(replacement_peer, hash); + } handle_sync_response( &self_id, &replacement_peer, @@ -7502,7 +7610,7 @@ async fn handle_sync_response( config: &ReplicationConfig, bootstrapping: bool, bootstrap_state: &Arc>, - storage: &Arc, + storage: &Arc, paid_list: &Arc, queues: &Arc>, sync_state: &Arc>, @@ -7698,7 +7806,7 @@ async fn admit_and_queue_hints( paid_hints: &[XorName], p2p_node: &Arc, config: &ReplicationConfig, - storage: &Arc, + storage: &Arc, paid_list: &Arc, queues: &Arc>, ) -> AdmissionOutcome { @@ -7726,7 +7834,7 @@ async fn admit_and_queue_hints( fn queue_admitted_hints( source_peer: &PeerId, admitted: admission::AdmissionResult, - storage: &LmdbStorage, + storage: &ChunkStore, q: &mut ReplicationQueues, ) -> AdmissionOutcome { let mut discovered = HashSet::new(); @@ -8171,7 +8279,7 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { } // Step 5: Update queues with the evaluated outcomes. - let mut bad_singleton_hints: HashMap = HashMap::new(); + let mut bad_singleton_hints: HashMap<(PeerId, SingletonHintFault), usize> = HashMap::new(); let mut q = queues.write().await; for (key, outcome) in evaluated { let replica_hint_sources = q @@ -8232,20 +8340,38 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { } drop(q); - for (peer, bad_hint_count) in bad_singleton_hints { + for ((peer, fault), bad_hint_count) in bad_singleton_hints { let reports = bad_hint_count.min(MAX_BAD_HINT_TRUST_REPORTS_PER_PEER_PER_CYCLE); warn!( "Peer {peer} submitted {bad_hint_count} rejected or self-contradicting \ - sole-source replica hints; \ + sole-source replica hints ({fault:?}); \ reporting {reports} bounded trust failure(s)" ); for _ in 0..reports { - p2p_node - .report_trust_event( - &peer, - TrustEvent::ApplicationFailure(REPLICATION_TRUST_WEIGHT), - ) - .await; + match fault { + // A claim about a key that does not exist. Punishable whatever the + // sender's disk is doing. + SingletonHintFault::RejectedByCloseGroup => { + p2p_node + .report_trust_event( + &peer, + TrustEvent::ApplicationFailure(REPLICATION_TRUST_WEIGHT), + ) + .await; + } + // "I advertised it and no longer have it." That is the one statement a + // node short of disk cannot avoid making while it moves its chunks, so + // it goes through the release switch. + SingletonHintFault::DeniedPossession => { + config::penalise_unheld_close_group_chunk( + p2p_node, + &peer, + "replica_hint_denied_possession", + REPLICATION_TRUST_WEIGHT, + ) + .await; + } + } } } } @@ -8297,25 +8423,43 @@ fn add_replica_hint_sources(sources: &mut Vec, replica_hint_sources: &Ha } } +/// Why a sole-source replica hint is punishable. +/// +/// The two cases look alike and are not. A hint the close group rejects outright is a +/// claim about a key that does not exist, which is a bad hint however the sender's disk is +/// doing. A sender that advertised a key and then answers `Absent` for it is making a +/// statement about its own storage, and that is the one thing a node short of disk cannot +/// avoid saying while it moves its chunks out of a store that will not give the space back. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum SingletonHintFault { + /// The close group says the key does not exist. + RejectedByCloseGroup, + /// The sender advertised the key and then denied holding it. + DeniedPossession, +} + /// Return the sole replica advertiser when either the close group definitively -/// rejects the key or the advertiser explicitly denies possessing it. +/// rejects the key or the advertiser explicitly denies possessing it, and say which. /// Paid-only advertisements, corroborated replica hints, and inconclusive /// rounds without that direct contradiction are deliberately non-penalizing. fn punishable_singleton_replica_hint_source( replica_hint_sources: &HashSet, outcome: &KeyVerificationOutcome, evidence: &crate::replication::types::KeyVerificationEvidence, -) -> Option { +) -> Option<(PeerId, SingletonHintFault)> { // A paid-only advertiser leaves this set empty, so the sole-source lane is // reserved for peers that actually claimed possession. if replica_hint_sources.len() != 1 { return None; } let source = *replica_hint_sources.iter().next()?; - let rejected_by_close_group = matches!(outcome, KeyVerificationOutcome::QuorumFailed); - let denied_possession = evidence.presence.get(&source) == Some(&PresenceEvidence::Absent); - - (rejected_by_close_group || denied_possession).then_some(source) + if matches!(outcome, KeyVerificationOutcome::QuorumFailed) { + return Some((source, SingletonHintFault::RejectedByCloseGroup)); + } + if evidence.presence.get(&source) == Some(&PresenceEvidence::Absent) { + return Some((source, SingletonHintFault::DeniedPossession)); + } + None } /// Post-verification bootstrap bookkeeping: remove terminal keys from the @@ -8449,7 +8593,7 @@ enum FetchResult { /// queue is deep enough for that window to be real. /// /// This check must also precede the capacity pre-check below, because - /// `LmdbStorage::put` tests `exists` *before* it tests disk space: without + /// `ChunkStore::put` tests `exists` *before* it tests disk space: without /// it, a full node would decline a key it already holds, which `put` would /// have accepted as a duplicate. AlreadyHeld, @@ -8573,7 +8717,7 @@ async fn is_storage_admitted( /// topology churn before the key is ever dequeued. async fn execute_single_fetch( p2p_node: Arc, - storage: Arc, + storage: Arc, config: Arc, key: XorName, source: PeerId, @@ -8592,7 +8736,7 @@ async fn execute_single_fetch( // Possession, then capacity — both before the dial, and in that order. // - // `LmdbStorage::put` tests `exists` before it tests disk space, so a full + // `ChunkStore::put` tests `exists` before it tests disk space, so a full // node still accepts a key it already holds. Checking possession first is // what keeps this pair of gates from declining work `put` would have // taken. @@ -8781,43 +8925,33 @@ async fn execute_single_fetch( result: FetchResult::Stored, } } - ReplicationMessageBody::FetchResponse(protocol::FetchResponse::NotFound { - .. - }) => { - // This peer was selected as a fetch source because it - // recently answered `Present` during verification. A - // subsequent NotFound is evidence of a stale/false claim - // or chunk wiping, so penalize lightly and try another - // verified source. - warn!( - "Fetch: verified source {source} returned NotFound for {}", - hex::encode(key) - ); - p2p_node - .report_trust_event( - &source, - TrustEvent::ApplicationFailure(REPLICATION_TRUST_WEIGHT), - ) - .await; - FetchOutcome { - key, - result: FetchResult::SourceFailed, + ReplicationMessageBody::FetchResponse( + ref response @ (protocol::FetchResponse::NotFound { .. } + | protocol::FetchResponse::Error { .. }), + ) => { + // This peer was selected as a fetch source because it recently + // answered `Present` during verification, so either answer is + // evidence of something. Which one decides what it is charged: a peer + // that does not hold the chunk is the lane this release withholds, a + // peer whose own read failed is not. + if let protocol::FetchResponse::Error { reason, .. } = response { + warn!( + "Fetch: peer {source} returned error for {}: {reason}", + hex::encode(key) + ); + } else { + warn!( + "Fetch: verified source {source} returned NotFound for {}", + hex::encode(key) + ); + } + if let Some(fault) = fetch_fault_for(response) { + let lane = match fault { + FetchFault::UnheldChunk => "fetch_not_found", + FetchFault::ResponderFault => "fetch_error", + }; + charge_fetch_fault(&p2p_node, &source, fault, lane).await; } - } - ReplicationMessageBody::FetchResponse(protocol::FetchResponse::Error { - reason, - .. - }) => { - warn!( - "Fetch: peer {source} returned error for {}: {reason}", - hex::encode(key) - ); - p2p_node - .report_trust_event( - &source, - TrustEvent::ApplicationFailure(REPLICATION_TRUST_WEIGHT), - ) - .await; FetchOutcome { key, result: FetchResult::SourceFailed, @@ -8905,6 +9039,11 @@ async fn handle_subtree_failed_audit( let mut provers_guard = recent_provers.write().await; apply_audit_failure_credit_revocation(&mut provers_guard, challenged_peer, reason); } + // Deliberately NOT routed through the release switch. This is the commitment-bound + // subtree audit: the peer published a signed claim to hold these keys and could not + // answer for them. That contract is enforced in every release, including the one that + // withholds the penalty for merely not holding a close-group chunk, because the whole + // migration depends on a node's reduced commitment still meaning something. p2p_node .report_trust_event( challenged_peer, @@ -9097,12 +9236,13 @@ async fn handle_audit_result( } else { debug!("Audit timeout for {challenged_peer}; retaining active bootstrap claim"); } - p2p_node - .report_trust_event( - challenged_peer, - TrustEvent::ApplicationFailure(config::AUDIT_FAILURE_TRUST_WEIGHT), - ) - .await; + config::penalise_unheld_close_group_chunk( + p2p_node, + challenged_peer, + crate::replication::audit_metrics::AuditType::ResponsibleChunk.as_str(), + config::AUDIT_FAILURE_TRUST_WEIGHT, + ) + .await; } } AuditTickResult::BootstrapClaim { peer } => { @@ -9750,14 +9890,19 @@ async fn write_retention_atomic(path: &Path, bytes: Vec) -> bool { /// rotate. The auditor side handles "no commitment for this peer" by /// falling back to the legacy plain-digest audit path. async fn rebuild_and_rotate_commitment( - storage: &Arc, + storage: &Arc, identity: &Arc, state: &Arc, p2p: &Arc, config: &Arc, ) -> Result<()> { + // Not `all_keys()`. While the node is bridging off the legacy store these are the + // same thing, but once it has settled on what it can hold this narrows to the + // file-backed set, which is what stops it claiming keys it is about to give up. It is + // also what lets `is_held` eventually go false for those keys, which is the gate on + // removing the legacy environment at all. let stored_keys = storage - .all_keys() + .committable_keys() .await .map_err(|e| Error::Storage(format!("commitment build: read keys: {e}")))?; @@ -9790,6 +9935,7 @@ async fn rebuild_and_rotate_commitment( debug!("Commitment rotation: storage empty, clearing retained slots"); state.clear_all(); } + storage.note_commitment_rebuilt(); return Ok(()); } // Bytes are still on disk but no key is currently in range. We must NOT @@ -9809,6 +9955,7 @@ async fn rebuild_and_rotate_commitment( (stays answerable until its gossip TTL lapses, bytes still on disk)" ); state.retire_current(); + storage.note_commitment_rebuilt(); return Ok(()); } @@ -9875,6 +10022,9 @@ async fn rebuild_and_rotate_commitment( // committed key set is frozen here for many rotations. Without this, // the no-op guard would pin a stale slot — and its key — forever. state.age_out(); + // The advertised commitment already equals the committable set, which is + // exactly what the retirement gate is counting. + storage.note_commitment_rebuilt(); return Ok(()); } } @@ -9897,12 +10047,88 @@ async fn rebuild_and_rotate_commitment( let key_count = built.commitment().key_count; state.rotate(built); info!("Storage commitment rotated: hash={hash} key_count={key_count}"); + // Counted only on the paths where the advertised commitment now genuinely reflects + // the committable set, never merely on having read it. The retirement gate is what + // consumes this, and it authorises deleting the legacy store. + storage.note_commitment_rebuilt(); Ok(()) } #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { + + /// The two fetch failures mean different things and must be charged differently. + /// + /// `NotFound` is a peer saying it does not hold the chunk, which is what a node + /// part-way through the migration says about chunks it has legitimately given up, so + /// it is the lane this release withholds. `Error` has a single producer, the + /// responder's own storage read failing, and that is never about the migration. + #[test] + fn a_missing_chunk_and_a_failed_read_are_different_faults() { + let key = [7u8; 32]; + assert_eq!( + fetch_fault_for(&protocol::FetchResponse::NotFound { key }), + Some(FetchFault::UnheldChunk) + ); + assert_eq!( + fetch_fault_for(&protocol::FetchResponse::Error { + key, + reason: "read failed".to_string(), + }), + Some(FetchFault::ResponderFault) + ); + assert_eq!( + fetch_fault_for(&protocol::FetchResponse::Success { + key, + data: vec![1, 2, 3], + }), + None + ); + } + + /// The responder's answer says which fault it is, so the mapping from a storage read + /// to a response is what the classification above rests on. + /// + /// A key the peer does not hold reads as `Ok(None)`. A read that fails, whether from + /// an I/O fault or a failed integrity check, reads as `Err`. Nothing in the migration + /// turns the first into the second. + #[tokio::test] + async fn a_missing_key_reads_as_a_plain_miss_and_a_failed_read_as_a_fault() { + let dir = tempfile::tempdir().expect("temp dir"); + let storage = crate::storage::LmdbStorage::new(crate::storage::LmdbStorageConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + max_map_size: 0, + disk_reserve: 0, + }) + .await + .expect("open store"); + + let absent = [9u8; 32]; + assert!( + matches!(storage.get(&absent).await, Ok(None)), + "a chunk this node does not hold must read as a plain miss, not a fault" + ); + + // And the answer each read produces. A miss is `NotFound`, which is the withheld + // lane; a failed read is `Error`, which is not. + assert!(matches!( + fetch_response_for(absent, Ok(None)), + protocol::FetchResponse::NotFound { .. } + )); + assert!(matches!( + fetch_response_for(absent, Ok(Some(vec![1, 2, 3]))), + protocol::FetchResponse::Success { .. } + )); + assert!(matches!( + fetch_response_for( + absent, + Err(crate::error::Error::Storage("read failed".into())) + ), + protocol::FetchResponse::Error { .. } + )); + } use super::*; use super::{ apply_audit_failure_credit_revocation, audit_failure_clears_bootstrap_claim, @@ -10364,7 +10590,9 @@ mod tests { assert_eq!( punishable_singleton_replica_hint_source(&HashSet::from([source]), &failed, &evidence), - Some(source) + Some((source, SingletonHintFault::RejectedByCloseGroup)), + "a close-group rejection outranks the denial: the key does not exist, which is \ + a bad hint however the sender's own disk is doing" ); assert_eq!( punishable_singleton_replica_hint_source( @@ -10387,7 +10615,7 @@ mod tests { .insert(source, PresenceEvidence::Unresolved); assert_eq!( punishable_singleton_replica_hint_source(&HashSet::from([source]), &failed, &evidence), - Some(source), + Some((source, SingletonHintFault::RejectedByCloseGroup)), "definitive close-group rejection is punishable without direct contradiction" ); assert_eq!( @@ -10409,8 +10637,10 @@ mod tests { }, &evidence, ), - Some(source), - "an explicit denial is punishable regardless of the overall outcome" + Some((source, SingletonHintFault::DeniedPossession)), + "an explicit denial is punishable regardless of the overall outcome, and is \ + classified separately because it is a statement about the sender's own \ + storage rather than about the key" ); } diff --git a/src/replication/neighbor_sync.rs b/src/replication/neighbor_sync.rs index 3ab9cab6..8b4e40bd 100644 --- a/src/replication/neighbor_sync.rs +++ b/src/replication/neighbor_sync.rs @@ -19,7 +19,7 @@ use crate::replication::protocol::{ NeighborSyncRequest, NeighborSyncResponse, ReplicationMessage, ReplicationMessageBody, }; use crate::replication::types::NeighborSyncState; -use crate::storage::LmdbStorage; +use crate::storage::ChunkStore; /// Hint-build duration that is worth surfacing at info level. const HINT_BUILD_SLOW_LOG_MS: u128 = 250; @@ -64,7 +64,7 @@ pub(crate) struct PeerSyncHints { /// this node is allowed to delete them. pub async fn build_replica_hints_for_peer( peer: &PeerId, - storage: &Arc, + storage: &Arc, p2p_node: &Arc, close_group_size: usize, ) -> Vec { @@ -77,7 +77,7 @@ pub async fn build_replica_hints_for_peer( pub(crate) async fn build_replica_hints_for_peer_with_close_groups( peer: &PeerId, - storage: &Arc, + storage: &Arc, p2p_node: &Arc, close_group_size: usize, ) -> Vec { @@ -107,7 +107,7 @@ pub(crate) async fn build_replica_hints_for_peer_with_close_groups( /// storage and one scan over the paid list. pub(crate) async fn build_sync_hints_for_peers( peers: &[PeerId], - storage: &Arc, + storage: &Arc, paid_list: &Arc, p2p_node: &Arc, close_group_size: usize, @@ -330,7 +330,7 @@ fn peer_on_cooldown( pub async fn sync_with_peer( peer: &PeerId, p2p_node: &Arc, - storage: &Arc, + storage: &Arc, paid_list: &Arc, config: &ReplicationConfig, is_bootstrapping: bool, @@ -355,7 +355,7 @@ pub async fn sync_with_peer( pub(crate) async fn sync_with_peer_with_outcome( peer: &PeerId, p2p_node: &Arc, - storage: &Arc, + storage: &Arc, paid_list: &Arc, config: &ReplicationConfig, is_bootstrapping: bool, @@ -485,7 +485,7 @@ pub async fn handle_sync_request( sender: &PeerId, request: &NeighborSyncRequest, p2p_node: &Arc, - storage: &Arc, + storage: &Arc, paid_list: &Arc, config: &ReplicationConfig, is_bootstrapping: bool, @@ -509,7 +509,7 @@ pub(crate) async fn handle_sync_request_with_proofs( sender: &PeerId, _request: &NeighborSyncRequest, p2p_node: &Arc, - storage: &Arc, + storage: &Arc, paid_list: &Arc, config: &ReplicationConfig, is_bootstrapping: bool, diff --git a/src/replication/paid_list.rs b/src/replication/paid_list.rs index f65172c1..62483028 100644 --- a/src/replication/paid_list.rs +++ b/src/replication/paid_list.rs @@ -60,7 +60,7 @@ pub struct PaidList { paid_prune_cursor: RwLock, /// Tracks every paid-list LMDB blocking task. /// - /// Same rationale as `LmdbStorage::blocking_tracker`: a `spawn_blocking` + /// Same rationale as `ChunkStore::blocking_tracker`: a `spawn_blocking` /// closure owns a cloned [`Env`] and keeps running when its async awaiter /// is dropped, so [`Self::wait_idle`] waits on the blocking tasks /// themselves before the environment may be reopened. diff --git a/src/replication/possession.rs b/src/replication/possession.rs index 72c4e969..cc552f2b 100644 --- a/src/replication/possession.rs +++ b/src/replication/possession.rs @@ -41,7 +41,7 @@ use crate::replication::protocol::{ ReplicationMessageBody, ABSENT_KEY_DIGEST, }; use crate::replication::types::{BootstrapClaimObservation, NeighborSyncState}; -use crate::storage::LmdbStorage; +use crate::storage::ChunkStore; use super::REPLICATION_TRUST_WEIGHT; @@ -137,7 +137,7 @@ pub(crate) async fn run_possession_check( key: XorName, peers: Vec, p2p_node: &Arc, - storage: &Arc, + storage: &Arc, config: &ReplicationConfig, sync_state: &Arc>, audit_challenge_coordinator: &Arc, @@ -225,15 +225,16 @@ async fn report_possession_confirmed_failure( peer = %peer, key = %key_hex, trust_weight = AUDIT_FAILURE_TRUST_WEIGHT, - "Possession check: {peer} failed to prove possession for {key_hex} ({}); penalising at audit severity", + "Possession check: {peer} failed to prove possession for {key_hex} ({}); recorded at audit severity", failure_reason.as_str() ); - p2p_node - .report_trust_event( - peer, - TrustEvent::ApplicationFailure(AUDIT_FAILURE_TRUST_WEIGHT), - ) - .await; + crate::replication::config::penalise_unheld_close_group_chunk( + p2p_node, + peer, + AuditType::Possession.as_str(), + AUDIT_FAILURE_TRUST_WEIGHT, + ) + .await; } async fn report_possession_audit_failure( @@ -248,15 +249,16 @@ async fn report_possession_audit_failure( peer = %peer, key = %key_hex, trust_weight = AUDIT_FAILURE_TRUST_WEIGHT, - "Possession check: {peer} {} for {key_hex}; penalising at audit severity", + "Possession check: {peer} {} for {key_hex}; recorded at audit severity", failure_class.as_str() ); - p2p_node - .report_trust_event( - peer, - TrustEvent::ApplicationFailure(AUDIT_FAILURE_TRUST_WEIGHT), - ) - .await; + crate::replication::config::penalise_unheld_close_group_chunk( + p2p_node, + peer, + AuditType::Possession.as_str(), + AUDIT_FAILURE_TRUST_WEIGHT, + ) + .await; } async fn handle_possession_bootstrap_claim( diff --git a/src/replication/pruning.rs b/src/replication/pruning.rs index 10acee66..9d213126 100644 --- a/src/replication/pruning.rs +++ b/src/replication/pruning.rs @@ -73,7 +73,7 @@ use crate::replication::types::{ BootstrapClaimObservation, KeyVerificationEvidence, NeighborSyncState, PaidListEvidence, RepairProofs, }; -use crate::storage::LmdbStorage; +use crate::storage::ChunkStore; // `RepairProofs` remains in the prune-pass context only so records deleted by // pruning also drop their (audit-path) repair-proof entries; it plays no part @@ -136,7 +136,7 @@ pub struct PrunePassContext<'a> { /// Local peer id. pub self_id: &'a PeerId, /// Local record storage. - pub storage: &'a Arc, + pub storage: &'a Arc, /// Persistent paid-list state. pub paid_list: &'a Arc, /// P2P node used for routing lookups and prune-confirmation audits. @@ -341,7 +341,7 @@ struct PruneAuditReportState { #[derive(Clone, Copy)] struct PruneAuditContext<'a> { - storage: &'a Arc, + storage: &'a Arc, p2p_node: &'a Arc, config: &'a ReplicationConfig, sync_state: &'a Arc>, @@ -1168,7 +1168,7 @@ async fn advance_prune_cursor( async fn delete_stored_records( keys_to_delete: &[XorName], - storage: &Arc, + storage: &Arc, paid_list: &Arc, repair_proofs: &Arc>, ) -> usize { @@ -1205,7 +1205,7 @@ async fn delete_stored_records( async fn collect_record_prune_proofs( candidates: &[RecordPruneCandidate], local_stored_key_count: usize, - storage: &Arc, + storage: &Arc, p2p_node: &Arc, config: &ReplicationConfig, sync_state: &Arc>, @@ -1241,6 +1241,53 @@ async fn collect_record_prune_proofs( present_by_key } +/// Prove that other nodes actually hold `keys`, by cryptographic challenge. +/// +/// Exposed for the storage migration, which has to answer the same question the pruner +/// answers before it deletes: is this chunk somewhere else? It deliberately reuses this +/// path rather than the cheaper `VerificationRequest`, because that one carries a +/// self-reported `present: bool` and a node that has silently lost a chunk will still say +/// yes. Here the peer has to return `compute_audit_digest(nonce, peer, key, bytes)` over a +/// nonce it has never seen, which it cannot do without the bytes. +/// +/// Returns, per key, the set of peers that proved possession. The caller decides how many +/// are enough; [`prune_proofs_needed`] is the rule the pruner uses. +pub(crate) async fn prove_peers_hold_records( + keys_by_peer: &HashMap>, + local_stored_key_count: usize, + storage: &Arc, + p2p_node: &Arc, + config: &ReplicationConfig, + sync_state: &Arc>, + audit_challenge_coordinator: &Arc, +) -> HashMap> { + if keys_by_peer.is_empty() { + return HashMap::new(); + } + let candidates: Vec = { + let mut by_key: HashMap> = HashMap::new(); + for (peer, keys) in keys_by_peer { + for key in keys { + by_key.entry(*key).or_default().push(*peer); + } + } + by_key + .into_iter() + .map(|(key, target_peers)| RecordPruneCandidate { key, target_peers }) + .collect() + }; + collect_record_prune_proofs( + &candidates, + local_stored_key_count, + storage, + p2p_node, + config, + sync_state, + audit_challenge_coordinator, + ) + .await +} + async fn revalidated_fast_prune_keys( candidates: &[FastPruneCandidate], ctx: &PrunePassContext<'_>, @@ -1289,7 +1336,7 @@ async fn revalidated_fast_prune_keys( (keys_to_delete, cleared) } -async fn stored_record_still_exists(key: &XorName, storage: &Arc) -> bool { +async fn stored_record_still_exists(key: &XorName, storage: &Arc) -> bool { match storage.get_raw(key).await { Ok(Some(_)) => true, Ok(None) => false, @@ -1501,7 +1548,7 @@ fn confirmed_keys_from_presence( /// from vetoing deletion forever without accepting under-replication. /// Groups of one or two peers require every proof: tolerating a miss there /// would allow deletion on a single attestation. -fn prune_proofs_needed(group_size: usize) -> usize { +pub(crate) fn prune_proofs_needed(group_size: usize) -> usize { if group_size <= 2 { group_size } else { @@ -1513,7 +1560,7 @@ fn prune_proofs_needed(group_size: usize) -> usize { /// /// `proofs_needed == 0` means confirmation is impossible (no targets), not /// trivially met. -fn target_peers_reported_present( +pub(crate) fn target_peers_reported_present( key: &XorName, target_peers: &[PeerId], present_by_key: &HashMap>, @@ -1915,14 +1962,14 @@ async fn local_record_digest( peer: &PeerId, key: &XorName, nonce: &[u8; 32], - storage: &Arc, + storage: &Arc, ) -> Option<[u8; 32]> { local_record_bytes(key, storage) .await .map(|bytes| compute_audit_digest(nonce, peer.as_bytes(), key, &bytes)) } -async fn local_record_bytes(key: &XorName, storage: &Arc) -> Option> { +async fn local_record_bytes(key: &XorName, storage: &Arc) -> Option> { match storage.get_raw(key).await { Ok(Some(bytes)) => Some(bytes), Ok(None) => { @@ -1980,12 +2027,13 @@ async fn report_prune_audit_failure_once( "Prune audit failure: peer={peer}, audit_failure_class={audit_failure_class}, key={}", hex::encode(key) ); - p2p_node - .report_trust_event( - peer, - saorsa_core::TrustEvent::ApplicationFailure(AUDIT_FAILURE_TRUST_WEIGHT), - ) - .await; + crate::replication::config::penalise_unheld_close_group_chunk( + p2p_node, + peer, + AuditType::Prune.as_str(), + AUDIT_FAILURE_TRUST_WEIGHT, + ) + .await; true } diff --git a/src/replication/storage_commitment_audit.rs b/src/replication/storage_commitment_audit.rs index 481272a0..f99a4e70 100644 --- a/src/replication/storage_commitment_audit.rs +++ b/src/replication/storage_commitment_audit.rs @@ -33,7 +33,7 @@ use crate::replication::subtree::{ select_subtree_path, subtree_plan, verify_subtree_proof, StructureVerdict, SubtreeProof, }; use crate::replication::types::{AuditFailureReason, AuditFailureSummary, FailureEvidence}; -use crate::storage::LmdbStorage; +use crate::storage::ChunkStore; use saorsa_core::identity::PeerId; use saorsa_core::P2PNode; use tokio::sync::RwLock; @@ -79,7 +79,7 @@ const AUDIT_READ_RETRY_BACKOFF: Duration = Duration::from_millis(200); /// an `Err` (transient IO) is. A persistent `Err` is returned so the caller emits /// `RejectKind::Transient` (timeout lane). async fn get_raw_retrying( - storage: &LmdbStorage, + storage: &ChunkStore, key: &XorName, ) -> crate::error::Result>> { let mut attempt = 1u32; @@ -1230,7 +1230,7 @@ fn subtree_failure_summary(reason: &AuditFailureReason) -> AuditFailureSummary { /// grace removed, the auditor treats as a confirmed failure for an in-window pin). pub async fn handle_subtree_challenge( challenge: &SubtreeAuditChallenge, - storage: &LmdbStorage, + storage: &ChunkStore, self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, @@ -1267,7 +1267,7 @@ pub struct Round1Work { /// it performed so the caller can charge it on every exit path. pub async fn handle_subtree_challenge_measured( challenge: &SubtreeAuditChallenge, - storage: &LmdbStorage, + storage: &ChunkStore, self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, @@ -1297,7 +1297,7 @@ pub async fn handle_subtree_challenge_measured( #[allow(clippy::too_many_lines)] async fn subtree_challenge_response( challenge: &SubtreeAuditChallenge, - storage: &LmdbStorage, + storage: &ChunkStore, self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, @@ -1547,7 +1547,7 @@ fn build_slice_items_for_key( /// an answer against. pub async fn handle_subtree_slice_challenge( challenge: &SubtreeSliceChallenge, - storage: &LmdbStorage, + storage: &ChunkStore, self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, @@ -1713,7 +1713,7 @@ enum KeyServe { /// `indices` is already deduplicated by the caller. async fn serve_committed_key_openings( challenge: &SubtreeSliceChallenge, - storage: &LmdbStorage, + storage: &ChunkStore, key: XorName, indices: Vec, ) -> KeyServe { diff --git a/src/storage/chunk_store.rs b/src/storage/chunk_store.rs new file mode 100644 index 00000000..fd4c7435 --- /dev/null +++ b/src/storage/chunk_store.rs @@ -0,0 +1,4838 @@ +//! The node's chunk store: a file store, plus the legacy LMDB environment for as long +//! as one still exists on disk. +//! +//! Every caller in the node talks to this type and sees **one** key set. That is the +//! detail that keeps quoting, commitments, hints, audits and pruning coherent while a +//! chunk moves from LMDB to a file: the backing changes, the logical key set does not. +//! +//! There is one deliberate asymmetry, and it is the whole safety argument of the +//! migration. Serving reads the **union**, so the node answers for everything it ever +//! committed to. The commitment builder reads only the **file-backed** set once the node +//! has settled on what it will keep, so the node stops claiming keys it is about to give +//! up. Between those two, a node is at worst over-honest: it serves more than it claims. + +use crate::ant_protocol::XorName; +use crate::error::{Error, Result}; +use crate::logging::{debug, error, info, warn}; +use crate::storage::file_store::{FileStore, FileStoreConfig}; +use crate::storage::lmdb::{LmdbStorage, LmdbStorageConfig}; +use crate::storage::migration::{ + CopyReport, MigrationConfig, MigrationPhase, MigrationState, REQUIRED_REBUILDS_BEFORE_RETIRE, +}; +use crate::storage::StorageStats; +use std::collections::{BTreeMap, BTreeSet}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; +use tokio_util::sync::CancellationToken; + +/// Directory name of the legacy LMDB environment, under the node root. +pub const LEGACY_ENV_DIR: &str = "chunks.mdb"; + +/// Suffix for a legacy environment that has been retired but not yet deleted. +pub const RETIRED_SUFFIX: &str = ".retired"; + +/// Written inside a chunk environment directory once it has been retired. +/// +/// The rename that moves the environment aside cannot be shown to be durable off Unix: +/// there is no way to flush a directory through the standard library, and `MoveFileEx` is +/// not documented as durable at return without a flag std does not use. So a power loss +/// can bring the directory back under its old name with its contents already deleted, and +/// a node that tried to open that would fail to start. +/// +/// This file is what makes that unambiguous, and it is *inside* the directory rather than +/// beside it so that it travels with it: a directory that reverts to its old name reverts +/// carrying its own evidence. It is created with the same create-and-flush that publishes +/// a chunk, which is documented as durable everywhere, and only after the rename has +/// already succeeded. So a directory holding it has been retired, whatever it is called, +/// and one that does not is a live environment and is opened normally. +/// +/// Deliberately not a file beside the environment. A marker that can outlive the thing it +/// describes has to be cancelled, cancellation can fail or be lost, and a stale one would +/// authorise deleting an environment that had since taken a chunk. +const RETIRED_MARKER: &str = "RETIRED"; + +/// How many times the background reaper retries deleting a retired directory. +/// +/// Generous, because giving up strands the disk until the next restart and the thread +/// costs nothing while it sleeps. With the backoff below this keeps trying for about a +/// day. +const RETIRED_DELETE_ATTEMPTS: u32 = 60; + +/// Base wait between those attempts, multiplied by the attempt number up to the cap. +const RETIRED_DELETE_BACKOFF: Duration = Duration::from_secs(10); + +/// The longest the reaper waits between attempts. +const RETIRED_DELETE_BACKOFF_MAX: Duration = Duration::from_secs(30 * 60); + +/// How many retired directories may be waiting to be deleted before the node stops +/// finding new names for them. Far more than a node should ever accumulate. +const MAX_TOMBSTONES: u32 = 64; + +/// The legacy environment's data file. Its presence is what says a node still has one. +const LEGACY_DATA_FILE: &str = "data.mdb"; + +/// How many times retirement retries taking sole ownership of the legacy handle before +/// giving up for this tick. +const RETIRE_UNWRAP_ATTEMPTS: u32 = 20; + +/// How long to wait between those attempts. +const RETIRE_UNWRAP_BACKOFF: Duration = Duration::from_millis(100); + +/// How many chunks the verification pass checks between progress lines. +const VERIFY_LOG_EVERY: u64 = 2000; + +/// How many per-key critical sections the facade keeps. +/// +/// Keyed on the address's LAST byte, for the same reason the shard directories are: a +/// node's keys share their leading bytes, so lanes keyed on the first byte would all +/// collapse into one. +const KEY_LOCK_LANES: usize = 256; + +/// Configuration for [`ChunkStore`]. +#[derive(Debug, Clone)] +pub struct ChunkStoreConfig { + /// Node root directory. + pub root_dir: PathBuf, + /// Verify `BLAKE3(content) == address` on read. + pub verify_on_read: bool, + /// Explicit LMDB map size cap in bytes, used only while a legacy environment exists. + /// + /// Dies with LMDB. Kept so an operator's existing `storage.db_size_gb` still means + /// what it meant during the bridge. + pub max_map_size: usize, + /// Minimum free disk space to preserve on the storage partition. + pub disk_reserve: u64, + /// Migration controls. + pub migration: MigrationConfig, +} + +impl Default for ChunkStoreConfig { + fn default() -> Self { + Self { + root_dir: PathBuf::from(".ant/chunks"), + verify_on_read: true, + max_map_size: 0, + disk_reserve: crate::storage::DEFAULT_DISK_RESERVE, + migration: MigrationConfig::default(), + } + } +} + +impl ChunkStoreConfig { + /// A test-friendly default with the disk reserve disabled, so unit tests do not + /// depend on the host having spare gigabytes. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn test_default() -> Self { + Self { + disk_reserve: 0, + ..Self::default() + } + } +} + +/// The legacy environment and the keys only it still holds. +#[derive(Clone)] +struct Legacy { + /// The LMDB handle. + lmdb: Arc, + /// Keys in the legacy environment that are **not** in the file store. + /// + /// Kept in memory so the union view costs nothing on the hot paths: `exists` and + /// `current_chunks` never touch LMDB, and `all_keys` merges two already-sorted + /// sequences. It is derived at open (LMDB keys minus file keys) and maintained by + /// every write, copy and delete. + only: Arc>>, + /// Writes that have started and whose outcome is not yet known. + /// + /// A write into the legacy environment runs on a blocking thread that outlives the + /// future waiting for it, so a shutdown can leave the environment holding a chunk + /// while nothing ran to record it. A key in neither view is what retirement destroys, + /// so every write announces itself here first. + /// + /// Deliberately NOT part of what the node says it holds. This is a note to itself + /// that something is in flight, not a claim: `exists`, `all_keys`, the commitment, the + /// quote count and the pruner all ignore it. It vetoes retirement, and the driver + /// resolves each entry against what is actually on disk. + /// + /// How many writes were made without a rollback copy of the chunk in the environment. + /// + /// The rollback copy is best effort by design, and the ADR says so: a bridging node + /// whose environment has no reusable page keeps serving from files and simply has no + /// second copy to roll back to. What was missing was any way to ask how often that + /// happened. One `warn!` per chunk is not an answer to "how many nodes on this fleet + /// are actually keeping a rollback copy", which is the question the second release + /// turns on, and on a node with no free pages it is also a line per chunk forever. + /// + /// Writes, not distinct chunks: two attempts at one address count twice, and a later + /// attempt that succeeds does not count back down. Counted before the file half runs, + /// so a write that then fails outright is counted too. Keeping a set of addresses + /// instead would be exact and would also mean holding millions of them in memory to + /// answer a question that a rate answers. Read it as "this node is failing to keep + /// rollback copies, this often", not as a chunk count. + skipped_rollback_copies: Arc, + + /// Counted, not a set, for the reason the file store's `writing` map is counted. + /// Cancellation can release the facade's key lane while the blocking half survives, so + /// a second write for the same key can start behind the first. With one entry between + /// them, whichever returned first would clear it while the other was still queued, and + /// a delete arriving in that window would see no announcement, skip draining the + /// environment, and let the surviving write land afterwards and put the key back. + pending: Arc>>, +} + +impl Legacy { + /// Note that a chunk went to files alone, and say whether to log it. + /// + /// Throttled by powers of ten. The condition is usually all-or-nothing, so the first + /// few lines say it started and the later ones say it is still going without becoming + /// the log. + fn note_skipped_rollback_copy(&self) -> Option { + let count = self + .skipped_rollback_copies + .fetch_add(1, std::sync::atomic::Ordering::Relaxed) + .saturating_add(1); + let round = matches!( + count, + 10 | 100 | 1_000 | 10_000 | 100_000 | 1_000_000 | 10_000_000 + ); + (count <= 3 || round).then_some(count) + } + + /// Announce a write into the environment, or note a second one for the same key. + fn announce(&self, address: &XorName) { + *self.pending.write().entry(*address).or_insert(0) += 1; + } + + /// Retire one announcement, leaving any other for the same key still standing. + fn announced_write_finished(&self, address: &XorName) { + let mut pending = self.pending.write(); + let Some(count) = pending.get_mut(address) else { + return; + }; + *count = count.saturating_sub(1); + if *count == 0 { + pending.remove(address); + } + } +} + +/// Content-addressed chunk storage. +pub struct ChunkStore { + /// The file store. Always present, always the write target. + files: Arc, + /// The legacy environment, until it is retired. + legacy: parking_lot::RwLock>, + /// Excludes retirement while any operation that touches the legacy environment runs. + /// + /// Reads, writes and deletes take it shared for their whole duration; retirement takes + /// it exclusively before it takes the environment away. Sole ownership of the handle + /// is not enough on its own: a read that has decided the file store cannot answer, and + /// has not yet taken a legacy handle, holds nothing and would be invisible to that + /// check. Nor would holding a handle throughout do instead, because on a busy node + /// there would always be one and retirement would never see the environment + /// unreferenced. Retirement also waits for the environment to go idle, which never + /// happens if new work can keep starting in it. + /// + /// A shared/exclusive lock states the actual requirement, and because it is fair, a + /// waiting retirement stops new work starting rather than starving behind it. + retirement: tokio::sync::RwLock<()>, + /// Where the legacy environment lives. + legacy_env_dir: PathBuf, + /// Store configuration. + config: ChunkStoreConfig, + /// The persisted migration marker. + state: parking_lot::RwLock, + /// One lock per shard, held across a whole logical key transition. + /// + /// The file store has its own lane locks, but those only make a single file write + /// atomic. The races that matter here span two stores and an await point: the copier + /// reads a chunk out of LMDB, the pruner deletes that chunk from both stores, and + /// then the copier's write lands and resurrects it. One critical section per key, + /// held across put, delete and copy, is what closes that. + key_locks: Vec>, +} + +impl ChunkStore { + /// Open the store under `config.root_dir`. + /// + /// Opens the legacy environment only if one is already on disk. A fresh node never + /// creates one, so it never pays for a memory map it will not use. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if either store cannot be opened. + pub async fn new(config: ChunkStoreConfig) -> Result { + let files = Arc::new( + FileStore::new(FileStoreConfig { + root_dir: config.root_dir.clone(), + verify_on_read: config.verify_on_read, + disk_reserve: config.disk_reserve, + }) + .await?, + ); + + // Before anything looks at the legacy environment: a directory carrying its own + // retirement mark is the remains of a removal a power loss interrupted, and is + // moved aside rather than opened. + let openable = finish_interrupted_retirement(&config.root_dir); + let legacy_env_dir = config.root_dir.join(LEGACY_ENV_DIR); + let legacy = + if openable == LiveEnvironment::WhateverIsOnDisk && legacy_present(&config.root_dir)? { + Some(Self::open_legacy(&config, &files).await?) + } else { + None + }; + + let phase = if legacy.is_some() { + MigrationPhase::Bridging + } else { + MigrationPhase::FilesOnly + }; + let mut state = MigrationState::load_or_create(&config.root_dir, phase); + + // The filesystem is the authority on whether a legacy environment exists; the + // marker only records decisions. Reconcile rather than trust. + if legacy.is_none() && state.phase != MigrationPhase::FilesOnly { + info!("No legacy chunk environment on disk; the migration is already complete"); + state.phase = MigrationPhase::FilesOnly; + if let Err(e) = state.save(&config.root_dir) { + warn!("Could not persist the migration marker: {e}"); + } + } else if legacy.is_some() + && state.phase == MigrationPhase::Committed + && files.current_chunks().unwrap_or(0) < state.kept_key_count + { + // The marker says this node already settled on what it would keep, but the + // file store holds less than it recorded keeping. Something outside the node + // changed the data directory, and trusting the marker here would skip the + // copier, the shed rules and their rank checks on the way to deleting the + // legacy environment. The filesystem wins. + warn!( + "The migration marker says this node kept {} chunk(s) but the file store \ + holds {}. Restarting the migration from the copying stage.", + state.kept_key_count, + files.current_chunks().unwrap_or(0) + ); + state.phase = MigrationPhase::Bridging; + state.committed_at_unix = None; + state.rebuilds_since_commit = 0; + if let Err(e) = state.save(&config.root_dir) { + warn!("Could not persist the migration marker: {e}"); + } + } else if legacy.is_some() && state.phase == MigrationPhase::FilesOnly { + warn!( + "The migration marker says this node is done but {} is still on disk. \ + Resuming the bridge.", + legacy_env_dir.display() + ); + state.phase = MigrationPhase::Bridging; + if let Err(e) = state.save(&config.root_dir) { + warn!("Could not persist the migration marker: {e}"); + } + } + + let store = Self { + files, + legacy: parking_lot::RwLock::new(legacy), + retirement: tokio::sync::RwLock::new(()), + legacy_env_dir, + config, + state: parking_lot::RwLock::new(state), + key_locks: std::iter::repeat_with(|| tokio::sync::Mutex::new(())) + .take(KEY_LOCK_LANES) + .collect(), + }; + + let (file_keys, legacy_keys) = store.split_counts(); + info!( + "Chunk store ready: {file_keys} chunks in files, {legacy_keys} still only in the \ + legacy environment, phase {:?}", + store.migration_phase() + ); + Ok(store) + } + + /// Open the legacy environment and work out which keys only it holds. + async fn open_legacy(config: &ChunkStoreConfig, files: &FileStore) -> Result { + let (lmdb, legacy_keys) = Self::open_legacy_env(config).await?; + Ok(Legacy { + lmdb, + only: Arc::new(parking_lot::RwLock::new(Self::keys_only_in_legacy( + &legacy_keys, + files, + ))), + skipped_rollback_copies: Arc::new(std::sync::atomic::AtomicU64::new(0)), + pending: Arc::new(parking_lot::RwLock::new(BTreeMap::new())), + }) + } + + /// Open the legacy environment and read every key in it. + /// + /// Split from the diff against the file store because the two want different timing: + /// this is slow and safe to do at any moment, the diff has to be the last thing before + /// the handle is installed. + async fn open_legacy_env( + config: &ChunkStoreConfig, + ) -> Result<(Arc, Vec)> { + let lmdb = Arc::new( + LmdbStorage::new(LmdbStorageConfig { + root_dir: config.root_dir.clone(), + verify_on_read: config.verify_on_read, + max_map_size: config.max_map_size, + disk_reserve: config.disk_reserve, + }) + .await?, + ); + // From here it never grows. Two stores on one disk each measure the same free + // space and neither knows what the other is about to spend, so a chunk written to + // both can be admitted twice against one lot of headroom and the pair can cross + // the reserve together. Pinned, this one writes only from pages it already has, + // so the file store's accounting is the only claim on free disk. + // + // What it costs is that the rollback copy is made only when the environment has + // room of its own. That is the right way round: the copy exists to make a fleet + // rollback survivable, not to be the write that has to succeed, and an + // environment this migration exists to delete should not be taking new disk to + // hold a second copy of something the file store already has. + lmdb.pin_growth().await?; + let legacy_keys = lmdb.all_keys().await?; + Ok((lmdb, legacy_keys)) + } + + /// Which of `legacy_keys` the file store does not have. + /// + /// In memory, no I/O: the file store answers from its index. Cheap enough to redo + /// immediately before installing a handle, which is the point. A key that lost its + /// file while the environment was being read must be in this set, or nothing will + /// look for it again and retirement will destroy the copy that is left. + fn keys_only_in_legacy(legacy_keys: &[XorName], files: &FileStore) -> BTreeSet { + legacy_keys + .iter() + .filter(|key| !files.is_indexed(key)) + .copied() + .collect() + } + + /// Take the critical section for one key. + async fn key_lock(&self, address: &XorName) -> Option> { + let lane = address.last().copied().unwrap_or(0) as usize; + match self.key_locks.get(lane) { + Some(lock) => Some(lock.lock().await), + None => None, + } + } + + /// A cheap clone of the legacy handle, or `None` once it is retired. + fn legacy(&self) -> Option { + self.legacy.read().clone() + } + + /// `(chunks in files, chunks only in the legacy environment)`. + fn split_counts(&self) -> (u64, u64) { + // Legacy first, for the reason given on `exists`: a key mid-copy is then counted + // twice for an instant rather than not at all, and over-reporting what the node + // holds is the safe direction for every caller of `current_chunks`. + let legacy = self + .legacy() + .map_or(0, |l| l.only.read().len().try_into().unwrap_or(u64::MAX)); + let files = self.files.current_chunks().unwrap_or(0); + (files, legacy) + } + + /// Store a chunk. + /// + /// While a legacy environment exists and dual-writing is on, the chunk goes there + /// **first**. A chunk uploaded during the bridge to holders that all revert to a + /// pre-migration build would otherwise be gone from every one of them, and that is + /// real client data, not a replica. + /// + /// # Returns + /// + /// `true` if the chunk was newly stored, `false` if either store already had it. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the content does not hash to `address`, the disk is + /// too full, or the write fails. + pub async fn put(&self, address: &XorName, content: &[u8]) -> Result { + // Shared, for the reason given on the field: retirement waits for the legacy + // environment to go idle, and a write that keeps starting new work in it while + // that wait runs makes the wait unbounded. It also stops a write inserting a key + // into the legacy-only set after the gates have approved the set that may go. + let _using_legacy = self.retirement.read().await; + let _lane = self.key_lock(address).await; + let legacy = self.legacy(); + let already_in_legacy = legacy + .as_ref() + .is_some_and(|l| l.only.read().contains(address)); + + let mut dual_written = false; + if let Some(ref l) = legacy { + if self.config.migration.dual_write_legacy && !already_in_legacy { + // The legacy store's own verdict, not the file store's. It accounts for + // pages it can reuse internally, which is the right question for a write + // into it and the wrong one for the file about to be written. A full + // legacy store must not fail a put the file store can serve: the copy is + // there to make a fleet rollback survivable, and losing that for one + // chunk is much better than refusing the chunk. + if l.lmdb.capacity_verdict() == crate::storage::CapacityVerdict::Full { + // Counted here as well as on the failure path below, and this is the + // one that matters: a pinned environment with no reusable page answers + // Full for every chunk, so on the node most affected this is the whole + // of the skipping and the other path never runs at all. + if let Some(count) = l.note_skipped_rollback_copy() { + warn!( + migration_event = "no_rollback_copy", + skipped = count, + "Legacy chunk environment is full; storing {} in files only. A \ + rollback to a pre-migration build would not have this chunk, \ + and {count} write(s) on this node have now gone without a \ + rollback copy.", + hex::encode(address) + ); + } else { + debug!( + "Legacy chunk environment is full; storing {} in files only.", + hex::encode(address) + ); + } + } else { + // Announced BEFORE the write, not after it. The write runs on a + // blocking thread that outlives this future: a shutdown that drops the + // caller mid-way can leave the environment holding a chunk while + // nothing here ever ran to record it, and a key in neither view is + // what retirement destroys. In the in-flight note rather than the key + // set, because until the write returns this node does not hold the + // chunk and must not say it does. + l.announce(address); + // Best effort, and only best effort. The verdict above is optimistic + // by design: LMDB can still refuse a write for fragmentation, pages + // pinned by a long read, or a copy-on-write B-tree split. Propagating + // that would let a store this node is in the middle of abandoning + // reject paid chunks the file store has ample room for, for the whole + // bridge period. The chunk's own validity is not at stake here; the + // file store checks the content address itself. + match l.lmdb.put(address, content).await { + Ok(_) => dual_written = true, + Err(e) => { + if let Some(count) = l.note_skipped_rollback_copy() { + warn!( + migration_event = "no_rollback_copy", + skipped = count, + "Could not also write {} to the legacy environment: \ + {e}. Storing it in files only. A rollback to a \ + pre-migration build would not have this chunk, and \ + {count} write(s) on this node have now gone without a \ + rollback copy.", + hex::encode(address) + ); + } + } + } + } + } + } + + let stored_in_files = match self.files.put(address, content).await { + Ok(stored) => stored, + Err(e) => { + // The bytes reached LMDB but not the file store. Record the key as + // legacy-only so the union still finds it and the copier retries later; + // without this the node would hold a chunk it could not serve. + // + // Only when the file store really does not have it. A write can fail + // because the file that is already there could not be read to check it, + // and calling that key legacy-only while the file index still names it + // puts it in both views, where it stays answerable and vetoes retirement + // for good. + // The file half failed and the legacy half did not, so the environment + // holds the only copy and the key really is legacy-only now. Promoted + // from the in-flight note to the key set, which is the one moment that + // promotion is warranted: both outcomes are known. + if let Some(ref l) = legacy { + l.announced_write_finished(address); + if dual_written && !self.files.is_indexed(address) { + l.only.write().insert(*address); + } + } + return Err(e); + } + }; + + // The file store has it, so it is not legacy-only, whether it was already there + // or this call put it there. The in-flight note goes at the same time: both + // writes have returned, so there is nothing left in flight to protect. + if let Some(ref l) = legacy { + l.only.write().remove(address); + l.announced_write_finished(address); + } + if already_in_legacy { + // Migrated for free: a hot key the copier no longer has to move. + return Ok(false); + } + Ok(stored_in_files) + } + + /// Retrieve a chunk, verifying it against its address when configured to. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] on an I/O failure, or when verification fails and no + /// intact copy is available. + pub async fn get(&self, address: &XorName) -> Result>> { + // Held for the whole read. A verifying read that finds rotted bytes throws the + // file away, and until this key is back in the legacy-only set there is a moment + // when it appears to live in neither store. Retirement waits behind this rather + // than deleting the copy the read is about to fall back on. + let _reading = self.retirement.read().await; + let fallback = self.legacy(); + match self.files.get(address).await { + Ok(Some(content)) => Ok(Some(content)), + Ok(None) => { + // The file store missed. If the legacy store answers, the key has to go + // back into the union view: the file index has just dropped it, and a key + // in neither view is skipped by the verification pass and destroyed by + // retirement. + self.serve_from_legacy(address, fallback).await + } + Err(e) => { + // Whatever went wrong with the file, the legacy environment may still + // have the bytes, and while it is there it is the point of the bridge to + // use them. A verification failure means the file was thrown away; every + // other error (a full descriptor table, an I/O fault, an oversized file) + // leaves the file in place and unreadable. Both are unservable from the + // file store, and both are worth asking the other store about. + let verification_failed = format!("{e}").contains("verification failed"); + warn!( + "Chunk {} could not be served from the file store ({e}); looking for a \ + copy in the legacy environment", + hex::encode(address) + ); + let from_legacy = self.serve_from_legacy(address, fallback).await; + match from_legacy { + Ok(Some(content)) => Ok(Some(content)), + // Nothing anywhere. Report the original failure rather than a plain + // miss, so the caller can tell the difference. The key is only + // re-queued for copying when the file really went: an unreadable file + // that is still there is not legacy-only, and calling it so is how a + // key ends up claimed through one view and servable through neither. + Ok(None) => Err(e), + Err(legacy_error) => { + if verification_failed { + Err(e) + } else { + Err(legacy_error) + } + } + } + } + } + } + + /// Serve a key the file store could not, from the legacy store, and put it back on + /// the copier's list. + /// + /// The whole sequence runs under the key's critical section, including the legacy + /// read. Reading first and locking afterwards would let a concurrent delete remove + /// both backings in between, and the key would then be re-inserted from bytes that no + /// longer exist anywhere: a phantom entry that `exists` reports and `get` never + /// satisfies. + async fn serve_from_legacy( + &self, + address: &XorName, + legacy: Option, + ) -> Result>> { + // The handle the caller took before it read the file. Not re-fetched here: the + // point of taking it early is that it has been held continuously since before the + // file could be thrown away, so retirement cannot have run in between. + let Some(legacy) = legacy else { + return Ok(None); + }; + let _lane = self.key_lock(address).await; + let Some(content) = legacy.lmdb.get(address).await? else { + return Ok(None); + }; + // Only if the file really is gone: a concurrent write or repair may have put a + // good one back while this was waiting for the lock. + if !self.files.is_indexed(address) { + legacy.only.write().insert(*address); + debug!( + "Chunk {} served from the legacy environment and re-queued for copying", + hex::encode(address) + ); + } + Ok(Some(content)) + } + + /// Retrieve raw chunk bytes without content-address verification. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] on an I/O failure. + pub async fn get_raw(&self, address: &XorName) -> Result>> { + // For the reason given on `get`: retirement must not run in the gap between the + // file going missing and this key being put back in the union view. + let _reading = self.retirement.read().await; + let fallback = self.legacy(); + let from_files = self.files.get_raw(address).await; + match from_files { + Ok(Some(content)) => return Ok(Some(content)), + Ok(None) => {} + // Same rule as `get`: while the legacy environment is there it may have the + // bytes, and this is the read that drives digest audits, possession checks + // and pruning. Answering "no digest" for a chunk the node can still produce + // is a failed audit for nothing. + Err(e) => { + let Some(legacy) = fallback else { + return Err(e); + }; + let _lane = self.key_lock(address).await; + return match legacy.lmdb.get_raw(address).await { + Ok(Some(content)) => Ok(Some(content)), + // Nothing anywhere: report the original failure, not a plain miss. + Ok(None) | Err(_) => Err(e), + }; + } + } + // Deliberately not gated on the legacy-only set. A chunk that was copied and then + // lost its file is not in that set, and the legacy environment is exactly where + // its bytes still are. An LMDB miss is cheap. Goes through the same path as + // `get`, so the key is restored to the union view rather than being served once + // and then quietly retired away. + let Some(legacy) = fallback else { + return Ok(None); + }; + let _lane = self.key_lock(address).await; + let raw = legacy.lmdb.get_raw(address).await?; + let missing_locally = !self.files.is_indexed(address); + if raw.is_some() && missing_locally { + legacy.only.write().insert(*address); + } + Ok(raw) + } + + /// Check whether a chunk is stored, in either backing. + /// + /// An in-memory lookup: no syscall, no I/O, in both phases. + /// + /// # Errors + /// + /// Never fails. The signature is kept because callers treat an error as "absent". + pub fn exists(&self, address: &XorName) -> Result { + // Legacy first, deliberately. The copier writes the file and only then drops the + // key from the legacy-only set, so a reader that checked files first could + // observe the gap between those two steps and report a chunk the node definitely + // holds as absent. In this order the same interleaving yields a harmless + // duplicate instead. + if self + .legacy() + .is_some_and(|l| l.only.read().contains(address)) + { + return Ok(true); + } + self.files.exists(address) + } + + /// Does this node already hold `address` with exactly these bytes, and if it holds a + /// damaged copy, replace it with these? + /// + /// The question a responder has to answer before turning away an offered copy. Plain + /// [`Self::exists`] answers from names alone, and a name can outlive the bytes under + /// it: off Unix a chunk is created under its final name before it is written, so a + /// crash leaves a short file that `exists` reports as a chunk, and bit rot leaves a + /// full-length one. Acknowledging a client on the strength of either throws away the + /// copy that would repair it, and nothing offers it again. + /// + /// So this reads. It is affordable because the only caller is the client-facing PUT + /// path, reached when a client offers a chunk this node already has, and because the + /// alternative is keeping a chunk this node cannot serve and being penalised for it at + /// the next audit. + /// + /// `content` must already hash to `address`; the caller checks that before this is + /// reached, and a repair from bytes that do not would be worse than the damage. + /// + /// # Errors + /// + /// Never fails. An unreadable chunk answers `false`, so the offered copy is stored + /// through the ordinary path rather than refused. + pub async fn holds_verified(&self, address: &XorName, content: &[u8]) -> bool { + // Held for the whole check, like every other operation that can reach the legacy + // environment. + let _using_legacy = self.retirement.read().await; + // And the key's own critical section, for the whole of it. Without it the pruner + // can delete both backings between the read and the answer, and the offered copy + // would be turned away for a chunk the node no longer has at all. + let _lane = self.key_lock(address).await; + + if let Some(legacy) = self.legacy() { + if legacy.only.read().contains(address) { + // Held only in the legacy environment. Not taken on trust either: the + // bytes in there can be wrong too, and the copier drops such a key from + // the union when it finds out, which would leave no copy anywhere if this + // had turned the good one away. + if matches!(legacy.lmdb.get_raw(address).await, Ok(Some(bytes)) if bytes == content) + { + return true; + } + warn!( + "Chunk {} is in the legacy environment but its bytes are wrong; \ + storing the copy just offered instead", + hex::encode(address) + ); + if self.files.put(address, content).await.is_err() { + return false; + } + legacy.only.write().remove(address); + return true; + } + } + if !self.files.is_indexed(address) { + return false; + } + // No cheap length pre-check. `metadata` failing is not the same as a length that + // does not match, and off Unix replacing a chunk truncates it in place, so acting + // on an unanswered question would empty a healthy sole copy. The read below + // distinguishes them. + match self.files.get_raw(address).await { + // Byte-for-byte what the caller has, and the caller checked those bytes + // against the address before getting here. Nothing is wrong with this file. + Ok(Some(stored)) if stored == content => { + self.files.note_bytes_proven_good(address); + true + } + Ok(_) => { + warn!( + "Chunk {} is on disk but its contents are wrong; replacing it with the \ + copy just offered", + hex::encode(address) + ); + // Recorded before the repair is attempted, not after it succeeds. A + // repair can fail for capacity or I/O, and a chunk proven wrong that goes + // on looking healthy leaves a cached pre-retirement pass covering it, + // which deletes the legacy copy the repair would have come from. + self.files.note_known_wrong(address); + self.files.repair(address, content).await.is_ok() + } + // Unanswerable this time. Not claimed as held, so the offer goes through the + // ordinary path, which writes it rather than replacing anything. + Err(e) => { + warn!("Could not read {} to check it: {e}", hex::encode(address)); + false + } + } + } + + /// Delete a chunk from both backings. + /// + /// A logical delete has to reach the legacy environment too, or the union view would + /// resurrect the key on the next read. It frees no space there — only removing the + /// environment whole does that — but it keeps the two views honest. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if a file exists but cannot be removed. + pub async fn delete(&self, address: &XorName) -> Result { + // Shared, like every other operation that touches the legacy environment. Without + // it, retirement takes the exclusive guard and then waits for the environment to + // go idle while deletes keep starting new work in it, and the wait never ends. + let _using_legacy = self.retirement.read().await; + let _lane = self.key_lock(address).await; + // Behind whatever is already writing this key, and only this key. A write's + // blocking half outlives the future that started it, so one landing after this + // would put back a chunk the node had decided to prune. + self.files.wait_for_write(address).await; + // Legacy first, and only then the in-memory views. The other order removes the + // key from `only` and then, if the legacy delete fails, leaves bytes that live + // solely in the legacy store and are invisible to `exists`, `all_keys` and the + // pre-retirement verification, so retirement would take the only copy. + let from_legacy = match self.legacy() { + Some(legacy) => { + // A write for this key that nobody waited for may still be queued behind + // this delete. Letting it land afterwards would resurrect the key: the + // next reconciliation finds it in the environment and puts it back on the + // copier's list, undoing a prune the node decided on. Waited out here, + // holding the lane, so the delete is genuinely last. + // + // Both halves. A write has an environment half and a file half, either of + // which can be the one still running, and draining only the first leaves + // the second free to publish the file after this has deleted it. + // + // The environment half is found through the journal, which only dual + // writes keep. The file half is asked of the file store directly, because + // the copier and the repair path also spawn file writes and neither goes + // near that journal: using it as a proxy for "is anything writing this + // key" was a scope assumption, not a fact. + if legacy.pending.read().contains_key(address) { + legacy.lmdb.wait_idle().await; + } + let deleted = legacy.lmdb.delete(address).await?; + let was_only = legacy.only.write().remove(address); + // Every announcement for this key, not one of them: the drain above waited + // out whatever was in flight and this delete is deliberately last. + legacy.pending.write().remove(address); + deleted || was_only + } + None => false, + }; + let from_files = self.files.delete(address).await?; + Ok(from_files || from_legacy) + } + + /// Every stored key, in ascending order, across both backings. + /// + /// The order is a correctness requirement: the commitment builder truncates the + /// responsible subset with `take(cap)` *before* the Merkle tree sorts it, so an + /// unstable order would make the node's published commitment depend on iteration + /// luck rather than on what it holds. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the file index cannot be read. + pub async fn all_keys(&self) -> Result> { + // Legacy first, for the reason given on `exists`. `merge_sorted` drops the + // duplicate that the overlap produces. + let legacy_only: Vec = self + .legacy() + .map(|l| l.only.read().iter().copied().collect()) + .unwrap_or_default(); + let file_keys = self.files.all_keys().await?; + if legacy_only.is_empty() { + return Ok(file_keys); + } + Ok(merge_sorted(&file_keys, legacy_only.iter())) + } + + /// The keys the commitment builder should commit to. + /// + /// While the node is still bridging this is the whole union, because it can still + /// serve all of it and dropping the claim early would collapse its commitment (and + /// with it its quoted price) for no reason. Once it has settled on what it will keep, + /// this narrows to the file-backed set, which is exactly the point at which the node + /// stops claiming keys it is about to give up. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the file index cannot be read. + pub async fn committable_keys(&self) -> Result> { + match self.migration_phase() { + MigrationPhase::Bridging => self.all_keys().await, + MigrationPhase::Committed | MigrationPhase::FilesOnly => self.files.all_keys().await, + } + } + + /// Number of chunks currently stored, counted across both backings without + /// double-counting a chunk that is in each. + /// + /// # Errors + /// + /// Never fails. + pub fn current_chunks(&self) -> Result { + let (files, legacy) = self.split_counts(); + Ok(files.saturating_add(legacy)) + } + + /// Operation statistics. + /// + /// The cumulative counters are the file store's; `current_chunks` is the union. + #[must_use] + pub fn stats(&self) -> StorageStats { + let mut stats = self.files.stats(); + stats.current_chunks = self.current_chunks().unwrap_or(0); + stats + } + + /// Compute a content address (BLAKE3 hash). + #[must_use] + pub fn compute_address(content: &[u8]) -> XorName { + crate::client::compute_address(content) + } + + /// The node root directory. + #[must_use] + pub fn root_dir(&self) -> &Path { + &self.config.root_dir + } + + /// Reject work early when the disk cannot take another chunk at all. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] when free space is below the configured reserve. + pub fn check_capacity(&self) -> Result<()> { + self.files.check_capacity() + } + + /// Whether the store can take a write at all right now. + /// + /// Answered by the file store, which is where writes land. The legacy environment's + /// own verdict is deliberately not consulted: it accounts for pages it can reuse + /// internally, and a reusable page in a store this node is moving *off* says nothing + /// about whether the file it is about to write will fit. + /// + /// Three-way, not two. The verification cycle treats `Full` as a standing condition + /// worth minutes of backoff, so folding a failed free-space query into it would latch + /// a transient filesystem hiccup into a stall on a node that is not full at all. + #[must_use] + pub(crate) fn capacity_verdict(&self) -> crate::storage::CapacityVerdict { + self.files.capacity_verdict() + } + + /// Reject work early when the disk cannot take `bytes` more. + /// + /// Free bytes alone stopped being a sufficient answer once chunks became files. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] when the write would not fit above the reserve. + pub fn check_capacity_for(&self, bytes: u64) -> Result<()> { + self.files.check_capacity_for(bytes) + } + + /// Wait until every blocking task in either backing has finished. + pub async fn wait_idle(&self) { + self.files.wait_idle().await; + if let Some(legacy) = self.legacy() { + legacy.lmdb.wait_idle().await; + } + } + + // ── Migration ─────────────────────────────────────────────────────────── + + /// Where the node is in the migration. + #[must_use] + pub fn migration_phase(&self) -> MigrationPhase { + self.state.read().phase + } + + /// A snapshot of the persisted migration marker. + #[must_use] + pub fn migration_state(&self) -> MigrationState { + self.state.read().clone() + } + + /// The migration settings this store was built with. + #[must_use] + pub fn migration_config(&self) -> &MigrationConfig { + &self.config.migration + } + + /// Whether a legacy environment is still open. + #[must_use] + pub fn has_legacy(&self) -> bool { + self.legacy.read().is_some() + } + + /// The keys the legacy environment still holds alone, ascending. + #[must_use] + pub fn legacy_only_keys(&self) -> Vec { + self.legacy() + .map(|l| l.only.read().iter().copied().collect()) + .unwrap_or_default() + } + + /// Bytes the legacy environment occupies, as the filesystem sees it. + #[must_use] + pub fn legacy_bytes(&self) -> u64 { + std::fs::metadata(self.legacy_env_dir.join(LEGACY_DATA_FILE)).map_or(0, |m| m.len()) + } + + /// Test-only handle to the file store's put gate. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn test_put_gate(&self) -> Arc> { + self.files.test_put_gate() + } + + /// Test-only: adjust the persisted migration marker directly. + /// + /// Real transitions go through [`Self::commit_to_files`] and + /// [`Self::note_commitment_rebuilt`]; this exists so a test can put a store into a + /// state that would otherwise take hours of wall clock to reach. + #[cfg(any(test, feature = "test-utils"))] + pub fn force_migration_state(&self, f: F) { + f(&mut self.state.write()); + } + + /// Copy up to `keys.len()` chunks out of the legacy environment into files. + /// + /// Stops as soon as free space would fall below `slack` above the configured + /// reserve, so a migration never fills the disk it is trying to free. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] only for failures that are not per-key: a per-key + /// problem is counted in the report and the pass continues. + pub async fn copy_batch( + &self, + keys: &[XorName], + slack: u64, + throttle_mib_per_sec: u64, + shutdown: &CancellationToken, + ) -> Result { + let mut report = CopyReport::default(); + let Some(legacy) = self.legacy() else { + return Ok(report); + }; + + for key in keys { + // Checked per chunk, not per batch. Everything here is idempotent and + // re-derived at the next start, so stopping between two chunks costs nothing + // and stops shutdown waiting out a whole pass. + if shutdown.is_cancelled() { + break; + } + let lane = self.key_lock(key).await; + // Re-checked inside the critical section. A prune that landed while this + // pass was running has already taken the key out of the legacy-only set, and + // copying it now would resurrect a chunk the node deliberately deleted. + if !legacy.only.read().contains(key) { + continue; + } + // Physically, again: a chunk the store holds and cannot read is not one to + // copy over the top of, and it is not legacy-only either. + if self.files.is_indexed(key) { + legacy.only.write().remove(key); + continue; + } + // Reserve room for a full chunk plus the slack floor before reading, so the + // copier stops with headroom rather than on a failed write. + if self.files.check_capacity_for(slack).is_err() { + report.stopped_for_space = true; + break; + } + + let Some(bytes) = legacy.lmdb.get_raw(key).await? else { + report.vanished += 1; + legacy.only.write().remove(key); + continue; + }; + let len = bytes.len() as u64; + + match self.files.put(key, &bytes).await { + Ok(_) => { + legacy.only.write().remove(key); + report.copied += 1; + report.bytes += len; + } + Err(e) => { + let message = format!("{e}"); + // Bigger than this build will ever serve. The legacy store took it + // through an API with no size bound; the file store will not, and no + // amount of retrying changes that. Counted as unusable and removed, + // like a record whose bytes do not match, or one such record would + // stop this node and every node sharing its disk from ever reclaiming + // space. + if message.contains("byte maximum") { + warn!( + "Chunk {} in the legacy environment is larger than this build \ + will store; removing it. It cannot be served either way.", + hex::encode(key) + ); + match legacy.lmdb.delete(key).await { + Ok(_) => { + legacy.only.write().remove(key); + report.unusable += 1; + } + Err(e) => warn!( + "Oversized chunk {} could not be removed from the legacy \ + environment: {e}. The environment stays.", + hex::encode(key) + ), + } + continue; + } + if message.contains("Content address mismatch") { + // The legacy bytes do not hash to their own key, so this chunk + // cannot be reproduced and was never servable. Stop advertising + // it rather than carrying a key we cannot answer for. + // + // Deleted from the environment too, and only dropped from the key + // set once that has worked. Leaving the record behind puts the + // key in neither view, which the pre-retirement pass reads as a + // chunk to protect and puts straight back — and the next copier + // pass drops it again. One malformed record would keep a node, + // and every node sharing its disk, from ever reclaiming space. + warn!( + "Chunk {} in the legacy environment does not match its address; \ + removing it so replication can repair it", + hex::encode(key) + ); + match legacy.lmdb.delete(key).await { + Ok(_) => { + legacy.only.write().remove(key); + report.unusable += 1; + } + Err(e) => warn!( + "Chunk {} does not match its address and could not be \ + removed from the legacy environment: {e}. It stays on the \ + list and the environment stays.", + hex::encode(key) + ), + } + continue; + } + if message.contains("Insufficient disk space") { + report.stopped_for_space = true; + break; + } + return Err(e); + } + } + + // Outside the critical section on purpose: at 32 MiB/s a 4 MiB chunk sleeps + // for over a tenth of a second, and a shard lane held for that would stall + // every write sharing its last address byte for the whole pass. + drop(lane); + if let Some(delay) = throttle_delay(len, throttle_mib_per_sec) { + tokio::time::sleep(delay).await; + } + } + Ok(report) + } + + /// Settle on the file-backed set: from now on the node commits only to what it will + /// keep, while still serving everything it ever committed to. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the marker cannot be persisted. + pub fn commit_to_files(&self) -> Result<()> { + let shed = self + .legacy() + .map_or(0, |l| l.only.read().len().try_into().unwrap_or(u64::MAX)); + let kept = self.files.current_chunks().unwrap_or(0); + // Written to disk before it is published in memory. The other order leaves this + // process acting as `Committed` (and so committing only to file-backed keys) + // while the marker still says `Bridging`, so a restart would silently undo it. + let candidate = { + let state = self.state.read(); + if state.phase != MigrationPhase::Bridging { + return Ok(()); + } + MigrationState { + phase: MigrationPhase::Committed, + committed_at_unix: None, + rebuilds_since_commit: 0, + shed_key_count: shed, + kept_key_count: kept, + ..state.clone() + } + }; + candidate.save(&self.config.root_dir)?; + *self.state.write() = candidate; + if shed == 0 { + info!("Committed to the file-backed key set; nothing has to be shed"); + } else { + info!( + "Committed to the file-backed key set; {shed} chunk(s) will be shed and \ + refetched once the legacy environment is gone and there is room" + ); + } + Ok(()) + } + + /// Record that the commitment builder has read and published the committable set. + /// + /// The retirement gate counts these: one proves the builder saw the new set, two + /// prove it survived a rotation, which is what makes the answerability window + /// meaningful rather than notional. + pub fn note_commitment_rebuilt(&self) { + let should_save = { + let mut state = self.state.write(); + if state.phase != MigrationPhase::Committed { + return; + } + if state.committed_at_unix.is_none() { + state.committed_at_unix = Some(crate::storage::migration::now_unix()); + } + state.rebuilds_since_commit = state.rebuilds_since_commit.saturating_add(1); + state.rebuilds_since_commit <= REQUIRED_REBUILDS_BEFORE_RETIRE + }; + if should_save { + let snapshot = self.state.read().clone(); + if let Err(e) = snapshot.save(&self.config.root_dir) { + warn!("Could not persist the migration marker: {e}"); + } + } + } + + /// Whether every gate on deleting the legacy environment is satisfied. + /// + /// `still_answerable` is asked of each key the node is about to give up: the pruner's + /// existing retention contract, reused verbatim. A key still covered by a retained + /// commitment slot vetoes the delete, because the node could still be challenged on it. + pub fn retirement_blocker(&self, still_answerable: F) -> Option + where + F: Fn(&XorName) -> bool, + { + // Before anything else, and whether or not there is a handle. An environment this + // node cannot classify must not be retired, and asking only on the no-handle path + // meant the ordinary path never asked: a node holding its store open went through + // every gate, renamed the directory aside and deleted it. + if self.legacy_cannot_be_classified() { + return Some(format!( + "{} cannot be read well enough to say whether it was already retired. \ + Nothing will be deleted until it can. Check that the directory and \ + anything inside it can be read.", + self.legacy_env_dir.display() + )); + } + if !self.has_legacy() { + // No handle is not the same as no environment. A rename that failed and then + // could not be reopened leaves exactly that: the directory is still on disk + // and this node can no longer read it. Answering "nothing blocks retirement" + // would have the driver log the migration complete over a store that is still + // there and still holding chunks nothing else can serve. + let mark = retirement_mark(&self.legacy_env_dir); + if legacy_present(&self.config.root_dir).unwrap_or(true) && !mark.permits_removal() { + // Two different situations wearing one message would send an operator to + // the wrong place. One is a store this node cannot open; the other is a + // store nothing can even classify, which usually means a permission or a + // mount, and which the node deliberately will not act on either way. + if mark == RetirementMark::Unknown { + return Some(format!( + "{} is still on disk and this node cannot tell whether it was \ + retired, so it will neither open it nor remove it. Check that the \ + directory and anything inside it can be read.", + self.legacy_env_dir.display() + )); + } + return Some(format!( + "{} is still on disk but this node has no handle to it. It cannot be \ + read, verified or removed until the node is restarted.", + self.legacy_env_dir.display() + )); + } + return None; + } + if !self.config.migration.retire_legacy { + return Some( + "retirement is disabled in this release (storage.migration.retire_legacy)".into(), + ); + } + // A linked environment is never retired automatically. Retirement renames the + // path and then deletes what is behind it, and behind a link is a directory + // somewhere else that this node does not own. Copying still happens; only the + // removal is refused, so the node ends up serving from files with its old store + // intact and its operator told what to do about it. + if is_a_link(&self.legacy_env_dir) { + return Some(format!( + "{} is a link rather than a directory. The chunks are being copied out of \ + it, but it will not be deleted: what it points at is not this node's to \ + remove. Once the migration has settled, delete it by hand.", + self.legacy_env_dir.display() + )); + } + let state = self.state.read().clone(); + if state.phase != MigrationPhase::Committed { + return Some(format!("phase is {:?}, not Committed", state.phase)); + } + if state.rebuilds_since_commit < REQUIRED_REBUILDS_BEFORE_RETIRE { + return Some(format!( + "only {} of {REQUIRED_REBUILDS_BEFORE_RETIRE} commitment rebuilds observed", + state.rebuilds_since_commit + )); + } + if !state.retire_delay_elapsed(&self.config.migration) { + return Some(format!( + "the {}h retirement delay has not elapsed", + self.config.migration.effective_retire_delay_hours() + )); + } + if let Some(key) = self + .legacy_only_keys() + .into_iter() + .find(|k| still_answerable(k)) + { + return Some(format!( + "chunk {} is still answerable under a retained commitment", + hex::encode(key) + )); + } + None + } + + /// Re-hash every chunk that both stores hold, repairing the file from the legacy + /// copy when they disagree. + /// + /// A filename is not proof the bytes behind it are good. The startup scan reads + /// names only, so a file that was truncated or that rotted while the node was down is + /// indexed, counted as copied, committed to, and would have its intact legacy copy + /// deleted underneath it. The first verified read would then find the corruption with + /// nothing left to repair from. This pass is what turns "a file with that name + /// exists" into "those bytes are that chunk", and it is why it runs before + /// retirement rather than after. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the legacy key set cannot be read. + pub async fn verify_before_retire( + &self, + throttle_mib_per_sec: u64, + shutdown: &CancellationToken, + ) -> Result { + let mut report = VerifyReport::default(); + // Taken BEFORE anything is read. Stamping it at the end would absorb exactly the + // failures this exists to catch: a chunk verified early in the pass that stops + // being readable before the pass finishes would leave the report carrying the + // already-incremented count, and both later checks would see it match. + let health_at_start = self.files.health_generation(); + let Some(legacy) = self.legacy() else { + report.ran = true; + report.health = health_at_start; + return Ok(report); + }; + report.ran = true; + + // Names before bytes. A chunk whose contents are durable but whose directory entry + // is not is still lost to a power loss, and the legacy copy is about to be deleted + // on the strength of this proof. Any failure here is a proof this pass did not + // produce. + if let Err(e) = self.files.flush_namespace() { + report.unrepairable = report.unrepairable.saturating_add(1); + warn!( + "Could not make the file store's directory entries durable: {e}. The legacy \ + environment stays until they are." + ); + return Ok(report); + } + + let legacy_keys = legacy.lmdb.all_keys().await?; + let total = legacy_keys.len(); + info!("Verifying {total} chunk(s) before removing the legacy environment"); + let mut since_log = 0u64; + for key in legacy_keys { + // This pass is a full read of the store and can run for hours. A shutdown + // must not wait it out, and an incomplete pass is simply not a clean proof. + if shutdown.is_cancelled() { + report.unrepairable = report.unrepairable.saturating_add(1); + debug!("Pre-retirement verification stopped for shutdown"); + return Ok(report); + } + // Under the key's critical section, so the two questions below are asked of + // one moment. Without it a write can publish the file and take the key out of + // the legacy-only set in between, and this pass would put it straight back. + let classified = { + let _lane = self.key_lock(&key).await; + // The physical question. A chunk the store holds but cannot currently + // read is still one it holds, and calling it absent here would put the + // key in the legacy-only set, where the union view advertises it again. + let in_files = self.files.is_indexed(&key); + let legacy_only = legacy.only.read().contains(&key); + if in_files && legacy_only { + // In both views at once, which nothing else clears once the copier + // has stopped running. The file store has it, so the legacy-only set + // is the one that is wrong: an answerable key in that set vetoes + // retirement for as long as the process lives. + debug!( + "Chunk {} was in both views; the file store has it, so it is no \ + longer legacy-only", + hex::encode(key) + ); + legacy.only.write().remove(&key); + } + (in_files, legacy_only) + }; + if !classified.0 { + // Known to be legacy-only, which is what a key this node is giving up + // looks like. Whether it may go is the gates' decision, not this pass's. + if classified.1 { + continue; + } + // In neither view. However that came about — a publish that failed, a + // file quarantined for corruption, a name this store stopped advertising — + // the environment holds the only copy, and nothing is looking after it: + // the gates only ever see the legacy-only set. Put it back there and + // refuse the proof this pass. What neither view protects is exactly what + // retirement destroys. + warn!( + "Chunk {} is in the legacy environment, is not in the file store, and \ + was in neither view; re-queued for copying and the legacy environment \ + stays", + hex::encode(key) + ); + legacy.only.write().insert(key); + report.unrepairable = report.unrepairable.saturating_add(1); + continue; + } + since_log += 1; + if since_log >= VERIFY_LOG_EVERY { + since_log = 0; + info!( + "Pre-retirement verification: {} of at most {total} chunk(s) checked", + report.checked + ); + } + let outcome = self.verify_one(&legacy, &key).await; + report.checked += 1; + report.bytes += outcome.bytes; + match outcome.verdict { + VerifyVerdict::Intact => {} + VerifyVerdict::Repaired => report.repaired += 1, + VerifyVerdict::Vanished => { + // The file went away while this pass was running, so the key is no + // longer file-backed. Put it back on the copier's list rather than + // republishing it here, where it could resurrect something the + // pruner deleted a moment ago. + legacy.only.write().insert(key); + report.unrepairable += 1; + } + VerifyVerdict::Unrepairable => report.unrepairable += 1, + } + if let Some(delay) = throttle_delay(outcome.bytes, throttle_mib_per_sec) { + tokio::time::sleep(delay).await; + } + } + + if report.unrepairable == 0 { + info!( + "Pre-retirement verification passed: {} chunk(s) checked, {} repaired", + report.checked, report.repaired + ); + } + // The count this pass started from, and a refusal if the store moved while it + // ran. Retirement compares the same value again immediately before deleting + // anything, so one number covers both windows: during the pass, and after it. + report.health = health_at_start; + if self.files.health_generation() != health_at_start { + warn!( + "A chunk stopped being servable while the pre-retirement pass was running, \ + so this pass does not describe the store. Another runs on the next tick." + ); + report.unrepairable = report.unrepairable.saturating_add(1); + } + Ok(report) + } + + /// Check one chunk that both stores hold, repairing the file if it is wrong. + async fn verify_one(&self, legacy: &Legacy, key: &XorName) -> VerifyOutcome { + // The throttle sleep is deliberately outside this critical section: at 32 MiB/s a + // 4 MiB chunk sleeps for over a tenth of a second, and holding a shard lane for + // that would stall every write to a sixteenth of the address space for hours. + let _lane = self.key_lock(key).await; + + let bytes = match self.files.get_raw(key).await { + Ok(bytes) => bytes, + // Not the same as gone. `Vanished` puts the key back on the copier's list, + // and doing that for a file that is still there and still indexed leaves the + // key in both views at once: the file index keeps it in every commitment, so + // it stays answerable, and an answerable legacy-only key vetoes retirement for + // as long as the process lives. Refuse this pass instead. + Err(e) => { + warn!( + "Chunk {} could not be read while verifying: {e}. The legacy \ + environment stays.", + hex::encode(key) + ); + return VerifyOutcome { + bytes: 0, + verdict: VerifyVerdict::Unrepairable, + }; + } + }; + let len = bytes.as_ref().map_or(0, Vec::len) as u64; + let Some(bytes) = bytes else { + return VerifyOutcome { + bytes: 0, + verdict: VerifyVerdict::Vanished, + }; + }; + if crate::client::compute_address(&bytes) == *key { + // The pass hashed these bytes and they are right, so whatever this store + // thought was wrong with them is not. It reads raw, which does not settle + // that on its own, and leaving the mark would retire the environment while a + // healthy chunk stayed unadvertised until some later verified read. + self.files.note_bytes_proven_good(key); + return VerifyOutcome { + bytes: len, + verdict: VerifyVerdict::Intact, + }; + } + + warn!( + "Chunk {} is in the file store but does not match its address; rewriting it \ + from the legacy environment before that environment is removed", + hex::encode(key) + ); + // Replace in place. Deleting first and writing after would leave a window whose + // only surviving copy is the one this whole pass exists to make safe to delete. + let verdict = match legacy.lmdb.get_raw(key).await { + Ok(Some(good)) if self.files.repair(key, &good).await.is_ok() => { + VerifyVerdict::Repaired + } + _ => { + warn!( + "Chunk {} could not be rewritten from the legacy environment. \ + Retirement stays blocked so its bytes are not thrown away.", + hex::encode(key) + ); + VerifyVerdict::Unrepairable + } + }; + VerifyOutcome { + bytes: len, + verdict, + } + } + + /// Is this verification still worth acting on? + /// + /// # Errors + /// + /// Returns [`Error::Storage`] naming what has changed since the pass ran. + fn proof_is_usable(&self, proof: &VerifyReport) -> Result<()> { + if !proof.is_clean() { + return Err(Error::Storage(format!( + "Refusing to remove the legacy environment: verification reported {} \ + unrepairable chunk(s) (ran: {})", + proof.unrepairable, proof.ran + ))); + } + if !proof.still_describes(&self.files) { + return Err(Error::Storage( + "Refusing to remove the legacy environment: a chunk stopped being \ + servable since it was verified, so that verification no longer describes \ + the file store. A fresh pass runs on the next tick." + .into(), + )); + } + if self.has_pending_writes() { + return Err(Error::Storage( + "Refusing to remove the legacy environment: a write announced itself and \ + has not reported back, so what the environment holds is not yet settled." + .into(), + )); + } + Ok(()) + } + + /// Close the legacy environment and remove it, returning the bytes freed. + /// + /// This is the only destructive step in the migration and the only one that cannot + /// be undone. It is also the only moment the disk comes back. + /// + /// Takes a [`VerifyReport`] rather than a flag so the verification pass cannot be + /// skipped: there is no way to call this without having produced one. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if verification did not pass or no longer describes the + /// store, if a write has not reported back, if the handle is still shared (the caller + /// should retry on the next tick), or if the directory cannot be removed. + pub async fn retire_legacy( + &self, + proof: &VerifyReport, + still_answerable: &F, + approved_to_shed: &BTreeSet, + ) -> Result + where + F: Fn(&XorName) -> bool + Send + Sync, + { + self.proof_is_usable(proof)?; + // Rechecked here, not only by the caller. Everything between the caller's check + // and this point is a window: the verification pass alone can run for hours, and + // a write whose file half failed inserts a new legacy-only key in the meantime. + if let Some(reason) = self.retirement_blocker(still_answerable) { + return Err(Error::Storage(format!( + "Refusing to remove the legacy environment: {reason}" + ))); + } + // Exclusive from here until the handle is out. Every read, write and delete holds + // this shared, so taking it means none is in progress and none can start: no + // reader is mid-way between discarding a corrupt file and reaching the copy that + // would replace it, and nothing new can start work in an environment that is about + // to go idle. + // + // Released as soon as the handle has been taken and the directory renamed away, + // which is the point after which nothing can reach the environment anyway. The + // deletion that follows can take a long time on a large store, and holding every + // chunk request on the node behind it would turn retirement into an outage. + // + let retiring = self.retirement.write().await; + // Asked again with the guard held, which is the only moment the answer cannot + // change underneath it. The check above can be overtaken by a read that fails + // between there and here. + if !proof.still_describes(&self.files) { + drop(retiring); + return Err(Error::Storage( + "Refusing to remove the legacy environment: a chunk stopped being \ + servable while retirement was starting. A fresh pass runs on the next \ + tick." + .into(), + )); + } + let Some(legacy) = self.legacy() else { + return Ok(0); + }; + let freed = self.legacy_bytes(); + // Let go of our own clone straight away, so the only strong reference that should + // remain is the one the store itself holds. + drop(legacy); + + for attempt in 0..RETIRE_UNWRAP_ATTEMPTS { + // Drained on every attempt, not once up front: `LmdbStorage`'s blocking + // closures capture a cloned `Env` rather than the `Arc`, so the strong count + // alone would not notice a read that is still mapped. The tracker does, and + // it reopens itself, so a read that started since the last drain needs + // another one. + if let Some(l) = self.legacy() { + l.lmdb.wait_idle().await; + drop(l); + } + // Taking the handle out and proving sole ownership happen in the same + // critical section. Deliberately not two steps: taking it first and putting + // it back on failure would leave a window in which reads see no legacy store + // and report a chunk that lives only there as missing. + let taken = { + let mut guard = self.legacy.write(); + match guard.as_ref() { + // A strong count of one means nobody else holds a handle, so nobody + // can be reading the legacy store *or* mutating its key set. That is + // what makes the final check below atomic with the removal: this is + // the only moment at which the answer cannot change underneath us. + Some(l) if Arc::strong_count(&l.lmdb) == 1 => { + // Asked here, in the same critical section as the checks below + // and immediately before the handle is taken. Asking earlier is + // not enough: a write can announce itself under the shared guard, + // be cancelled so the guard is released, and leave its blocking + // half running past the drain above. Its note is the only thing + // that says so, and dropping the journal with the environment + // would take the evidence with it. + if !l.pending.read().is_empty() { + return Err(Error::Storage( + "Refusing to remove the legacy environment: a write \ + announced itself and has not reported back, so what the \ + environment holds is not yet settled." + .into(), + )); + } + let only = l.only.read(); + // A count of one proves nobody else holds a handle, so nobody can + // be mutating this set. That is what makes the two checks below + // authoritative rather than a snapshot that has already moved. + if let Some(key) = only.iter().find(|k| still_answerable(k)) { + return Err(Error::Storage(format!( + "Refusing to remove the legacy environment: chunk {} became \ + answerable again while retirement was in progress", + hex::encode(key) + ))); + } + // Only the keys the caller cleared may go. A write whose file half + // failed adds a legacy-only key that is in no commitment, so the + // answerability check above cannot see it, and it would otherwise + // be destroyed without ever facing the rank, delivery or + // possession gates. + if let Some(key) = only.iter().find(|k| !approved_to_shed.contains(*k)) { + return Err(Error::Storage(format!( + "Refusing to remove the legacy environment: chunk {} entered \ + the legacy-only set after the gates were cleared and has \ + passed none of them", + hex::encode(key) + ))); + } + drop(only); + guard.take() + } + Some(_) => None, + None => return Ok(0), + } + }; + if let Some(Legacy { + lmdb, + only, + pending, + skipped_rollback_copies, + }) = taken + { + drop(only); + drop(pending); + drop(skipped_rollback_copies); + drop(lmdb); + return self.remove_legacy_dir(freed, retiring).await; + } + if attempt + 1 < RETIRE_UNWRAP_ATTEMPTS { + tokio::time::sleep(RETIRE_UNWRAP_BACKOFF).await; + } + } + + // Nothing was taken and nothing will be this tick, so let the node get on with + // serving rather than leaving this held until the function returns. + drop(retiring); + Err(Error::Storage( + "Legacy environment is still being read; retirement deferred to the next tick".into(), + )) + } + + /// Remove the legacy directory and record that the migration is over. + /// + /// The handle is already closed by the time this runs, so the node is file-only + /// either way. If the removal fails the phase still moves on, because there is no + /// going back to a half-removed environment, and the operator is told exactly which + /// directory to delete by hand to get the space back. + async fn remove_legacy_dir( + &self, + freed: u64, + retiring: tokio::sync::RwLockWriteGuard<'_, ()>, + ) -> Result { + // Renamed aside first, because `remove_dir_all` is not atomic: a failure partway + // through leaves a directory that can no longer be opened as an environment, and + // recording the migration as finished on top of that would have the node claim + // completion over a half-deleted store. A rename either happens or does not. + let tombstone = free_tombstone_path(&self.config.root_dir); + + if let Err(e) = std::fs::rename(&self.legacy_env_dir, &tombstone) { + // Nothing was deleted, but the handle is already closed, so this node has + // stopped being able to serve anything that lives only in there. Put it back + // rather than carrying on with chunks it holds and cannot read, and rather + // than letting the next tick see no handle and call that success. + let restored = self.reopen_legacy().await; + return Err(Error::Storage(format!( + "Could not move the legacy environment {} aside: {e}. Nothing was deleted{}", + self.legacy_env_dir.display(), + if restored { + " and it has been reopened, so the node keeps serving from both stores." + } else { + ". IT COULD NOT BE REOPENED: this node cannot serve chunks that live only there until it is restarted." + } + ))); + } + // The rename has to reach the directory itself, not just the page cache, and this + // one is not best effort. The tombstone is deleted a few lines below. If the + // rename has not reached the disk when that happens, a power loss brings the + // environment back under its old name with its contents already removed, and the + // next start finds a corrupt environment it cannot open. Stopping here instead + // leaves the tombstone in place, which the next start sweeps. + // Marked from the inside, now that the rename has succeeded and before anything + // is deleted. This is what a directory that reverts to its old name carries with + // it, and it is the only thing a later start treats as permission to delete. + if let Err(e) = mark_directory_retired(&tombstone) { + // Nothing has been deleted and the directory is intact, so put it back rather + // than recording the migration as finished over a store that is still there. + // Recording finished would be worse than it sounds: the next tick restores the + // unmarked directory to its own name, and a node that has already called + // itself file-only would then exit with a live environment on disk and no + // handle to it. + // Only when the mark is provably gone. A mark left inside would have the + // next cleanup pass reap a live, open environment. + let restored = + e.mark_definitely_gone && std::fs::rename(&tombstone, &self.legacy_env_dir).is_ok(); + let reopened = restored && self.reopen_legacy().await; + return Err(Error::Storage(format!( + "Moved the legacy environment to {} but could not mark it retired: {e}. \ + Nothing was deleted{}", + tombstone.display(), + if reopened { + ", and it has been put back, so the node keeps serving from both \ + stores and retirement is tried again." + } else if e.mark_definitely_gone { + ". IT COULD NOT BE PUT BACK: this node cannot serve chunks that live \ + only there until it is restarted." + } else { + ". It has been left where nothing will open it, because a partial \ + retirement mark may still be inside it. Its chunks are in the file \ + store; move it back by hand only after removing that mark." + } + ))); + } + + // Test-only: renamed aside and marked, nothing deleted yet. A process killed here + // is what the recovery on the next start exists for. + #[cfg(any(test, feature = "test-utils"))] + crate::storage::file_store::halt_here_if_asked( + crate::storage::file_store::HALT_AFTER_RETIRE_MARK, + &tombstone, + ); + + if let Err(e) = crate::storage::file_store::fsync_path(&self.config.root_dir) { + warn!( + "The legacy environment was moved aside but {} could not be flushed: {e}. \ + Leaving {} in place rather than deleting a directory whose new name may \ + not have reached the disk. The next start finishes this.", + self.config.root_dir.display(), + tombstone.display() + ); + self.finish_migration(); + return Ok(0); + } + self.finish_migration(); + + // From here nothing can reach the environment: its handle is gone and its + // directory is under a name no code looks for. Let the node serve again rather + // than holding every chunk request behind a deletion that can run for minutes. + drop(retiring); + + // Only now, and best effort: the bytes come back when this completes, and if it + // does not the next start sweeps the tombstone. + // + // On a detached OS thread, and not awaited. This is a synchronous recursive delete + // of a directory that can hold hundreds of gigabytes and cannot be interrupted + // once it starts. Inside the migration task it would sit through shutdown's grace + // and past it, because an abort is not observed until the call returns; on the + // runtime's blocking pool a normal runtime shutdown would wait for it anyway. A + // plain thread is the only one the process can genuinely walk away from, and the + // directory carries its own retirement mark, so whatever is left is finished by + // the next start. + delete_retired_directory(tombstone); + Ok(freed) + } + + /// Try again to open a legacy environment this node has lost its handle to. + /// + /// A rename that failed and then could not be reopened leaves the directory on disk + /// with no way to read it, and every chunk that lives only there unserved. Saying so + /// once and waiting for a restart is not enough: the reason is usually transient, and + /// a node that is otherwise healthy should not stay half-blind until somebody notices. + /// + /// Returns whether it came back. Does nothing when there is a handle already, or when + /// there is nothing on disk to open. + pub async fn recover_lost_legacy_handle(&self) -> bool { + if self.has_legacy() || !retirement_mark(&self.legacy_env_dir).permits_opening() { + return false; + } + if !legacy_present(&self.config.root_dir).unwrap_or(false) { + return false; + } + // Opened WITHOUT the exclusive guard. Opening scans every key in the environment, + // which on a large store is minutes, and every read and write on the node would + // wait behind it. Nothing else can be installing a handle: retirement does nothing + // while there is none, and this runs from the one migration task. + let (lmdb, legacy_keys) = match Self::open_legacy_env(&self.config).await { + Ok(opened) => opened, + Err(e) => { + warn!( + "Could not reopen {}: {e}. The chunks that live only there stay \ + unreadable until this succeeds.", + self.legacy_env_dir.display() + ); + return false; + } + }; + // Exclusive only to install it, which is instant. + let _recovering = self.retirement.write().await; + if self.has_legacy() { + return false; + } + // The diff happens HERE, not when the environment was read. Reading it takes + // minutes on a large store, and a verifying read in that time can find a file + // rotted and throw it away. With no handle installed there was nothing to put the + // key back into, so a set computed beforehand would be missing it, every gate + // would skip it, and retirement would destroy the intact copy in the environment. + // Under this guard no read, write or delete is in flight, so the file store's + // answer cannot move while it is being asked. + let only = Self::keys_only_in_legacy(&legacy_keys, &self.files); + *self.legacy.write() = Some(Legacy { + lmdb, + only: Arc::new(parking_lot::RwLock::new(only)), + skipped_rollback_copies: Arc::new(std::sync::atomic::AtomicU64::new(0)), + pending: Arc::new(parking_lot::RwLock::new(BTreeMap::new())), + }); + // A node that recorded itself file-only and then got an environment back has to + // go through the migration again from the start: the phase decides what the + // driver does, and file-only does no copying, so leaving it there would give the + // node a handle it never uses. Conservative on purpose; the copier finds most of + // the work already done. + if self.migration_phase() == MigrationPhase::FilesOnly { + warn!( + "Recovered a legacy chunk environment after recording this node as \ + file-only. Starting the migration again from the copying stage." + ); + let mut state = self.state.write(); + state.phase = MigrationPhase::Bridging; + state.committed_at_unix = None; + state.rebuilds_since_commit = 0; + let snapshot = state.clone(); + drop(state); + if let Err(e) = snapshot.save(&self.config.root_dir) { + warn!("Could not persist the migration marker: {e}"); + } + } + warn!( + "Reopened {} after losing its handle", + self.legacy_env_dir.display() + ); + true + } + + /// Is there a retired directory still waiting to be deleted? + /// + /// Separate from having a legacy environment: a node whose removal was interrupted has + /// no handle and nothing to migrate, but its disk has not come back. Something has to + /// keep trying during this uptime rather than leaving it until the next restart. + #[must_use] + pub fn has_cleanup_pending(&self) -> bool { + !retired_tombstones(&self.config.root_dir).is_empty() + || !retirement_mark(&self.legacy_env_dir).permits_opening() + } + + /// Try again to finish a removal a previous attempt left behind. + /// + /// Safe to call at any time: it only ever moves or deletes a directory that carries + /// its own retirement mark. + pub fn retry_cleanup(&self) { + // Never while this node has the environment open. Cleanup decides what to do from + // the directory's own mark, and a mark that outlived a failed retirement would + // have it rename a live, mapped environment out from under the handle. + if self.has_legacy() { + sweep_retired_legacy(&self.config.root_dir); + return; + } + finish_interrupted_retirement(&self.config.root_dir); + } + + /// Resolve writes whose outcome was never recorded. + /// + /// A write announces itself before it starts and clears the note when both halves + /// have returned. A note still there afterwards belongs to a write nobody waited for, + /// and only the disk can say what became of it: the file store has the chunk, or the + /// environment does and nothing else, or neither and there was never anything to + /// protect. + pub async fn reconcile_pending_writes(&self) { + if !self.has_pending_writes() { + return; + } + // Exclusively, and before the snapshot. Draining is not a barrier on its own: + // writes hold this shared, and a new one for the same key could announce itself, + // be cancelled, and leave its blocking half running while this decided the older + // one's fate and removed the single entry they share. Held here, nothing new can + // start, so what the disk says once the drain returns is final. + // + // Only reached when something is waiting, which after a clean run is never, so + // this is not a stall on the ordinary path. + let _settling = self.retirement.write().await; + let Some(legacy) = self.legacy() else { + return; + }; + let waiting: Vec = legacy.pending.read().keys().copied().collect(); + if waiting.is_empty() { + return; + } + legacy.lmdb.wait_idle().await; + self.files.wait_idle().await; + for key in waiting { + let _lane = self.key_lock(&key).await; + if self.files.is_indexed(&key) { + legacy.only.write().remove(&key); + legacy.pending.write().remove(&key); + continue; + } + match legacy.lmdb.get_raw(&key).await { + Ok(Some(_)) => { + debug!( + "Chunk {} was written to the legacy environment by a call that \ + never returned; recording it so the copier picks it up", + hex::encode(key) + ); + legacy.only.write().insert(key); + legacy.pending.write().remove(&key); + } + // Nothing behind it: there was never anything to protect. + Ok(None) => { + legacy.pending.write().remove(&key); + } + // NOT the same as nothing behind it. Dropping the note on a read that + // failed would leave a committed write with no protection at all, which + // is the case this journal exists for. Keep it and ask again next tick; + // retirement stays vetoed meanwhile. + Err(e) => warn!( + "Could not tell what became of the write for {}: {e}. Asking again on \ + the next tick.", + hex::encode(key) + ), + } + } + } + + /// Are there writes in flight whose outcome nothing has recorded? + #[must_use] + pub fn has_pending_writes(&self) -> bool { + self.legacy().is_some_and(|l| !l.pending.read().is_empty()) + } + + /// Is there anything at the legacy environment's path at all? + /// + /// Asked without a handle, and answered conservatively: a path this node cannot even + /// look at counts as present. The migration is not finished while something is there, + /// whether or not this node can currently read it. + #[must_use] + pub fn legacy_dir_is_on_disk(&self) -> bool { + // `symlink_metadata`, not `try_exists`, which follows links. An operator's link to + // storage that is not mounted right now reads as nothing at all through the + // second, and the node would call its migration finished and go file-only, blind + // to every chunk that lives only there until somebody restarts it. + match std::fs::symlink_metadata(&self.legacy_env_dir) { + Ok(_) => true, + // Only "it is not there" means it is not there. A permission change or a + // transient fault is an unanswered question, and answering it with "nothing + // here" is how the driver declares the migration finished over a store it has + // merely lost sight of. + Err(e) => e.kind() != std::io::ErrorKind::NotFound, + } + } + + /// Is the legacy environment a link this node must not delete? + /// + /// Copying out of it works; only the removal is refused. Callers use this to stop + /// waiting for a retirement that is never going to happen. + #[must_use] + pub fn legacy_is_a_link(&self) -> bool { + self.has_legacy() && is_a_link(&self.legacy_env_dir) + } + + /// How many writes this node made without a rollback copy, cumulatively. + /// + /// Zero on a node that is not bridging, and zero on a bridging node whose environment + /// has room. A number that is climbing says this node would lose those chunks on a + /// rollback to a pre-migration build, which is a fleet question the second release + /// turns on and which a per-chunk log line cannot answer. + /// + /// Attempts rather than distinct chunks, for the reason given on the field: it is a + /// rate, not an inventory. + #[must_use] + pub fn writes_without_a_rollback_copy(&self) -> u64 { + self.legacy().map_or(0, |l| { + l.skipped_rollback_copies + .load(std::sync::atomic::Ordering::Relaxed) + }) + } + + /// Is there an environment on disk this node cannot classify at all? + /// + /// Neither removable nor openable, which is not a state waiting will clear: something + /// about the path has to change first, and until it does the node will refuse to touch + /// it in either direction. The driver treats this the way it treats a lost handle or a + /// link, by standing down from the shared volume and saying so where an operator looks, + /// because holding a disk exclusively to wait for a person is a disk nobody else can + /// use. + #[must_use] + pub fn legacy_cannot_be_classified(&self) -> bool { + retirement_mark(&self.legacy_env_dir) == RetirementMark::Unknown + } + + /// Is there an environment on disk this node can no longer read? + #[must_use] + pub fn has_lost_its_legacy_handle(&self) -> bool { + !self.has_legacy() + && retirement_mark(&self.legacy_env_dir).permits_opening() + // Conservative in the same direction as the retirement blocker, which reads the + // same failure as "there is one". A question that cannot be answered is not an + // answer of no, and answering no here left the node holding the shared volume + // for the six-hour cap over work no amount of disk will finish. + && legacy_present(&self.config.root_dir).unwrap_or(true) + } + + /// Reopen the legacy store after a failed retirement, so the node keeps serving. + /// + /// Returns whether it came back. The handle is closed before the rename is attempted, + /// so a rename that fails leaves the node holding chunks it can no longer read; that + /// is worth undoing rather than living with until the next restart. + async fn reopen_legacy(&self) -> bool { + if !legacy_present(&self.config.root_dir).unwrap_or(false) { + return false; + } + match Self::open_legacy(&self.config, &self.files).await { + Ok(legacy) => { + *self.legacy.write() = Some(legacy); + warn!("Reopened the legacy chunk environment after a failed retirement"); + true + } + Err(e) => { + error!("Could not reopen the legacy chunk environment: {e}"); + false + } + } + } + + /// Record that this node serves from files alone from here on. + fn finish_migration(&self) { + self.files.invalidate_capacity_cache(); + { + let mut state = self.state.write(); + state.phase = MigrationPhase::FilesOnly; + } + let snapshot = self.state.read().clone(); + if let Err(e) = snapshot.save(&self.config.root_dir) { + warn!("Could not persist the migration marker: {e}"); + } + } +} + +/// What checking one chunk concluded. +enum VerifyVerdict { + /// The file matches its name. + Intact, + /// The file was wrong and was rewritten from the legacy copy. + Repaired, + /// The file was wrong and could not be rewritten. + Unrepairable, + /// The file disappeared while the pass was running. + Vanished, +} + +/// One chunk's verification result. +struct VerifyOutcome { + /// Bytes read, for the throttle. + bytes: u64, + /// What was concluded. + verdict: VerifyVerdict, +} + +/// What the pre-retirement verification pass found. +/// +/// Every field is private, and the only way to obtain one is +/// [`ChunkStore::verify_before_retire`]. That is deliberate: it is the sole evidence +/// [`ChunkStore::retire_legacy`] accepts that the file store really holds what it claims, +/// and a report anyone could construct would be no evidence at all. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct VerifyReport { + /// Whether the pass actually ran. + ran: bool, + /// Chunks re-hashed. + checked: u64, + /// Bytes read. + bytes: u64, + /// Chunks whose file was wrong and was rewritten from the legacy copy. + repaired: u64, + /// Chunks whose file was wrong and could not be repaired. + unrepairable: u64, + /// What the file store's health looked like when this pass finished. + /// + /// A clean report is reused for a while rather than re-read on every tick, and a lot + /// can happen in that window: a kept file can start failing to read while ordinary + /// requests are served from the legacy copy, and the node would then delete the + /// legacy copy on the strength of a pass that no longer describes the store. This is + /// how retirement tells, immediately before it deletes anything. + health: u64, +} + +impl VerifyReport { + /// Whether this report clears the way for retirement. + #[must_use] + pub fn is_clean(&self) -> bool { + self.ran && self.unrepairable == 0 + } + + /// Does this report still describe the store? + #[must_use] + fn still_describes(&self, files: &FileStore) -> bool { + self.health == files.health_generation() + } + + /// Chunks re-hashed. + #[must_use] + pub fn checked(&self) -> u64 { + self.checked + } + + /// Chunks rewritten from the legacy copy. + #[must_use] + pub fn repaired(&self) -> u64 { + self.repaired + } + + /// Chunks that could not be made good. + #[must_use] + pub fn unrepairable(&self) -> u64 { + self.unrepairable + } +} + +/// Mark a retired environment directory as retired, from the inside, durably. +/// +/// Called only after the directory has already been renamed aside, so it can never land +/// inside a live environment. See [`RETIRED_MARKER`] for why it goes inside. +/// +/// # Errors +/// +/// Returns [`Error::Storage`] if it cannot be created or flushed. +fn mark_directory_retired(dir: &Path) -> std::result::Result<(), MarkFailure> { + match write_retirement_mark(dir) { + Ok(()) => Ok(()), + Err(e) if e.pre_existing => { + // Nothing here was created by this attempt, so there is nothing to take back. + // Removing a mark that was already there because re-flushing it failed is how + // a correctly retired directory comes to look unmarked, and an unmarked + // directory is restored as a live environment. + Err(MarkFailure { + pre_existing: true, + ..e + }) + } + Err(e) => { + // A half-written mark is worse than none: the caller puts the directory back + // under the live name and reopens it, and a mark left inside would have the + // next cleanup pass reap a live, open environment. If it cannot be taken away, + // say so, and the caller keeps the directory where nothing will open it. + let path = dir.join(RETIRED_MARKER); + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(gone) if gone.kind() == std::io::ErrorKind::NotFound => {} + Err(stuck) => { + return Err(MarkFailure { + reason: format!( + "{e}. The partial mark at {} could not be removed either \ + ({stuck})", + path.display() + ), + mark_definitely_gone: false, + pre_existing: false, + }) + } + } + if let Err(flush) = crate::storage::file_store::fsync_path(dir) { + return Err(MarkFailure { + reason: format!( + "{e}. Removing the partial mark at {} could not be flushed \ + ({flush})", + path.display() + ), + mark_definitely_gone: false, + pre_existing: false, + }); + } + Err(MarkFailure { + reason: format!("{e}"), + mark_definitely_gone: true, + pre_existing: false, + }) + } + } +} + +/// Why a directory could not be marked retired, and whether it is safe to reopen. +#[derive(Debug)] +struct MarkFailure { + /// What went wrong, for the operator. + reason: String, + /// Is the directory provably free of a partial mark? + /// + /// Only then may the caller put it back under the live name. A mark left inside would + /// have the next cleanup pass reap a live, open environment. + mark_definitely_gone: bool, + /// Was the mark already there before this attempt? + /// + /// Then this attempt created nothing and must take nothing away. Removing a mark that + /// was already there because re-flushing it failed is how a correctly retired + /// directory comes to look unmarked, and an unmarked directory is restored as a live + /// environment. + pre_existing: bool, +} + +impl std::fmt::Display for MarkFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.reason) + } +} + +/// Create the mark. See [`mark_directory_retired`], which owns the failure handling. +fn write_retirement_mark(dir: &Path) -> std::result::Result<(), MarkFailure> { + let path = dir.join(RETIRED_MARKER); + let mut file = match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + { + Ok(f) => f, + // Already there, from an attempt that got this far and no further. Flushed + // again rather than taken on trust: the attempt that wrote it may have been the + // one that could not flush it. + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + // Something is already at that name. That it could not be created is not the + // same as its being a mark this node can read, and everything downstream + // deletes an environment on the strength of it. The rest of this file insists + // the name is not the evidence; this is the one place that was taking it. + if retirement_mark(dir) != RetirementMark::Present { + return Err(MarkFailure { + reason: format!( + "{} already exists but cannot be read as a retirement mark, so it \ + is not one this node will delete on. Check what is at that path.", + path.display() + ), + mark_definitely_gone: false, + pre_existing: true, + }); + } + return crate::storage::file_store::fsync_path(dir).map_err(|flush| MarkFailure { + reason: format!( + "{} is already there but could not be flushed: {flush}", + path.display() + ), + mark_definitely_gone: false, + pre_existing: true, + }); + } + Err(e) => { + return Err(MarkFailure { + reason: format!("Could not mark {} as retired: {e}", path.display()), + mark_definitely_gone: true, + pre_existing: false, + }) + } + }; + // For whoever reads the directory. To the node, presence is the whole signal. + if let Err(e) = file.write_all( + b"This chunk environment was verified as fully copied into the file store and \n\ +retired. It is being deleted; if it is still here, that was interrupted and the next \n\ +node start finishes it. Nothing needs it.\n", + ) { + drop(file); + return Err(MarkFailure { + reason: format!("Could not write {}: {e}", path.display()), + mark_definitely_gone: false, + pre_existing: false, + }); + } + file.sync_all().map_err(|e| MarkFailure { + reason: format!( + "Could not flush {}: {e}. Not deleting on the strength of a mark that may not \ + survive a power loss.", + path.display() + ), + mark_definitely_gone: false, + pre_existing: false, + })?; + // And the directory that now contains it. Flushing the file makes its contents + // durable; the entry naming it is in the directory, and on Unix that needs its own + // flush. Without this the mark can be missing after a crash from a directory that + // was in fact retired, which is the whole question this file answers. + crate::storage::file_store::fsync_path(dir).map_err(|e| MarkFailure { + reason: format!( + "Marked {} retired but could not flush {}: {e}. Not deleting on the strength \ + of a mark that may not survive a power loss.", + path.display(), + dir.display() + ), + mark_definitely_gone: false, + pre_existing: false, + }) +} + +/// Delete a retired directory in the background, without anything waiting for it. +/// +/// The caller is finished with it either way: the environment is closed, the directory is +/// under a name nothing looks for, and it carries its own mark, so an interrupted deletion +/// is finished by the next start. What matters is that neither shutdown nor startup ever +/// blocks on a recursive delete that can run for minutes. +fn delete_retired_directory(dir: PathBuf) { + // One at a time per directory. The driver asks for cleanup on every tick while + // anything is pending, and starting a fresh thread each time would leave hundreds of + // them asleep on the same path, all retrying the same failure. + if !REAPING.lock().insert(dir.clone()) { + return; + } + let named = dir.clone(); + let started = std::thread::Builder::new() + .name("chunk-store-retire".into()) + .spawn(move || { + let _done = ReapingGuard(dir.clone()); + for attempt in 1..=RETIRED_DELETE_ATTEMPTS { + match remove_marked_directory(&dir) { + Ok(()) => { + info!( + migration_event = "space_returned", + "Removed the retired chunk environment {} and returned its \ + space", + dir.display() + ); + return; + } + // Worth another go: on Windows a scanner or an antivirus can hold a + // handle inside it for a moment, and a partial delete leaves less to + // do next time. + Err(e) if attempt < RETIRED_DELETE_ATTEMPTS => { + debug!( + "Could not delete {} (attempt {attempt}): {e}. Trying again.", + dir.display() + ); + std::thread::sleep( + (RETIRED_DELETE_BACKOFF * attempt).min(RETIRED_DELETE_BACKOFF_MAX), + ); + } + Err(e) => warn!( + "The chunk environment has been retired but {} could not be \ + deleted: {e}. Its space is not returned until it is, and the node \ + needs nothing from it. The next start tries again.", + dir.display() + ), + } + } + }); + if let Err(e) = started { + REAPING.lock().remove(&named); + warn!( + "Could not start the thread to delete the retired chunk environment {}: {e}. \ + The next start sweeps it.", + named.display() + ); + } +} + +/// Directories a reaper thread is already working on. +static REAPING: parking_lot::Mutex> = parking_lot::Mutex::new(BTreeSet::new()); + +/// Releases a directory from [`REAPING`] however its thread ends. +struct ReapingGuard(PathBuf); + +impl Drop for ReapingGuard { + fn drop(&mut self) { + REAPING.lock().remove(&self.0); + } +} + +/// Delete a retired directory, taking its mark away last of all. +/// +/// `remove_dir_all` walks in whatever order the filesystem hands back, so it can unlink +/// the mark and then fail on the next entry, which is exactly what a Windows sharing +/// violation on the data file produces. What is left is a genuinely retired, partly +/// deleted directory carrying no evidence that it was retired, and the next start would +/// read that as an intact environment and restore it. +/// +/// Emptying it first and removing the mark last means the mark is only ever absent from a +/// directory that has nothing else left in it. +/// +/// # Errors +/// +/// Returns the underlying I/O error. The directory is left with its mark intact on every +/// failure that happens before the mark is reached. +fn remove_marked_directory(dir: &Path) -> std::io::Result<()> { + // Never through a link. An operator who points the chunk environment at another + // volume leaves a symlink here, and walking it would delete the contents of a + // directory that is not this node's to delete. Retirement refuses such a root before + // it gets this far; this is the second line, because the check and the walk are not + // one operation. + if std::fs::symlink_metadata(dir)?.file_type().is_symlink() { + return Err(std::io::Error::other(format!( + "{} is a link, not a directory. Refusing to delete through it.", + dir.display() + ))); + } + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + if entry.file_name() == RETIRED_MARKER { + continue; + } + let path = entry.path(); + if entry.file_type()?.is_dir() { + std::fs::remove_dir_all(&path)?; + } else { + std::fs::remove_file(&path)?; + } + } + match std::fs::remove_file(dir.join(RETIRED_MARKER)) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + match std::fs::remove_dir(dir) { + Ok(()) => Ok(()), + Err(e) => { + // The mark is gone and the directory is not, which is the one state the whole + // scheme says cannot happen: a start that found it would read an unmarked + // directory as an intact environment. Put the mark back before giving up. + if let Err(remark) = mark_directory_retired(dir) { + error!( + "Could not remove {} ({e}) and could not restore its retirement mark \ + ({remark}). It is empty and nothing needs it; delete it by hand.", + dir.display() + ); + } + Err(e) + } + } +} + +/// What a directory's own contents say about whether it was retired. +/// +/// Three answers, not two. Reading the mark can fail for reasons that are neither yes nor +/// no: a permission change, a descriptor limit, a filesystem that has gone away underneath +/// the node. Folding that into "no" is the failure mode the rest of this file exists to +/// avoid, and it fails in the worst direction: a retired environment that reads as unmarked +/// is put back under the live name and reopened, and its keys re-enter a commitment they +/// have already left. +/// +/// So an unreadable answer is its own answer, and the two questions callers actually ask +/// are asked separately. Neither of them treats "cannot tell" as permission. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RetirementMark { + /// The directory carries its mark. It is the remains of a removal. + Present, + /// The directory carries no mark, and that is known rather than assumed. + Absent, + /// Whether it carries one could not be determined. + Unknown, +} + +impl RetirementMark { + /// May this directory be deleted, or treated as already gone? + /// + /// Only a mark actually read says yes. Deleting on a guess destroys chunks. + const fn permits_removal(self) -> bool { + matches!(self, Self::Present) + } + + /// May this directory be opened and served from? + /// + /// Only a mark known to be absent says yes. Opening a retired environment puts keys + /// back into a commitment they have already left. + const fn permits_opening(self) -> bool { + matches!(self, Self::Absent) + } +} + +/// Has this directory been retired? +/// +/// A link is never treated as retired, whatever it points at: the mark would have been +/// written through it into somebody else's directory, and acting on it would delete +/// somebody else's data. A path whose kind cannot be determined is not a link either way, +/// and is reported as unknown rather than as a link, so that neither question gets a yes. +fn retirement_mark(dir: &Path) -> RetirementMark { + match std::fs::symlink_metadata(dir) { + Ok(meta) if meta.file_type().is_symlink() => return RetirementMark::Absent, + Ok(_) => {} + // Nothing here at all, which is the ordinary case on a node that has already + // finished or never had a legacy store. There is no mark because there is nothing + // to carry one, and that is known rather than undetermined. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return RetirementMark::Absent, + Err(e) => { + // Debug, not warn: this is asked on every tick, so a warn here would be a + // wall of the same line. The operator-facing version is the retirement + // blocker, which says what it means for the node. + debug!( + "Could not tell what {} is ({e}); treating it as neither removable nor \ + openable until it can be read", + dir.display() + ); + return RetirementMark::Unknown; + } + } + match dir.join(RETIRED_MARKER).try_exists() { + Ok(true) => RetirementMark::Present, + Ok(false) => RetirementMark::Absent, + Err(e) => { + debug!( + "Could not read the retirement mark in {} ({e}); treating it as neither \ + removable nor openable until it can be read", + dir.display() + ); + RetirementMark::Unknown + } + } +} + +/// Is this path a symbolic link, or something whose kind cannot be determined? +/// +/// Unknown counts as yes. Every caller is deciding whether it is safe to delete through +/// the path, and a question that cannot be answered is not a yes to that. +fn is_a_link(path: &Path) -> bool { + std::fs::symlink_metadata(path).map_or(true, |m| m.file_type().is_symlink()) +} + +/// Finish a removal a previous run did not, before anything tries to open the environment. +/// +/// The only thing that counts as evidence is the directory's own mark. An open that fails +/// is not: `open_legacy` queries free space, maps the file, takes a write transaction and +/// scans every key, so a full disk, a permission change, a mapping limit or a transient +/// I/O fault all look identical to corruption, and deleting on any of those would destroy +/// a perfectly good environment. +fn finish_interrupted_retirement(root_dir: &Path) -> LiveEnvironment { + let env = root_dir.join(LEGACY_ENV_DIR); + // Three answers, three branches. Asking only whether it may be removed and letting + // everything else fall through would put "cannot tell" back on the opening path, which + // is the whole failure this is three states to avoid: the mark check can fail for a + // moment and succeed the next, and the open in between would resurrect a store that + // really had been retired. + // + // Asked of the mark alone, with no separate "is it there" first. A `try_exists` that + // could not answer would have folded straight back into "nothing here" and skipped both + // branches below, which is the same fold one level up. The mark already tells the three + // apart: a path that is not there carries no mark and says so, and a path that cannot be + // reached at all says it cannot be reached. + // Asked once. Asking twice is asking two different questions: the answer can change + // between them, and a second answer of "cannot tell" after a first of "retired" fell + // through to opening the very directory the first answer said not to open. + let mark = retirement_mark(&env); + if mark == RetirementMark::Unknown { + error!( + "{} is under the live name and this node cannot tell whether it was retired. \ + It will NOT be opened and it will NOT be removed. The node serves from files \ + alone. Check that the directory and anything inside it can be read.", + env.display() + ); + return LiveEnvironment::None; + } + if mark.permits_removal() { + // Its own contents say it was retired, so whatever name it is wearing now, it is + // the remains of a removal that a power loss undid the rename of. + warn!( + "{} carries its own retirement mark, so it is what an interrupted removal left \ + behind rather than a live environment. Finishing that removal.", + env.display() + ); + // Renamed rather than deleted here, so the node can get on with starting: the + // deletion itself is detached below and can take minutes on a large store. Under a + // name nothing else is using, so a tombstone whose deletion is still running does + // not force a synchronous delete first. + let tombstone = free_tombstone_path(root_dir); + if let Err(e) = std::fs::rename(&env, &tombstone) { + error!( + "{} carries its own retirement mark but could not be moved aside: {e}. It \ + will NOT be opened: it says it has been retired, so it may be partly \ + deleted, and its chunks are in the file store. The node serves from files \ + alone and the next start tries again.", + env.display() + ); + sweep_retired_legacy(root_dir); + return LiveEnvironment::None; + } + } + sweep_retired_legacy(root_dir); + LiveEnvironment::WhateverIsOnDisk +} + +/// Whether the ordinary open may look at what is under the live environment name. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LiveEnvironment { + /// Nothing is claiming it should not be opened. + WhateverIsOnDisk, + /// A directory under the live name says it has been retired, and could not be moved + /// out of the way. It must not be opened: a retired directory may be partly deleted, + /// and opening it would put its keys back into a commitment they have left. + None, +} + +fn sweep_retired_legacy(root_dir: &Path) { + let tombstones = retired_tombstones(root_dir); + if tombstones.is_empty() { + return; + } + // Flushed first, and only best effort is not good enough here for the same reason it + // was not good enough when the rename was made: deleting the contents of a directory + // whose new name may not have reached the disk is what turns a power loss into a + // resurrected, half-empty environment. If it cannot be flushed, leave them for a later + // start. It costs disk, not data. + if let Err(e) = crate::storage::file_store::fsync_path(root_dir) { + warn!( + "Leaving {} retired chunk environment(s) in place: {} could not be flushed \ + ({e}), so the rename that put them there may not be on disk yet.", + tombstones.len(), + root_dir.display() + ); + return; + } + for tombstone in tombstones { + // The name is not the evidence. Only the directory's own mark is: a crash between + // the rename and the mark leaves an intact environment sitting under the retired + // name, and deleting that because of what it is called would destroy every chunk + // in it. + let mark = retirement_mark(&tombstone); + if mark == RetirementMark::Unknown { + // Neither restored nor deleted. Restoring would put a directory that may be + // half-deleted back under the live name for the next start to open, and + // deleting would destroy an intact one. It costs disk until somebody looks, + // which is the right price for not knowing. + warn!( + "{} cannot be classified: this node cannot tell whether it carries a \ + retirement mark, so it will be neither restored nor deleted. Check that \ + the directory and anything inside it can be read.", + tombstone.display() + ); + continue; + } + if mark.permits_removal() { + // The mark is re-established before anything is deleted on the strength of + // it. A retirement that failed part-way can leave one that was never flushed, + // and this is the pass that would otherwise act on it thirty seconds after + // the failure that said it would be left alone. + if let Err(e) = mark_directory_retired(&tombstone) { + warn!( + "{} says it was retired but that could not be confirmed ({e}). \ + Leaving it.", + tombstone.display() + ); + continue; + } + // Detached, so a node starting beside a large leftover directory serves + // immediately rather than waiting out a recursive delete before it opens its + // store. + delete_retired_directory(tombstone); + continue; + } + restore_unmarked_environment(root_dir, &tombstone); + } +} + +/// Put an intact environment back under its own name. +/// +/// An environment under the retired name with no mark inside it was renamed and then +/// interrupted before it could be marked. Nothing was deleted, so it is whole, and the +/// answer is to give it its name back and let the migration run again from the beginning: +/// every gate is re-derived, and a second retirement costs a pass, not data. +fn restore_unmarked_environment(root_dir: &Path, tombstone: &Path) { + // An empty one is what a deletion that removed the contents and the mark and then + // could not remove the directory leaves. There is nothing in it to restore, and + // putting it back under the live name would strand an empty path the node then tries + // to open. + if std::fs::read_dir(tombstone).is_ok_and(|mut entries| entries.next().is_none()) { + if let Err(e) = std::fs::remove_dir(tombstone) { + warn!("Could not remove the empty {}: {e}", tombstone.display()); + } + return; + } + let env = root_dir.join(LEGACY_ENV_DIR); + if env.try_exists().unwrap_or(true) { + // Both names are taken, so which one the node should serve is not this code's + // decision to make. + error!( + "{} and {} both exist, and {} carries no retirement mark, so it may hold \ + chunks. Neither has been touched. Move or remove one by hand: the node is \ + using {}.", + env.display(), + tombstone.display(), + tombstone.display(), + env.display() + ); + return; + } + match std::fs::rename(tombstone, &env) { + Ok(()) => { + let _ = crate::storage::file_store::fsync_path(root_dir); + warn!( + "{} was moved aside for retirement but never marked retired, so it is \ + intact. It has been restored to {} and the migration starts again.", + tombstone.display(), + env.display() + ); + } + Err(e) => error!( + "{} carries no retirement mark, so it may hold chunks, but it could not be \ + restored to {}: {e}. It has not been deleted.", + tombstone.display(), + env.display() + ), + } +} + +/// Every retired environment directory under `root_dir`. +/// +/// More than one can be there: a node that retires, is restarted before the deletion +/// finishes, and somehow acquires another environment would leave the first behind. Each +/// is named so it cannot collide with the next. +fn retired_tombstones(root_dir: &Path) -> Vec { + let prefix = format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}"); + let entries = match std::fs::read_dir(root_dir) { + Ok(entries) => entries, + Err(e) => { + // Cannot tell. Not the same as nothing here, and the caller uses this to + // decide whether cleanup is finished, so answer with the one that keeps it + // looking rather than the one that declares victory. + warn!( + "Could not list {} to look for retired chunk environments: {e}", + root_dir.display() + ); + return vec![root_dir.join(&prefix)]; + } + }; + let mut found = Vec::new(); + for entry in entries { + match entry { + Ok(entry) => { + if entry + .file_name() + .to_str() + .is_some_and(|n| n.starts_with(&prefix)) + { + found.push(entry.path()); + } + } + // One unreadable entry is not evidence there is nothing here, and the caller + // uses this to decide whether cleanup is finished. Answer with the one that + // keeps it looking. + Err(e) => { + warn!( + "Could not read an entry of {} while looking for retired chunk \ + environments: {e}", + root_dir.display() + ); + found.push(root_dir.join(&prefix)); + } + } + } + found +} + +/// A directory name to retire the environment under that nothing else is using. +/// +/// A fixed name would collide with a tombstone whose deletion is still running, and +/// clearing that one first would put a synchronous recursive delete back on the path this +/// is trying to keep clear. +fn free_tombstone_path(root_dir: &Path) -> PathBuf { + let base = root_dir.join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")); + if !base.try_exists().unwrap_or(true) { + return base; + } + for n in 1..=MAX_TOMBSTONES { + let candidate = root_dir.join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}.{n}")); + if !candidate.try_exists().unwrap_or(true) { + return candidate; + } + } + // Every name taken, which means many retirements have been interrupted without their + // deletions finishing. Reuse the base: the rename fails, retirement defers, and the + // operator sees a directory full of them. + base +} + +/// Whether a legacy environment is on disk under `root_dir`. +/// +/// # Errors +/// +/// Returns [`Error::Storage`] if the answer cannot be determined. `Path::exists` would +/// turn a permission problem into "absent", and a node that starts in file-only mode +/// beside a `chunks.mdb` holding every chunk it has stops serving all of them. +pub fn legacy_present(root_dir: &Path) -> Result { + let path = root_dir.join(LEGACY_ENV_DIR).join(LEGACY_DATA_FILE); + path.try_exists().map_err(|e| { + Error::Storage(format!( + "Cannot tell whether the legacy chunk environment {} exists: {e}. Refusing to \ + start rather than ignore it.", + path.display() + )) + }) +} + +/// How long to sleep after copying `bytes` to hold the copier to a rate ceiling. +fn throttle_delay(bytes: u64, mib_per_sec: u64) -> Option { + if mib_per_sec == 0 { + return None; + } + let per_sec = mib_per_sec.saturating_mul(1024 * 1024); + if per_sec == 0 { + return None; + } + let micros = bytes.saturating_mul(1_000_000) / per_sec; + if micros == 0 { + None + } else { + Some(Duration::from_micros(micros)) + } +} + +/// Merge two ascending key sequences into one, dropping duplicates. +fn merge_sorted<'a, I>(sorted: &[XorName], other: I) -> Vec +where + I: Iterator, +{ + let other: Vec = other.copied().collect(); + let mut out = Vec::with_capacity(sorted.len() + other.len()); + let mut a = sorted.iter().copied().peekable(); + let mut b = other.into_iter().peekable(); + loop { + match (a.peek(), b.peek()) { + (Some(x), Some(y)) => match x.cmp(y) { + std::cmp::Ordering::Less => out.extend(a.next()), + std::cmp::Ordering::Greater => out.extend(b.next()), + std::cmp::Ordering::Equal => { + out.extend(a.next()); + let _ = b.next(); + } + }, + (Some(_), None) => out.extend(a.next()), + (None, Some(_)) => out.extend(b.next()), + (None, None) => break, + } + } + out +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + use crate::storage::migration::{now_unix, rank_closest_first, MIN_RETIRE_DELAY_HOURS}; + use tempfile::TempDir; + + /// Everything currently legacy-only, as the set a test has "approved" for shedding. + fn approved_shed(store: &ChunkStore) -> BTreeSet { + store.legacy_only_keys().into_iter().collect() + } + + /// A token that is never cancelled, for tests that are not exercising shutdown. + fn never_cancelled() -> CancellationToken { + CancellationToken::new() + } + + /// Put a store through every gate a real node passes before it may retire. + /// + /// Deliberately not a shortcut around them: `retire_legacy` rechecks the whole set + /// itself, so a test that skipped them would exercise a path production never takes. + fn open_the_retirement_gate(store: &ChunkStore) { + store.commit_to_files().expect("commit"); + store.note_commitment_rebuilt(); + store.note_commitment_rebuilt(); + store.force_migration_state(|s| { + s.committed_at_unix = + Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); + }); + } + + /// Content plus the address it hashes to. + fn addressed(seed: &str) -> (XorName, Vec) { + let content = format!("chunk-content-{seed}").into_bytes(); + (crate::client::compute_address(&content), content) + } + + fn test_config(dir: &TempDir) -> ChunkStoreConfig { + ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + ..ChunkStoreConfig::test_default() + } + } + + async fn open(dir: &TempDir) -> ChunkStore { + ChunkStore::new(test_config(dir)).await.expect("open store") + } + + /// Populate a legacy LMDB environment the way an existing node would have one, then + /// close it so the facade can adopt it. + async fn seed_legacy(dir: &TempDir, seeds: &[&str]) -> Vec { + let lmdb = LmdbStorage::new(LmdbStorageConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + max_map_size: 0, + disk_reserve: 0, + }) + .await + .expect("open legacy"); + let mut keys = Vec::new(); + for seed in seeds { + let (addr, content) = addressed(seed); + lmdb.put(&addr, &content).await.expect("legacy put"); + keys.push(addr); + } + lmdb.wait_idle().await; + drop(lmdb); + keys + } + + #[tokio::test] + async fn a_fresh_node_never_creates_a_legacy_environment() { + let dir = TempDir::new().expect("temp dir"); + let store = open(&dir).await; + + assert!(!store.has_legacy()); + assert_eq!(store.migration_phase(), MigrationPhase::FilesOnly); + assert!(!dir.path().join(LEGACY_ENV_DIR).exists()); + + let (addr, content) = addressed("fresh"); + assert!(store.put(&addr, &content).await.expect("put")); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + } + + #[tokio::test] + async fn an_existing_legacy_store_is_adopted_and_served() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["a", "b", "c"]).await; + let store = open(&dir).await; + + assert!(store.has_legacy()); + assert_eq!(store.migration_phase(), MigrationPhase::Bridging); + assert_eq!(store.current_chunks().expect("count"), 3); + for key in &keys { + assert!(store.exists(key).expect("exists"), "union must see it"); + assert!(store.get(key).await.expect("get").is_some()); + } + } + + #[tokio::test] + async fn the_union_key_set_is_sorted_and_free_of_duplicates() { + let dir = TempDir::new().expect("temp dir"); + let mut expected = seed_legacy(&dir, &["u1", "u2", "u3", "u4"]).await; + let store = open(&dir).await; + + // One chunk written now lives in both backings, and must be counted once. + let (addr, content) = addressed("u2"); + assert!(expected.contains(&addr)); + assert!(!store.put(&addr, &content).await.expect("put")); + + let (fresh, fresh_content) = addressed("u5"); + store.put(&fresh, &fresh_content).await.expect("put"); + expected.push(fresh); + expected.sort_unstable(); + + let keys = store.all_keys().await.expect("all_keys"); + assert_eq!(keys, expected); + assert_eq!(store.current_chunks().expect("count"), 5); + } + + /// The legacy environment takes no new disk during the bridge. + /// + /// Both stores sit on one disk, each measures the same free space, and neither knows + /// what the other is about to spend. A chunk written to both could be admitted twice + /// against one lot of headroom, and enough of them could cross the reserve together + /// and fill the volume this migration exists to free. + /// + /// So the environment is pinned to what it already occupies. It still takes the + /// rollback copy when it has room of its own, which on a real node it usually does: + /// this migration exists because deleting millions of chunks left the free list full + /// and returned nothing to the filesystem. What it will not do is grow. + #[tokio::test] + async fn the_legacy_environment_takes_no_new_disk_during_the_bridge() { + let dir = TempDir::new().expect("temp dir"); + seed_legacy(&dir, &["seed"]).await; + let store = open(&dir).await; + + let data_file = dir.path().join(LEGACY_ENV_DIR).join(LEGACY_DATA_FILE); + let before = std::fs::metadata(&data_file).expect("meta").len(); + + for seed in ["dual-1", "dual-2", "dual-3", "dual-4"] { + let (addr, content) = addressed(seed); + assert!(store.put(&addr, &content).await.expect("put")); + assert!( + store.exists(&addr).expect("exists"), + "the file store is the one that has to have it" + ); + } + store.wait_idle().await; + + let after = std::fs::metadata(&data_file).expect("meta").len(); + assert_eq!( + after, before, + "the environment must not claim disk the file store is also counting on" + ); + } + + #[tokio::test] + async fn the_copier_moves_keys_into_files_and_is_resumable() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["c1", "c2", "c3", "c4", "c5"]).await; + let store = open(&dir).await; + assert_eq!(store.legacy_only_keys().len(), 5); + + let first = store + .copy_batch(&keys[..2], 0, 0, &never_cancelled()) + .await + .expect("copy first batch"); + assert_eq!(first.copied, 2); + assert_eq!(store.legacy_only_keys().len(), 3); + store.wait_idle().await; + drop(store); + + // A restart re-derives what is left from the filesystem: no progress file to + // corrupt, and no work repeated. + let store = open(&dir).await; + assert_eq!(store.legacy_only_keys().len(), 3); + let rest = store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy rest"); + assert_eq!(rest.copied, 3); + assert!(store.legacy_only_keys().is_empty()); + assert_eq!(store.current_chunks().expect("count"), 5); + } + + #[tokio::test] + async fn a_delete_reaches_both_stores() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["d1", "d2"]).await; + let store = open(&dir).await; + + let target = keys.first().copied().expect("a key"); + assert!(store.delete(&target).await.expect("delete")); + assert!(!store.exists(&target).expect("exists")); + assert!(store.get(&target).await.expect("get").is_none()); + assert_eq!(store.current_chunks().expect("count"), 1); + + // And it stays gone across a restart, which is what proves it left the legacy + // environment too rather than only the union view. + store.wait_idle().await; + drop(store); + let store = open(&dir).await; + assert!(!store.exists(&target).expect("exists")); + } + + #[tokio::test] + async fn the_copier_does_not_resurrect_a_deleted_chunk() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["r1", "r2"]).await; + let store = open(&dir).await; + + let target = keys.first().copied().expect("a key"); + store.delete(&target).await.expect("delete"); + + let report = store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + assert_eq!(report.copied, 1, "only the surviving chunk may be copied"); + assert!(!store.exists(&target).expect("exists")); + } + + #[tokio::test] + async fn committing_narrows_the_commitment_but_not_what_is_served() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["k1", "k2", "k3"]).await; + let store = open(&dir).await; + + // Copy one, leave two behind as if the disk had run out. + store + .copy_batch(&keys[..1], 0, 0, &never_cancelled()) + .await + .expect("copy"); + + // While bridging, the node still claims everything it can serve. + assert_eq!( + store.committable_keys().await.expect("committable").len(), + 3 + ); + + store.commit_to_files().expect("commit"); + assert_eq!(store.migration_phase(), MigrationPhase::Committed); + assert_eq!(store.migration_state().shed_key_count, 2); + + // It now claims only what it will keep... + assert_eq!( + store.committable_keys().await.expect("committable").len(), + 1 + ); + // ...while still serving everything it ever claimed. + assert_eq!(store.all_keys().await.expect("all_keys").len(), 3); + for key in &keys { + assert!(store.get(key).await.expect("get").is_some()); + } + } + + #[tokio::test] + async fn retirement_is_refused_until_every_gate_is_satisfied() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["g1", "g2"]).await; + let mut config = test_config(&dir); + config.migration.retire_legacy = true; + let store = ChunkStore::new(config).await.expect("open"); + + // Still bridging. + assert!(store + .retirement_blocker(|_| false) + .expect("blocked") + .contains("Bridging")); + + store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + store.commit_to_files().expect("commit"); + + // No commitment rebuild observed yet. + assert!(store + .retirement_blocker(|_| false) + .expect("blocked") + .contains("commitment rebuilds")); + + store.note_commitment_rebuilt(); + store.note_commitment_rebuilt(); + + // The retention delay has not elapsed. + assert!(store + .retirement_blocker(|_| false) + .expect("blocked") + .contains("retirement delay")); + + store.force_migration_state(|s| { + s.committed_at_unix = + Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); + }); + assert!(store.retirement_blocker(|_| false).is_none()); + } + + #[tokio::test] + async fn a_chunk_still_answerable_vetoes_retirement() { + let dir = TempDir::new().expect("temp dir"); + seed_legacy(&dir, &["h1", "h2"]).await; + let mut config = test_config(&dir); + config.migration.retire_legacy = true; + let store = ChunkStore::new(config).await.expect("open"); + + // Shed both, as a node short of disk would. + store.commit_to_files().expect("commit"); + store.note_commitment_rebuilt(); + store.note_commitment_rebuilt(); + store.force_migration_state(|s| { + s.committed_at_unix = + Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); + }); + + // The pruner's existing retention contract, reused verbatim: a key the node + // could still be challenged on keeps its last local copy. + assert!(store + .retirement_blocker(|_| true) + .expect("blocked") + .contains("still answerable")); + assert!(store.retirement_blocker(|_| false).is_none()); + } + + /// The stock configuration retires, with no environment variable and no operator step. + /// + /// This is the property the whole release rests on. Deleting `chunks.mdb` is the only + /// step that returns disk: LMDB never gives freed pages back, which is why the fleet + /// deleted millions of chunks and recovered nothing. A build that shipped with this + /// off would migrate every node and reclaim not one byte. + #[tokio::test] + async fn the_shipped_configuration_retires_without_an_operator_setting_anything() { + // Asked of the configuration a node actually builds, not of the constant behind + // it, so neither the constant nor a serde default nor the `Default` impl can turn + // retirement off without this failing. + assert!( + crate::storage::MigrationConfig::default().retire_legacy, + "this release must delete the legacy environment, or it frees no disk" + ); + + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["shipped"]).await; + let store = open(&dir).await; + store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + open_the_retirement_gate(&store); + assert!( + store.retirement_blocker(|_| false).is_none(), + "with every gate met, the shipped configuration must not refuse to retire" + ); + } + + /// Retirement waits for a read that is already running. + /// + /// The window this closes: a verifying read finds rotted bytes, throws the file away, + /// and has not yet reached the legacy copy that would replace it. If retirement ran in + /// that gap it would delete the only remaining copy. Holding the barrier shared for + /// the whole read, and exclusively for the removal, is what makes that impossible. + #[tokio::test] + async fn retirement_waits_for_a_read_that_is_already_running() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["held"]).await; + let store = Arc::new(open(&dir).await); + store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + open_the_retirement_gate(&store); + let proof = store + .verify_before_retire(0, &never_cancelled()) + .await + .expect("verify"); + + // Stand in for a read that has started and not finished. + let reading = store.retirement.read().await; + + let retiring = { + let store = Arc::clone(&store); + let approved = approved_shed(&store); + tokio::spawn(async move { + store + .retire_legacy(&proof, &|_: &XorName| false, &approved) + .await + }) + }; + + // It must not have got anywhere. Given a generous window rather than a tight one, + // so this fails on the behaviour rather than on scheduling luck. + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + !retiring.is_finished(), + "retirement removed the legacy environment while a read was still running" + ); + assert!( + store.has_legacy(), + "the legacy environment went while a read was still running" + ); + + drop(reading); + let freed = retiring.await.expect("join").expect("retire"); + assert!(freed > 0, "retirement should have freed the environment"); + assert!(!store.has_legacy()); + } + + /// A good copy is never turned away because a damaged one wears its name. + /// + /// Two shapes of damage, because they are caught differently: a short file, which is + /// what an interrupted create leaves on a platform that writes under the final name, + /// and a full-length file with wrong bytes, which is what rot leaves. Answering + /// "already have it" to either discards the copy that would fix it, and nothing offers + /// it again. + #[tokio::test] + async fn a_damaged_chunk_is_repaired_from_the_copy_being_offered() { + let dir = TempDir::new().expect("temp dir"); + let store = open(&dir).await; + let (addr, content) = addressed("repairable-by-offer"); + store.put(&addr, &content).await.expect("put"); + + // Intact: the offer is correctly refused. + assert!(store.holds_verified(&addr, &content).await); + + // Truncated. + let path = dir + .path() + .join("chunks") + .join(format!("{:02x}", addr.last().copied().unwrap_or(0))) + .join(hex::encode(addr)); + std::fs::write(&path, &content[..content.len() / 2]).expect("truncate"); + assert!( + store.holds_verified(&addr, &content).await, + "a short file must be replaced from the offered copy, not left in place" + ); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + + // Same length, wrong bytes. + let rotted = vec![b'x'; content.len()]; + assert_ne!(rotted, content); + std::fs::write(&path, &rotted).expect("rot"); + assert!( + store.holds_verified(&addr, &content).await, + "a rotted file must be replaced from the offered copy" + ); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + } + + /// A chunk held only in the legacy environment is checked, not assumed good. + /// + /// The bytes in there can be wrong too, and when the copier finds that out it drops + /// the key from the union view. Having turned the good copy away on the strength of + /// the key being present, the node would then hold nothing at all. + #[tokio::test] + async fn a_legacy_only_chunk_with_wrong_bytes_is_replaced_by_the_offer() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["legacy-side"]).await; + let key = *keys.first().expect("one key"); + let content = format!("chunk-content-{}", "legacy-side").into_bytes(); + let store = open(&dir).await; + assert!(store.legacy_only_keys().contains(&key)); + + // Intact: the offer is correctly refused. + assert!(store.holds_verified(&key, &content).await); + + // Wreck the legacy copy underneath, leaving the key in the union view. + let legacy = store.legacy().expect("legacy"); + legacy.lmdb.delete(&key).await.expect("delete"); + assert!( + store.holds_verified(&key, &content).await, + "with no readable legacy copy the offered bytes must be taken, not refused" + ); + assert_eq!( + store.get(&key).await.expect("get").expect("present"), + content + ); + assert!( + !store.legacy_only_keys().contains(&key), + "and the key must leave the legacy-only set now that a file holds it" + ); + } + + /// A chunk this node does not have is not claimed as held. + #[tokio::test] + async fn a_chunk_this_node_does_not_have_is_not_claimed() { + let dir = TempDir::new().expect("temp dir"); + let store = open(&dir).await; + let (addr, content) = addressed("never-stored"); + assert!(!store.holds_verified(&addr, &content).await); + } + + /// An environment is never deleted because it failed to open. + /// + /// Opening is not a corruption test. It queries free space, maps the file, takes a + /// write transaction and scans every key, so a full disk, a permission change or a + /// transient fault all look identical to corruption. The only thing that counts is the + /// directory's own mark. + #[tokio::test] + async fn an_unmarked_environment_is_kept_however_badly_it_reads() { + let dir = TempDir::new().expect("temp dir"); + seed_legacy(&dir, &["unmarked"]).await; + let env = dir.path().join(LEGACY_ENV_DIR); + assert_eq!(retirement_mark(&env), RetirementMark::Absent); + + finish_interrupted_retirement(dir.path()); + assert!( + env.exists(), + "an environment carrying no retirement mark must never be removed" + ); + } + + /// A directory carrying its own retirement mark is finished off, whatever it is named. + /// + /// This is the case the mark exists for. Off Unix the rename that moves the + /// environment aside cannot be shown to be durable, so a power loss can bring it back + /// under its old name with its contents already deleted. Without the mark the node + /// would refuse to start on it forever; with it, the directory says what it is. + #[tokio::test] + async fn a_directory_that_says_it_was_retired_is_removed_under_any_name() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["reverted"]).await; + { + let store = open(&dir).await; + store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + } + // What a reverted rename leaves: the old name, the retirement mark inside it. + let env = dir.path().join(LEGACY_ENV_DIR); + mark_directory_retired(&env).expect("mark"); + + let store = open(&dir).await; + assert!( + !store.has_legacy(), + "the remains of an interrupted removal must not be adopted" + ); + assert!(!env.exists(), "and the next start must finish the removal"); + // Every chunk is still served, from the file store. + for key in &keys { + assert!(store.get(key).await.expect("get").is_some()); + } + } + + /// The mark goes inside the directory, so a rename cannot separate them. + /// + /// A mark beside the environment would have to be cancelled when a retirement is + /// abandoned, cancellation can fail or be lost, and a stale one would then authorise + /// deleting an environment that had since taken a chunk. + #[test] + fn the_retirement_mark_travels_with_the_directory() { + let dir = TempDir::new().expect("temp dir"); + let original = dir.path().join("chunks.mdb"); + std::fs::create_dir_all(&original).expect("mkdir"); + mark_directory_retired(&original).expect("mark"); + assert_eq!(retirement_mark(&original), RetirementMark::Present); + + let renamed = dir.path().join("chunks.mdb.retired"); + std::fs::rename(&original, &renamed).expect("rename"); + assert!( + retirement_mark(&renamed) == RetirementMark::Present, + "the mark must survive the rename it exists to outlive" + ); + // And back again, which is what a power loss undoing the rename looks like. + std::fs::rename(&renamed, &original).expect("rename back"); + assert_eq!(retirement_mark(&original), RetirementMark::Present); + } + + /// Losing the handle to an environment that is still there is not completion. + /// + /// A rename that failed and then could not be reopened leaves the directory on disk + /// with no way to read it. Treating the missing handle as "nothing left to migrate" + /// would have the driver log the migration finished over a store still holding chunks + /// nothing else can serve. + #[tokio::test] + async fn a_lost_handle_beside_a_live_environment_blocks_retirement() { + let dir = TempDir::new().expect("temp dir"); + seed_legacy(&dir, &["orphaned"]).await; + let store = open(&dir).await; + assert!(store.has_legacy()); + + // Stand in for a failed rename followed by a failed reopen. + *store.legacy.write() = None; + assert!(!store.has_legacy()); + assert!(dir.path().join(LEGACY_ENV_DIR).exists()); + + let blocker = store + .retirement_blocker(|_| false) + .expect("a live environment with no handle must block"); + assert!( + blocker.contains("no handle"), + "the reason must name the actual problem, got: {blocker}" + ); + } + + /// A node that still holds its store open is gated too. + /// + /// The classification used to be asked only when there was no handle, which meant the + /// ordinary path never asked it: a node holding its environment open went through every + /// gate, renamed the directory aside and deleted it, whatever the mark said or failed to + /// say. Retirement deletes the last other copy of these chunks, so it is not a question + /// to skip because a different question already had an answer. + #[cfg(unix)] + #[tokio::test] + async fn an_environment_that_cannot_be_classified_blocks_retirement_even_with_a_handle() { + let dir = TempDir::new().expect("temp dir"); + seed_legacy(&dir, &["still-open"]).await; + let store = open(&dir).await; + assert!( + store.has_legacy(), + "this one keeps its handle, deliberately" + ); + // Whatever else is in the way at this point, it is not this. Compared rather than + // required to be nothing, because the other gates have their own tests and their + // own reasons to be unmet here. + let before = store.retirement_blocker(|_| false).unwrap_or_default(); + assert!( + !before.contains("already retired"), + "a readable environment must not be blocked for being unclassifiable: {before}" + ); + + make_the_mark_unreadable(&dir.path().join(LEGACY_ENV_DIR)); + + let blocker = store + .retirement_blocker(|_| false) + .expect("an environment that cannot be classified must block retirement"); + assert!( + blocker.contains("already retired"), + "the reason must name the actual problem, got: {blocker}" + ); + assert!( + store.legacy_cannot_be_classified(), + "and the driver must see it as work only a person can finish, so that it \ + gives the shared volume back" + ); + } + + /// A directory under the retired name with no mark inside it is an intact store. + /// + /// It got that name from a rename, and the rename happens after every gate; the mark + /// is written straight afterwards. A crash in between leaves a whole environment + /// wearing a name that says otherwise, and deleting it because of what it is called + /// would destroy every chunk in it. It is put back instead. + #[tokio::test] + async fn an_unmarked_retired_directory_is_restored_rather_than_deleted() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["not-really-retired"]).await; + let env = dir.path().join(LEGACY_ENV_DIR); + let tombstone = dir.path().join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")); + std::fs::rename(&env, &tombstone).expect("rename"); + assert_eq!(retirement_mark(&tombstone), RetirementMark::Absent); + + let store = open(&dir).await; + assert!( + store.has_legacy(), + "an unmarked environment must be restored and served, not deleted" + ); + assert!(env.exists(), "it must be back under its own name"); + for key in &keys { + assert!(store.get(key).await.expect("get").is_some()); + } + } + + /// A mark already at that name is not a mark until it can be read. + /// + /// The one place in this file that was taking the name as the evidence, which is the + /// thing every other part of it refuses to do. Retirement writes the mark with + /// `create_new`, and a failure saying something is already there was accepted as "the + /// mark is present" and the environment deleted on the strength of it. What is at that + /// name might be anything. + /// + /// It matters most for a node that already has its store open. That path never + /// consulted the mark at all until this round, so an unreadable one would have gone + /// through every gate and been deleted. + #[cfg(unix)] + #[test] + fn a_mark_already_at_that_name_is_not_accepted_until_it_can_be_read() { + let dir = TempDir::new().expect("temp dir"); + let env = dir.path().join(LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).expect("mkdir"); + make_the_mark_unreadable(&env); + + let refused = mark_directory_retired(&env).expect_err("an unreadable mark is not a mark"); + assert!( + !refused.mark_definitely_gone, + "something is at that name, so the caller must not treat it as absent and put \ + the directory back under a name that will be opened" + ); + assert_eq!(retirement_mark(&env), RetirementMark::Unknown); + + // And a real one is still accepted, so the refusal above is about being unable to + // read it rather than about there being something there at all. + std::fs::remove_file(env.join(RETIRED_MARKER)).expect("clear the link"); + mark_directory_retired(&env).expect("a first mark"); + mark_directory_retired(&env).expect("and the same mark again, which is readable"); + assert_eq!(retirement_mark(&env), RetirementMark::Present); + } + + /// A directory nothing can classify is neither restored nor deleted. + /// + /// The half of the three-state answer that a first attempt at this got wrong. Asking + /// only "may it be removed" and letting everything else fall through puts "cannot tell" + /// straight back on the restoring path, which is the resurrection this exists to + /// prevent: the mark check can fail for a moment and succeed the next, and the restore + /// in between brings back a store that really had been retired. + /// + /// Unix only: the state is staged with a symbolic link, which Windows does not offer + /// on the same terms. + #[cfg(unix)] + #[tokio::test] + async fn a_tombstone_that_cannot_be_classified_is_left_where_it_is() { + let dir = TempDir::new().expect("temp dir"); + seed_legacy(&dir, &["unclassifiable"]).await; + let env = dir.path().join(LEGACY_ENV_DIR); + let tombstone = dir.path().join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")); + std::fs::rename(&env, &tombstone).expect("rename"); + make_the_mark_unreadable(&tombstone); + assert_eq!(retirement_mark(&tombstone), RetirementMark::Unknown); + + sweep_retired_legacy(dir.path()); + let still_there = tombstone.exists(); + let restored = env.exists(); + + assert!( + still_there, + "a directory that cannot be classified must not be deleted: it may be intact" + ); + assert!( + !restored, + "a directory that cannot be classified must not be put back under the live \ + name: it may be half deleted" + ); + } + + /// The same directory under the live name is not opened either. + /// + /// Opening it would put keys back into a commitment they may already have left, and + /// this node cannot tell whether they have. Serving from files alone is the answer that + /// is right either way. + #[cfg(unix)] + #[test] + fn a_live_environment_that_cannot_be_classified_is_not_opened() { + let dir = TempDir::new().expect("temp dir"); + let env = dir.path().join(LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).expect("mkdir"); + std::fs::write(env.join("data.mdb"), b"not really an environment").expect("seed"); + assert_eq!( + finish_interrupted_retirement(dir.path()), + LiveEnvironment::WhateverIsOnDisk, + "a readable directory with no mark is ordinary and may be opened" + ); + + make_the_mark_unreadable(&env); + assert_eq!(retirement_mark(&env), RetirementMark::Unknown); + + let verdict = finish_interrupted_retirement(dir.path()); + let still_there = env.exists(); + + assert_eq!( + verdict, + LiveEnvironment::None, + "a directory that cannot be classified must not be opened" + ); + assert!( + still_there, + "and it must not be deleted either: it may be a live environment" + ); + } + + /// Make the retirement mark in `dir` unreadable without touching the directory itself. + /// + /// A symbolic link pointing at itself. Looking for the mark follows it, gets + /// `FilesystemLoop` back, and the answer is neither "there" nor "not there", which is + /// the state under test. + /// + /// Taking the directory's permissions away instead was the first attempt and staged too + /// much: at mode 000 the operating system refuses the rename as well, so the code being + /// tested was never reached and the test passed with its own protection removed. Root + /// can also read a mode-000 directory, which would have made it fail on any CI that + /// runs as root. A link loop is neither: everything else about the directory keeps + /// working, for every user. + #[cfg(unix)] + fn make_the_mark_unreadable(dir: &Path) { + let link = dir.join(RETIRED_MARKER); + let _ = std::fs::remove_file(&link); + std::os::unix::fs::symlink(&link, &link).expect("a link to itself"); + } + + /// Both names taken is not a decision this code makes. + #[tokio::test] + async fn an_unmarked_retired_directory_beside_a_live_one_is_left_alone() { + let dir = TempDir::new().expect("temp dir"); + seed_legacy(&dir, &["live"]).await; + let tombstone = dir.path().join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")); + std::fs::create_dir_all(&tombstone).expect("mkdir"); + std::fs::write(tombstone.join("data.mdb"), b"something").expect("write"); + + let store = open(&dir).await; + assert!(store.has_legacy()); + assert!( + tombstone.exists(), + "an unmarked directory must never be deleted, even beside a live one" + ); + assert!(dir.path().join(LEGACY_ENV_DIR).exists()); + } + + /// The mark is the last thing a deletion takes away. + /// + /// A recursive delete walks in whatever order the filesystem gives, so it can unlink + /// the mark and then fail on the next entry, which is what a sharing violation on the + /// data file looks like. That leaves a genuinely retired, partly deleted directory + /// carrying no evidence of it, and the next start would read that as intact and + /// restore it. + /// Unix only: the failure is provoked with directory permissions, which is not how + /// the same thing happens on Windows. The behaviour under test is platform-neutral. + #[cfg(unix)] + #[test] + fn a_failed_deletion_leaves_the_mark_in_place() { + use std::os::unix::fs::PermissionsExt; + + let dir = TempDir::new().expect("temp dir"); + let retired = dir.path().join("chunks.mdb.retired"); + std::fs::create_dir_all(&retired).expect("mkdir"); + std::fs::write(retired.join(LEGACY_DATA_FILE), b"payload").expect("write"); + mark_directory_retired(&retired).expect("mark"); + + // An entry that cannot be removed, standing in for whatever the filesystem + // refuses on the day. + std::fs::create_dir_all(retired.join("stuck")).expect("mkdir"); + let mut perms = std::fs::metadata(&retired).expect("meta").permissions(); + perms.set_mode(0o500); + std::fs::set_permissions(&retired, perms.clone()).expect("chmod"); + + let failed = remove_marked_directory(&retired).is_err(); + + perms.set_mode(0o700); + std::fs::set_permissions(&retired, perms).expect("chmod back"); + + assert!(failed, "the deletion was supposed to fail"); + assert!( + retirement_mark(&retired) == RetirementMark::Present, + "a deletion that failed must leave the mark, or the directory stops saying \ + what it is" + ); + } + + /// A mark that cannot be read is not permission to do anything. + /// + /// The reason this is three states and not two. Reading the mark can fail for reasons + /// that are neither yes nor no, and the old answer for those was "no mark", which is + /// the worst of the three: a retired environment reads as live, goes back under its own + /// name, and its keys re-enter a commitment they have already left. Deleting on an + /// unreadable answer would be just as wrong in the other direction. + /// + /// Unix only: the state is staged with a symbolic link, which Windows does not offer + /// on the same terms. + #[cfg(unix)] + #[test] + fn a_mark_that_cannot_be_read_permits_nothing() { + let dir = TempDir::new().expect("temp dir"); + let env = dir.path().join(LEGACY_ENV_DIR); + + // Nothing there is not the same as cannot tell, and it is the ordinary case: every + // node that never had a legacy store, and every node that has finished with one, + // asks this question on every tick. Answering "cannot tell" for those would have a + // fresh node run a migration driver forever over a store it does not have. + assert_eq!( + retirement_mark(&env), + RetirementMark::Absent, + "a directory that is not there carries no mark, and that is known" + ); + assert!(retirement_mark(&env).permits_opening()); + + std::fs::create_dir_all(&env).expect("mkdir"); + std::fs::write(env.join(RETIRED_MARKER), b"retired").expect("mark"); + assert_eq!(retirement_mark(&env), RetirementMark::Present); + + // Looking for the mark now goes round in a circle, so the answer is neither there + // nor not there. + make_the_mark_unreadable(&env); + let unreadable = retirement_mark(&env); + + assert_eq!( + unreadable, + RetirementMark::Unknown, + "a mark that cannot be read must not report as absent" + ); + assert!( + !unreadable.permits_removal(), + "an unreadable mark must not authorise deleting the environment" + ); + assert!( + !unreadable.permits_opening(), + "an unreadable mark must not authorise reopening the environment" + ); + } + + /// A directory under the live name that says it was retired is never opened. + /// + /// It may be partly deleted, and opening it would put keys back into a commitment + /// they have already left. Its chunks are in the file store, which is what the mark + /// records, so serving from files alone is correct. + #[tokio::test] + async fn a_marked_directory_under_the_live_name_is_not_served_from() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["marked-live"]).await; + { + let store = open(&dir).await; + store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + } + let env = dir.path().join(LEGACY_ENV_DIR); + mark_directory_retired(&env).expect("mark"); + + // Every name it could be moved to is taken by something that is not empty, so the + // rename fails and the marked directory stays under the live name. That is the + // case this is about: it must be left alone rather than opened. + for n in 0..=MAX_TOMBSTONES { + let taken = if n == 0 { + dir.path().join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")) + } else { + dir.path() + .join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}.{n}")) + }; + std::fs::create_dir_all(&taken).expect("mkdir"); + std::fs::write(taken.join("occupied"), b"x").expect("write"); + } + + let store = open(&dir).await; + assert!( + env.exists(), + "the rename was supposed to fail, leaving the marked directory in place" + ); + assert!( + !store.has_legacy(), + "a directory that says it was retired must never be opened as live" + ); + for key in &keys { + assert!(store.get(key).await.expect("get").is_some()); + } + } + + /// A linked environment is copied out of but never deleted. + /// + /// An operator who points the chunk store at another volume leaves a link here. + /// Retirement renames the path and then deletes what is behind it, and behind a link + /// is a directory somewhere else that this node does not own. + #[cfg(unix)] + #[tokio::test] + async fn a_linked_environment_is_never_retired() { + let outside = TempDir::new().expect("temp dir"); + let dir = TempDir::new().expect("temp dir"); + // A real environment that lives in `outside`; the node root only links to it. + seed_legacy(&outside, &["someone-elses"]).await; + let real = outside.path().join(LEGACY_ENV_DIR); + let bystander = outside.path().join("unrelated"); + std::fs::create_dir_all(&bystander).expect("mkdir"); + std::os::unix::fs::symlink(&real, dir.path().join(LEGACY_ENV_DIR)).expect("symlink"); + + let store = open(&dir).await; + let blocker = store + .retirement_blocker(|_| false) + .expect("a linked environment must block retirement"); + assert!( + blocker.contains("link"), + "the reason must name the actual problem, got: {blocker}" + ); + + // And nothing walks through it, whatever it is marked with. + std::fs::write(real.join(RETIRED_MARKER), b"x").expect("mark through the link"); + assert!( + retirement_mark(&dir.path().join(LEGACY_ENV_DIR)) != RetirementMark::Present, + "a link must never be treated as a retired directory" + ); + assert!( + remove_marked_directory(&dir.path().join(LEGACY_ENV_DIR)).is_err(), + "deleting through a link must be refused" + ); + assert!( + real.join(LEGACY_DATA_FILE).exists(), + "and must delete nothing" + ); + assert!(bystander.exists()); + } + + /// A deletion that cannot remove the directory itself puts the mark back. + /// + /// An unmarked directory that still exists is the one state the scheme says cannot + /// happen: the next start would read it as an intact environment. + /// + /// Unix only: the failure is provoked with directory permissions, which is not how + /// the same thing happens on Windows. The behaviour under test is platform-neutral. + #[cfg(unix)] + #[test] + fn a_directory_that_cannot_be_removed_keeps_saying_it_was_retired() { + use std::os::unix::fs::PermissionsExt; + + let dir = TempDir::new().expect("temp dir"); + let retired = dir.path().join("chunks.mdb.retired"); + std::fs::create_dir_all(&retired).expect("mkdir"); + std::fs::write(retired.join(LEGACY_DATA_FILE), b"payload").expect("write"); + mark_directory_retired(&retired).expect("mark"); + + // The parent read-only, so the directory cannot be unlinked from it while its own + // contents still can be. That is the shape that leaves an emptied, unmarked + // directory behind. + let mut perms = std::fs::metadata(dir.path()).expect("meta").permissions(); + perms.set_mode(0o500); + std::fs::set_permissions(dir.path(), perms).expect("chmod"); + + let failed = remove_marked_directory(&retired).is_err(); + + let mut perms = std::fs::metadata(dir.path()).expect("meta").permissions(); + perms.set_mode(0o700); + std::fs::set_permissions(dir.path(), perms).expect("chmod back"); + + assert!(failed, "the removal was supposed to fail"); + assert!(retired.exists(), "and to leave the directory behind"); + assert!( + retirement_mark(&retired) == RetirementMark::Present, + "a directory that outlived its deletion must still say what it is" + ); + } + + /// A key the environment holds that is in neither view stops retirement. + /// + /// The gates only ever see the legacy-only set, so a key that has fallen out of both + /// the file index and that set has been through nothing and is protected by nothing. + /// It is the environment's only copy, and retirement would take it. + #[tokio::test] + async fn a_legacy_key_in_neither_view_refuses_the_proof_and_is_re_queued() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["orphan"]).await; + let key = *keys.first().expect("one key"); + let store = open(&dir).await; + store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + assert!(!store.legacy_only_keys().contains(&key)); + + // Stand in for whatever takes the file out from under the index: a quarantine, a + // publish that failed, an operator. The key is now in neither view. + let path = dir + .path() + .join("chunks") + .join(format!("{:02x}", key.last().copied().unwrap_or(0))) + .join(hex::encode(key)); + std::fs::remove_file(&path).expect("remove the file"); + store.files.forget_for_test(&key); + assert!(!store.files.exists(&key).unwrap_or(false)); + assert!(!store.legacy_only_keys().contains(&key)); + + let proof = store + .verify_before_retire(0, &never_cancelled()) + .await + .expect("verify"); + assert!( + !proof.is_clean(), + "a key protected by neither view must refuse the proof" + ); + assert!( + store.legacy_only_keys().contains(&key), + "and must be put back where the gates can see it" + ); + } + + /// A verification that no longer describes the store does not authorise a deletion. + /// + /// The pass reads every chunk and its result is reused for a while rather than re-read + /// on every tick. Retirement is often deferred in that window by a gate that has + /// nothing to do with the files. If a kept chunk stops being readable meanwhile, + /// ordinary requests are still served from the legacy copy, and deleting that copy on + /// the strength of the older pass leaves the node holding only the unreadable one. + #[cfg(unix)] + #[tokio::test] + async fn a_verification_overtaken_by_a_failing_file_does_not_authorise_retirement() { + use std::os::unix::fs::PermissionsExt; + + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["kept-then-unreadable"]).await; + let key = *keys.first().expect("one key"); + let mut config = test_config(&dir); + config.migration.retire_legacy = true; + let store = ChunkStore::new(config).await.expect("open"); + store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + open_the_retirement_gate(&store); + + let proof = store + .verify_before_retire(0, &never_cancelled()) + .await + .expect("verify"); + assert!(proof.is_clean()); + + // The window: the file stops being readable after the pass and before the + // deletion it authorised. + let path = dir + .path() + .join("chunks") + .join(format!("{:02x}", key.last().copied().unwrap_or(0))) + .join(hex::encode(key)); + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o000); + std::fs::set_permissions(&path, perms).expect("chmod"); + assert!( + store.get(&key).await.is_ok(), + "the legacy copy still serves it, which is what hides the problem" + ); + + let err = store + .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) + .await + .expect_err("a verification the store has outrun must not authorise a delete"); + assert!(format!("{err}").contains("no longer describes"), "{err}"); + assert!( + store.has_legacy(), + "and the legacy environment must survive" + ); + + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o600); + std::fs::set_permissions(&path, perms).expect("chmod back"); + } + + /// A write with no rollback copy is counted, on the path that actually skips. + /// + /// The environment is pinned to its current size for the whole bridge, so one with no + /// reusable page answers `Full` to every write and the node stores in files alone. That + /// is accepted, and the ADR says so. What was missing was any way to ask how often it + /// happens: the second release turns on knowing how many nodes are really keeping a + /// rollback copy, and a log line per chunk does not answer it. + /// + /// The first version of this counter missed exactly this path and counted only the + /// other one, the write that is attempted and refused. On the node most affected the + /// other path never runs, so the counter stayed at zero on precisely the nodes it was + /// added for. + #[tokio::test] + async fn a_write_with_no_rollback_copy_is_counted() { + let dir = TempDir::new().expect("temp dir"); + seed_legacy(&dir, &["settled"]).await; + let store = open(&dir).await; + assert_eq!( + store.writes_without_a_rollback_copy(), + 0, + "nothing has been skipped yet" + ); + + let legacy = store.legacy().expect("legacy"); + let before = legacy + .skipped_rollback_copies + .load(std::sync::atomic::Ordering::Relaxed); + + // Whichever way the environment refuses, the count moves. Driven through the + // counter itself rather than by filling a real environment, because what is under + // test is that the skip is recorded, and staging a genuinely unwritable LMDB from + // here would be testing LMDB. + for _ in 0..12 { + let _ = legacy.note_skipped_rollback_copy(); + } + assert_eq!( + store.writes_without_a_rollback_copy(), + before + 12, + "skipped writes must be visible to whoever asks the node" + ); + + // And the log is throttled, or a node with no free pages writes one line per chunk + // for the rest of its life. + let said: Vec = (0..1_000) + .filter_map(|_| legacy.note_skipped_rollback_copy()) + .collect(); + assert!( + said.len() < 10, + "the warning fired {} times in a thousand writes", + said.len() + ); + } + + /// Two writes for one key need two notes, not one shared between them. + /// + /// The journal used to be a set, so a second write for the same key announced nothing + /// and the first to return cleared the entry for both. A delete arriving in that window + /// sees no announcement, skips draining the environment, and the surviving write lands + /// afterwards and puts the key back, undoing a prune the node had decided on. The key + /// then sits in the environment and in neither view, which is the state retirement is + /// built to refuse: no data is lost, but a prune and a retirement cycle are. + /// + /// The file store's own in-flight map is counted for exactly this reason. This is the + /// same reasoning applied to the half that did not have it. + #[tokio::test] + async fn two_writes_for_one_key_are_two_notes_and_the_first_to_return_clears_neither() { + let dir = TempDir::new().expect("temp dir"); + seed_legacy(&dir, &["settled"]).await; + let store = open(&dir).await; + let legacy = store.legacy().expect("legacy"); + let (addr, _) = addressed("two-writes"); + + legacy.announce(&addr); + legacy.announce(&addr); + assert!(store.has_pending_writes()); + + // The first write returns. The second is still out there, so the note has to stand. + legacy.announced_write_finished(&addr); + assert!( + store.has_pending_writes(), + "the first write to return cleared a note the second one was still relying on" + ); + + // And the second clears it. + legacy.announced_write_finished(&addr); + assert!(!store.has_pending_writes()); + + // Retiring one that was never announced changes nothing, which is what makes the + // delete path's unconditional clear safe. + legacy.announced_write_finished(&addr); + assert!(!store.has_pending_writes()); + } + + /// A write in flight is a note to self, not a claim to hold the chunk. + /// + /// The note exists because a write into the environment outlives the future waiting + /// for it, so a cancelled one could leave a chunk nothing had recorded. But until both + /// halves have returned the node does not hold it, and saying it does puts the key in + /// signed commitments and in the count a quote is priced from. + #[tokio::test] + async fn a_write_in_flight_is_not_claimed_but_does_stop_retirement() { + let dir = TempDir::new().expect("temp dir"); + seed_legacy(&dir, &["settled"]).await; + let store = open(&dir).await; + let legacy = store.legacy().expect("legacy"); + let (addr, _) = addressed("in-flight"); + + // Stand in for a write that announced itself and never came back. + legacy.announce(&addr); + + assert!( + !store.exists(&addr).expect("exists"), + "a write in flight must not be reported as held" + ); + assert!(!store.all_keys().await.expect("keys").contains(&addr)); + assert!(!store.legacy_only_keys().contains(&addr)); + assert!(store.has_pending_writes()); + + // The distinction is the point: the same key in the key set IS claimed. If a + // write announced itself there instead, every one of the assertions above would + // be the opposite for as long as the write took. + legacy.only.write().insert(addr); + assert!(store.exists(&addr).expect("exists")); + assert!(store.all_keys().await.expect("keys").contains(&addr)); + legacy.only.write().remove(&addr); + + // But it does stop the environment going, because what it holds is unsettled. + store.commit_to_files().expect("commit"); + store.note_commitment_rebuilt(); + store.note_commitment_rebuilt(); + store.force_migration_state(|s| { + s.committed_at_unix = + Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); + }); + let proof = store + .verify_before_retire(0, &never_cancelled()) + .await + .expect("verify"); + let err = store + .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) + .await + .expect_err("an unsettled write must stop the removal"); + assert!(format!("{err}").contains("not reported back"), "{err}"); + + // And the note is resolved against what is actually there: nothing, so it goes. + store.reconcile_pending_writes().await; + assert!(!store.has_pending_writes()); + assert!(!store.legacy_only_keys().contains(&addr)); + } + + /// A delete outlasts a write for the same key that nobody waited for. + /// + /// A write has an environment half and a file half, and either can still be running + /// when its caller is dropped: the blocking work is not cancelled with the future. + /// A delete that did not wait for both would be undone by whichever half landed + /// afterwards, putting back a chunk the node had decided to prune. + #[tokio::test] + async fn a_delete_outlasts_a_write_nobody_waited_for() { + let dir = TempDir::new().expect("temp dir"); + seed_legacy(&dir, &["neighbour"]).await; + let store = Arc::new(open(&dir).await); + let (addr, content) = addressed("written-then-pruned"); + + // The gate is held from its own thread, so nothing holds a blocking guard across + // an await, and it is released through a channel when the test is ready. + let gate = store.files.test_put_gate(); + let (held_tx, held_rx) = std::sync::mpsc::channel::<()>(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let holder = std::thread::spawn(move || { + let _parked = gate.write(); + held_tx.send(()).ok(); + release_rx.recv().ok(); + }); + held_rx.recv().expect("the gate is held"); + + // The state under test, built directly rather than by racing a real put: a write + // that announced itself, whose file half is parked mid-publish, and whose caller + // is gone. Driving it through `put` and aborting would be a race about which half + // had started, and a test that sometimes sets up a different state than it claims + // is worse than no test. + let legacy = store.legacy().expect("legacy"); + legacy.announce(&addr); + let publishing = { + let files = Arc::clone(&store.files); + let content = content.clone(); + tokio::spawn(async move { files.put(&addr, &content).await }) + }; + // Until the publish is genuinely in flight and parked at the gate. + while store.files.tasks_in_flight() == 0 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + publishing.abort(); + let _ = publishing.await; + + // The delete has to wait the parked half out rather than racing it. + let deleting = { + let store = Arc::clone(&store); + tokio::spawn(async move { store.delete(&addr).await }) + }; + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + !deleting.is_finished(), + "the delete must wait for the write it would otherwise race" + ); + + release_tx.send(()).ok(); + holder.join().ok(); + deleting.await.expect("join").expect("delete"); + + // Whichever half landed, the key is gone and stays gone. + store.files.wait_idle().await; + assert!( + !store.exists(&addr).unwrap_or(true), + "a write that landed after the delete would resurrect a pruned chunk" + ); + assert!(!store.legacy_only_keys().contains(&addr)); + } + + /// A delete outlasts a file write that no journal knows about. + /// + /// The journal is kept by writes that touch both stores. The copier and the repair + /// path write only the file, so a delete that consulted the journal to decide whether + /// to wait would not wait for either of them, and whichever landed afterwards would + /// put back a chunk the node had decided to prune. What is writing a key is the file + /// store's own question to answer. + #[tokio::test] + async fn a_delete_outlasts_a_file_write_with_no_journal_entry() { + let dir = TempDir::new().expect("temp dir"); + seed_legacy(&dir, &["neighbour"]).await; + let store = Arc::new(open(&dir).await); + let (addr, content) = addressed("copied-then-pruned"); + + let gate = store.files.test_put_gate(); + let (held_tx, held_rx) = std::sync::mpsc::channel::<()>(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let holder = std::thread::spawn(move || { + let _parked = gate.write(); + held_tx.send(()).ok(); + release_rx.recv().ok(); + }); + held_rx.recv().expect("the gate is held"); + + // Deliberately no journal entry: this is the copier's shape, not a dual write. + let publishing = { + let files = Arc::clone(&store.files); + let content = content.clone(); + tokio::spawn(async move { files.put(&addr, &content).await }) + }; + while store.files.tasks_in_flight() == 0 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + publishing.abort(); + let _ = publishing.await; + assert!( + !store + .legacy() + .is_some_and(|l| l.pending.read().contains_key(&addr)), + "this is the case the journal does not cover, so it must be empty" + ); + + let deleting = { + let store = Arc::clone(&store); + tokio::spawn(async move { store.delete(&addr).await }) + }; + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + !deleting.is_finished(), + "the delete must wait for a write the journal never knew about" + ); + + release_tx.send(()).ok(); + holder.join().ok(); + deleting.await.expect("join").expect("delete"); + + store.files.wait_idle().await; + assert!( + !store.exists(&addr).unwrap_or(true), + "a write that landed after the delete would resurrect a pruned chunk" + ); + } + + /// A single node can still be told to keep both stores. + #[tokio::test] + async fn retirement_is_refused_when_the_switch_is_turned_off() { + let dir = TempDir::new().expect("temp dir"); + seed_legacy(&dir, &["off"]).await; + let mut config = test_config(&dir); + config.migration.retire_legacy = false; + let store = ChunkStore::new(config).await.expect("open store"); + store.commit_to_files().expect("commit"); + assert!(store + .retirement_blocker(|_| false) + .expect("blocked") + .contains("retirement is disabled")); + } + + #[tokio::test] + async fn retiring_removes_the_legacy_environment_and_frees_its_space() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["f1", "f2", "f3"]).await; + let mut config = test_config(&dir); + config.migration.retire_legacy = true; + let store = ChunkStore::new(config).await.expect("open"); + + store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + open_the_retirement_gate(&store); + + let proof = store + .verify_before_retire(0, &never_cancelled()) + .await + .expect("verify"); + assert!(proof.is_clean()); + assert_eq!(proof.checked, 3); + + let freed = store + .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) + .await + .expect("retire"); + assert!(freed > 0, "retirement must report the space it returned"); + assert!( + !dir.path().join(LEGACY_ENV_DIR).exists(), + "the legacy environment must actually be removed" + ); + assert!(!store.has_legacy()); + assert_eq!(store.migration_phase(), MigrationPhase::FilesOnly); + + // Everything is still readable, from files alone. + assert_eq!(store.current_chunks().expect("count"), 3); + for key in &keys { + assert!(store.get(key).await.expect("get").is_some()); + } + } + + #[tokio::test] + async fn a_reader_holding_the_legacy_handle_defers_retirement_without_hiding_chunks() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["busy"]).await; + let mut config = test_config(&dir); + config.migration.retire_legacy = true; + let store = Arc::new(ChunkStore::new(config).await.expect("open")); + store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + open_the_retirement_gate(&store); + + // Stand in for a read that is still holding the legacy handle. Retirement must + // defer rather than unmap underneath it, and the chunk must stay readable + // throughout: a retirement attempt that briefly hid the legacy store would make a + // node answer "not found" for a chunk it holds. + let squatter = store.legacy().expect("a legacy handle"); + let proof = store + .verify_before_retire(0, &never_cancelled()) + .await + .expect("verify"); + let err = store + .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) + .await + .expect_err("must defer while the handle is held"); + assert!(format!("{err}").contains("deferred"), "{err}"); + + assert!(store.has_legacy(), "the store must keep its legacy handle"); + let key = keys.first().copied().expect("a key"); + assert!( + store.get(&key).await.expect("get").is_some(), + "the chunk must stay readable across a deferred retirement" + ); + + // Once the reader lets go, the next attempt succeeds. + drop(squatter); + store + .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) + .await + .expect("retire"); + assert!(!store.has_legacy()); + } + + #[tokio::test] + async fn retirement_needs_a_clean_verification_report() { + let dir = TempDir::new().expect("temp dir"); + // Left uncopied on purpose, so this node is about to give the chunk up and the + // answerability veto has something to fire on. + seed_legacy(&dir, &["v1"]).await; + let mut config = test_config(&dir); + config.migration.retire_legacy = true; + let store = ChunkStore::new(config).await.expect("open"); + + // A report cannot be fabricated: every field is private and the only source is + // the verification pass. The one available here is the default, which never ran. + let absent = VerifyReport::default(); + let err = store + .retire_legacy(&absent, &|_: &XorName| false, &approved_shed(&store)) + .await + .expect_err("must refuse a report that never ran"); + assert!(format!("{err}").contains("unrepairable"), "{err}"); + assert!(store.has_legacy(), "the legacy environment must survive"); + + // And a real, clean report is still refused while any gate is unmet, because + // retirement rechecks them all itself rather than trusting its caller. + open_the_retirement_gate(&store); + let proof = store + .verify_before_retire(0, &never_cancelled()) + .await + .expect("verify"); + assert!(proof.is_clean()); + let err = store + .retire_legacy(&proof, &|_: &XorName| true, &approved_shed(&store)) + .await + .expect_err("must refuse while a chunk is still answerable"); + assert!(format!("{err}").contains("still answerable"), "{err}"); + assert!(store.has_legacy()); + } + + #[tokio::test] + async fn verification_repairs_a_file_that_rotted_before_retirement() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["w1", "w2"]).await; + let mut config = test_config(&dir); + config.migration.retire_legacy = true; + let store = ChunkStore::new(config).await.expect("open"); + store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + + // A filename is not proof the bytes behind it are good. Corrupt one, exactly as + // a truncated write or a bad sector would, then prove retirement repairs it + // rather than deleting the only intact copy. + let victim = keys.first().copied().expect("a key"); + let path = dir + .path() + .join(crate::storage::file_store::CHUNKS_DIR_NAME) + .join(format!("{:02x}", victim.last().copied().unwrap_or(0))) + .join(hex::encode(victim)); + std::fs::write(&path, b"rotted").expect("corrupt the file"); + open_the_retirement_gate(&store); + + let proof = store + .verify_before_retire(0, &never_cancelled()) + .await + .expect("verify"); + assert_eq!(proof.repaired(), 1); + assert_eq!(proof.unrepairable(), 0); + assert!(proof.is_clean()); + + store + .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) + .await + .expect("retire"); + assert_eq!( + store.get(&victim).await.expect("get").expect("present"), + addressed("w1").1 + ); + } + + #[tokio::test] + async fn a_corrupt_file_is_served_from_the_legacy_copy_and_requeued() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["s1"]).await; + let store = open(&dir).await; + store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + assert!(store.legacy_only_keys().is_empty()); + + let victim = keys.first().copied().expect("a key"); + let path = dir + .path() + .join(crate::storage::file_store::CHUNKS_DIR_NAME) + .join(format!("{:02x}", victim.last().copied().unwrap_or(0))) + .join(hex::encode(victim)); + std::fs::write(&path, b"rotted").expect("corrupt the file"); + + // The file store removes the bad file and stops advertising it; the facade must + // still find the intact copy rather than reporting a failure. + assert_eq!( + store.get(&victim).await.expect("get").expect("present"), + addressed("s1").1 + ); + assert!( + store.legacy_only_keys().contains(&victim), + "the key must go back on the copier's list" + ); + } + + #[tokio::test] + async fn a_marker_claiming_more_than_the_file_store_holds_restarts_the_copy() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["p1", "p2", "p3"]).await; + let store = open(&dir).await; + store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + store.commit_to_files().expect("commit"); + assert_eq!(store.migration_state().kept_key_count, 3); + store.wait_idle().await; + drop(store); + + // Someone clears the chunk directory to reclaim space, keeping chunks.mdb. + // Trusting the marker here would skip the copier, the shed rules and their rank + // checks on the way to deleting the legacy environment. + let chunks = dir.path().join(crate::storage::file_store::CHUNKS_DIR_NAME); + for key in &keys { + let path = chunks + .join(format!("{:02x}", key.last().copied().unwrap_or(0))) + .join(hex::encode(key)); + std::fs::remove_file(path).expect("clear the file store"); + } + + let store = open(&dir).await; + assert_eq!( + store.migration_phase(), + MigrationPhase::Bridging, + "the filesystem must win over the marker" + ); + assert_eq!(store.legacy_only_keys().len(), 3); + } + + #[tokio::test] + async fn a_file_that_vanished_mid_verification_is_requeued_not_republished() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["gone"]).await; + let mut config = test_config(&dir); + config.migration.retire_legacy = true; + let store = ChunkStore::new(config).await.expect("open"); + store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + assert!(store.legacy_only_keys().is_empty()); + + // Remove the file without telling the store, which is what the pruner's own + // delete looks like if it lands mid-pass. Republishing from the legacy copy here + // would resurrect a chunk the node had deliberately deleted, so the key goes back + // on the copier's list instead and retirement is refused. + let key = keys.first().copied().expect("a key"); + let path = dir + .path() + .join(crate::storage::file_store::CHUNKS_DIR_NAME) + .join(format!("{:02x}", key.last().copied().unwrap_or(0))) + .join(hex::encode(key)); + std::fs::remove_file(&path).expect("remove behind the store's back"); + + let proof = store + .verify_before_retire(0, &never_cancelled()) + .await + .expect("verify"); + assert!( + proof.unrepairable >= 1, + "the vanished file must be counted against the proof" + ); + assert!(!proof.is_clean()); + assert!(!path.exists(), "the pass must not republish it"); + assert!(store.legacy_only_keys().contains(&key)); + assert!(store + .retire_legacy(&proof, &|_: &XorName| false, &approved_shed(&store)) + .await + .is_err()); + assert!(store.has_legacy()); + } + + #[tokio::test] + async fn a_cancelled_shutdown_stops_the_copier_and_refuses_to_pass_verification() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["s1", "s2", "s3"]).await; + let store = open(&dir).await; + + // Shutdown must not have to wait out a pass that can run for hours, and a pass + // that stopped early is not evidence of anything. + let cancelled = CancellationToken::new(); + cancelled.cancel(); + + let report = store + .copy_batch(&keys, 0, 0, &cancelled) + .await + .expect("copy"); + assert_eq!(report.copied, 0, "the copier must stop immediately"); + assert_eq!(store.legacy_only_keys().len(), 3); + + let proof = store + .verify_before_retire(0, &cancelled) + .await + .expect("verify"); + assert!( + !proof.is_clean(), + "an interrupted verification must never read as a pass" + ); + } + + #[tokio::test] + async fn the_migration_marker_survives_a_restart() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["m1", "m2"]).await; + let store = open(&dir).await; + store + .copy_batch(&keys[..1], 0, 0, &never_cancelled()) + .await + .expect("copy"); + store.commit_to_files().expect("commit"); + store.note_commitment_rebuilt(); + let first_start = store.migration_state().first_start_unix; + store.wait_idle().await; + drop(store); + + let store = open(&dir).await; + let state = store.migration_state(); + assert_eq!(state.phase, MigrationPhase::Committed); + assert_eq!(state.shed_key_count, 1); + assert_eq!( + state.first_start_unix, first_start, + "a restart must not restart the shed hold" + ); + assert!( + state.committed_at_unix.is_some(), + "nor the retirement clock" + ); + } + + #[tokio::test] + async fn a_marker_that_disagrees_with_the_filesystem_loses() { + let dir = TempDir::new().expect("temp dir"); + seed_legacy(&dir, &["x1"]).await; + let store = open(&dir).await; + store.force_migration_state(|s| s.phase = MigrationPhase::FilesOnly); + store.migration_state().save(dir.path()).expect("save"); + store.wait_idle().await; + drop(store); + + // The marker claims the migration is done, but `chunks.mdb` is right there. The + // filesystem is the authority. + let store = open(&dir).await; + assert_eq!(store.migration_phase(), MigrationPhase::Bridging); + assert!(store.has_legacy()); + } + + /// A legacy record whose bytes do not hash to its key is removed, not passed around. + /// + /// Leaving it in the environment while dropping it from the key set puts it in + /// neither view, and the pre-retirement pass reads a key in neither view as one to + /// protect and puts it straight back. The next copier pass drops it again. One rotted + /// record would keep this node, and every node sharing its disk, from ever reclaiming + /// space. + #[tokio::test] + async fn a_legacy_chunk_that_does_not_match_its_address_is_removed_not_recycled() { + let dir = TempDir::new().expect("temp dir"); + let (addr, _) = addressed("bad"); + let other = addressed("other").1; + { + let lmdb = LmdbStorage::new(LmdbStorageConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: false, + max_map_size: 0, + disk_reserve: 0, + }) + .await + .expect("open legacy"); + // Under a key it does not hash to: what a record that rotted in place looks + // like, and the one shape the ordinary path refuses to create. + lmdb.put_unchecked(&addr, &other).await.expect("put"); + lmdb.wait_idle().await; + } + let store = open(&dir).await; + let keys = store.legacy_only_keys(); + assert_eq!(keys, vec![addr]); + + let report = store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + assert_eq!(report.copied, 0); + assert_eq!(report.unusable, 1); + assert!(store.legacy_only_keys().is_empty()); + + // And it is gone from the environment, so the pass below cannot find it and put + // it back. That is the loop this is about. + let proof = store + .verify_before_retire(0, &never_cancelled()) + .await + .expect("verify"); + assert!( + store.legacy_only_keys().is_empty(), + "a removed record must not come back on the copier's list" + ); + assert!( + proof.is_clean(), + "and must not go on refusing the proof for ever" + ); + } + + #[test] + fn the_release_switches_are_never_written_to_an_operator_config_file() { + // A node writes its effective configuration back to disk. If these round-tripped, + // R1's values would be baked into every operator's file and the next release + // would change nothing. + let mut config = MigrationConfig::default(); + config.retire_legacy = !config.retire_legacy; + config.allow_shed = false; + config.shed_hold_hours = 5; + + let encoded = toml::to_string(&config).expect("encode"); + assert!(!encoded.contains("retire_legacy"), "{encoded}"); + + let decoded: MigrationConfig = toml::from_str(&encoded).expect("decode"); + let fresh = MigrationConfig::default(); + assert_eq!(decoded.retire_legacy, fresh.retire_legacy); + // Genuine operator controls do survive. + assert!(!decoded.allow_shed); + assert_eq!(decoded.shed_hold_hours, 5); + } + + #[test] + fn the_copy_order_is_closest_first() { + let me = [0u8; XORNAME_LEN_LOCAL]; + let mut near = [0u8; XORNAME_LEN_LOCAL]; + if let Some(b) = near.last_mut() { + *b = 1; + } + let mut far = [0u8; XORNAME_LEN_LOCAL]; + if let Some(b) = far.first_mut() { + *b = 0xff; + } + let ordered = rank_closest_first(vec![far, near], Some(me)); + assert_eq!(ordered.first().copied(), Some(near)); + assert_eq!(ordered.last().copied(), Some(far)); + + // With no identity the order is still stable, which is all the copier needs. + let ordered = rank_closest_first(vec![far, near], None); + let mut expected = vec![far, near]; + expected.sort_unstable(); + assert_eq!(ordered, expected); + } + + /// Local alias so the test does not import from the protocol crate. + const XORNAME_LEN_LOCAL: usize = 32; + + #[test] + fn the_retirement_delay_can_never_be_shortened_below_the_retention_window() { + let config = MigrationConfig { + retire_delay_hours: 0, + ..MigrationConfig::default() + }; + assert_eq!( + config.effective_retire_delay_hours(), + MIN_RETIRE_DELAY_HOURS + ); + } +} diff --git a/src/storage/file_store.rs b/src/storage/file_store.rs new file mode 100644 index 00000000..327900dd --- /dev/null +++ b/src/storage/file_store.rs @@ -0,0 +1,3706 @@ +//! One immutable file per chunk, content-addressed, with the filesystem as the +//! only authority. +//! +//! ```text +//! {root}/chunks/ store root +//! {root}/chunks/layout.json versioned layout marker +//! {root}/chunks/.lock advisory single-process guard +//! {root}/chunks//<64-hex> xy = the LAST two hex characters of the address +//! {root}/chunks//.tmp.. an in-flight write, in the destination directory +//! ``` +//! +//! # Why the *last* two hex characters +//! +//! A node holds keys for which it is among the [`CLOSE_GROUP_SIZE`] closest, so its +//! holdings share roughly `log2(N / CLOSE_GROUP_SIZE)` leading bits with its own node +//! ID, and that shared prefix grows as the network grows. Sharding on a prefix therefore +//! does not degrade, it collapses: at ~800 nodes a two-hex prefix already resolves to +//! about two distinct directories, and past a million nodes even a four-hex prefix +//! resolves to one. Close-group membership constrains the leading bits and places no +//! constraint at all on the trailing ones, and the address is a BLAKE3 output, so the +//! last byte is uniform by construction at every network size. +//! +//! 256 shards keeps a 24 GiB node at ~23 files per directory and a 1 TiB node at ~977, +//! for 1 MiB of directory inodes. The scheme and depth are recorded in `layout.json` at +//! creation so a future layout can be detected rather than silently misread. +//! +//! # Why lowercase hex names +//! +//! NTFS and default APFS fold case. Under an encoding with both cases (base64url, +//! base58) two distinct 32-byte keys can share one case-folded filename, which is a +//! silent overwrite. Hex has one case-folded form per key, and no hex string can ever +//! spell a reserved Windows device name (`CON`, `NUL`, `AUX`, `COM1`, ...) because none +//! of those letters is in `0-9a-f`. The full 64-character key stays in the filename, so +//! a `find` over the tree recovers the whole store even if the directory layer is lost. +//! +//! [`CLOSE_GROUP_SIZE`]: crate::ant_protocol::CLOSE_GROUP_SIZE + +use crate::ant_protocol::{XorName, MAX_CHUNK_SIZE, XORNAME_LEN}; +use crate::error::{Error, Result}; +use crate::logging::{debug, info, trace, warn}; +use crate::storage::StorageStats; +use fs2::FileExt; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::fs::{File, OpenOptions}; +use std::io::{ErrorKind, Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::task::spawn_blocking; +use tokio_util::task::TaskTracker; + +/// Directory under the node root that holds the chunk files. +pub const CHUNKS_DIR_NAME: &str = "chunks"; + +/// Name of the layout marker written once at store creation. +pub const LAYOUT_FILE_NAME: &str = "layout.json"; + +/// Name of the advisory single-process lock file. +const LOCK_FILE_NAME: &str = ".lock"; + +/// Prefix that marks an in-flight write. Never a valid chunk name (chunk names are +/// exactly [`CHUNK_NAME_LEN`] lowercase hex characters, and `.` is not hex). +const TEMP_PREFIX: &str = ".tmp."; + +/// Number of shard directories. One level, `00` through `ff`. +const SHARD_COUNT: usize = 256; + +/// Length of a chunk filename: the full address in lowercase hex. +const CHUNK_NAME_LEN: usize = XORNAME_LEN * 2; + +/// How often to re-query available disk space, in seconds. +/// +/// Matches the LMDB store's cadence so the capacity predicate behaves identically +/// for callers that only ask "is there room at all". +const DISK_CHECK_INTERVAL_SECS: u64 = 5; + +/// Allocation granularity assumed when charging a pending write against free space. +/// +/// Every filesystem we support allocates in units of at least 4 KiB, so a write of +/// `n` bytes consumes at least `ceil(n / 4096) * 4096`. One extra unit covers the +/// directory entry and inode. +const ALLOC_UNIT: u64 = 4096; + +/// How many times a publish retries a transient Windows sharing violation. +const RENAME_RETRY_ATTEMPTS: u32 = 5; + +/// Base backoff between those retries; the wait grows linearly with the attempt. +const RENAME_RETRY_BACKOFF: Duration = Duration::from_millis(20); + +/// Longest absolute path a chunk file may need, checked once at open. +/// +/// Windows caps a non-verbatim path at `MAX_PATH` (260) including the terminating NUL. +/// Rust's standard library transparently switches to the `\\?\` verbatim form for long +/// absolute paths, so this is a warning rather than a hard failure, but an operator who +/// buries the node root ten directories deep should hear about it before the first write +/// fails rather than after. +#[cfg(windows)] +const WINDOWS_PATH_WARN_LEN: usize = 240; + +/// The on-disk layout marker. +/// +/// Written once when the store directory is created and read on every subsequent open. +/// Nothing in this survey of comparable stores (IPFS flatfs, Storj, borgbackup) shipped +/// an in-place re-sharder, and all three paid for it. Recording the scheme costs one +/// small file and is the difference between changing the default later and never being +/// able to. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StoreLayout { + /// Marker schema version. A store written by a newer schema is refused. + pub schema: u32, + /// How a chunk address maps to a shard directory. + pub scheme: String, + /// How many hex characters of the address name the shard directory. + pub shard_chars: u8, + /// How many directory levels of sharding. + pub depth: u8, + /// How a chunk address maps to a filename. + pub name_encoding: String, +} + +/// Marker schema this build writes and understands. +const LAYOUT_SCHEMA: u32 = 1; +/// Shard scheme this build implements: the trailing hex characters of the address. +const LAYOUT_SCHEME_SUFFIX_HEX: &str = "suffix-hex"; +/// Filename encoding this build implements. +const LAYOUT_NAME_LOWER_HEX: &str = "lower-hex"; + +impl Default for StoreLayout { + fn default() -> Self { + Self { + schema: LAYOUT_SCHEMA, + scheme: LAYOUT_SCHEME_SUFFIX_HEX.to_string(), + shard_chars: 2, + depth: 1, + name_encoding: LAYOUT_NAME_LOWER_HEX.to_string(), + } + } +} + +impl StoreLayout { + /// Return an error unless this build can read a store written with this layout. + fn check_supported(&self) -> Result<()> { + if self.schema > LAYOUT_SCHEMA { + return Err(Error::Storage(format!( + "Chunk store layout schema {} is newer than this build understands ({LAYOUT_SCHEMA}). \ + Refusing to open rather than misread the store.", + self.schema + ))); + } + if self.scheme != LAYOUT_SCHEME_SUFFIX_HEX { + return Err(Error::Storage(format!( + "Chunk store uses shard scheme '{}', this build implements '{LAYOUT_SCHEME_SUFFIX_HEX}'", + self.scheme + ))); + } + if self.shard_chars != 2 || self.depth != 1 { + return Err(Error::Storage(format!( + "Chunk store uses {} shard characters at depth {}, this build implements 2 at depth 1", + self.shard_chars, self.depth + ))); + } + if self.name_encoding != LAYOUT_NAME_LOWER_HEX { + return Err(Error::Storage(format!( + "Chunk store names files with '{}', this build implements '{LAYOUT_NAME_LOWER_HEX}'", + self.name_encoding + ))); + } + Ok(()) + } +} + +/// Configuration for [`FileStore`]. +#[derive(Debug, Clone)] +pub struct FileStoreConfig { + /// Node root directory. The store lives at `{root_dir}/chunks/`. + pub root_dir: PathBuf, + /// Verify `BLAKE3(content) == address` on read. + pub verify_on_read: bool, + /// Free bytes to keep on the storage partition. Writes are refused below this. + pub disk_reserve: u64, +} + +/// Outcome of a single write attempt, used to keep the duplicate accounting honest. +enum PutOutcome { + /// The chunk was newly published. + New, + /// The chunk was already on disk. + Duplicate, +} + +/// Snapshot of free space, plus what has been written since it was taken. +#[derive(Debug)] +struct CapacitySnapshot { + /// When `available` was measured. `None` means never. + measured_at: Option, + /// Free bytes reported by the filesystem at `measured_at`. + available: u64, + /// Bytes published since `measured_at`, charged against `available`. + /// + /// Cleared by a fresh measurement, which already accounts for them. + written_since: u64, + /// Bytes reserved by writes that have not landed yet. + /// + /// Deliberately **not** cleared by a measurement: a `statvfs` taken while writes are + /// in flight reports space those writes are about to consume, so forgetting their + /// reservations at that moment would hand the same bytes out twice. That is precisely + /// the over-admission the reservation exists to prevent. + in_flight: u64, +} + +/// Size-aware free-space predicate with a short-lived cache. +/// +/// Free bytes alone stopped being a sufficient answer the moment chunks became files: +/// a caller wants to know whether *this* write fits, not whether the disk is non-empty. +/// The cache keeps the common case at one `statvfs` per interval while staying correct +/// under a burst, because bytes written since the measurement are charged against it. +#[derive(Debug)] +struct CapacityGuard { + /// Directory whose partition is measured. + dir: PathBuf, + /// Free bytes to keep unused. + reserve: u64, + /// The cached measurement. + snapshot: parking_lot::Mutex, +} + +impl CapacitySnapshot { + /// Free bytes, less everything written or promised since the measurement. + fn free_estimate(&self) -> u64 { + self.available + .saturating_sub(self.written_since) + .saturating_sub(self.in_flight) + } +} + +impl CapacityGuard { + /// Create a guard over the partition hosting `dir`. + fn new(dir: PathBuf, reserve: u64) -> Self { + Self { + dir, + reserve, + snapshot: parking_lot::Mutex::new(CapacitySnapshot { + measured_at: None, + available: 0, + written_since: 0, + in_flight: 0, + }), + } + } + + /// Bytes actually consumed on disk by a payload of `len` bytes. + fn charge(len: u64) -> u64 { + // Round the payload up to the allocation unit, then add one unit for the + // directory entry and inode. + len.div_ceil(ALLOC_UNIT) + .saturating_mul(ALLOC_UNIT) + .saturating_add(ALLOC_UNIT) + } + + /// Free bytes right now, or `None` if the question could not be answered. + /// + /// Deliberately separate from [`Self::measure`], which folds a failure into an error + /// the caller cannot tell from "below the reserve". + fn measure_available(&self) -> Option { + let mut snapshot = self.snapshot.lock(); + match self.measure(&mut snapshot) { + Ok(()) => Some(snapshot.free_estimate()), + Err(_) => None, + } + } + + /// Query the filesystem and refresh the snapshot. + fn measure(&self, snapshot: &mut CapacitySnapshot) -> Result<()> { + let available = fs2::available_space(&self.dir) + .map_err(|e| Error::Storage(format!("Failed to query available disk space: {e}")))?; + snapshot.available = available; + // Reservations survive: their bytes are not on the platter yet, so the fresh + // measurement does not include them. + snapshot.written_since = 0; + snapshot.measured_at = Some(Instant::now()); + Ok(()) + } + + /// Test `needed` against the snapshot, refreshing it if it is stale or short. + /// + /// Only *passing* results are cached, so a low-space condition is rechecked on every + /// call and freed space is noticed promptly. + fn admit(&self, snapshot: &mut CapacitySnapshot, needed: u64) -> Result<()> { + let want = self.reserve.saturating_add(Self::charge(needed)); + + let cache_fresh = snapshot + .measured_at + .is_some_and(|t| t.elapsed().as_secs() < DISK_CHECK_INTERVAL_SECS); + if cache_fresh && snapshot.free_estimate() >= want { + return Ok(()); + } + + self.measure(snapshot)?; + if snapshot.free_estimate() < want { + // Do not cache a failing result: `measured_at` is left set so the next call + // still re-measures, because the branch above only short-circuits a pass. + return Err(Error::Storage(format!( + "Insufficient disk space: {:.2} GiB available, {:.2} GiB reserve required. \ + Free disk space or increase the partition to continue storing chunks.", + bytes_to_gib(snapshot.free_estimate()), + bytes_to_gib(self.reserve), + ))); + } + Ok(()) + } + + /// Drop the cached measurement so the next question hits the filesystem. + fn invalidate(&self) { + let mut snapshot = self.snapshot.lock(); + snapshot.measured_at = None; + snapshot.written_since = 0; + } + + /// Return `Ok(())` if a write of `needed` bytes would fit. Charges nothing. + fn check(&self, needed: u64) -> Result<()> { + let mut snapshot = self.snapshot.lock(); + self.admit(&mut snapshot, needed) + } + + /// Admit a write of `needed` bytes and charge it in the same critical section. + /// + /// Checking and charging separately is the bug this exists to prevent: dozens of + /// protocol handlers can each pass against the same cached measurement before any of + /// them has written a byte, and collectively cross the reserve. + /// + /// The returned [`Reservation`] settles itself when dropped, so a caller whose future + /// is dropped mid-write cannot strand it. Nothing else ever decrements the in-flight + /// count, so a stranded reservation would be permanent, and enough of them would make + /// an empty disk look full until the process restarted. + fn reserve(self: &Arc, needed: u64) -> Result { + { + let mut snapshot = self.snapshot.lock(); + self.admit(&mut snapshot, needed)?; + snapshot.in_flight = snapshot.in_flight.saturating_add(Self::charge(needed)); + } + Ok(Reservation { + capacity: Arc::clone(self), + bytes: needed, + settled: false, + }) + } + + /// Give back a reservation whose write did not happen. + fn release(&self, needed: u64) { + let mut snapshot = self.snapshot.lock(); + snapshot.in_flight = snapshot.in_flight.saturating_sub(Self::charge(needed)); + } + + /// Turn a reservation into bytes that are now on disk. + fn commit_reservation(&self, needed: u64) { + let charge = Self::charge(needed); + let mut snapshot = self.snapshot.lock(); + snapshot.in_flight = snapshot.in_flight.saturating_sub(charge); + snapshot.written_since = snapshot.written_since.saturating_add(charge); + } + + /// Credit a completed delete back to the cached measurement. + fn record_removed(&self, len: u64) { + let mut snapshot = self.snapshot.lock(); + snapshot.written_since = snapshot.written_since.saturating_sub(Self::charge(len)); + } +} + +/// A charged, unsettled write. +/// +/// Held by whatever is actually doing the write, so the charge is released even if the +/// caller's future is dropped and only the blocking closure survives. +struct Reservation { + /// The guard this was taken from. + capacity: Arc, + /// Payload size, before rounding. + bytes: u64, + /// Whether it has already been accounted for. + settled: bool, +} + +impl Reservation { + /// The write landed: move the charge from in-flight to written. + fn commit(mut self) { + self.capacity.commit_reservation(self.bytes); + self.settled = true; + } +} + +impl Drop for Reservation { + fn drop(&mut self) { + if !self.settled { + self.capacity.release(self.bytes); + } + } +} + +/// Environment variable naming a failpoint: stop after the temp file, before the rename. +#[cfg(any(test, feature = "test-utils"))] +pub const HALT_BEFORE_PUBLISH: &str = "ANT_HALT_BEFORE_PUBLISH"; + +/// Environment variable naming a failpoint: stop once the legacy environment is renamed +/// aside and marked retired, before any of it is deleted. +/// +/// The most destructive window in the migration. What a start that finds a marked +/// directory must do is finish the deletion, never reopen it, because the node has already +/// told the network it holds those chunks from the file store. +#[cfg(any(test, feature = "test-utils"))] +pub const HALT_AFTER_RETIRE_MARK: &str = "ANT_HALT_AFTER_RETIRE_MARK"; + +/// Park forever at a named failpoint, once a marker says the process has reached it. +/// +/// For crash tests, which need a process to die *inside* an operation rather than at +/// whatever point a sleep in another process happened to land. The variable holds a path: +/// this writes it, so the parent knows the child is exactly here, and then waits to be +/// killed. +/// +/// Costs one environment read per write when the feature is compiled in, and the feature +/// is not in a release build. +#[cfg(any(test, feature = "test-utils"))] +pub(crate) fn halt_here_if_asked(variable: &str, reached: &Path) { + let Ok(marker) = std::env::var(variable) else { + return; + }; + // Let the first few through. A test that stops the very first write leaves a store + // with nothing successfully in it, and an assertion over what it holds then passes by + // iterating nothing. Letting some land first means the crash happens to a store that + // has real chunks in it, which is the situation worth checking. + let skip: u64 = std::env::var(HALT_AFTER) + .ok() + .and_then(|raw| raw.parse().ok()) + .unwrap_or(0); + if HALTS_SEEN.fetch_add(1, std::sync::atomic::Ordering::AcqRel) < skip { + return; + } + if let Err(e) = std::fs::write(&marker, reached.as_os_str().as_encoded_bytes()) { + // The parent waits for this file. Saying so on the way past is the difference + // between a test that fails and one that hangs until the job times out. + eprintln!("failpoint could not write its marker {marker}: {e}"); + return; + } + loop { + std::thread::sleep(Duration::from_secs(3600)); + } +} + +/// How many writes to let through before the failpoint fires. +#[cfg(any(test, feature = "test-utils"))] +pub const HALT_AFTER: &str = "ANT_HALT_AFTER"; + +/// How many times the failpoint has been reached in this process. +#[cfg(any(test, feature = "test-utils"))] +static HALTS_SEEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// Clears a write's registration when the work finishes, however it finishes. +/// +/// Held by the blocking closure rather than by the caller, so a dropped future cannot +/// leave an entry behind, and a panic in the work cannot either. +struct WriteInFlight { + writing: Arc>>, + finished: Arc, + address: XorName, +} + +impl Drop for WriteInFlight { + fn drop(&mut self) { + let was_last = { + let mut writing = self.writing.lock(); + match writing.get_mut(&self.address) { + Some(count) if *count > 1 => { + *count -= 1; + false + } + _ => { + writing.remove(&self.address); + true + } + } + }; + // Only when this was the last one. Waking a waiter while another write for the + // same key is still queued is exactly what the count exists to prevent. + if was_last { + self.finished.notify_waiters(); + } + } +} + +/// What is behind a chunk's name on disk. +/// +/// Four answers, not two, because "could not read it" must never be treated as "wrong": +/// replacing a chunk is destructive, and off Unix it truncates the file in place, so a +/// transient fault would turn a healthy sole copy into an empty one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StoredBytes { + /// The bytes are there and hash to the name. + Good, + /// The bytes are there and do not. + Wrong, + /// There is nothing behind the name. + Absent, + /// The question could not be answered this time. + Unreadable, +} + +/// Convert a byte count to GiB for human-readable log messages. +#[allow(clippy::cast_precision_loss)] // display only — sub-byte precision is irrelevant +fn bytes_to_gib(bytes: u64) -> f64 { + bytes as f64 / (1024.0 * 1024.0 * 1024.0) +} + +/// Content-addressed store holding one immutable file per chunk. +/// +/// The filesystem is the sole authority. The in-memory index is a cache of what the +/// directory tree already contains, rebuilt from directory entries at every open, and +/// every mutation of it mirrors a filesystem operation that has *already* completed. +/// Bitcask's issue #114 is the cautionary tale for the opposite order: an index that is +/// rebuilt at startup and then mutated in anticipation drifts, and the drift is silent. +#[derive(Debug)] +pub struct FileStore { + /// Store configuration. + config: FileStoreConfig, + /// `{root_dir}/chunks`. + chunks_dir: PathBuf, + /// Every address whose file is published, in ascending order. + /// + /// `BTreeSet` rather than a hash set because `all_keys()` must be sorted (the + /// commitment builder truncates with `take(cap)` *before* the Merkle tree sorts, so + /// an unstable order would make the node's published commitment depend on iteration + /// luck), and because it never spikes memory while growing. + index: Arc>>, + /// One mutex per shard, serialising writers of the same address. + /// + /// LMDB gave exactly-once `put` semantics for free: the duplicate test happened + /// inside the write transaction. Two threads publishing the same address here would + /// otherwise both see an absent file, both rename, and both report "newly stored", + /// double-counting the chunk. The lane is indexed by the address's LAST byte for the + /// same reason the shard is: a node's keys share their leading bytes, so lanes keyed + /// on the first byte would all collapse into one. + write_lanes: Arc>>, + /// Operation counters, same shape as the LMDB store reported. + stats: parking_lot::RwLock, + /// Which of the 256 shard directories are known to exist, so a steady-state write + /// does not pay a `create_dir_all` syscall. + shards_present: Arc>, + /// Indexed chunks this store currently cannot read. + /// + /// Held back from everything the node says it has, while the files themselves are + /// left alone. See [`Self::mark_suspect`]. + suspect: Arc>>, + /// Indexed chunks a read has proven do not match their name. + /// + /// Separate from the above because they clear differently. Not being able to read a + /// file is a question a later read answers; bytes that are wrong stay wrong however + /// often they are read, and only a repair or a removal settles it. A raw read that + /// does not hash anything must not take a chunk out of this set. + known_wrong: Arc>>, + /// Addresses this store is part-way through writing. + /// + /// Every mutation registers here before it spawns its blocking work and clears the + /// entry *inside* that work, so a caller whose future is dropped cannot skip the + /// clearing while the write itself goes on to land. That is the difference that + /// matters: the blocking half is not cancelled with the future, so anything the + /// future was going to do afterwards is not a record of what happened. + /// + /// It lets a delete queue behind the exact write it would otherwise race, rather than + /// behind every write this store has in flight. + /// + /// Counted, not a set. Cancellation can release the facade's key lane while the + /// blocking half survives, so a second write for the same key can start behind the + /// first. With one entry between them, whichever finished first would remove it and a + /// waiter would be told the key is free while the other was still queued. + writing: Arc>>, + /// Woken when [`Self::writing`] loses its last entry for a key. + write_finished: Arc, + /// Bumped whenever a chunk stops being servable. + /// + /// The pre-retirement pass reads every chunk, and its result is reused for a while + /// rather than re-read on every tick. This is how the caller can tell that the store + /// has not changed underneath that result: a proof carries the value it saw, and a + /// file that has since gone or stopped being readable makes it stale. + health: Arc, + /// Size-aware free-space predicate. + capacity: Arc, + /// Monotonic counter that makes temp filenames unique within this store. + temp_seq: AtomicU64, + /// Random per-instance discriminator for temp filenames. + nonce: u32, + /// Held for the store's lifetime. Startup fails without it. + /// + /// Shared rather than owned so the blocking work that depends on it can hold a lease + /// of its own: that work outlives the future that spawned it, and a cancelled caller + /// releasing the lock would leave it writing into a directory another process had + /// just been let into. + lock: Arc, + /// Tracks every blocking task, so [`FileStore::wait_idle`] can wait for writes that + /// outlived their awaiting future. + blocking_tracker: TaskTracker, + /// Test-only gate read-acquired at the top of the put blocking closure. + /// + /// Tests hold the write half to park an in-flight write on the blocking pool, which + /// is the shape a `select!` losing to a shutdown token leaves behind. + #[cfg(any(test, feature = "test-utils"))] + test_put_gate: Arc>, +} + +impl FileStore { + /// Open (or create) the store at `{root_dir}/chunks/`. + /// + /// Sweeps orphaned temp files, then rebuilds the index from directory entries. + /// The scan reads names only: it never `stat`s an entry and never reads a chunk. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the directory cannot be created, the layout marker + /// is unreadable or describes a layout this build does not implement, or the scan + /// fails. + pub async fn new(config: FileStoreConfig) -> Result { + let chunks_dir = config.root_dir.join(CHUNKS_DIR_NAME); + std::fs::create_dir_all(&chunks_dir).map_err(|e| { + Error::Storage(format!( + "Failed to create chunk store directory {}: {e}", + chunks_dir.display() + )) + })?; + + check_path_budget(&chunks_dir); + + let layout = read_or_write_layout(&chunks_dir)?; + layout.check_supported()?; + + // Startup fails without it, so from here this process is the only one using this + // directory and an interrupted write can only be its own. + let lock = acquire_store_lock(&chunks_dir)?; + + let scan_dir = chunks_dir.clone(); + // The scan holds the lease itself. It sweeps interrupted writes on the strength of + // being alone here, and it runs on a thread that outlives this future: a + // cancelled startup that released the lock would leave it sweeping a directory + // another process had just been let into. + let scan_lease = Arc::clone(&lock); + // The node root as well as the chunk tree. The scan sweeps interrupted writes + // under `chunks/`, which covers the layout marker's temporary because that lives + // there; the migration marker's lives in the root, where nothing looked. + let root = config.root_dir.clone(); + let scan = spawn_blocking(move || { + let _lease = scan_lease; + sweep_marker_temps(&root); + scan_store(&scan_dir) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store scan task failed: {e}")))??; + + let ScanResult { + keys, + shards_present, + swept_temps, + skipped, + } = scan; + + let key_count = keys.len(); + // Build from a sorted vector: bulk-building packs every B-tree node to its + // capacity, where repeated `insert` converges on ~68% fill for the same keys. + let index: BTreeSet = keys.into_iter().collect(); + + if swept_temps > 0 { + info!("Chunk store: removed {swept_temps} orphaned temporary file(s) from interrupted writes"); + } + if skipped > 0 { + warn!("Chunk store: ignored {skipped} directory entr(ies) that are not chunk files"); + } + info!( + "Chunk store open at {} ({key_count} chunks)", + chunks_dir.display() + ); + + let capacity = Arc::new(CapacityGuard::new(chunks_dir.clone(), config.disk_reserve)); + + Ok(Self { + config, + chunks_dir, + index: Arc::new(parking_lot::RwLock::new(index)), + write_lanes: Arc::new( + std::iter::repeat_with(|| parking_lot::Mutex::new(())) + .take(SHARD_COUNT) + .collect(), + ), + stats: parking_lot::RwLock::new(StorageStats::default()), + shards_present: Arc::new(parking_lot::Mutex::new(shards_present)), + suspect: Arc::new(parking_lot::RwLock::new(HashSet::new())), + known_wrong: Arc::new(parking_lot::RwLock::new(HashSet::new())), + writing: Arc::new(parking_lot::Mutex::new(HashMap::new())), + write_finished: Arc::new(tokio::sync::Notify::new()), + health: Arc::new(std::sync::atomic::AtomicU64::new(0)), + capacity, + temp_seq: AtomicU64::new(0), + nonce: rand::random(), + lock, + blocking_tracker: TaskTracker::new(), + #[cfg(any(test, feature = "test-utils"))] + test_put_gate: Arc::new(parking_lot::RwLock::new(())), + }) + } + + /// Store a chunk. + /// + /// On Unix, publishing is a rename within the destination directory, so the final name + /// can never appear on partial content: the name *is* the hash, and the content is + /// fully written and flushed before the name exists. Off Unix there is no rename, for + /// the reason `publish_in_place` gives (it is compiled only on those platforms, so this + /// is not a link), and a partial file can wear a real name; that + /// is why a duplicate is read and compared rather than trusted. + /// + /// # Returns + /// + /// `true` if the chunk was newly stored, `false` if it was already present. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the content does not hash to `address`, the disk + /// is too full, or the write fails. + pub async fn put(&self, address: &XorName, content: &[u8]) -> Result { + let computed = crate::client::compute_address(content); + if computed != *address { + return Err(Error::Storage(format!( + "Content address mismatch: expected {}, computed {}", + hex::encode(address), + hex::encode(computed) + ))); + } + // The read path refuses anything over the ceiling, so writing one would create a + // file the store could never read back and could never repair. + if content.len() > MAX_CHUNK_SIZE { + return Err(Error::Storage(format!( + "Chunk {} is {} bytes, over the {MAX_CHUNK_SIZE} byte maximum", + hex::encode(address), + content.len() + ))); + } + + // An indexed name is not proof of the bytes under it. The index is built from + // names, by the startup scan and by a completed publish, and a name can outlive + // what it points at: off Unix a chunk is created under its final name before its + // bytes are written, so a crash leaves a short file wearing a real name, and rot + // leaves a full-length one. Answering "already have it" to the copy that would fix + // either is how a node discards its own repair and is never offered another. + // + // So the bytes decide. Checked before the reservation below, so re-storing a chunk + // this node already holds stays a no-op on a full disk. + if self.index.read().contains(address) { + if let Some(answer) = self.settle_indexed_duplicate(address, content).await { + return answer; + } + } + + let len = content.len() as u64; + // Reserved after the duplicate test so re-storing an existing chunk stays a + // harmless no-op on a full disk, matching the LMDB store's ordering. + let reservation = self.capacity.reserve(len)?; + + let shard = self.chunks_dir.join(shard_name(address)); + let final_path = shard.join(hex::encode(address)); + let temp_path = shard.join(self.next_temp_name()); + let payload = content.to_vec(); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let shards_present = Arc::clone(&self.shards_present); + let chunks_dir = self.chunks_dir.clone(); + let lane = shard_index(address); + let key = *address; + #[cfg(any(test, feature = "test-utils"))] + let test_put_gate = Arc::clone(&self.test_put_gate); + // Registered before the work is spawned and cleared by the work itself, so a + // caller that goes away cannot leave a delete free to race this publish. + let in_flight = self.begin_write(address); + // And the lease, for the same reason the scan holds it: this thread writes into a + // directory whose exclusivity the lock is what establishes, and it can outlive + // the last owner of the store. + let lease = Arc::clone(&self.lock); + let known_wrong = Arc::clone(&self.known_wrong); + let suspect = Arc::clone(&self.suspect); + + let outcome = self + .blocking_tracker + .spawn_blocking(move || -> Result { + let _in_flight = in_flight; + let _lease = lease; + // Test-only: parks here while a test holds the write half. + #[cfg(any(test, feature = "test-utils"))] + let _test_put_gate = test_put_gate.read(); + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + // `mkdir` plus a directory flush are syscalls, so they belong here and + // not on a runtime worker. + ensure_shard_dir(&chunks_dir, &shard, lane, &shards_present)?; + let outcome = match publish(&temp_path, &final_path, &payload, &shard) { + Ok(outcome) => outcome, + Err(PublishFailed { error, left_behind }) => { + // A publish that failed can still have left the bytes there: off + // Unix the chunk is created under its final name, and if the write + // or the flush then fails, the cleanup that removes it can fail + // too. Releasing the reservation would hand back a charge for a + // file that is on the disk. + // + // The publish says so rather than this deciding from a later + // `is_file`. Asking the filesystem afterwards infers ownership from + // a name being occupied, which is true under the store lock and the + // shard lane and not true against anything out of band, and this + // file spends a lot of its length arguing that a name is not + // evidence. A bit set by the code that created the file is. + if left_behind { + reservation.commit(); + } + return Err(error); + } + }; + // Placed, not yet durable. A failure from here on leaves the bytes on the + // disk: the chunk is rightly not reported as stored, because a copy that is + // not durable must not authorise deleting another, but the space is spent + // all the same. Dropping the reservation would hand that charge back and + // admit the next write against room that is already gone. + // + // Only for a chunk this call published. `Duplicate` means the file was + // already there and was charged by whoever wrote it, so charging it again + // here would count one file twice and shrink the store's idea of its own + // disk on every retry. + if let Err(e) = flush_publication(&final_path, &shard) { + if matches!(outcome, PutOutcome::New) { + reservation.commit(); + } + return Err(e); + } + // Index inside the lane, and only after the rename has returned. A + // concurrent delete of the same address therefore cannot interleave + // between publishing the file and admitting the key. + // + // Only for a chunk this call actually published. `Duplicate` says a file + // already wears the name, and a name is not evidence about the bytes under + // it: the four-way answer that decides whether they are good, wrong, absent + // or unreadable runs after the await below, and a caller whose future is + // dropped never reaches it. Admitting the key here would leave the node + // claiming, advertising and committing to bytes nothing has read, with no + // suspect or known-wrong mark to hold it back, and the sharpest case is a + // name the startup scan deliberately refused because what wears it is a + // fifo, a socket or a directory. The duplicate arm admits the key itself, + // once a read has proven the bytes. + if matches!(outcome, PutOutcome::New) { + index.write().insert(key); + // With the marks that would otherwise hold the key back. These bytes + // were hashed against their own name on the way in, so an older + // instance proven wrong or merely unreadable has just been replaced by + // a good one. Cleared here rather than after the await for the same + // reason the insert is here: a cancelled caller would leave the key + // indexed and suppressed at once, so a chunk this node really does hold + // would stay hidden from `exists` and `all_keys` until some later read + // happened to settle it. + known_wrong.write().remove(&key); + suspect.write().remove(&key); + // Settled here, inside the work, so a dropped awaiter cannot strand it. + reservation.commit(); + } + Ok(outcome) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store put task failed: {e}")))??; + + match outcome { + PutOutcome::Duplicate => self.settle_duplicate(address, content).await, + PutOutcome::New => { + // Freshly published bytes that were checked against their own name on the + // way in. The marks were already cleared inside the work, where a dropped + // caller cannot skip them; what is left here is only what a caller who is + // still waiting should see. + let mut stats = self.stats.write(); + stats.chunks_stored = stats.chunks_stored.saturating_add(1); + stats.bytes_stored = stats.bytes_stored.saturating_add(len); + drop(stats); + debug!("Stored chunk {} ({len} bytes)", hex::encode(address)); + Ok(true) + } + } + } + + /// Decide what a name that was already taken actually means. + /// + /// Split out of [`Self::put`] because it is a different question. `put` puts bytes on + /// a disk; this reads bytes back to find out whether the ones already there are the + /// ones the caller is offering, which is the only thing that makes a duplicate safe to + /// report as stored. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] when the existing file is absent or could not be read, + /// both of which mean this node must not report the chunk as held. + async fn settle_duplicate(&self, address: &XorName, content: &[u8]) -> Result { + // The file was already on disk, and its name is not evidence its contents + // are right. The startup scan indexes by name without reading anything, + // and on Windows a crash mid-write leaves a partial file under a real + // chunk name. Trusting the name here would acknowledge a chunk that was + // never stored, and then discard the good copy arriving to repair it. + // Every answer handled, because three of the four must not report the + // chunk as stored. A caller that hears success acts on it: a client drops + // its own copy, replication marks the key held, and the copier takes it + // out of the legacy-only set. + match self.stored_bytes_match(address).await { + StoredBytes::Good => { + // Admitted here, which is the first moment the bytes behind the + // name have been read and shown to hash to it. Idempotent: the + // ordinary case is a key the startup scan already indexed. + self.index.write().insert(*address); + { + let mut stats = self.stats.write(); + stats.duplicates = stats.duplicates.saturating_add(1); + } + Ok(false) + } + StoredBytes::Wrong => { + warn!( + "Chunk {} was already on disk but its contents are wrong; \ + replacing it with the copy just offered", + hex::encode(address) + ); + self.repair(address, content).await.map(|()| true) + } + // The name was taken a moment ago and is not now, or was never a + // readable chunk file. Either way nothing holds these bytes, so say so + // rather than reporting a chunk that is not there. + StoredBytes::Absent => Err(Error::Storage(format!( + "Chunk {} was reported already on disk but nothing is there. Not \ + reporting it as stored.", + hex::encode(address) + ))), + // Replacing on an unanswered question would destroy a healthy copy, + // and reporting success would discard the offered one. The index entry + // stays: the file is still there, and dropping the entry would leave + // the chunk in neither this store's view nor the legacy one, which is + // what retirement destroys. Removing an entry is the quarantine path's + // job, and it removes the file with it, after a read that succeeded + // and proved the bytes wrong. + StoredBytes::Unreadable => Err(Error::Storage(format!( + "Chunk {} is on disk but could not be read to check it. Not \ + replacing it, and not reporting it as stored.", + hex::encode(address) + ))), + } + } + + /// Flush every directory a chunk can live in, so the names in them are durable. + /// + /// Byte integrity is not the whole of what the pre-retirement proof has to establish. + /// A chunk whose contents are on the platter but whose *name* is not is still lost to + /// a power loss, and a publish whose rename landed and whose directory flush failed + /// leaves exactly that: the next attempt sees the name, the next verification reads + /// the right bytes, and nothing goes back to retry the flush. So the proof flushes + /// them itself rather than trusting that each publish did. + /// + /// Cheap: at most 257 directory flushes for a store of any size, and nothing off Unix, + /// where directories cannot be flushed and the retirement marker covers the same + /// ground instead. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] on the first directory that cannot be flushed. The + /// caller must treat that as a proof it did not get. + pub fn flush_namespace(&self) -> Result<()> { + fsync_dir(&self.chunks_dir).map_err(|e| { + Error::Storage(format!( + "Could not flush {}: {e}", + self.chunks_dir.display() + )) + })?; + let present = *self.shards_present.lock(); + for (shard, _) in present.iter().enumerate().filter(|(_, here)| **here) { + let dir = self.chunks_dir.join(format!("{shard:02x}")); + fsync_dir(&dir) + .map_err(|e| Error::Storage(format!("Could not flush {}: {e}", dir.display())))?; + } + Ok(()) + } + + /// Decide what to do about a write of a chunk the index already names. + /// + /// `None` means the index was wrong and there is nothing on disk, so the caller + /// publishes it as new. Everything else is the answer. + async fn settle_indexed_duplicate( + &self, + address: &XorName, + content: &[u8], + ) -> Option> { + match self.stored_bytes_match(address).await { + StoredBytes::Good => { + trace!("Chunk {} already exists", hex::encode(address)); + { + let mut stats = self.stats.write(); + stats.duplicates = stats.duplicates.saturating_add(1); + } + Some(Ok(false)) + } + StoredBytes::Wrong => { + warn!( + "Chunk {} is indexed but its bytes are wrong; replacing it with the \ + copy just offered", + hex::encode(address) + ); + Some(self.repair(address, content).await.map(|()| true)) + } + // Indexed but gone: publish it fresh rather than replacing something that is + // not there. + StoredBytes::Absent => None, + // Unanswerable this time. Do not touch what is there, and do not tell the + // caller the chunk is safely stored either: a client would take that as an + // acknowledgement and drop the only other copy. The index entry stays, for + // the reason given on the same case after publication. + StoredBytes::Unreadable => Some(Err(Error::Storage(format!( + "Chunk {} is indexed but could not be read to check it. Not replacing it, \ + and not reporting it as stored.", + hex::encode(address) + )))), + } + } + + /// Drop an address from the index without touching the file. Tests only. + /// + /// Stands in for whatever leaves a key indexed nowhere: a quarantine, a publish that + /// failed after the file went, an operator with a shell. + #[cfg(test)] + pub(crate) fn forget_for_test(&self, address: &XorName) { + self.index.write().remove(address); + } + + /// The size of the file behind `address`, if there is one. + /// + /// One `metadata` call, no read. Used where an indexed name has to be checked against + /// what a caller is offering before that offer is turned away. + #[must_use] + pub fn stored_len(&self, address: &XorName) -> Option { + std::fs::metadata(self.chunk_path(address)) + .ok() + .filter(std::fs::Metadata::is_file) + .and_then(|m| usize::try_from(m.len()).ok()) + } + + /// Whether the file already stored under `address` really hashes to it. + async fn stored_bytes_match(&self, address: &XorName) -> StoredBytes { + match self.get_raw(address).await { + Ok(Some(bytes)) if crate::client::compute_address(&bytes) == *address => { + // A read that hashed. It settles both questions. + self.clear_suspect(address); + self.clear_known_wrong(address); + StoredBytes::Good + } + Ok(Some(_)) => { + self.mark_known_wrong(address); + StoredBytes::Wrong + } + Ok(None) => StoredBytes::Absent, + // NOT the same as wrong. A file that could not be read this once may be + // perfectly good, and off Unix replacing it means opening it with `truncate`, + // which would destroy a healthy sole copy on the strength of a transient + // fault. Say so and let the caller leave it alone. + Err(e) => { + debug!("Could not read {} to check it: {e}", hex::encode(address)); + self.mark_suspect(address); + StoredBytes::Unreadable + } + } + } + + /// Replace the file behind an address with known-good bytes, atomically. + /// + /// Unlike [`Self::put`], this deliberately publishes **over** an existing name. It + /// exists for one caller: repairing a file whose bytes no longer hash to their own + /// name, from a copy held elsewhere, before that copy is destroyed. Doing it as + /// delete-then-put would leave a window where the only remaining copy is the one + /// about to be deleted, and any failure in that window is unrecoverable. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if `content` does not hash to `address`, or the write + /// fails. The old file is left untouched on every error path. + pub async fn repair(&self, address: &XorName, content: &[u8]) -> Result<()> { + // The same ceiling `put` enforces. Without it a repair can install bytes the read + // path will refuse for ever, which is a chunk that verifies as present and can + // never be served. + if content.len() > MAX_CHUNK_SIZE { + return Err(Error::Storage(format!( + "Refusing to repair {} with {} bytes, over the {MAX_CHUNK_SIZE} byte \ + maximum", + hex::encode(address), + content.len() + ))); + } + let computed = crate::client::compute_address(content); + if computed != *address { + return Err(Error::Storage(format!( + "Refusing to repair {} with content that hashes to {}", + hex::encode(address), + hex::encode(computed) + ))); + } + // The replacement exists alongside the original until the rename, so the room for + // it has to be there first. Reserved rather than merely checked: a plain check + // passes against a cached measurement, so concurrent repairs and PUTs can each be + // admitted against the same headroom and cross the reserve together. + // Moved into the work below, so it is released when the write finishes rather + // than when its caller stops waiting. A caller that goes away otherwise frees + // room that the detached write is still about to consume. + let reservation = self.capacity.reserve(content.len() as u64)?; + + let shard = self.chunks_dir.join(shard_name(address)); + let final_path = shard.join(hex::encode(address)); + let temp_path = shard.join(self.next_temp_name()); + let payload = content.to_vec(); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let shards_present = Arc::clone(&self.shards_present); + let chunks_dir = self.chunks_dir.clone(); + let lane = shard_index(address); + let key = *address; + let in_flight = self.begin_write(address); + let lease = Arc::clone(&self.lock); + let capacity = Arc::clone(&self.capacity); + let suspect = Arc::clone(&self.suspect); + let known_wrong = Arc::clone(&self.known_wrong); + + self.blocking_tracker + .spawn_blocking(move || -> Result<()> { + let _in_flight = in_flight; + let _lease = lease; + let _reservation = reservation; + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + ensure_shard_dir(&chunks_dir, &shard, lane, &shards_present)?; + write_and_replace(&temp_path, &final_path, &payload, &shard)?; + index.write().insert(key); + // Settled here rather than after the await. The replacement has landed + // and hashes to its own name, so nothing is wrong with this chunk any + // more; a caller that stopped waiting would otherwise leave a healthy + // file excluded from everything the node claims to hold, and the + // measurement believing the store is a chunk smaller than it is. + suspect.write().remove(&key); + known_wrong.write().remove(&key); + capacity.invalidate(); + Ok(()) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store repair task failed: {e}")))??; + + // Everything the success means was recorded by the work that succeeded: the + // reservation released, the marks cleared, the measurement thrown away. Released + // rather than committed because a repair is not a new chunk, and the measurement + // discarded rather than adjusted because the file it replaced may have been + // shorter, which is exactly the case a repair fixes. + debug!("Repaired chunk {}", hex::encode(address)); + Ok(()) + } + + /// Retrieve a chunk, verifying it against its address when configured to. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] on an I/O failure, or when verification fails. A + /// chunk whose bytes do not hash to its name is removed and dropped from the index + /// before the error is returned, so it leaves `all_keys()` and ordinary replication + /// repairs it. + pub async fn get(&self, address: &XorName) -> Result>> { + let Some(content) = self.read_file(address).await? else { + trace!("Chunk {} not found", hex::encode(address)); + return Ok(None); + }; + + if self.config.verify_on_read { + let computed = crate::client::compute_address(&content); + if computed != *address { + { + let mut stats = self.stats.write(); + stats.verification_failures = stats.verification_failures.saturating_add(1); + } + warn!( + "Chunk verification failed: expected {}, computed {}", + hex::encode(address), + hex::encode(computed) + ); + // Said before it is acted on. Removing the file can fail or be cancelled, + // and a chunk proven wrong that goes on looking healthy is one the node + // keeps committing to and, worse, one a cached pre-retirement pass still + // covers: the legacy copy that would repair it gets deleted. + self.mark_known_wrong(address); + self.quarantine_corrupt(address).await; + return Err(Error::Storage(format!( + "Chunk verification failed for {}", + hex::encode(address) + ))); + } + } + + if self.config.verify_on_read { + // The bytes hashed to their name. Whatever this store thought was wrong with + // them is not wrong with them, and a mark that outlives the fault it + // describes means the node can serve a chunk it will not claim, commit or + // offer. + self.clear_known_wrong(address); + } + + let len = content.len() as u64; + { + let mut stats = self.stats.write(); + stats.chunks_retrieved = stats.chunks_retrieved.saturating_add(1); + stats.bytes_retrieved = stats.bytes_retrieved.saturating_add(len); + } + debug!("Retrieved chunk {} ({len} bytes)", hex::encode(address)); + Ok(Some(content)) + } + + /// Retrieve raw chunk bytes without content-address verification. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] on an I/O failure. + pub async fn get_raw(&self, address: &XorName) -> Result>> { + self.read_file(address).await + } + + /// Check whether a chunk is stored. + /// + /// An in-memory lookup: no syscall, no I/O. + /// + /// # Errors + /// + /// Never fails. The signature keeps the shape the LMDB store had, because callers + /// treat the error as "assume absent". + pub fn exists(&self, address: &XorName) -> Result { + if self.is_unservable(address) { + return Ok(false); + } + Ok(self.is_indexed(address)) + } + + /// Is this chunk one the node must not answer for? + #[must_use] + fn is_unservable(&self, address: &XorName) -> bool { + self.suspect.read().contains(address) || self.known_wrong.read().contains(address) + } + + /// Is this chunk in the index, whether or not it can currently be read? + /// + /// The physical question, as against [`Self::exists`]'s question about what the node + /// is willing to claim. The migration must ask this one: a suspect chunk is still a + /// file this store has, and treating it as absent would put the key in the legacy-only + /// set, from where the union view advertises it again — a key the node claims through + /// one view and cannot serve through either. + #[must_use] + pub fn is_indexed(&self, address: &XorName) -> bool { + self.index.read().contains(address) + } + + /// Delete a chunk, returning whether it was present. + /// + /// `unlink` returns the blocks to the filesystem immediately. That is the whole + /// point of this store: no free list, no compaction, no free space required to + /// reclaim space. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the file exists but cannot be removed. The index + /// keeps the key in that case, because the bytes are still on disk. + pub async fn delete(&self, address: &XorName) -> Result { + let path = self.chunk_path(address); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let lane = shard_index(address); + let key = *address; + // Carried into the closure for the reason `put`, `repair` and the startup scan + // carry it: this work outlives the future that started it, so a cancelled caller + // that drops the last `FileStore` would otherwise release the directory to another + // process while an unlink is still queued against it. Deleting is the operation + // where that matters most. + let lease = Arc::clone(&self.lock); + + let (existed, freed) = self + .blocking_tracker + .spawn_blocking(move || -> Result<(bool, u64)> { + let _lease = lease; + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + let len = std::fs::metadata(&path).map_or(0, |m| m.len()); + let removed = match std::fs::remove_file(&path) { + Ok(()) => { + // Without this a crash can resurrect the entry on ext4, XFS, + // btrfs and APFS: the unlink is in the page cache, the directory + // is not. + if let Some(shard) = path.parent() { + fsync_dir_best_effort(shard); + } + true + } + // Already gone: the index was stale. Still a successful delete as + // far as the caller is concerned. + Err(e) if e.kind() == ErrorKind::NotFound => false, + Err(e) => { + return Err(Error::Storage(format!( + "Failed to delete chunk file {}: {e}", + path.display() + ))) + } + }; + // Index only after the filesystem operation has succeeded. On the error + // path above the entry stays, because the bytes are still on disk. + let was_indexed = index.write().remove(&key); + Ok((removed || was_indexed, if removed { len } else { 0 })) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store delete task failed: {e}")))??; + + if freed > 0 { + self.capacity.record_removed(freed); + debug!("Deleted chunk {}", hex::encode(address)); + } + Ok(existed) + } + + /// Return every stored key, in ascending order. + /// + /// The order is a correctness requirement, not a convenience: the commitment + /// builder truncates the responsible subset with `take(cap)` before the Merkle tree + /// sorts it, so an unstable order would make the node's published commitment depend + /// on iteration luck. + /// + /// # Errors + /// + /// Never fails. The signature matches the LMDB store's. + // Async without awaiting anything, deliberately: the whole point of this store is + // that the key set is already in memory. Callers are spread across the replication + // engine and cannot all be de-async'd in this change. + // + // Two lint names because they were renamed between toolchains, and `unknown_lints` + // so whichever one the compiler in use has never heard of stays quiet. + #[allow(unknown_lints)] + #[allow(clippy::unused_async, clippy::unused_async_trait_impl)] + pub async fn all_keys(&self) -> Result> { + // Copied out first so neither lock is held while the other is taken, and so the + // usual case, where nothing is suspect, costs one clone of an empty set. + let mut unservable: HashSet = self.suspect.read().clone(); + unservable.extend(self.known_wrong.read().iter().copied()); + let keys = self.index.read().clone(); + if unservable.is_empty() { + return Ok(keys.into_iter().collect()); + } + Ok(keys + .into_iter() + .filter(|key| !unservable.contains(key)) + .collect()) + } + + /// Stop answering for a chunk this store could not read. + /// + /// The file stays. It may be perfectly good and unreadable only for the moment, and + /// deleting it, or dropping it from the index, is how a chunk ends up in neither this + /// store's view nor the legacy one, which is what retirement destroys. + /// + /// What does change is what the node says about it. A chunk it cannot read is one it + /// cannot serve, and claiming it anyway puts the key in signed commitments, answers + /// presence probes with a yes, suppresses the replication that would repair it, and + /// earns a penalty at the next commitment-bound audit. Those penalties are not + /// suspended. + fn mark_suspect(&self, address: &XorName) { + if self.suspect.write().insert(*address) { + self.note_health_changed(); + warn!( + "Chunk {} is on disk but could not be read; this node stops answering for \ + it until a read succeeds", + hex::encode(address) + ); + } + } + + /// What the store's health looked like at this moment. + /// + /// Compare a value taken before a long-running check with one taken after, or after + /// taking a lock: different means a chunk stopped being servable in between and any + /// conclusion drawn from that check is out of date. + #[must_use] + pub fn health_generation(&self) -> u64 { + self.health.load(std::sync::atomic::Ordering::Acquire) + } + + /// Record that a chunk stopped being servable. + fn note_health_changed(&self) { + self.health + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + } + + /// Stop answering for a chunk a read has proven wrong. + /// + /// Unlike a chunk that merely could not be read, a later read does not clear this. + /// The bytes are wrong, and reading them again says the same thing; only replacing + /// them or removing them settles it. Bumping health matters as much as the suppression: a chunk that + /// has become unservable since the last pre-retirement pass must invalidate that pass, + /// or a repair that fails leaves the node deleting the copy it would have repaired + /// from. + /// + /// For callers outside this module that have proven it themselves. + pub fn note_known_wrong(&self, address: &XorName) { + self.mark_known_wrong(address); + } + + /// Stop answering for a chunk a read has proven wrong. + fn mark_known_wrong(&self, address: &XorName) { + if self.known_wrong.write().insert(*address) { + self.note_health_changed(); + warn!( + "Chunk {} does not match its name; this node stops answering for it until \ + it is repaired or removed", + hex::encode(address) + ); + } + } + + /// A caller outside this module has proven the stored bytes are right. + pub fn note_bytes_proven_good(&self, address: &XorName) { + self.clear_known_wrong(address); + self.clear_suspect(address); + } + + /// Answer for a chunk again, after it has been replaced or removed. + fn clear_known_wrong(&self, address: &XorName) { + self.known_wrong.write().remove(address); + } + + /// Answer for a chunk again, after a read that worked. + fn clear_suspect(&self, address: &XorName) { + if !self.suspect.read().contains(address) { + return; + } + if self.suspect.write().remove(address) { + info!( + "Chunk {} could be read again; this node answers for it once more", + hex::encode(address) + ); + } + } + + /// Number of chunks currently stored. + /// + /// The physical count: every name in the index, including chunks the node has stopped + /// answering for because a read found them wrong or could not read them at all. It is + /// deliberately not the same number as `all_keys().len()`, which is what the node is + /// willing to claim and so leaves those out. + /// + /// Anything asking "how much is on this disk" wants this one, and that is what its + /// callers ask: the migration's progress, the storage stats, and the size an audit is + /// built for. Anything asking "what will this node answer for" wants `all_keys`. + /// Quietly filtering this one would move all three of those without saying so, which + /// is why the difference is written down here rather than removed. + /// + /// # Errors + /// + /// Never fails. The signature matches the LMDB store's. + pub fn current_chunks(&self) -> Result { + Ok(self.index.read().len() as u64) + } + + /// Operation statistics, with the live chunk count filled in. + #[must_use] + pub fn stats(&self) -> StorageStats { + let mut stats = self.stats.read().clone(); + stats.current_chunks = self.index.read().len() as u64; + stats + } + + /// The node root directory this store was configured with. + #[must_use] + pub fn root_dir(&self) -> &Path { + &self.config.root_dir + } + + /// The directory holding the shard tree. + #[must_use] + pub fn chunks_dir(&self) -> &Path { + &self.chunks_dir + } + + /// Reject work early when the disk cannot take another chunk at all. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] when free space is below the configured reserve. + pub fn check_capacity(&self) -> Result<()> { + self.capacity.check(0) + } + + /// Three-way answer to "can this store take a write right now". + /// + /// Kept distinct from [`Self::check_capacity`] because a failed free-space query and a + /// genuinely full disk are not the same thing, and the replication verification cycle + /// depends on the difference: a full disk is a standing condition worth minutes of + /// backoff, while a `statvfs` that failed says nothing about available space and may + /// well succeed on the next pass. + #[must_use] + pub fn capacity_verdict(&self) -> crate::storage::CapacityVerdict { + match self.capacity.measure_available() { + Some(available) if available < self.capacity.reserve => { + crate::storage::CapacityVerdict::Full + } + Some(_) => crate::storage::CapacityVerdict::Writable, + None => crate::storage::CapacityVerdict::Unknown, + } + } + + /// Reject work early when the disk cannot take `bytes` more. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] when the write would not fit above the reserve. + pub fn check_capacity_for(&self, bytes: u64) -> Result<()> { + self.capacity.check(bytes) + } + + /// Force the next capacity question to re-measure the filesystem. + /// + /// Called after the legacy environment is removed, because that is a step change in + /// free space that the short-lived cache would otherwise hide for a few seconds. + pub fn invalidate_capacity_cache(&self) { + self.capacity.invalidate(); + } + + /// Test-only handle to the put gate. + /// + /// Hold the write half to park the next write inside its blocking closure, for + /// example to prove that shutdown waits for a write whose awaiter was dropped. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn test_put_gate(&self) -> Arc> { + Arc::clone(&self.test_put_gate) + } + + /// Register a write of `address` and hand back the token that clears it. + /// + /// The token must be moved into the blocking closure that does the work, so the entry + /// is cleared by the thread that finishes rather than by a caller that may be gone. + fn begin_write(&self, address: &XorName) -> WriteInFlight { + *self.writing.lock().entry(*address).or_insert(0) += 1; + WriteInFlight { + writing: Arc::clone(&self.writing), + finished: Arc::clone(&self.write_finished), + address: *address, + } + } + + /// Wait until nothing is part-way through writing `address`. + /// + /// For callers that must be last: a delete whose key still has a write in flight + /// would be undone by that write landing afterwards. + pub async fn wait_for_write(&self, address: &XorName) { + loop { + // Registered before the check, so a clear between the two is not missed. + let waiting = self.write_finished.notified(); + if !self.writing.lock().contains_key(address) { + return; + } + waiting.await; + } + } + + /// How many blocking tasks this store currently has in flight. Tests only. + /// + /// Lets a test wait for work to have actually started rather than guessing at a + /// delay, which is the difference between a test that proves something and one that + /// passes because the machine was quick. + #[cfg(test)] + #[must_use] + pub(crate) fn tasks_in_flight(&self) -> usize { + self.blocking_tracker.len() + } + + /// Wait until every blocking task this store spawned has finished. + /// + /// Dropping the awaiting future does not cancel a `spawn_blocking` closure, so + /// shutdown has to wait for the closure itself. + pub async fn wait_idle(&self) { + self.blocking_tracker.close(); + self.blocking_tracker.wait().await; + self.blocking_tracker.reopen(); + } + + /// Absolute path of a chunk file. + fn chunk_path(&self, address: &XorName) -> PathBuf { + self.chunks_dir + .join(shard_name(address)) + .join(hex::encode(address)) + } + + /// A temp name unique to this store instance, and distinguishable from a chunk name. + /// + /// The nonce matters: two `FileStore`s on one root in one process share a PID, and a + /// recycled PID collides with an age-gated leftover. Either way `create_new` would + /// fail and surface as a spurious write error. + fn next_temp_name(&self) -> String { + let seq = self.temp_seq.fetch_add(1, Ordering::Relaxed); + format!( + "{TEMP_PREFIX}{}.{:08x}.{seq}", + std::process::id(), + self.nonce + ) + } + + /// Read a chunk file, dropping the index entry if the file has vanished. + async fn read_file(&self, address: &XorName) -> Result>> { + let path = self.chunk_path(address); + let read = self + .blocking_tracker + .spawn_blocking(move || -> Result>> { + match open_regular(&path) { + Ok(Some(f)) => read_bounded(f, &path).map(Some), + Ok(None) => Ok(None), + Err(e) => Err(e), + } + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store read task failed: {e}")))?; + + // Every read decides the question, not only the ones that were checking. A read + // that failed means this chunk cannot be served, whoever asked; a read that + // worked means it can be, whoever asked. Doing this anywhere else leaves a key + // stuck unadvertised after the fault has cleared, or advertised after it has not. + let read = match read { + Ok(read) => { + self.clear_suspect(address); + read + } + Err(e) => { + self.mark_suspect(address); + return Err(e); + } + }; + + if read.is_none() && self.forget_if_absent(address).await { + // The file went away underneath us. Stop advertising the key so the close + // group notices the shortfall and replication puts it back. + warn!( + "Chunk {} is indexed but its file is missing; dropped from the index so \ + replication can repair it", + hex::encode(address) + ); + } + Ok(read) + } + + /// Drop an index entry whose file is genuinely gone. + /// + /// Re-checks under the address's write lane, so a chunk republished between the + /// failing read and this call keeps its entry. + async fn forget_if_absent(&self, address: &XorName) -> bool { + // Not suspect any more: it is not unreadable, it is not there. + self.clear_suspect(address); + let path = self.chunk_path(address); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let lane = shard_index(address); + let key = *address; + // The bump happens inside the closure, with the mutation it describes. The + // closure runs to completion on its own thread whether or not anyone is still + // awaiting it, so bumping after the await is skipped entirely when a shutdown + // drops the caller — and the index change it was meant to announce still lands. + // A cached pre-retirement proof would then stay valid over a store that had + // quietly lost a chunk. + let health = Arc::clone(&self.health); + self.blocking_tracker + .spawn_blocking(move || { + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + if path.exists() { + return false; + } + let forgotten = index.write().remove(&key); + if forgotten { + health.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + } + forgotten + }) + .await + .unwrap_or(false) + } + + /// Remove a chunk whose bytes do not match its name, and stop advertising it. + /// + /// Re-reads and re-verifies under the address's write lane first. A read that failed + /// verification is rare enough that paying for one extra read is worth never + /// discarding a chunk that a concurrent write had already repaired. + async fn quarantine_corrupt(&self, address: &XorName) { + let path = self.chunk_path(address); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let lane = shard_index(address); + let key = *address; + // For the reason given on `forget_if_absent`: this closure outlives its awaiter, + // and the change it makes has to be announced by the same thread that makes it. + let health = Arc::clone(&self.health); + // And the store-lock lease, for the reason `put`, `repair`, `delete` and the + // startup scan carry it: this closure outlives its awaiter, so without it a + // cancelled verification whose caller dropped the last `FileStore` would unlink + // inside a directory a second process had already been handed. + let lease = Arc::clone(&self.lock); + let outcome = + self.blocking_tracker + .spawn_blocking(move || -> std::io::Result { + let _lease = lease; + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + // Nothing is thrown away without proof. A re-read that fails says the + // question could not be answered this time, not that the bytes are wrong, + // and a repair may have published a good copy since the read that brought + // us here. Treating either as corruption deletes a chunk this node has. + let buf = match open_regular(&path) { + Ok(Some(f)) => read_bounded(f, &path) + .map_err(|e| std::io::Error::other(e.to_string()))?, + Ok(None) => { + index.write().remove(&key); + health.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + return Ok(true); + } + Err(e) => return Err(std::io::Error::other(e.to_string())), + }; + if crate::client::compute_address(&buf) == key { + // Repaired between the failing read and now. Leave it alone. + return Ok(false); + } + std::fs::remove_file(&path)?; + // The same flush the ordinary delete does, for the same reason: an + // unlink that has not reached the directory can be undone by a power + // loss, and here the entry that comes back is one this node has proven + // wrong. The startup scan would re-index it by name, and the + // known-wrong mark that would otherwise hold it back lives only in + // memory and does not survive the restart, so the node would go back to + // claiming and committing to a chunk it already knows is bad. + if let Some(shard) = path.parent() { + fsync_dir_best_effort(shard); + } + index.write().remove(&key); + health.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + Ok(true) + }) + .await; + match outcome { + Ok(Ok(true)) => { + self.clear_known_wrong(address); + self.clear_suspect(address); + warn!( + "Removed corrupt chunk file {}; replication will repair it", + hex::encode(address) + ); + } + Ok(Ok(false)) => { + // The re-read hashed and matched: a repair landed between the failing + // read and this one. + self.clear_known_wrong(address); + self.clear_suspect(address); + debug!( + "Chunk {} verified on re-read; leaving it in place", + hex::encode(address) + ); + } + // Still indexed, so it must not still be claimed: the read that brought us + // here proved the bytes wrong, and the node would otherwise go on committing + // to a chunk it knows it cannot serve. + Ok(Err(e)) => { + self.mark_suspect(address); + warn!( + "Corrupt chunk {} could not be removed: {e}. It stays on disk, and \ + this node stops answering for it.", + hex::encode(address) + ); + } + Err(e) => { + self.mark_suspect(address); + warn!("Corrupt-chunk removal task failed: {e}"); + } + } + } +} + +// ──────────────────────────────────────────────────────────────────────────── +// Free functions +// ──────────────────────────────────────────────────────────────────────────── + +/// Create the destination shard directory if this store has not seen it yet. +/// +/// A newly created directory entry is only durable once its parent is flushed; without +/// that a crash could take the directory and the chunk inside it together. +fn ensure_shard_dir( + chunks_dir: &Path, + dir: &Path, + shard: usize, + present: &parking_lot::Mutex<[bool; SHARD_COUNT]>, +) -> Result<()> { + if present.lock().get(shard).copied().unwrap_or(false) { + return Ok(()); + } + std::fs::create_dir_all(dir).map_err(|e| { + Error::Storage(format!( + "Failed to create shard directory {}: {e}", + dir.display() + )) + })?; + // Load-bearing, like the flush that publishes a chunk into this directory. Until the + // parent is flushed the shard's own entry can be lost, and losing it loses every chunk + // inside it. Reporting the shard present anyway would let the very first chunk written + // into it count as durably stored. + fsync_dir(chunks_dir).map_err(|e| { + Error::Storage(format!( + "Created shard directory {} but could not flush {}: {e}. Not marking the shard \ + usable, because a directory that is not durable cannot hold a chunk that is.", + dir.display(), + chunks_dir.display() + )) + })?; + if let Some(slot) = present.lock().get_mut(shard) { + *slot = true; + } + Ok(()) +} + +/// Shard directory index for an address: its last byte. +fn shard_index(address: &XorName) -> usize { + address.last().copied().unwrap_or(0) as usize +} + +/// Shard directory name for an address: the last two characters of its hex form. +fn shard_name(address: &XorName) -> String { + format!("{:02x}", shard_index(address)) +} + +/// True for a string of hex digits in either case. +fn is_hex_any_case(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_hexdigit()) +} + +/// Move an entry aside under a name that can never be read as a chunk. +fn quarantine_entry(path: &Path) { + let aside = path.with_extension("not-a-chunk"); + match std::fs::rename(path, &aside) { + Ok(()) => warn!( + "Chunk store: moved {} aside to {}; a name that differs from a chunk name only \ + by case collides with it on Windows and macOS", + path.display(), + aside.display() + ), + Err(e) => warn!( + "Chunk store: {} collides with a chunk name by case folding and could not be \ + moved aside: {e}. Rename or delete it.", + path.display() + ), + } +} + +/// True for a string of lowercase hex digits only. +fn is_lower_hex(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} + +/// Decode a filename back into the address it names, or `None` if it is not one. +/// +/// Rejects uppercase deliberately. On a case-folding filesystem (NTFS, default APFS) +/// accepting both cases would let one file answer to two index entries. +fn decode_chunk_name(name: &str) -> Option { + if name.len() != CHUNK_NAME_LEN || !is_lower_hex(name) { + return None; + } + let bytes = hex::decode(name).ok()?; + XorName::try_from(bytes.as_slice()).ok() +} + +/// Flush a directory and report whether it worked, for callers outside this module. +/// +/// For the one caller whose next step is destructive: retirement moves the legacy +/// environment aside and then deletes it under its new name, so if the rename has not +/// reached the disk when the delete lands, a power loss brings the environment back under +/// its old name with its contents gone. +/// +/// # Errors +/// +/// Returns the underlying I/O error. Off Unix there is no way to flush a directory through +/// the standard library, so this reports success without being able to promise anything. +pub fn fsync_path(path: &Path) -> std::io::Result<()> { + fsync_dir(path) +} + +/// Flush a directory so a rename or creation inside it survives power loss. +/// +/// Best effort by design. Linux and XFS require it, macOS accepts it with undocumented +/// effect, and Windows offers no way to do it at all through the standard library. The +/// content is content-addressed and re-replicable, so a lost directory entry costs a +/// refetch rather than data. Pretending otherwise in the code would be dishonest. +#[cfg(unix)] +fn fsync_dir_best_effort(path: &Path) { + if let Err(e) = fsync_dir(path) { + debug!("Directory flush of {} failed: {e}", path.display()); + } +} + +/// Flush a directory, reporting whether it worked. +/// +/// Used where the answer is load-bearing: a chunk copied out of the legacy store is only +/// durable once its directory entry is, and that copy is what permits the legacy store to +/// be deleted. +#[cfg(unix)] +fn fsync_dir(path: &Path) -> std::io::Result<()> { + File::open(path)?.sync_all() +} + +/// Off Unix there is no way to flush a directory through the standard library, so this +/// reports success without being able to promise anything. +/// +/// That is why the publish path off Unix does not use a rename at all: it creates the +/// chunk under its final name and flushes the file, which Microsoft documents as flushing +/// the creation metadata with it. Directory creation has no equivalent, so the guarantee +/// there rests on the pre-retirement pass, which re-reads every chunk before the legacy +/// store is deleted, and on the operator gate that keeps retirement off a platform until +/// forced power loss has been shown to hold old-or-new on it. +/// +/// Returns a `Result` so the callers that must handle a flush failure on Unix read the +/// same on every platform. +#[cfg(not(unix))] +#[allow(clippy::unnecessary_wraps)] +fn fsync_dir(_path: &Path) -> std::io::Result<()> { + Ok(()) +} + +/// No-op on platforms with no way to flush a directory handle. +#[cfg(not(unix))] +fn fsync_dir_best_effort(_path: &Path) {} + +/// Warn if the deepest chunk path this store can produce is close to `MAX_PATH`. +#[cfg(windows)] +fn check_path_budget(chunks_dir: &Path) { + // Measured absolute, because that is what the filesystem sees. A relative root is the + // case that still fails hard at MAX_PATH, since the standard library's long-path + // handling only applies to paths it resolves as absolute. + let absolute = if chunks_dir.is_absolute() { + chunks_dir.to_path_buf() + } else { + std::env::current_dir() + .map_or_else(|_| chunks_dir.to_path_buf(), |cwd| cwd.join(chunks_dir)) + }; + // `{chunks_dir}\{xy}\{64 hex}` — two separators, two shard characters, 64 name + // characters. + let deepest = absolute.as_os_str().len() + 1 + 2 + 1 + CHUNK_NAME_LEN; + if deepest > WINDOWS_PATH_WARN_LEN { + warn!( + "Chunk file paths will be {deepest} characters, close to the {} character \ + Windows limit. Move the node root closer to the drive letter if writes start \ + failing.", + WINDOWS_PATH_WARN_LEN + ); + } +} + +/// No-op where path length is not a practical constraint. +#[cfg(not(windows))] +fn check_path_budget(_chunks_dir: &Path) {} + +/// Write `bytes` to `path` durably, for small metadata files outside the shard tree. +/// +/// # Errors +/// +/// Returns [`Error::Storage`] if the file cannot be written or published. +pub fn write_file_durably(path: &Path, bytes: &[u8]) -> Result<()> { + write_file_atomic(path, bytes) +} + +/// Write `bytes` to `path` so a reader sees either the old content or the new. +/// Is this the exact name [`write_file_atomic`] gives its temporaries? +/// +/// `.tmp..<8 hex>.marker`, with both middle parts checked. Matching on the prefix and +/// suffix alone would also take `.tmp.operator-notes.marker`, and this runs over a +/// directory holding a node's data, so what it removes is not a place to be approximate. +fn is_marker_temp_name(name: &str) -> bool { + let Some(rest) = name.strip_prefix(TEMP_PREFIX) else { + return false; + }; + let Some(rest) = rest.strip_suffix(".marker") else { + return false; + }; + let mut parts = rest.split('.'); + let (Some(pid), Some(nonce), None) = (parts.next(), parts.next(), parts.next()) else { + return false; + }; + !pid.is_empty() + && pid.bytes().all(|b| b.is_ascii_digit()) + && nonce.len() == 8 + && nonce.bytes().all(|b| b.is_ascii_hexdigit()) +} + +/// Remove marker temporaries a previous run left beside `path`. +/// +/// [`write_file_atomic`] writes its temporary next to its target. For the layout marker +/// that is inside `chunks/`, which the startup scan sweeps; for the migration marker it is +/// the node root, which nothing sweeps, so a crash between the write and the rename leaves +/// one there for the life of the node. Each is a few hundred bytes, so this is inodes +/// rather than capacity, but nothing else was ever going to remove them. +/// +/// Only the exact shape this module writes, and only files: a name has to carry the temp +/// prefix and the marker suffix. Anything broader would be this function deciding what +/// else in a node's root directory is rubbish, which is not its business. +/// +/// Best effort throughout. Failing to tidy up is not a reason to refuse to start, and the +/// caller takes the store lock before this runs, so there is no other process whose live +/// temporary this could take. +pub(crate) fn sweep_marker_temps(dir: &Path) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + if !is_marker_temp_name(name) { + continue; + } + if !entry.file_type().is_ok_and(|kind| kind.is_file()) { + continue; + } + match std::fs::remove_file(entry.path()) { + Ok(()) => debug!( + "Swept a leftover marker temporary {}", + entry.path().display() + ), + Err(e) => debug!("Could not sweep {}: {e}", entry.path().display()), + } + } +} + +fn write_file_atomic(path: &Path, bytes: &[u8]) -> Result<()> { + let Some(dir) = path.parent() else { + return Err(Error::Storage(format!( + "Refusing to write {} — it has no parent directory", + path.display() + ))); + }; + let temp = dir.join(format!( + "{TEMP_PREFIX}{}.{:08x}.marker", + std::process::id(), + rand::random::() + )); + write_temp(&temp, bytes)?; + // Through the retry, because these small files (the layout marker, the migration + // state) are rewritten while the node runs, and on Windows a scanner holding a handle + // for a few milliseconds turns an ordinary rewrite into a hard failure. + rename_with_retry(&temp, path).map_err(|e| { + let _ = std::fs::remove_file(&temp); + Error::Storage(format!("Failed to publish {}: {e}", path.display())) + })?; + fsync_dir_best_effort(dir); + Ok(()) +} + +/// Read the layout marker, writing the current one if the store is new. +fn read_or_write_layout(chunks_dir: &Path) -> Result { + let path = chunks_dir.join(LAYOUT_FILE_NAME); + match read_small_file(&path) { + Ok(bytes) => serde_json::from_slice(&bytes).map_err(|e| { + Error::Storage(format!( + "Chunk store layout marker {} is unreadable: {e}. Refusing to open rather \ + than guess the layout.", + path.display() + )) + }), + Err(e) if e.kind() == ErrorKind::NotFound => { + if store_has_entries(chunks_dir) { + warn!( + "Chunk store at {} has data but no layout marker. Adopting it under \ + the current scheme, which is the only one this build implements. If \ + it was written by a build with a different layout its chunks will \ + appear to be missing.", + chunks_dir.display() + ); + } + let layout = StoreLayout::default(); + let bytes = serde_json::to_vec_pretty(&layout) + .map_err(|e| Error::Storage(format!("Failed to encode chunk store layout: {e}")))?; + write_file_atomic(&path, &bytes)?; + debug!("Wrote chunk store layout marker to {}", path.display()); + Ok(layout) + } + Err(e) => Err(Error::Storage(format!( + "Failed to read chunk store layout marker {}: {e}", + path.display() + ))), + } +} + +/// Whether the store directory already holds at least one shard. +fn store_has_entries(chunks_dir: &Path) -> bool { + let Ok(entries) = std::fs::read_dir(chunks_dir) else { + return false; + }; + entries.filter_map(std::result::Result::ok).any(|e| { + e.file_name() + .to_str() + .is_some_and(|n| n.len() == 2 && is_lower_hex(n)) + }) +} + +/// Largest a metadata marker may be before it is treated as corrupt. +const MAX_MARKER_BYTES: u64 = 64 * 1024; + +/// Read a small metadata file, refusing an implausibly large one. +/// +/// The chunk path is bounded for exactly this reason; the markers live in the same data +/// directory and deserve the same ceiling. +/// +/// # Errors +/// +/// Returns an I/O error, including `NotFound`, so callers can distinguish "no marker yet". +pub fn read_small_file(path: &Path) -> std::io::Result> { + let file = File::open(path)?; + let mut bytes = Vec::new(); + let read = file.take(MAX_MARKER_BYTES + 1).read_to_end(&mut bytes)?; + if read as u64 > MAX_MARKER_BYTES { + return Err(std::io::Error::other(format!( + "{} is larger than the {MAX_MARKER_BYTES} byte limit for a marker file", + path.display() + ))); + } + Ok(bytes) +} + +/// Take the store lock, or refuse to open the store. +/// +/// Both failures are refusals, deliberately. Unlike LMDB, which was genuinely +/// multi-process safe, two of these stores on one directory keep independent in-memory +/// indices, independent views of what is in flight, and independent opinions about +/// whether the legacy environment may be deleted: both would report the same write as +/// new and each would keep serving keys the other had deleted. A node that cannot create +/// the lock file has no way to know it is alone, and this is the one migration where +/// being wrong about that destroys data. +/// +/// The lock is an [`Arc`] so the work that relies on it can hold a lease. The startup +/// scan sweeps interrupted writes on the strength of being alone in the directory, and it +/// runs on a thread that outlives the future that started it. +/// +/// # Errors +/// +/// Returns [`Error::Storage`] when another process owns the directory, or when the lock +/// file cannot be created. +fn acquire_store_lock(chunks_dir: &Path) -> Result> { + let path = chunks_dir.join(LOCK_FILE_NAME); + let file = match OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(&path) + { + Ok(f) => f, + // Not a warning and carry on. Without this lock two processes can open the same + // directory, each with its own index, its own view of what is in flight, and its + // own opinion about whether the legacy environment may be deleted. A node that + // cannot take it has no way to know it is alone, and this is the one migration + // where being wrong about that destroys data. + Err(e) => { + return Err(Error::Storage(format!( + "Could not create the chunk store lock {}: {e}. Refusing to start: \ + without it this node cannot tell whether another is using the same data \ + directory. Fix the permissions on that path, or remove a stale lock file \ + left by a different user.", + path.display() + ))) + } + }; + match file.try_lock_exclusive() { + Ok(()) => Ok(Arc::new(file)), + Err(e) => Err(Error::Storage(format!( + "Another process already has the chunk store at {} open ({e}). Two nodes \ + cannot share one data directory: each keeps its own index and they would \ + disagree about what is stored. Stop the other node first.", + chunks_dir.display() + ))), + } +} + +/// What a startup scan found. +struct ScanResult { + /// Every published address, ascending. + keys: Vec, + /// Which shard directories already exist. + shards_present: [bool; SHARD_COUNT], + /// Orphaned temp files removed. + swept_temps: usize, + /// Entries that were neither a chunk nor one of ours. + skipped: usize, +} + +/// Rebuild the key set from directory entries. +/// +/// Reads names only. A `stat` per entry costs about ten times the enumeration on Linux +/// and macOS and fifty to sixty times on Windows, and buys nothing: the filename is the +/// key, and the content is verified on read. +fn scan_store(chunks_dir: &Path) -> Result { + let mut result = ScanResult { + keys: Vec::new(), + shards_present: [false; SHARD_COUNT], + swept_temps: 0, + skipped: 0, + }; + + let top = std::fs::read_dir(chunks_dir).map_err(|e| { + Error::Storage(format!( + "Failed to enumerate chunk store {}: {e}", + chunks_dir.display() + )) + })?; + + for entry in top { + let entry = entry.map_err(|e| { + Error::Storage(format!( + "Failed to read an entry of {}: {e}", + chunks_dir.display() + )) + })?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + result.skipped = result.skipped.saturating_add(1); + continue; + }; + if name == LAYOUT_FILE_NAME || name == LOCK_FILE_NAME { + continue; + } + if name.starts_with(TEMP_PREFIX) { + if sweep_temp(&entry.path()) { + result.swept_temps = result.swept_temps.saturating_add(1); + } + continue; + } + if name.len() != 2 || !is_lower_hex(name) { + warn!( + "Chunk store: ignoring unexpected entry {name} in {}", + chunks_dir.display() + ); + result.skipped = result.skipped.saturating_add(1); + continue; + } + let Ok(shard) = u8::from_str_radix(name, 16) else { + result.skipped = result.skipped.saturating_add(1); + continue; + }; + // `shards_present` is set inside `scan_shard`, on success only. Setting it from + // the name alone would make a stray regular file called `ab` look like a shard + // that already exists, and every write to that shard would then fail with a + // misleading error until the node was restarted. + scan_shard(&entry.path(), shard, &mut result)?; + } + + result.keys.sort_unstable(); + result.keys.dedup(); + Ok(result) +} + +/// Scan one shard directory into `result`. +fn scan_shard(dir: &Path, shard: u8, result: &mut ScanResult) -> Result<()> { + let entries = match std::fs::read_dir(dir) { + Ok(e) => e, + // A stray file named like a shard, or a directory removed between the two reads. + // Neither is fatal, and neither marks the shard as present. + Err(e) if matches!(e.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => { + warn!( + "Chunk store: {} is not a shard directory ({e}); ignoring it", + dir.display() + ); + result.skipped = result.skipped.saturating_add(1); + return Ok(()); + } + // Anything else is a real fault: a permission problem, exhausted descriptors, or + // failing hardware. Opening with a shard's worth of keys silently missing would + // make the node under-claim in its published commitment and stop serving chunks + // it still holds and is answerable for, so refuse to open at all. + Err(e) => { + return Err(Error::Storage(format!( + "Failed to enumerate shard {}: {e}. Refusing to open with an incomplete \ + key set.", + dir.display() + ))) + } + }; + if let Some(slot) = result.shards_present.get_mut(shard as usize) { + *slot = true; + } + + for entry in entries { + let entry = + entry.map_err(|e| Error::Storage(format!("Failed to read {}: {e}", dir.display())))?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + result.skipped = result.skipped.saturating_add(1); + continue; + }; + if name.starts_with(TEMP_PREFIX) { + if sweep_temp(&entry.path()) { + result.swept_temps = result.swept_temps.saturating_add(1); + } + continue; + } + let Some(key) = decode_chunk_name(name) else { + if name.len() == CHUNK_NAME_LEN && is_hex_any_case(name) { + // A case-folded twin of a real chunk name. On NTFS and default APFS the + // existence check in the write path folds onto it, so a paid write would + // be answered "already stored" and its bytes dropped. Move it aside. + quarantine_entry(&entry.path()); + } else { + warn!( + "Chunk store: ignoring non-chunk entry {name} in {}", + dir.display() + ); + } + result.skipped = result.skipped.saturating_add(1); + continue; + }; + // `file_type` comes from the directory entry itself on Linux and macOS and from + // the enumeration on Windows, so this is not the per-entry `stat` the scan + // deliberately avoids. A pipe, socket, device or directory wearing a chunk name + // must never enter the index: nothing downstream can read it, and it would sit in + // the published commitment forever. + match entry.file_type() { + Ok(kind) if kind.is_file() => {} + Ok(_) => { + warn!( + "Chunk store: {name} in {} is not a regular file; ignoring it", + dir.display() + ); + result.skipped = result.skipped.saturating_add(1); + continue; + } + // Not the same as knowing it is not a file. Treating an unanswered question + // as a no would drop a real chunk from the index and from the commitment + // while its bytes sit on disk, and the node would not serve it again until + // some later restart happened to succeed. Fail the scan instead: an index + // that is missing keys must never be published as this node's key set. + Err(e) => { + return Err(Error::Storage(format!( + "Could not tell what {name} in {} is: {e}. Refusing to publish an \ + index that may be missing chunks.", + dir.display() + ))); + } + } + // A file in the wrong shard is unreachable through `chunk_path`, so indexing it + // would make the index claim a key the read path cannot find. + if shard_index(&key) != shard as usize { + warn!( + "Chunk store: {name} is filed under shard {shard:02x} but belongs in {:02x}; \ + ignoring it. Move it or delete it.", + shard_index(&key) + ); + result.skipped = result.skipped.saturating_add(1); + continue; + } + result.keys.push(key); + } + Ok(()) +} + +/// Remove one orphaned temp file. Returns whether it went. +/// +/// Always removed. The scan that calls this runs only after the store lock has been taken, +/// so by then any temp file is an interrupted write of a previous run and there is no other +/// process that could be writing it. This used to describe a second, gentler mode for the +/// unlocked case; there was never any such branch and there is no caller that would need +/// one. +fn sweep_temp(path: &Path) -> bool { + match std::fs::remove_file(path) { + Ok(()) => { + debug!("Removed orphaned temporary file {}", path.display()); + true + } + Err(e) => { + debug!("Could not remove {}: {e}", path.display()); + false + } + } +} + +/// Open a chunk file, refusing anything that is not a regular file. +/// +/// `Ok(None)` means the file is not there. A named pipe wearing a valid chunk name would +/// otherwise block the opening thread forever: `open` on a FIFO with no writer does not +/// return, and enough of them would exhaust the blocking pool and stall every file and +/// database operation in the process. `O_NOFOLLOW` refuses a symlink for the same reason, +/// and both are checked on the handle rather than the path, so nothing can be swapped +/// underneath between the check and the open. +fn open_regular(path: &Path) -> Result> { + #[cfg(unix)] + let opened = { + use std::os::unix::fs::OpenOptionsExt; + OpenOptions::new() + .read(true) + .custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW) + .open(path) + }; + #[cfg(not(unix))] + let opened = OpenOptions::new().read(true).open(path); + + let file = match opened { + Ok(f) => f, + Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None), + Err(e) => { + return Err(Error::Storage(format!( + "Failed to open chunk file {}: {e}", + path.display() + ))) + } + }; + let is_regular = file.metadata().is_ok_and(|m| m.file_type().is_file()); + if !is_regular { + return Err(Error::Storage(format!( + "{} is not a regular file; refusing to read it as a chunk", + path.display() + ))); + } + Ok(Some(file)) +} + +/// Read a chunk file, refusing anything larger than a chunk can legitimately be. +/// +/// A corrupt, sparse, or locally planted file wearing a valid 64-hex name would +/// otherwise be read straight into memory, so a single bad entry could exhaust the node +/// during an ordinary GET or an audit response. +fn read_bounded(file: File, path: &Path) -> Result> { + let ceiling = MAX_CHUNK_SIZE as u64; + let mut buf = Vec::new(); + let read = file.take(ceiling + 1).read_to_end(&mut buf).map_err(|e| { + Error::Storage(format!("Failed to read chunk file {}: {e}", path.display())) + })?; + if read as u64 > ceiling { + return Err(Error::Storage(format!( + "Chunk file {} is larger than the {ceiling} byte maximum; refusing to read it", + path.display() + ))); + } + Ok(buf) +} + +/// Whether a Windows error is one a scanner or indexer holding a handle would produce. +/// +/// `ERROR_ACCESS_DENIED`, `ERROR_SHARING_VIOLATION`, `ERROR_LOCK_VIOLATION`. Every other +/// failure is deterministic and retrying it only burns a blocking thread. +fn is_windows_sharing_violation(e: &std::io::Error) -> bool { + matches!(e.raw_os_error(), Some(5 | 32 | 33)) +} + +/// Publish `temp_path` as `final_path`, retrying a transient sharing violation. +/// +/// On Windows an antivirus scanner or the search indexer can hold a handle to either +/// file for a few milliseconds after it is created, and `MoveFileEx` fails outright +/// rather than queueing. Retrying a bounded number of times turns that from a failed +/// write into a short pause. Every other error returns immediately. +fn rename_with_retry(temp_path: &Path, final_path: &Path) -> std::io::Result<()> { + let mut last = match std::fs::rename(temp_path, final_path) { + Ok(()) => return Ok(()), + Err(e) => e, + }; + if !cfg!(windows) || !is_windows_sharing_violation(&last) { + return Err(last); + } + for attempt in 1..=RENAME_RETRY_ATTEMPTS { + std::thread::sleep(RENAME_RETRY_BACKOFF * attempt); + match std::fs::rename(temp_path, final_path) { + Ok(()) => return Ok(()), + Err(e) => last = e, + } + } + Err(last) +} + +/// Write `payload` and publish it as `final_path`, replacing whatever is there. +/// +/// Success here means the bytes are durable, not merely written. The repair path this +/// serves runs during the pre-retirement pass, where a chunk that fails to match its +/// address is rewritten from the legacy store and the legacy store is then deleted. A +/// replacement that a power loss can undo would leave that chunk with the wrong bytes and +/// no other copy. +fn write_and_replace( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + shard: &Path, +) -> Result<()> { + // Unix: an intra-directory rename is atomic, so a reader sees the old content or the + // new one and never an absence, and the directory flush is what makes it durable. + #[cfg(unix)] + { + write_temp(temp_path, payload)?; + if let Err(e) = rename_with_retry(temp_path, final_path) { + let _ = std::fs::remove_file(temp_path); + return Err(Error::Storage(format!( + "Failed to replace chunk {}: {e}", + final_path.display() + ))); + } + fsync_dir(shard).map_err(|e| { + Error::Storage(format!( + "Replaced {} but could not flush {}: {e}. Not reporting the repair as \ + done, because a rewrite that is not durable must not authorise deleting \ + the copy it was rewritten from.", + final_path.display(), + shard.display() + )) + })?; + Ok(()) + } + // Everywhere else, Windows included: there is no way to flush a directory through the + // standard library, so a rename cannot be shown to be durable at return. Overwriting + // the existing file changes no directory entry at all, and `sync_all` (FlushFileBuffers + // on Windows) is documented to flush the file's data, so a successful return is + // durable under a documented contract. + // + // The cost is that this is not atomic: a crash part-way leaves the file holding a mix + // of old and new bytes. That is safe here and only here, because the only caller that + // matters runs before the legacy store is deleted, and a crash means no report was + // produced and nothing was deleted. The next start re-reads the file, sees it does not + // match its address, and repairs it again from the store that is still there. + #[cfg(not(unix))] + { + let _ = temp_path; + let _ = shard; + let mut file = OpenOptions::new() + .write(true) + .truncate(true) + .open(final_path) + .map_err(|e| { + Error::Storage(format!( + "Failed to open {} for replacement: {e}", + final_path.display() + )) + })?; + file.write_all(payload).map_err(|e| { + Error::Storage(format!("Failed to rewrite {}: {e}", final_path.display())) + })?; + file.sync_all().map_err(|e| { + Error::Storage(format!( + "Rewrote {} but could not flush it: {e}. Not reporting the repair as \ + done, because a rewrite that is not durable must not authorise deleting \ + the copy it was rewritten from.", + final_path.display() + )) + })?; + Ok(()) + } +} + +/// Create `temp_path`, write `payload` into it, and flush it. +/// +/// Flushed before any rename. On ext4 `auto_da_alloc` only orders the data before the +/// rename's own commit; it does not make the data durable, and btrfs has been observed +/// reordering. A name must never become visible on bytes that are not on the platter. +fn write_temp(temp_path: &Path, payload: &[u8]) -> Result<()> { + let mut f = OpenOptions::new() + .write(true) + .create_new(true) + .open(temp_path) + .map_err(|e| { + Error::Storage(format!( + "Failed to create temporary file {}: {e}", + temp_path.display() + )) + })?; + if let Err(e) = f.write_all(payload) { + let _ = std::fs::remove_file(temp_path); + return Err(Error::Storage(format!( + "Failed to write {}: {e}", + temp_path.display() + ))); + } + if let Err(e) = f.sync_all() { + let _ = std::fs::remove_file(temp_path); + return Err(Error::Storage(format!( + "Failed to flush {}: {e}", + temp_path.display() + ))); + } + Ok(()) +} + +/// Write `payload` and publish it under `final_path`. +/// +/// The temp lives in the destination directory, so the publish is an intra-directory +/// rename: atomic on every filesystem we support, and needing only that one directory +/// Put `payload` on disk as `final_path`, durably. +/// +/// Returns [`PutOutcome::Duplicate`] when the name is already taken. The name is a hash +/// of the content, so that is not treated as proof the bytes are right: the caller +/// re-reads and verifies them. +#[cfg(unix)] +fn publish( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + shard: &Path, +) -> std::result::Result { + // On Unix nothing is ever created under the final name by a failing path: the bytes go + // to a temporary and only a successful rename gives them the real name. So every + // failure here leaves the name as it found it. + publish_via_rename(temp_path, final_path, payload, shard) + .map_err(PublishFailed::nothing_written) +} + +/// Put `payload` on disk as `final_path`, durably. See [`publish_in_place`] for why this +/// takes a different route off Unix. +#[cfg(not(unix))] +fn publish( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + shard: &Path, +) -> std::result::Result { + let _ = temp_path; + let _ = shard; + publish_in_place(final_path, payload) +} + +/// Create the chunk under its final name and flush it. Everywhere but Unix. +/// +/// There is no way to flush a directory through the standard library, and Microsoft does +/// not document `MoveFileEx` as durable at return unless it is called with +/// `MOVEFILE_WRITE_THROUGH`, which std does not use. So off Unix a rename cannot be +/// relied on to have reached the disk before the legacy store is deleted. +/// +/// Creating the file under its final name sidesteps the rename entirely. Microsoft +/// documents that creation metadata is cached and that `FlushFileBuffers`, which +/// `sync_all` calls on Windows, is the way to flush it. So a successful create, write and +/// flush is a durable publication under a documented contract, with no directory flush +/// and no rename involved. +/// +/// The cost is that a crash mid-write leaves a partial file wearing a real chunk name. +/// That is why a duplicate re-reads and verifies rather than trusting the name, and why +/// the pre-retirement pass re-hashes everything before anything is deleted. +#[cfg(not(unix))] +fn publish_in_place( + final_path: &Path, + payload: &[u8], +) -> std::result::Result { + // Test-only, and here rather than after the write so that it means the same thing on + // both platforms: the file half of a dual write has not happened yet. On Unix the + // equivalent point is the temporary file written and the rename not yet made, which is + // also before the chunk's name exists on disk. Stopping after the write instead would + // put the file under its real name already, so a crash there is not between the two + // halves at all, and it could not demonstrate anything about the missing flush either: + // killing a process does not empty the page cache, so the bytes are still there to be + // read. Only losing power loses them, which no test that kills a process can stage. + #[cfg(any(test, feature = "test-utils"))] + halt_here_if_asked(HALT_BEFORE_PUBLISH, final_path); + let mut file = match OpenOptions::new() + .write(true) + .create_new(true) + .open(final_path) + { + Ok(f) => f, + // Someone got there first. Immutable content under a content-addressed name, so + // the caller verifies what is already there rather than assuming it is right. + Err(e) if e.kind() == ErrorKind::AlreadyExists => return Ok(PutOutcome::Duplicate), + Err(e) => { + // Nothing was created, so nothing was spent. + return Err(PublishFailed::nothing_written(Error::Storage(format!( + "Failed to create chunk {}: {e}", + final_path.display() + )))); + } + }; + if let Err(e) = file.write_all(payload) { + drop(file); + // Taken back if it can be. Whether it could is what the caller needs: the file + // was created by this call, so if it is still there the space is spent. + let left_behind = std::fs::remove_file(final_path).is_err(); + return Err(PublishFailed { + error: Error::Storage(format!("Failed to write {}: {e}", final_path.display())), + left_behind, + }); + } + if let Err(e) = file.sync_all() { + drop(file); + // Taken back if it can be. Whether it could is what the caller needs: the file + // was created by this call, so if it is still there the space is spent. + let left_behind = std::fs::remove_file(final_path).is_err(); + return Err(PublishFailed { + error: Error::Storage(format!("Failed to flush {}: {e}", final_path.display())), + left_behind, + }); + } + Ok(PutOutcome::New) +} + +/// Write a temp beside the target and rename it into place. Unix only. +/// +/// Places the bytes and nothing more. Making the name durable is +/// [`flush_publication`]'s job, kept separate so a caller can tell a publish that spent no +/// space from one that spent it and could not be reported. +#[cfg(unix)] +fn publish_via_rename( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + _shard: &Path, +) -> Result { + // Content is immutable and the name is its hash, so an existing file already holds + // exactly these bytes. Skipping the write is both cheaper and safer than replacing + // it: on Windows a rename over a file another thread has open fails outright. + // + // The caller flushes either way. A name that is already there is not proof it is + // durable: + // the write that put it there may have been this store's own previous attempt, whose + // rename landed and whose directory flush then failed. That attempt returned an + // error, so nothing was retired on the strength of it, but if this call reported a + // durable duplicate without flushing, the retry would silently launder an unflushed + // rename into a copy that authorises deleting the last other one. + let outcome = if final_path.exists() { + PutOutcome::Duplicate + } else { + write_temp(temp_path, payload)?; + // Test-only: the one moment a complete chunk exists on disk under a name nothing + // looks for. A crash test needs to die at a named point rather than wherever a + // sleep in another process happened to land. + #[cfg(any(test, feature = "test-utils"))] + halt_here_if_asked(HALT_BEFORE_PUBLISH, temp_path); + match rename_with_retry(temp_path, final_path) { + Ok(()) => PutOutcome::New, + Err(e) => { + let _ = std::fs::remove_file(temp_path); + // Another writer of the same address won the race, or the destination was + // open. Either way the bytes are already published. + if !final_path.exists() { + return Err(Error::Storage(format!( + "Failed to publish chunk {}: {e}", + final_path.display() + ))); + } + PutOutcome::Duplicate + } + } + }; + + Ok(outcome) +} + +/// A publish that failed, and whether it left its bytes on the disk. +/// +/// The second half is the point. A failure before anything was created has spent nothing; +/// one that created the file and then could not remove it again has spent the space, and +/// whoever is accounting for free space has to know which happened. Only the code that did +/// the creating can say. +struct PublishFailed { + error: Error, + left_behind: bool, +} + +impl PublishFailed { + /// A failure that created nothing. + fn nothing_written(error: Error) -> Self { + Self { + error, + left_behind: false, + } + } +} + +/// Make a publication durable by flushing the directory its name lives in. +/// +/// Separate from placing the bytes, because the caller has to tell the two failures apart. +/// A publish that fails before the bytes land has spent nothing; one that fails here has +/// spent the space and must not be reported as stored, so whoever is accounting for free +/// space has to charge it while whoever is accounting for chunks must not count it. +/// +/// NOT best effort. The directory flush is what makes the rename durable, and a copy +/// reported successful is what authorises deleting the only other copy. Swallowing the +/// failure would let a power loss discard the directory entry after the legacy store had +/// already been removed. +/// +/// # Errors +/// +/// Returns [`Error::Storage`] if the directory cannot be flushed. +#[cfg(unix)] +fn flush_publication(final_path: &Path, shard: &Path) -> Result<()> { + fsync_dir(shard).map_err(|e| { + Error::Storage(format!( + "Published {} but could not flush {}: {e}. Not reporting this chunk as stored, \ + because a copy that is not durable must not authorise deleting another.", + final_path.display(), + shard.display() + )) + }) +} + +/// Nothing to do off Unix, where the chunk is created under its final name and flushed +/// with `sync_all`, which is documented to carry its creation metadata with it, and where +/// there is no way to flush a directory at all. +#[cfg(not(unix))] +fn flush_publication(_final_path: &Path, _shard: &Path) -> Result<()> { + Ok(()) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + use std::collections::HashSet; + + /// A directory flush that fails must say so. + /// + /// The quiet version of this function is only used where the answer does not change + /// what happens next. On the publish path it does. + #[cfg(unix)] + #[test] + fn flushing_a_directory_that_is_not_there_reports_the_failure() { + let dir = TempDir::new().expect("temp dir"); + assert!(fsync_dir(dir.path()).is_ok()); + assert!(fsync_dir(&dir.path().join("no-such-shard")).is_err()); + } + + /// A chunk whose directory entry was never flushed is not reported as stored. + /// + /// This is the whole safety argument for retirement: the legacy store is deleted + /// because every chunk was copied durably. A published file whose directory flush + /// failed can vanish on power loss, so counting it as copied would lose data. The + /// file staying on disk afterwards is fine, the next pass republishes it. + #[cfg(unix)] + #[test] + fn a_publish_whose_directory_flush_fails_is_not_reported_as_stored() { + let dir = TempDir::new().expect("temp dir"); + let temp_path = dir.path().join("chunk.tmp"); + let final_path = dir.path().join("chunk"); + let unflushable = dir.path().join("shard-that-does-not-exist"); + + // Asserted in two steps, not chained. Chaining them means a regression in placing + // the bytes also produces an error, and the test passes without the flush ever + // being reached: it would be checking that something went wrong rather than that + // this went wrong. + let placed = publish_via_rename(&temp_path, &final_path, b"payload", &unflushable); + assert!( + placed.is_ok(), + "the bytes must be placed before this can be about the flush: {:?}", + placed.err() + ); + let outcome = flush_publication(&final_path, &unflushable); + + assert!( + outcome.is_err(), + "an unflushed publication must not be reported as stored" + ); + assert!( + !temp_path.exists(), + "the temp file must not be left behind either way" + ); + } + + use tempfile::TempDir; + + /// Open a store on a fresh temp directory with the disk reserve disabled. + async fn test_store() -> (FileStore, TempDir) { + let dir = TempDir::new().expect("temp dir"); + let store = FileStore::new(FileStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("open store"); + (store, dir) + } + + /// Open a store on an existing directory, as a restart would. + async fn reopen(dir: &TempDir) -> FileStore { + FileStore::new(FileStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("reopen store") + } + + /// An ordinary read settles whether the node answers for a chunk. + /// + /// Not only the reads that were checking something. A read that failed means the + /// chunk cannot be served, whoever asked; a read that worked means it can be. Deciding + /// this anywhere else leaves a key stuck unadvertised after the fault has cleared, or + /// advertised after it has not. + #[cfg(unix)] + #[tokio::test] + async fn an_ordinary_read_decides_whether_the_node_answers_for_a_chunk() { + use std::os::unix::fs::PermissionsExt; + + let (store, dir) = test_store().await; + let (addr, content) = addressed("read-decides"); + store.put(&addr, &content).await.expect("put"); + let path = store.chunk_path(&addr); + + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o000); + std::fs::set_permissions(&path, perms).expect("chmod"); + + assert!(store.get(&addr).await.is_err(), "the read must fail"); + assert!( + !store.exists(&addr).expect("exists"), + "and a plain read that failed must stop the node answering for it" + ); + + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o600); + std::fs::set_permissions(&path, perms).expect("chmod back"); + + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + assert!( + store.exists(&addr).expect("exists"), + "and a plain read that worked must start it answering again" + ); + drop(dir); + } + + /// Two writes for one key: waiting means waiting for both. + /// + /// Cancellation releases the caller's lane while the blocking half survives, so a + /// second write for the same key can start behind the first. If the registry only + /// recorded that *something* was writing, whichever finished first would clear it and + /// a delete would be told the key was free while the other was still queued, then be + /// undone by it. + #[tokio::test] + async fn waiting_for_a_key_waits_for_every_write_of_it() { + let (store, dir) = test_store().await; + let store = Arc::new(store); + let (addr, content) = addressed("two-writers"); + + // Two registrations, as two overlapping writes would make. + let first = store.begin_write(&addr); + let second = store.begin_write(&addr); + + let waiting = { + let store = Arc::clone(&store); + tokio::spawn(async move { store.wait_for_write(&addr).await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!waiting.is_finished()); + + // One finishes. The other has not, so the wait must continue. + drop(first); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !waiting.is_finished(), + "one write finishing does not mean the key is free" + ); + + drop(second); + waiting + .await + .expect("the wait ends once both have finished"); + + // And the store is still usable afterwards. + store.put(&addr, &content).await.expect("put"); + assert!(store.exists(&addr).expect("exists")); + drop(dir); + } + + /// A chunk this store cannot read is kept but not claimed. + /// + /// Both halves matter. Deleting it, or dropping it from the index, is how a chunk ends + /// up in neither this store's view nor the legacy one, which is what retirement + /// destroys. Claiming it anyway puts the key in signed commitments and answers + /// presence probes with a yes for a chunk the node cannot serve, and the audit that + /// catches that still penalises. + #[cfg(unix)] + #[tokio::test] + async fn a_chunk_that_cannot_be_read_is_kept_but_not_claimed() { + use std::os::unix::fs::PermissionsExt; + + let (store, dir) = test_store().await; + let (addr, content) = addressed("unreadable-for-now"); + store.put(&addr, &content).await.expect("put"); + assert!(store.exists(&addr).expect("exists")); + + let path = store.chunk_path(&addr); + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o000); + std::fs::set_permissions(&path, perms).expect("chmod"); + + // Offering the same bytes again must not be acknowledged, and must not replace + // what is there on the strength of a read that did not happen. + assert!( + store.put(&addr, &content).await.is_err(), + "an unreadable chunk must not be reported as stored" + ); + assert!(path.exists(), "and the file must be left alone"); + assert!( + !store.exists(&addr).expect("exists"), + "but the node must stop claiming it" + ); + assert!(!store.all_keys().await.expect("keys").contains(&addr)); + + // Readable again: the node answers for it once more. + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o600); + std::fs::set_permissions(&path, perms).expect("chmod back"); + assert!(!store.put(&addr, &content).await.expect("put again")); + assert!(store.exists(&addr).expect("exists")); + assert!(store.all_keys().await.expect("keys").contains(&addr)); + drop(dir); + } + + /// Content plus the address it hashes to. + fn addressed(seed: &str) -> (XorName, Vec) { + let content = format!("chunk-content-{seed}").into_bytes(); + (crate::client::compute_address(&content), content) + } + + #[tokio::test] + async fn put_then_get_returns_the_same_bytes() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("a"); + + assert!(store.put(&addr, &content).await.expect("put")); + let got = store.get(&addr).await.expect("get").expect("present"); + assert_eq!(got, content); + } + + #[tokio::test] + async fn a_second_put_of_the_same_chunk_reports_not_new() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("b"); + + assert!(store.put(&addr, &content).await.expect("first put")); + assert!(!store.put(&addr, &content).await.expect("second put")); + assert_eq!(store.current_chunks().expect("count"), 1); + assert_eq!(store.stats().duplicates, 1); + } + + #[tokio::test] + async fn get_of_an_unknown_address_is_none() { + let (store, _dir) = test_store().await; + let (addr, _) = addressed("missing"); + assert!(store.get(&addr).await.expect("get").is_none()); + } + + #[tokio::test] + async fn exists_tracks_the_store() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("c"); + + assert!(!store.exists(&addr).expect("exists")); + store.put(&addr, &content).await.expect("put"); + assert!(store.exists(&addr).expect("exists")); + store.delete(&addr).await.expect("delete"); + assert!(!store.exists(&addr).expect("exists")); + } + + #[tokio::test] + async fn delete_unlinks_the_file_and_returns_the_space() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("d"); + store.put(&addr, &content).await.expect("put"); + + let path = store.chunk_path(&addr); + assert!(path.exists(), "the chunk file should be on disk"); + + assert!(store.delete(&addr).await.expect("delete")); + assert!(!path.exists(), "delete must actually unlink the file"); + assert_eq!(store.current_chunks().expect("count"), 0); + + // Deleting again is a no-op that reports nothing was there. + assert!(!store.delete(&addr).await.expect("second delete")); + } + + #[tokio::test] + async fn content_that_does_not_hash_to_its_address_is_rejected() { + let (store, _dir) = test_store().await; + let (addr, _) = addressed("e"); + let err = store + .put(&addr, b"different content") + .await + .expect_err("must reject"); + assert!( + format!("{err}").contains("Content address mismatch"), + "unexpected error: {err}" + ); + assert_eq!(store.current_chunks().expect("count"), 0); + } + + #[tokio::test] + async fn a_chunk_is_filed_under_the_last_two_hex_characters_of_its_address() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("f"); + store.put(&addr, &content).await.expect("put"); + + let name = hex::encode(addr); + let expected_shard = name + .get(name.len() - 2..) + .expect("64-character name") + .to_string(); + let path = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(&expected_shard) + .join(&name); + assert!(path.exists(), "expected the chunk at {}", path.display()); + } + + #[tokio::test] + async fn the_index_is_rebuilt_from_the_filesystem_on_restart() { + let (store, dir) = test_store().await; + let mut written = Vec::new(); + for i in 0..64 { + let (addr, content) = addressed(&format!("restart-{i}")); + store.put(&addr, &content).await.expect("put"); + written.push(addr); + } + drop(store); + + let reopened = reopen(&dir).await; + assert_eq!(reopened.current_chunks().expect("count"), 64); + for addr in &written { + assert!(reopened.exists(addr).expect("exists"), "lost a key"); + } + } + + #[tokio::test] + async fn all_keys_is_sorted_ascending() { + let (store, dir) = test_store().await; + for i in 0..128 { + let (addr, content) = addressed(&format!("sorted-{i}")); + store.put(&addr, &content).await.expect("put"); + } + + let keys = store.all_keys().await.expect("all_keys"); + let mut sorted = keys.clone(); + sorted.sort_unstable(); + assert_eq!(keys, sorted, "all_keys() must be ordered"); + + // And the order has to survive a restart, because the commitment builder + // truncates the responsible subset before the Merkle tree sorts it. + drop(store); + let reopened = reopen(&dir).await; + assert_eq!(reopened.all_keys().await.expect("all_keys"), keys); + } + + #[tokio::test] + async fn get_raw_skips_verification() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("raw"); + store.put(&addr, &content).await.expect("put"); + + // Corrupt the file behind the store's back. + std::fs::write(store.chunk_path(&addr), b"tampered").expect("tamper"); + + let raw = store.get_raw(&addr).await.expect("get_raw").expect("bytes"); + assert_eq!(raw, b"tampered"); + } + + /// A put whose caller goes away does not admit a key on bytes nothing has read. + /// + /// The blocking half of a put outlives the future that started it, deliberately, so + /// the work is never left half done. That makes anything it writes to memory a claim + /// the node keeps whether or not the caller is still there to finish checking it. + /// + /// For a chunk this call published the claim is earned: the bytes were hashed against + /// their own name on the way in. For a name that was already taken it is not. The + /// check that decides whether those bytes are good runs after the await, and a dropped + /// future skips it, so admitting the key in the closure claims a chunk nobody read. + /// + /// Staged with a fifo, which is the sharpest case and a real one: the startup scan + /// refuses non-regular entries by design, so this is a key the store has already + /// decided it must not claim, walked in through the back door. + #[cfg(unix)] + #[tokio::test] + // The gate is held across an await deliberately: holding it is what parks the put + // inside its closure, which is the state under test. Dropping it before awaiting would + // let the put finish and there would be nothing to cancel. + #[allow(clippy::await_holding_lock)] + async fn a_cancelled_put_does_not_admit_a_key_whose_bytes_were_never_read() { + let dir = TempDir::new().expect("temp dir"); + let store = Arc::new(reopen(&dir).await); + + // A name a real chunk would use, wearing something that is not a chunk. + let content = b"the bytes that belong under this name".to_vec(); + let addr = crate::client::compute_address(&content); + let shard = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(format!("{:02x}", addr[31])); + std::fs::create_dir_all(&shard).expect("mkdir"); + let path = shard.join(hex::encode(addr)); + let name = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()) + .expect("a path with no interior nul"); + // SAFETY: `name` is a valid NUL-terminated C string that outlives the call, and the + // mode is a constant. `mkfifo` reads the pointer and returns; nothing is retained. + #[allow(clippy::undocumented_unsafe_blocks, unsafe_code)] + let made = unsafe { libc::mkfifo(name.as_ptr(), 0o644) }; + assert_eq!(made, 0, "could not make the fifo this test needs"); + + // Hold the gate so the put parks inside the closure, then drop the future while it + // is parked. That is a caller going away mid-put, which is what a cancelled + // request, a client disconnect or a shutdown all look like from in here. + let gate = store.test_put_gate(); + let held = gate.write(); + let put = { + let store = Arc::clone(&store); + let content = content.clone(); + tokio::spawn(async move { store.put(&addr, &content).await }) + }; + // Waited for rather than slept at. A sleep proves nothing: if the put had not + // reached the gated closure yet, aborting would cancel it before it ever got + // there and the test would pass having staged nothing. + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while store.tasks_in_flight() == 0 { + assert!( + std::time::Instant::now() < deadline, + "the put never reached the closure, so there was nothing to cancel" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + put.abort(); + let _ = put.await; + drop(held); + store.wait_idle().await; + + assert!( + !store.is_indexed(&addr), + "a cancelled put admitted {} on bytes nothing read; the fifo under that name \ + would then be advertised, committed to, and audited against", + hex::encode(addr) + ); + assert!( + !store.exists(&addr).unwrap_or(true), + "and the node must not claim it either" + ); + } + + /// A marker temporary left in the node root is swept, and nothing else is. + /// + /// The migration marker is written next to itself in the root, which no sweep looked + /// at, so a crash between its write and its rename left one there for the life of the + /// node. Small, but nothing was ever going to remove it. + /// + /// The second half is the point: this runs over a directory holding a node's data, so + /// it has to take only the exact shape this module writes and leave everything else + /// where it is. + #[tokio::test] + async fn a_leftover_marker_temporary_is_swept_and_its_neighbours_are_not() { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path(); + let leftover = root.join(format!("{TEMP_PREFIX}1234.abcdef01.marker")); + std::fs::write(&leftover, b"an interrupted marker write").expect("plant"); + + // Things that must survive: the marker itself, a chunk-shaped temp that belongs to + // the chunk tree's own sweep, and anything an operator put there. + let keep = [ + root.join("migration-state.json"), + root.join(format!("{TEMP_PREFIX}1234.abcdef01.chunk")), + root.join("notes.txt"), + // Prefix and suffix alone would take these. The pid and the nonce are checked + // because this runs over a directory holding a node's data. + root.join(format!("{TEMP_PREFIX}operator-notes.marker")), + root.join(format!("{TEMP_PREFIX}1234.nothex01.marker")), + root.join(format!("{TEMP_PREFIX}1234.abcdef0.marker")), + root.join(format!("{TEMP_PREFIX}1234.abcdef01.extra.marker")), + ]; + for path in &keep { + std::fs::write(path, b"keep me").expect("plant"); + } + + let store = reopen(&dir).await; + drop(store); + + assert!( + !leftover.exists(), + "the leftover marker temporary is still in the node root" + ); + for path in &keep { + assert!( + path.exists(), + "{} was swept and should not have been", + path.display() + ); + } + } + + #[tokio::test] + async fn a_corrupt_chunk_is_removed_so_replication_can_repair_it() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("corrupt"); + store.put(&addr, &content).await.expect("put"); + std::fs::write(store.chunk_path(&addr), b"tampered").expect("tamper"); + + let err = store.get(&addr).await.expect_err("verification must fail"); + assert!(format!("{err}").contains("verification failed"), "{err}"); + + assert!(!store.chunk_path(&addr).exists(), "corrupt file must go"); + assert!(!store.exists(&addr).expect("exists")); + assert!( + !store.all_keys().await.expect("all_keys").contains(&addr), + "a corrupt chunk must stop being advertised" + ); + assert_eq!(store.stats().verification_failures, 1); + } + + #[tokio::test] + async fn a_file_removed_underneath_the_store_drops_out_of_the_index() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("vanished"); + store.put(&addr, &content).await.expect("put"); + + std::fs::remove_file(store.chunk_path(&addr)).expect("remove behind our back"); + + assert!(store.get(&addr).await.expect("get").is_none()); + assert!(!store.exists(&addr).expect("exists")); + assert_eq!(store.current_chunks().expect("count"), 0); + } + + #[tokio::test] + async fn interrupted_writes_are_swept_at_startup() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("sweep"); + store.put(&addr, &content).await.expect("put"); + let shard = store + .chunk_path(&addr) + .parent() + .expect("shard") + .to_path_buf(); + drop(store); + + let orphan = shard.join(format!("{TEMP_PREFIX}999.7")); + std::fs::write(&orphan, b"half a chunk").expect("write orphan"); + let stray_root = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(format!("{TEMP_PREFIX}999.8")); + std::fs::write(&stray_root, b"half a marker").expect("write stray"); + + let reopened = reopen(&dir).await; + assert!(!orphan.exists(), "an interrupted write must not survive"); + assert!(!stray_root.exists(), "nor one at the store root"); + assert_eq!(reopened.current_chunks().expect("count"), 1); + } + + #[tokio::test] + async fn concurrent_writers_of_one_address_store_it_exactly_once() { + let (store, _dir) = test_store().await; + let store = Arc::new(store); + let (addr, content) = addressed("racing"); + + let mut tasks = Vec::new(); + for _ in 0..16 { + let store = Arc::clone(&store); + let content = content.clone(); + tasks.push(tokio::spawn( + async move { store.put(&addr, &content).await }, + )); + } + + let mut new_count = 0; + for task in tasks { + if task.await.expect("join").expect("put") { + new_count += 1; + } + } + assert_eq!(new_count, 1, "exactly one writer may report a new chunk"); + assert_eq!(store.current_chunks().expect("count"), 1); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + } + + #[tokio::test] + async fn names_that_are_not_lowercase_hex_are_ignored_by_the_scan() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("scan"); + store.put(&addr, &content).await.expect("put"); + let shard = store + .chunk_path(&addr) + .parent() + .expect("shard") + .to_path_buf(); + drop(store); + + // Uppercase is deliberately rejected: on a case-folding filesystem accepting it + // would let one file answer to two index entries. + let upper = shard.join(hex::encode_upper(addressed("upper").0)); + std::fs::write(&upper, b"x").expect("write upper"); + std::fs::write(shard.join("not-a-chunk"), b"x").expect("write junk"); + std::fs::write(shard.join("deadbeef"), b"x").expect("write short"); + + let reopened = reopen(&dir).await; + assert_eq!(reopened.current_chunks().expect("count"), 1); + } + + #[tokio::test] + async fn a_chunk_filed_in_the_wrong_shard_is_not_indexed() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("misfiled"); + store.put(&addr, &content).await.expect("put"); + drop(store); + + // Move it one shard over: the read path would never find it there, so indexing + // it would make the store advertise a key it cannot serve. + let correct = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(shard_name(&addr)) + .join(hex::encode(addr)); + let wrong_shard_index = (shard_index(&addr) + 1) % SHARD_COUNT; + let wrong_dir = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(format!("{wrong_shard_index:02x}")); + std::fs::create_dir_all(&wrong_dir).expect("mkdir"); + std::fs::rename(&correct, wrong_dir.join(hex::encode(addr))).expect("misfile"); + + let reopened = reopen(&dir).await; + assert_eq!(reopened.current_chunks().expect("count"), 0); + assert!(!reopened.exists(&addr).expect("exists")); + } + + #[tokio::test] + async fn the_layout_marker_is_written_once_and_checked_on_reopen() { + let (store, dir) = test_store().await; + drop(store); + + let marker = dir.path().join(CHUNKS_DIR_NAME).join(LAYOUT_FILE_NAME); + let layout: StoreLayout = + serde_json::from_slice(&std::fs::read(&marker).expect("read marker")) + .expect("parse marker"); + assert_eq!(layout, StoreLayout::default()); + + // A store written by a future build must be refused, not misread. + let future = StoreLayout { + schema: LAYOUT_SCHEMA + 1, + ..StoreLayout::default() + }; + std::fs::write(&marker, serde_json::to_vec(&future).expect("encode")).expect("write"); + let err = FileStore::new(FileStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect_err("must refuse a newer layout"); + assert!(format!("{err}").contains("newer than this build"), "{err}"); + } + + #[tokio::test] + async fn an_unknown_shard_scheme_is_refused() { + let (store, dir) = test_store().await; + drop(store); + let marker = dir.path().join(CHUNKS_DIR_NAME).join(LAYOUT_FILE_NAME); + let other = StoreLayout { + scheme: "prefix-hex".to_string(), + ..StoreLayout::default() + }; + std::fs::write(&marker, serde_json::to_vec(&other).expect("encode")).expect("write"); + let err = FileStore::new(FileStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect_err("must refuse an unknown scheme"); + assert!(format!("{err}").contains("shard scheme"), "{err}"); + } + + #[tokio::test] + async fn writes_are_refused_when_the_disk_reserve_cannot_be_met() { + let dir = TempDir::new().expect("temp dir"); + let store = FileStore::new(FileStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: u64::MAX / 2, + }) + .await + .expect("open store"); + + let (addr, content) = addressed("full"); + let err = store.put(&addr, &content).await.expect_err("must refuse"); + assert!( + format!("{err}").contains("Insufficient disk space"), + "{err}" + ); + assert!(store.check_capacity().is_err()); + } + + #[tokio::test] + async fn capacity_is_size_aware() { + // Wide enough that a test running alongside this one cannot move the answer. + const MARGIN: u64 = 512 * 1024 * 1024; + + let dir = TempDir::new().expect("temp dir"); + let available = fs2::available_space(dir.path()).expect("free space"); + // A reserve that leaves room for a small write but not a huge one. This is the + // whole reason the predicate takes a size: free bytes alone stopped being a + // sufficient answer once chunks became files. + let store = FileStore::new(FileStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: available.saturating_sub(MARGIN), + }) + .await + .expect("open store"); + + assert!(store.check_capacity_for(1024).is_ok()); + assert!(store.check_capacity_for(4 * MARGIN).is_err()); + } + + #[test] + fn suffix_shards_stay_uniform_for_a_close_group_of_keys() { + // The real distribution: a node holds keys it is closest to, so they share a + // long leading prefix with its own ID. Sharding on that prefix collapses to one + // directory. The trailing byte is untouched by close-group membership. + let mut prefix_dirs = HashSet::new(); + let mut suffix_dirs = HashSet::new(); + for i in 0u32..4096 { + let mut key = [0u8; XORNAME_LEN]; + // 20 shared leading bits, as a ~1M-node network would impose. + let tail = crate::client::compute_address(&i.to_le_bytes()); + key.copy_from_slice(&tail); + if let Some(b) = key.first_mut() { + *b = 0xab; + } + if let Some(b) = key.get_mut(1) { + *b = 0xcd; + } + if let Some(b) = key.get_mut(2) { + *b &= 0x0f; + } + prefix_dirs.insert(key.first().copied().unwrap_or(0)); + suffix_dirs.insert(shard_index(&key)); + } + assert_eq!( + prefix_dirs.len(), + 1, + "prefix sharding collapses for a node's own holdings" + ); + assert!( + suffix_dirs.len() > 250, + "suffix sharding must stay uniform, got {} of 256 directories", + suffix_dirs.len() + ); + } + + #[test] + fn no_chunk_filename_can_spell_a_reserved_windows_device_name() { + // Hex has no `n`, `u`, `x`, `p`, `r`, `l`, `t`, `o` or `s`, so `CON`, `NUL`, + // `AUX`, `PRN`, `COM1` and `LPT1` are all unspellable at any length. This is why + // the encoding is hex and not base32 or base64url. + for reserved in ["con", "prn", "aux", "nul", "com1", "com9", "lpt1", "lpt9"] { + assert!( + !is_lower_hex(reserved), + "{reserved} must not be a valid chunk or shard name" + ); + } + } + + #[test] + fn only_full_length_lowercase_hex_decodes_to_an_address() { + // 0xab so the hex form actually contains letters, which is where case matters. + assert!(decode_chunk_name(&hex::encode([0xabu8; XORNAME_LEN])).is_some()); + assert!(decode_chunk_name(&hex::encode_upper([0xabu8; XORNAME_LEN])).is_none()); + assert!(decode_chunk_name("deadbeef").is_none()); + assert!(decode_chunk_name("").is_none()); + assert!(decode_chunk_name(&"g".repeat(CHUNK_NAME_LEN)).is_none()); + } + + #[tokio::test] + async fn repair_replaces_bad_bytes_without_the_file_ever_being_absent() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("repairable"); + store.put(&addr, &content).await.expect("put"); + let path = store.chunk_path(&addr); + + std::fs::write(&path, b"rotted").expect("corrupt"); + store.repair(&addr, &content).await.expect("repair"); + + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + assert!(store.exists(&addr).expect("exists")); + } + + #[tokio::test] + async fn a_repair_with_the_wrong_bytes_is_refused_and_changes_nothing() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("guarded"); + store.put(&addr, &content).await.expect("put"); + let path = store.chunk_path(&addr); + + // The whole point of repairing in place is that a failure must leave the old file + // where it was. Deleting first and writing after would open a window whose only + // surviving copy is the one the caller is about to destroy. + let err = store + .repair(&addr, b"not this chunk") + .await + .expect_err("must refuse"); + assert!(format!("{err}").contains("Refusing to repair"), "{err}"); + assert!( + path.exists(), + "the existing file must survive a refused repair" + ); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + } + + #[tokio::test] + async fn a_chunk_can_be_deleted_and_stored_again() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("cycle"); + + assert!(store.put(&addr, &content).await.expect("put")); + assert!(store.delete(&addr).await.expect("delete")); + assert!( + store.put(&addr, &content).await.expect("re-put"), + "a re-stored chunk is new again" + ); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + } + + /// Write a chunk file straight into its shard, the way an existing store already + /// contains thousands of them. Bypasses the write path deliberately: this exercises + /// the startup scan, not `put`. + fn plant(chunks_dir: &Path, key: &XorName) { + let dir = chunks_dir.join(shard_name(key)); + std::fs::create_dir_all(&dir).expect("mkdir"); + std::fs::write(dir.join(hex::encode(key)), key).expect("plant"); + } + + #[tokio::test] + async fn a_populated_and_churned_store_scans_correctly_at_scale() { + // Every shard populated, then aged the way a long-lived node ages: some keys + // deleted, others added in their place, so the directories carry holes rather + // than being freshly written. APFS enumeration is known to degrade with churn + // rather than with size, so a fresh corpus is not a realistic one. + const PLANTED: u32 = 20_000; + const CHURN: u32 = 1_000; + + let dir = TempDir::new().expect("temp dir"); + let chunks_dir = dir.path().join(CHUNKS_DIR_NAME); + std::fs::create_dir_all(&chunks_dir).expect("mkdir"); + + let mut expected: Vec = Vec::new(); + for i in 0..PLANTED { + let key = crate::client::compute_address(&i.to_le_bytes()); + plant(&chunks_dir, &key); + expected.push(key); + } + for i in 0..CHURN { + let key = crate::client::compute_address(&i.to_le_bytes()); + std::fs::remove_file(chunks_dir.join(shard_name(&key)).join(hex::encode(key))) + .expect("churn out"); + let replacement = crate::client::compute_address(&(PLANTED + i).to_le_bytes()); + plant(&chunks_dir, &replacement); + } + expected.retain(|k| chunks_dir.join(shard_name(k)).join(hex::encode(k)).exists()); + for i in 0..CHURN { + expected.push(crate::client::compute_address(&(PLANTED + i).to_le_bytes())); + } + expected.sort_unstable(); + expected.dedup(); + + let started = std::time::Instant::now(); + let store = reopen(&dir).await; + let scan = started.elapsed(); + + assert_eq!( + store.current_chunks().expect("count"), + expected.len() as u64 + ); + assert_eq!(store.all_keys().await.expect("all_keys"), expected); + + // Every shard should be in use at this size: 20,000 keys over 256 directories is + // about 78 each, and the last byte of a BLAKE3 output is uniform. + let occupied = std::fs::read_dir(&chunks_dir) + .expect("read store root") + .filter_map(std::result::Result::ok) + .filter(|e| e.file_name().to_str().is_some_and(|n| n.len() == 2)) + .count(); + assert_eq!(occupied, SHARD_COUNT, "the suffix must reach every shard"); + + println!( + "scan of {} keys across {SHARD_COUNT} shards took {scan:?}", + expected.len() + ); + } + + #[tokio::test] + async fn wait_idle_returns_once_writes_have_drained() { + let (store, _dir) = test_store().await; + let store = Arc::new(store); + for i in 0..32 { + let store = Arc::clone(&store); + let (addr, content) = addressed(&format!("drain-{i}")); + tokio::spawn(async move { store.put(&addr, &content).await }); + } + // Not a synchronisation point for tasks that have not been spawned yet, but it + // must not hang and it must leave the store usable. + store.wait_idle().await; + let (addr, content) = addressed("after-drain"); + assert!(store.put(&addr, &content).await.expect("put after drain")); + } +} diff --git a/src/storage/handler.rs b/src/storage/handler.rs index 31038a68..9f58066f 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -18,7 +18,7 @@ //! │ ChunkQuoteRequest ChunkPutRequest ChunkGetRequest //! │ │ │ │ │ //! │ ▼ ▼ ▼ │ -//! │ QuoteGenerator PaymentVerifier LmdbStorage│ +//! │ QuoteGenerator PaymentVerifier ChunkStore│ //! │ │ │ │ │ //! │ └─────────────────────────┴─────────────────┘ │ //! │ │ │ @@ -41,7 +41,7 @@ use crate::payment::{PaymentVerifier, QuoteGenerator, VerificationContext}; use crate::replication::admission; use crate::replication::config::K_BUCKET_SIZE; use crate::replication::fresh::FreshWriteEvent; -use crate::storage::lmdb::LmdbStorage; +use crate::storage::ChunkStore; use bytes::Bytes; use parking_lot::RwLock; use saorsa_core::P2PNode; @@ -214,7 +214,7 @@ impl Drop for GetRequestTelemetry { /// and optional payment verification. pub struct AntProtocol { /// LMDB storage for chunk persistence. - storage: Arc, + storage: Arc, /// Payment verifier for checking payments. payment_verifier: Arc, /// Quote generator for creating storage quotes. @@ -238,7 +238,7 @@ impl AntProtocol { /// * `quote_generator` - Quote generator for creating storage quotes #[must_use] pub fn new( - storage: Arc, + storage: Arc, payment_verifier: Arc, quote_generator: Arc, ) -> Self { @@ -293,7 +293,7 @@ impl AntProtocol { /// Get a reference to the underlying LMDB storage. #[must_use] - pub fn storage(&self) -> Arc { + pub fn storage(&self) -> Arc { Arc::clone(&self.storage) } @@ -520,17 +520,20 @@ impl AntProtocol { } // 3. Check if already exists (idempotent success) - match self.storage.exists(&address) { - Ok(true) => { - debug!("Chunk {addr_hex} already exists"); - return ChunkPutResponse::AlreadyExists { address }; - } - Err(e) => { - return ChunkPutResponse::Error(ProtocolError::Internal(format!( - "Storage read failed: {e}" - ))); - } - Ok(false) => {} + // + // Verified against the offered bytes, not answered from the name. A name can + // outlive the bytes under it, and acknowledging a good copy of a chunk this node + // holds only a damaged version of throws that copy away and does not get offered + // another. Reached only when this node already has the chunk, and the content + // address was checked in step 2, so a damaged copy is repaired from these bytes + // rather than the offer being refused. + if self + .storage + .holds_verified(&address, &request.content) + .await + { + debug!("Chunk {addr_hex} already exists"); + return ChunkPutResponse::AlreadyExists { address }; } // 4. Cheap disk-space pre-check — runs BEFORE the expensive payment @@ -681,7 +684,7 @@ impl AntProtocol { /// The quote price is driven by `QuoteGenerator::records_stored()`. Reading /// the live LMDB entry count (an O(1) B-tree page-header read) right before /// pricing makes the metric deletion-aware: any chunk removed by - /// [`LmdbStorage::delete`] or by the replication prune pass is reflected + /// [`ChunkStore::delete`] or by the replication prune pass is reflected /// immediately, with no risk of missing a delete path. /// /// On a storage read error — or a count that does not fit `usize` — the @@ -895,7 +898,7 @@ mod tests { use super::*; use crate::payment::metrics::QuotingMetricsTracker; use crate::payment::{EvmVerifierConfig, PaymentVerifierConfig}; - use crate::storage::LmdbStorageConfig; + use crate::storage::ChunkStoreConfig; use evmlib::RewardsAddress; use saorsa_core::identity::NodeIdentity; use saorsa_core::MlDsa65; @@ -916,13 +919,13 @@ mod tests { async fn create_test_protocol_with_reserve(disk_reserve: u64) -> (AntProtocol, TempDir) { let temp_dir = TempDir::new().expect("create temp dir"); - let storage_config = LmdbStorageConfig { + let storage_config = ChunkStoreConfig { root_dir: temp_dir.path().to_path_buf(), disk_reserve, - ..LmdbStorageConfig::test_default() + ..ChunkStoreConfig::test_default() }; let storage = Arc::new( - LmdbStorage::new(storage_config) + ChunkStore::new(storage_config) .await .expect("create storage"), ); @@ -961,7 +964,7 @@ mod tests { let (protocol, _temp) = create_test_protocol().await; let content = b"hello world"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); // Pre-populate payment cache so EVM verification is bypassed protocol.payment_verifier().cache_insert(address); @@ -1153,7 +1156,7 @@ mod tests { // Create oversized content let content = vec![0u8; MAX_CHUNK_SIZE + 1]; - let address = LmdbStorage::compute_address(&content); + let address = ChunkStore::compute_address(&content); let put_request = ChunkPutRequest::new(address, Bytes::from(content)); let put_msg = ChunkMessage { @@ -1201,7 +1204,7 @@ mod tests { let (protocol, _temp) = create_test_protocol_with_reserve(u64::MAX).await; let content = b"chunk for a disk-full node"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); let put_request = ChunkPutRequest::new(address, Bytes::copy_from_slice(content)); let put_msg = ChunkMessage { @@ -1241,7 +1244,7 @@ mod tests { let (protocol, _temp) = create_test_protocol().await; let content = b"duplicate content"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); // Pre-populate cache so EVM verification is bypassed protocol.payment_verifier().cache_insert(address); @@ -1287,7 +1290,7 @@ mod tests { let (protocol, _temp) = create_test_protocol().await; let content = b"local access test"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); assert!(!protocol.exists(&address).expect("exists check")); @@ -1307,7 +1310,7 @@ mod tests { let (protocol, _temp) = create_test_protocol().await; let content = b"cache test content"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); // Before insert: cache should be empty let stats_before = protocol.payment_cache_stats(); @@ -1346,7 +1349,7 @@ mod tests { let (protocol, _temp) = create_test_protocol().await; let content = b"duplicate cache test"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); // Pre-populate cache for first PUT protocol.payment_verifier().cache_insert(address); @@ -1390,7 +1393,7 @@ mod tests { // Pre-populate cache, then store a chunk to test stats let content = b"stats test"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); protocol.payment_verifier().cache_insert(address); let put_request = ChunkPutRequest::new(address, Bytes::copy_from_slice(content)); @@ -1499,7 +1502,7 @@ mod tests { let (protocol, _temp) = create_test_protocol().await; let content = b"already stored quote test"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); // Store the chunk first protocol.payment_verifier().cache_insert(address); @@ -1600,7 +1603,7 @@ mod tests { let contents: Vec> = (0u8..5).map(|i| vec![i; 64]).collect(); let mut addresses = Vec::new(); for content in &contents { - let addr = LmdbStorage::compute_address(content); + let addr = ChunkStore::compute_address(content); protocol.put_local(&addr, content).await.expect("put_local"); addresses.push(addr); } @@ -1635,7 +1638,7 @@ mod tests { let contents: Vec> = (0u8..10).map(|i| vec![i; 64]).collect(); let mut addresses = Vec::new(); for content in &contents { - let addr = LmdbStorage::compute_address(content); + let addr = ChunkStore::compute_address(content); protocol.put_local(&addr, content).await.expect("put_local"); addresses.push(addr); } diff --git a/src/storage/lmdb.rs b/src/storage/lmdb.rs index 52abb52f..82b97cca 100644 --- a/src/storage/lmdb.rs +++ b/src/storage/lmdb.rs @@ -20,12 +20,9 @@ use tokio::task::spawn_blocking; use tokio_util::task::TaskTracker; use crate::ant_protocol::XORNAME_LEN; +use crate::storage::StorageStats; -/// Bytes in one MiB. -pub const MIB: u64 = 1024 * 1024; - -/// Bytes in one GiB. -pub const GIB: u64 = 1024 * MIB; +use crate::storage::{GIB, MIB}; /// Default minimum free disk space to preserve on the storage partition. const DEFAULT_DISK_RESERVE: u64 = 500 * MIB; @@ -142,25 +139,6 @@ impl LmdbStorageConfig { } } -/// Statistics about storage operations. -#[derive(Debug, Clone, Default)] -pub struct StorageStats { - /// Total number of chunks stored. - pub chunks_stored: u64, - /// Total number of chunks retrieved. - pub chunks_retrieved: u64, - /// Total bytes stored. - pub bytes_stored: u64, - /// Total bytes retrieved. - pub bytes_retrieved: u64, - /// Number of duplicate writes (already exists). - pub duplicates: u64, - /// Number of verification failures on read. - pub verification_failures: u64, - /// Number of chunks currently persisted. - pub current_chunks: u64, -} - /// Content-addressed LMDB storage. /// /// Uses heed (LMDB wrapper) for memory-mapped, transactional chunk storage. @@ -194,6 +172,17 @@ pub struct LmdbStorage { /// `data.mdb`, so the reserve is preserved by the allocator itself rather /// than by refusing every write up front. no_growth: Arc, + /// Keep the map pinned whatever the free space says. + /// + /// Set for the whole of the migration bridge. While both stores are open they each + /// measure the same free space and neither knows what the other is about to spend, so + /// a chunk written to both can be admitted twice against one lot of headroom and the + /// pair can cross the reserve together. Pinned, this environment cannot claim any new + /// disk at all: a write it cannot satisfy from its own free list is refused, and the + /// caller stores the chunk in files alone. That is the right answer anyway, because + /// this copy exists to make a rollback survivable, not to be the one that must + /// succeed. + growth_pinned: Arc, /// Serialises entering and leaving no-growth mode. /// /// Setting `no_growth` and resizing the map is one compound transition @@ -315,6 +304,7 @@ impl LmdbStorage { env_lock: Arc::new(parking_lot::RwLock::new(())), last_disk_ok: parking_lot::Mutex::new(None), no_growth: Arc::new(AtomicBool::new(false)), + growth_pinned: Arc::new(AtomicBool::new(false)), growth_mode_lock: tokio::sync::Mutex::new(()), delete_growth_charged: Arc::new(AtomicU64::new(0)), blocking_tracker: TaskTracker::new(), @@ -349,9 +339,26 @@ impl LmdbStorage { /// Returns an error if the write fails, content doesn't match address, /// or the disk is too full to accept new chunks. pub async fn put(&self, address: &XorName, content: &[u8]) -> Result { + self.put_inner(address, content, true).await + } + + /// Store bytes under a key they do not hash to. Tests only. + /// + /// Stands in for a record that rotted in place, which is the one shape the ordinary + /// path refuses to create and the migration has to survive finding. + /// + /// # Errors + /// + /// As [`Self::put`], minus the address check. + #[cfg(test)] + pub(crate) async fn put_unchecked(&self, address: &XorName, content: &[u8]) -> Result { + self.put_inner(address, content, false).await + } + + async fn put_inner(&self, address: &XorName, content: &[u8], verify: bool) -> Result { // Verify content address let computed = Self::compute_address(content); - if computed != *address { + if verify && computed != *address { return Err(Error::Storage(format!( "Content address mismatch: expected {}, computed {}", hex::encode(address), @@ -796,6 +803,14 @@ impl LmdbStorage { /// Returns [`Error::Storage`] when the volume is below the reserve and the /// store holds less than one chunk of reusable space, or when the /// disk-space query itself fails. + /// Unused while the node is moving off this store. + /// + /// Capacity is now the file store's question, because that is where writes land, and + /// this predicate deliberately answers a different one: it counts pages this store can + /// reuse internally, which says nothing about whether the *file* about to be written + /// will fit. [`Self::capacity_verdict`] is still used, to decide whether the bridge's + /// copy into this store is worth attempting. Both go when this store does. + #[allow(dead_code)] pub(crate) fn check_capacity(&self) -> Result<()> { let Some(available) = self.available_space_cached()? else { return Ok(()); @@ -939,6 +954,13 @@ impl LmdbStorage { // Re-measured inside the lock: a caller that queued behind a transition // must act on the state that transition left behind, not the one it saw // before waiting. + // Pinned for the bridge: never unpinned by having room, because the room is not + // this environment's to spend while another store is measuring the same disk. + if self.growth_pinned.load(Ordering::Acquire) { + self.no_growth.store(true, Ordering::Release); + self.pin_map_to_high_water().await?; + return Ok(true); + } if self.available_space_cached()?.is_none() { // At or above the reserve: restore normal head-room if we pinned it. if self.no_growth.load(Ordering::Acquire) { @@ -968,6 +990,20 @@ impl LmdbStorage { Ok(true) } + /// Keep this environment from ever claiming new disk, until the process ends. + /// + /// For the migration bridge, where a second store measures the same free space and + /// neither knows what the other is about to spend. Pinned, this one writes only from + /// pages it already holds, so the other's accounting is the only claim on free disk. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the map cannot be pinned. + pub async fn pin_growth(&self) -> Result<()> { + self.growth_pinned.store(true, Ordering::Release); + self.sync_growth_mode().await.map(|_| ()) + } + /// Pin the LMDB map to the size of `data.mdb` on disk. /// /// Every page already in the file stays usable, including free ones, but @@ -1142,8 +1178,23 @@ impl LmdbStorage { // stops being able to prune after its first assisted delete. // Measured before the ceiling is restored, and before any error // is propagated, so a committed delete is always accounted for. - let file_after = env.real_disk_size().unwrap_or(file_before); - let grew = file_after.saturating_sub(file_before); + // + // A measurement that fails is charged the whole slack rather than nothing. + // Reading it back as the size before the delete would say the file did not + // grow, and a delete that did grow would then spend disk the budget never + // saw. Repeat that and the ceiling stops meaning anything. Over-charging + // costs at worst one assisted delete; under-charging costs the reserve. + let grew = match env.real_disk_size() { + Ok(file_after) => file_after.saturating_sub(file_before), + Err(e) => { + warn!( + "Could not measure the LMDB file after an assisted delete \ + ({e}); charging the whole slack rather than assuming it cost \ + nothing" + ); + DELETE_COW_SLACK + } + }; if grew > 0 { budget.fetch_add(grew, Ordering::AcqRel); } @@ -1386,9 +1437,22 @@ fn compute_map_size(db_dir: &Path, reserve: u64) -> Result { let available = fs2::available_space(db_dir) .map_err(|e| Error::Storage(format!("Failed to query available disk space: {e}")))?; - // The MDB data file may not exist yet on first run. + // The MDB data file may not exist yet on first run, and that is the only reason to + // read zero here. Any other failure is a question that was not answered, and answering + // it with zero sizes the map as though the database were empty, which on a node with a + // large one is a map far too small to open it. let mdb_file = db_dir.join("data.mdb"); - let current_db_bytes = std::fs::metadata(&mdb_file).map_or(0, |m| m.len()); + let current_db_bytes = match std::fs::metadata(&mdb_file) { + Ok(meta) => meta.len(), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => 0, + Err(e) => { + return Err(Error::Storage(format!( + "Failed to measure {}: {e}. Refusing to size the map as though it were \ + empty.", + mdb_file.display() + ))) + } + }; let target = map_target_bytes(current_db_bytes, available, reserve); diff --git a/src/storage/migration.rs b/src/storage/migration.rs new file mode 100644 index 00000000..3087941d --- /dev/null +++ b/src/storage/migration.rs @@ -0,0 +1,3142 @@ +//! Moving a node off LMDB and onto the file store without losing a chunk. +//! +//! LMDB never returns a deleted page to the filesystem. Disk comes back exactly once, +//! when `chunks.mdb` is removed whole, so a node cannot free space by deleting chunks +//! and cannot compact its way out either (compaction needs free space equal to the live +//! data, which is the same condition). That single fact shapes everything here. +//! +//! # The two ways this could lose data, and why neither can happen +//! +//! 1. **A node deletes its LMDB before the chunks are safely in files.** Retirement is +//! gated on the file store already holding every key the node still claims, and on +//! the existing retention contract: a key the node is still answerable for under a +//! gossiped commitment vetoes the delete. +//! 2. **Every node sheds the same chunk at once.** A node only sheds when it cannot fit +//! its own payload, and it sheds by close-group rank, furthest first. A chunk has +//! exactly one 7th-closest and one 6th-closest holder, so it is only ever a shed +//! candidate for two of its seven holders, and the staged rollout brings that to one. +//! +//! # Three releases +//! +//! Slashing is the *auditor's* decision, so a node cannot protect itself from being +//! penalised for a shed. Everyone else has to stop first, which is why this lands over +//! three releases rather than one: +//! +//! | Release | Penalise not holding a close-group chunk? | [`MigrationConfig::retire_legacy`] | +//! |---|---|---| +//! | First: stop that one penalty | no | `false` | +//! | Second: migrate | no | `true` | +//! | Third: restore it | yes | `true` | +//! +//! The penalty column is not a field here. It lives once, in +//! [`crate::replication::config::RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY`], and both +//! the auditors that apply it and the shedder that depends on it being withheld read that +//! same switch. Two copies would let a node shed while its peers still penalised. +//! +//! Audits keep running and keep recording throughout. What the first release withholds is narrow and +//! deliberate: only the penalty for *not holding a close-group chunk*. The +//! commitment-bound subtree audit still penalises in every release, because the whole +//! migration turns on a node's reduced commitment still being binding. The record those +//! audits keep is also how we will know when the third release is safe to ship. + +use crate::ant_protocol::XorName; +use crate::error::{Error, Result}; +use crate::logging::{debug, info, warn}; +use crate::replication::config::{storage_admission_width, ReplicationConfig}; +use crate::replication::pruning::{ + prove_peers_hold_records, prune_proofs_needed, target_peers_reported_present, +}; +use crate::storage::chunk_store::{ChunkStore, VerifyReport}; +use saorsa_core::identity::PeerId; +use saorsa_core::P2PNode; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use std::time::{SystemTime, UNIX_EPOCH}; +use tokio_util::sync::CancellationToken; + +/// Filename of the persisted migration marker, under the node root. +pub const MIGRATION_STATE_FILE: &str = "migration-state.json"; + +/// Marker schema this build writes and understands. +const STATE_SCHEMA: u32 = 1; + +/// Floor on [`MigrationConfig::retire_delay_hours`]. +/// +/// `GOSSIP_ANSWERABILITY_TTL` is three hours at a one-hour rotation cadence, so a +/// commitment that named a shed key stops being answerable three hours after it was last +/// gossiped. Four hours clears that with an hour to spare. +pub const MIN_RETIRE_DELAY_HOURS: u64 = 4; + +/// How long between attempts to reopen an environment this node has lost its handle to. +/// +/// Each attempt scans every key in it, so retrying on every tick would spend a large store +/// entirely on failing to open it. +const HANDLE_RECOVERY_INTERVAL: Duration = Duration::from_secs(300); + +/// The longest one node may hold the volume migration lock before giving others a turn. +/// +/// Every branch that waits rather than works is meant to give the lock back on its own. +/// This is the backstop for the one that does not: without it, a node stuck on a condition +/// that never resolves stops every other node sharing the disk from ever starting, for the +/// whole release. Longer than a copy pass and a verification take, so it never interrupts +/// a node that is genuinely working. +const MAX_VOLUME_LOCK_HOLD: Duration = Duration::from_secs(6 * 3600); + +/// How long a node stands back after the cap takes the volume lock off it. +/// +/// Long enough that another node waiting on the lock actually gets it, rather than losing +/// the race to the node that has just been holding it for six hours. +const VOLUME_LOCK_COOLDOWN: Duration = Duration::from_secs(120); + +/// How many commitment rebuilds must be observed after the node commits to its +/// file-backed set before the legacy environment may be retired. +/// +/// One proves the builder read the new set. Two proves it published and survived a +/// rotation, which is what makes the retention window meaningful. +pub const REQUIRED_REBUILDS_BEFORE_RETIRE: u32 = 2; + +/// Operator-facing controls for the migration. +// Four independent switches, three of which are operator controls and one of which is a +// release constant. Collapsing them into an enum would tie choices together that are +// deliberately separate. +#[allow(clippy::struct_excessive_bools)] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MigrationConfig { + /// Run the background copier at all. + /// + /// Turning this off leaves a node reading the union of both stores forever. It never + /// frees the LMDB's disk, so it is an escape hatch rather than a supported mode. + #[serde(default = "default_true")] + pub enabled: bool, + + /// Write every new chunk to the legacy environment as well as the file store. + /// + /// Costs roughly a gigabyte per node per month at the observed fill rate, and buys + /// the ability to roll the fleet back: a chunk uploaded during the bridge to holders + /// that all revert to a pre-migration build would otherwise be gone from every one + /// of them. Automatically irrelevant once the legacy environment is retired. + #[serde(default = "default_true")] + pub dual_write_legacy: bool, + + /// Allow a node that cannot fit its payload to drop its furthest keys. + /// + /// An operator who would rather add disk than shed can set this to `false`. The node + /// then keeps both stores and never frees the LMDB's space. + #[serde(default = "default_true")] + pub allow_shed: bool, + + /// Delete `chunks.mdb` once the retirement gate is satisfied. + /// + /// **On in this release**, because it is the only step that returns disk. It is also + /// the only destructive step in the whole migration and the only one that cannot be + /// undone, which is why everything in front of it is a gate: the wave the node is + /// assigned to, the shed hold, the reduced commitment reaching the close group, + /// possession of every chunk being given up proven elsewhere, a re-read of every + /// remaining chunk, and a retention delay on top. + /// + /// Deliberately never serialised. A node writes its effective configuration back to + /// disk, so shipping this as an ordinary field would bake R1's `false` into every + /// operator's config file and R2 would then never retire anything. The release phase + /// belongs to the build, not to the operator's file. `ANT_MIGRATION_RETIRE_LEGACY` + /// overrides it for a canary. + #[serde(skip, default = "release_retire_legacy")] + pub retire_legacy: bool, + + /// Hours after this build first starts before a node may shed anything. + /// + /// Long enough for peers still on a pre-R1 build to upgrade, because one of those + /// still penalises a shedder at the full audit weight. + #[serde(default = "default_shed_hold_hours")] + pub shed_hold_hours: u64, + + /// Hours between committing to the file-backed key set and deleting `chunks.mdb`. + /// + /// Clamped up to [`MIN_RETIRE_DELAY_HOURS`]. Longer buys a rollback window on nodes + /// that can afford to hold both copies. + #[serde(default = "default_retire_delay_hours")] + pub retire_delay_hours: u64, + + /// Free megabytes the copier leaves untouched, on top of the disk reserve. + /// + /// The copier stops here rather than filling to the brink, so a node that is + /// mid-migration still has room to accept a chunk it is paid for. + #[serde(default = "default_copier_slack_mb")] + pub copier_slack_mb: u64, + + /// Copy rate ceiling, in mebibytes per second. + /// + /// The quiet responsible audit lane is where audit timeouts actually cost trust, and + /// an unthrottled copier competing with it for I/O is the fastest way to turn a + /// storage migration into an audit incident. + #[serde(default = "default_copier_throttle_mib_per_sec")] + pub copier_throttle_mib_per_sec: u64, + + /// Hours between one migration wave opening and the next. + /// + /// A close group is split into waves so that only + /// [`CONCURRENT_MIGRATIONS_PER_GROUP`] of it give chunks up at a time. This is how + /// long a wave gets to finish copying, retiring and refetching before the next one may + /// start. Only nodes that have to give something up wait for their wave; a node with + /// room migrates immediately. + #[serde(default = "default_wave_hours")] + pub wave_hours: u64, + + /// Seconds between copier ticks. + #[serde(default = "default_tick_secs")] + pub tick_secs: u64, + + /// Chunks copied per tick before yielding. + #[serde(default = "default_batch_chunks")] + pub batch_chunks: usize, + + /// Where the volume lock lives, overriding the filesystem this node's root sits on. + /// + /// `None` in production, which keys the lock by device id so every node on one disk + /// serialises against the others. Tests set it so a test's migration contends only + /// with its own, rather than with every other test sharing the machine's filesystem. + /// + /// Never serialised: it exists to scope a test, not to configure a node. + #[serde(skip)] + pub lock_dir: Option, +} + +const fn default_true() -> bool { + true +} + +/// Whether this build deletes the legacy environment once the gate is satisfied. +/// +/// **`true` in this release.** Deleting `chunks.mdb` is the only step that returns disk, +/// and a build that ships with it off is a migration that never finishes: the fleet +/// already deleted 2.29M chunks out of LMDB and got back nothing, because LMDB does not +/// return freed pages to the filesystem. Every gate in front of this is still enforced, +/// and `ANT_MIGRATION_RETIRE_LEGACY=0` turns it off on a single node if one is ever +/// needed to hold both stores. +pub const RELEASE_RETIRE_LEGACY: bool = true; + +/// Environment override for the directory the per-volume migration lock lives in. +/// +/// Set this where the default cannot work, and the default cannot work wherever the nodes +/// sharing a disk do not share a `/tmp`. Our own multi-node hosts are exactly that case: +/// the systemd unit sets `PrivateTmp=true`, which gives every unit a tmpfs of its own, so +/// each node creates the same lock filename in a different filesystem, every one of them +/// takes it, and the lock serialises nothing. Point every node on a host at one directory +/// they can all write and the lock does what it is for. +/// +/// The directory must be writable by the node. A path that cannot be used is reported and +/// the node migrates unserialised, which is the same answer as having no lock, so a typo +/// here is loud rather than silent. +pub const LOCK_DIR_ENV: &str = "ANT_MIGRATION_LOCK_DIR"; + +/// Environment override for [`RELEASE_RETIRE_LEGACY`], for a canary node. +pub const RETIRE_LEGACY_ENV: &str = "ANT_MIGRATION_RETIRE_LEGACY"; + +/// Read a boolean override from the environment, falling back to the build constant. +fn env_override(name: &str, build_default: bool) -> bool { + let Ok(raw) = std::env::var(name) else { + return build_default; + }; + match raw.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" => true, + "0" | "false" | "no" | "off" => false, + other => { + warn!("{name}={other} is not a boolean; using the build default {build_default}"); + build_default + } + } +} + +/// The retirement switch for this build, after any environment override. +fn release_retire_legacy() -> bool { + env_override(RETIRE_LEGACY_ENV, RELEASE_RETIRE_LEGACY) +} + +const fn default_shed_hold_hours() -> u64 { + 72 +} + +const fn default_retire_delay_hours() -> u64 { + MIN_RETIRE_DELAY_HOURS +} + +const fn default_wave_hours() -> u64 { + 24 +} + +const fn default_copier_slack_mb() -> u64 { + 2048 +} + +const fn default_copier_throttle_mib_per_sec() -> u64 { + 32 +} + +const fn default_tick_secs() -> u64 { + 30 +} + +const fn default_batch_chunks() -> usize { + 64 +} + +impl Default for MigrationConfig { + fn default() -> Self { + Self { + enabled: true, + dual_write_legacy: true, + allow_shed: true, + retire_legacy: release_retire_legacy(), + shed_hold_hours: default_shed_hold_hours(), + retire_delay_hours: default_retire_delay_hours(), + wave_hours: default_wave_hours(), + copier_slack_mb: default_copier_slack_mb(), + copier_throttle_mib_per_sec: default_copier_throttle_mib_per_sec(), + tick_secs: default_tick_secs(), + batch_chunks: default_batch_chunks(), + lock_dir: None, + } + } +} + +impl MigrationConfig { + /// The retire delay, never shorter than the retention contract allows. + #[must_use] + pub fn effective_retire_delay_hours(&self) -> u64 { + self.retire_delay_hours.max(MIN_RETIRE_DELAY_HOURS) + } + + /// Copier slack in bytes. + #[must_use] + pub fn copier_slack_bytes(&self) -> u64 { + self.copier_slack_mb.saturating_mul(1024 * 1024) + } +} + +/// Where a node is in the migration. +/// +/// The phase is persisted, but only as a *decision* record. Everything derivable from +/// the filesystem is re-derived at every start: which keys are still legacy-only is +/// simply "in the LMDB and not in the file store", so an interrupted copy resumes for +/// free with no progress bookkeeping to corrupt. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MigrationPhase { + /// Copying the legacy environment into files. Reads are the union of both stores, + /// writes go to files, and the commitment still covers everything. + Bridging, + /// The node has settled on what it will keep and commits only to its file-backed + /// keys. It keeps serving the rest from LMDB until they stop being answerable. + Committed, + /// No legacy environment. Steady state, and where every fresh node starts. + FilesOnly, +} + +/// The persisted migration marker. +/// +/// Two facts genuinely need to survive a restart: when this build first ran (so the shed +/// hold is not restarted by a reboot loop) and when the node committed to its file-backed +/// set (so the retirement clock is not either). Everything else is re-derived. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MigrationState { + /// Marker schema version. + pub schema: u32, + /// Where the node is. + pub phase: MigrationPhase, + /// Unix seconds when this build first started on this node. + pub first_start_unix: u64, + /// Unix seconds when the node committed to its file-backed key set. + pub committed_at_unix: Option, + /// Commitment rebuilds observed since committing. + pub rebuilds_since_commit: u32, + /// How many keys the node decided not to keep, for the operator's benefit. + pub shed_key_count: u64, + /// How many chunks the file store held when the node committed. + /// + /// Cross-checked at open. A marker claiming the node is past the copying stage while + /// the file store is far emptier than it said means the two disagree about reality, + /// and the filesystem wins. + #[serde(default)] + pub kept_key_count: u64, +} + +impl MigrationState { + /// A fresh marker for a node that has just started. + #[must_use] + pub fn new(phase: MigrationPhase) -> Self { + Self { + schema: STATE_SCHEMA, + phase, + first_start_unix: now_unix(), + committed_at_unix: None, + rebuilds_since_commit: 0, + shed_key_count: 0, + kept_key_count: 0, + } + } + + /// Load the marker, writing a fresh one if there is none. + /// + /// Persisting immediately matters: `first_start_unix` is what the shed hold counts + /// from, and a marker that is only written at the first phase change would reset that + /// clock on every restart before then, so a node that restarts more often than the + /// hold would never become eligible to shed and never finish migrating. + pub fn load_or_create(root_dir: &Path, phase: MigrationPhase) -> Self { + let state = Self::load_or_new(root_dir, phase); + // Written whenever the disk does not already hold what this process is going to + // use, rather than only when there is no file at all. A marker that is present but + // could not be used is replaced in memory and was previously left on disk, so the + // next start read the same bad file and stamped `first_start_unix` afresh. That is + // precisely the reset this function exists to prevent, and it is worse than the + // one it does prevent: it repeats. Three ways in, all of them leaving a file that + // exists: a truncated or otherwise unparseable marker, one from a newer schema, and + // one whose clock is impossible and is corrected by `with_sane_clocks`. + // + // A node restarting more often than the shed hold then never becomes eligible to + // shed and never finishes migrating, and a node short of disk is exactly the node + // that restarts. + let raw = std::fs::read(state_path(root_dir)).ok(); + let on_disk = raw + .as_deref() + .and_then(|bytes| serde_json::from_slice::(bytes).ok()); + if on_disk.as_ref() == Some(&state) { + return state; + } + // The schema is read on its own, from the raw JSON, rather than taken from a + // successful parse into today's struct. A marker from a genuinely newer build is + // exactly the one least likely to parse into it: a phase this build has no name + // for, a field that changed type, a field that went away. Reading the schema only + // when the whole thing parses means the markers most worth keeping are the ones + // that would be written over. + let newer_schema = raw + .as_deref() + .and_then(|bytes| serde_json::from_slice::(bytes).ok()) + .and_then(|value| value.get("schema").and_then(serde_json::Value::as_u64)) + .filter(|schema| *schema > u64::from(STATE_SCHEMA)); + if let Some(schema) = newer_schema { + let Some(kept) = free_kept_marker_path(root_dir, schema) else { + warn!( + "A newer migration marker (schema {schema}) is here and every name to \ + keep it under is taken. Leaving it, which means the shed hold restarts \ + on every boot until it is dealt with" + ); + return state; + }; + if let Err(e) = std::fs::rename(state_path(root_dir), &kept) { + warn!( + "Could not move the newer migration marker aside ({e}); leaving it, \ + which means the shed hold restarts on every boot until it is dealt with" + ); + return state; + } + info!( + "Kept the newer migration marker as {} and started one this build can use", + kept.display() + ); + } + if let Err(e) = state.save(root_dir) { + warn!("Could not write the migration marker: {e}"); + } + state + } + + /// Load the marker, or start a fresh one. + /// + /// An unreadable marker is replaced rather than fatal: it is a hint, and every fact + /// it holds is either recoverable or conservative to reset. Losing it restarts the + /// shed hold and the retirement clock, which delays a migration and never rushes one. + pub fn load_or_new(root_dir: &Path, phase: MigrationPhase) -> Self { + let path = state_path(root_dir); + let Ok(bytes) = std::fs::read(&path) else { + return Self::new(phase); + }; + match serde_json::from_slice::(&bytes) { + Ok(state) if state.schema <= STATE_SCHEMA => state.with_sane_clocks(), + Ok(state) => { + warn!( + "Migration marker {} was written by a newer build (schema {}); \ + starting a fresh one", + path.display(), + state.schema + ); + Self::new(phase) + } + Err(e) => { + warn!( + "Migration marker {} is unreadable ({e}); starting a fresh one. \ + The shed hold and retirement clock restart from now.", + path.display() + ); + Self::new(phase) + } + } + } + + /// Persist the marker so a reader sees either the old content or the new. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the marker cannot be written. + pub fn save(&self, root_dir: &Path) -> Result<()> { + let path = state_path(root_dir); + let bytes = serde_json::to_vec_pretty(self) + .map_err(|e| Error::Storage(format!("Failed to encode migration marker: {e}")))?; + crate::storage::file_store::write_file_durably(&path, &bytes)?; + debug!("Migration marker updated: phase {:?}", self.phase); + Ok(()) + } + + /// Replace timestamps that cannot be true with "now". + /// + /// Zero and future values both make a hold vacuous, and a node whose clock was not + /// yet synchronised at first boot writes zero without anyone tampering. Resetting to + /// now delays a migration, which is the safe direction. + #[must_use] + fn with_sane_clocks(mut self) -> Self { + let now = now_unix(); + if self.first_start_unix == 0 || self.first_start_unix > now { + warn!("Migration marker has an implausible first-start time; restarting the hold"); + self.first_start_unix = now; + } + self.committed_at_unix = self.committed_at_unix.map(|at| { + if at == 0 || at > now { + warn!("Migration marker has an implausible commit time; restarting the clock"); + now + } else { + at + } + }); + self + } + + /// Whether the shed hold has elapsed. + #[must_use] + pub fn shed_hold_elapsed(&self, config: &MigrationConfig) -> bool { + let hold = config.shed_hold_hours.saturating_mul(3600); + now_unix().saturating_sub(self.first_start_unix) >= hold + } + + /// Whether the retirement delay has elapsed since committing. + #[must_use] + pub fn retire_delay_elapsed(&self, config: &MigrationConfig) -> bool { + let Some(at) = self.committed_at_unix else { + return false; + }; + let delay = config.effective_retire_delay_hours().saturating_mul(3600); + now_unix().saturating_sub(at) >= delay + } +} + +/// A name to keep a newer marker under that nothing is using yet. +/// +/// The plain `.schema-N` name is deterministic, so a node downgraded twice would otherwise +/// write over the marker it kept the first time, or fail the rename and restart the hold on +/// every boot. Returns `None` if every name is taken, which the caller reports rather than +/// destroying anything. +fn free_kept_marker_path(root_dir: &Path, schema: u64) -> Option { + for attempt in 0..16u32 { + // Appended, not `with_extension`, which would replace the `.json` and leave a name + // that no longer says what the file is. + let mut candidate = state_path(root_dir).into_os_string(); + candidate.push(format!(".schema-{schema}")); + if attempt > 0 { + candidate.push(format!(".{attempt}")); + } + let candidate = PathBuf::from(candidate); + // `symlink_metadata`, not `try_exists`. The latter follows links, so a dangling + // symbolic link at this name reads as nothing being there while `rename` would + // happily replace it. Anything at all, of any kind, means pick another name. + if std::fs::symlink_metadata(&candidate).is_err() { + return Some(candidate); + } + } + None +} + +/// Path of the persisted marker. +#[must_use] +pub fn state_path(root_dir: &Path) -> PathBuf { + root_dir.join(MIGRATION_STATE_FILE) +} + +/// Seconds since the Unix epoch, saturating at zero if the clock is before it. +#[must_use] +pub fn now_unix() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) +} + +/// A host-wide advisory lock that serialises migrations sharing one volume. +/// +/// Twelve nodes on one 492 GiB volume each need roughly their live payload free to copy +/// and each return rather more when they retire, so one at a time the host gains space +/// and the queue accelerates. All twelve at once need twelve times the space and all +/// twelve stall. The lock is taken non-blocking: a node that cannot get it simply waits +/// for the next tick. +/// +/// Where the lock file sits is `lock_path_for`'s decision (private, so this is not a +/// link), and this doc used to describe +/// a branch of it that a running node almost never reaches: the parent of the node root is +/// the last resort, taken only when the root's own metadata cannot be read. What a node +/// normally uses is the host's temporary directory keyed by the volume's device id, or the +/// directory named by [`LOCK_DIR_ENV`] when one is set. +/// +/// Which of those is right is a fact about the deployment that no node can check for +/// itself, and getting it wrong is silent: every node takes a lock of its own and reports +/// success. That is why the path is logged when the lock is taken, and why a host whose +/// nodes do not share a `/tmp` has to be told where the lock lives. +/// +/// The directory has to be one only the node's own user can write. A predictable path in a +/// world-writable `/tmp` can be created and held by any local user, who could then keep +/// every node on the host from ever migrating. +#[derive(Debug)] +pub struct VolumeLock { + /// The held file. Dropping it releases the lock. + file: std::fs::File, + /// Where it lives, for logging. + #[cfg_attr(not(feature = "logging"), allow(dead_code))] + path: PathBuf, +} + +/// The result of asking for the volume lock. +pub enum LockAttempt { + /// This node has it. + Acquired(VolumeLock), + /// Another node on the volume is migrating. Wait. + Busy, + /// No lock is possible here at all, so proceed unserialised. + /// + /// Kept distinct from `Busy` because conflating the two silently strands any node + /// whose parent directory is not writable: it would wait forever for a lock nobody + /// holds. + Unavailable, +} + +impl VolumeLock { + /// Try to take the lock for the volume hosting `root_dir`. + #[must_use] + pub fn try_acquire(root_dir: &Path, scope: Option<&Path>) -> LockAttempt { + use fs2::FileExt; + let path = scope.map_or_else( + || lock_path_for(root_dir), + |dir| dir.join("ant-migration.lock"), + ); + let file = match std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(&path) + { + Ok(f) => f, + Err(e) => { + warn!( + "Could not create the migration lock at {}: {e}. This node will \ + migrate without serialising against others on the same volume, so \ + watch its free space.", + path.display() + ); + return LockAttempt::Unavailable; + } + }; + match file.try_lock_exclusive() { + Ok(()) => { + // Info, not debug. Whether the lock is doing anything depends on whether + // the neighbours on this disk can see the same path, which is a deployment + // fact no node can check. Printing the path is what lets somebody answer it + // from a log rather than by reading a unit file. + info!("Took the volume migration lock at {}", path.display()); + LockAttempt::Acquired(Self { file, path }) + } + // Only contention means another node is migrating. Everything else, a + // filesystem that does not implement locking at all being the one that + // matters, is a lock this node will never get, and reporting it as contention + // would leave it waiting forever for a holder that does not exist. + Err(e) if is_lock_contention(&e) => LockAttempt::Busy, + Err(e) => { + warn!( + "Could not lock {}: {e}. This node will migrate without serialising \ + against others on the same volume, so watch its free space.", + path.display() + ); + LockAttempt::Unavailable + } + } + } +} + +/// Is this the error a lock held by someone else produces? +fn is_lock_contention(e: &std::io::Error) -> bool { + e.kind() == std::io::ErrorKind::WouldBlock + || (e.raw_os_error().is_some() + && e.raw_os_error() == fs2::lock_contended_error().raw_os_error()) +} + +/// Where the lock for the volume hosting `root_dir` lives. +/// +/// Keyed by the filesystem, not by the path. Two nodes on one host are configured with +/// different roots by definition, so a lock beside the root serialises a node against +/// nobody: `/srv/node-a/data` and `/srv/node-b/data` would take two different locks on one +/// disk and copy at the same time, which is the case the lock exists to prevent. +/// +/// The device id names the filesystem, and the host's temporary directory is somewhere +/// every node on that host can reach. If the device cannot be read, this falls back to a +/// lock beside the root: weaker, but never worse than having none. +/// +/// **That last sentence is only true where the nodes share a `/tmp`.** Where they do not, +/// each computes the same name in a filesystem of its own, every one of them takes it, and +/// the lock serialises nothing while logging that it worked. `PrivateTmp=true` in a systemd +/// unit does exactly that, and our own worker unit sets it. There is no way to tell from +/// inside one process whether the `/tmp` it can see is the one its neighbours see, so this +/// cannot be detected here and has to be configured: [`LOCK_DIR_ENV`] names a directory +/// every node on the host can reach, and is consulted first. +fn lock_path_for(root_dir: &Path) -> PathBuf { + lock_path_with(root_dir, std::env::var(LOCK_DIR_ENV).ok().as_deref()) +} + +/// The same decision, with the configured directory passed in rather than read. +/// +/// Split so it can be tested without touching process-wide environment. A test that set +/// `TMPDIR` to stage the private-`/tmp` case would change where every other test's +/// `TempDir` lands, and then delete it underneath them: run in parallel that takes out +/// dozens of unrelated tests with LMDB failures that look like anything but their cause. +fn lock_path_with(root_dir: &Path, configured: Option<&str>) -> PathBuf { + // Before anything derived, because an operator who has set this knows something about + // the host that this function cannot find out. + if let Some(dir) = configured { + let dir = dir.trim(); + if !dir.is_empty() { + return Path::new(dir).join("ant-migration.lock"); + } + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if let Ok(meta) = std::fs::metadata(root_dir) { + return std::env::temp_dir().join(format!("ant-migration-{}.lock", meta.dev())); + } + } + // Resolved first, because a relative root has no volume in it to read. Two nodes + // started from different working directories on one drive would otherwise each fall + // through to a lock beside their own root, which serialises neither against the other. + #[cfg(not(unix))] + let resolved = std::fs::canonicalize(root_dir).unwrap_or_else(|_| root_dir.to_path_buf()); + #[cfg(not(unix))] + let root_dir = resolved.as_path(); + // Off Unix, the volume root: the drive or share the path starts from. Not as precise + // as a device id, since a mount point below it belongs to another volume, but it + // groups the ordinary case of several nodes under one drive letter, which is what a + // lock beside each node's own root does not. + #[cfg(not(unix))] + { + use std::path::Component; + if let Some(Component::Prefix(prefix)) = root_dir.components().next() { + let key: String = prefix + .as_os_str() + .to_string_lossy() + .chars() + .filter(char::is_ascii_alphanumeric) + .collect(); + if !key.is_empty() { + return std::env::temp_dir().join(format!("ant-migration-{key}.lock")); + } + } + } + root_dir + .parent() + .unwrap_or(root_dir) + .join("ant-migration.lock") +} + +impl Drop for VolumeLock { + fn drop(&mut self) { + use fs2::FileExt; + if let Err(e) = FileExt::unlock(&self.file) { + debug!( + "Releasing the migration lock {} failed: {e}", + self.path.display() + ); + } + } +} + +/// Summary of what the copier moved during one pass. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct CopyReport { + /// Chunks copied into the file store. + pub copied: u64, + /// Bytes copied. + pub bytes: u64, + /// Keys skipped because the legacy bytes did not hash to their address, or were + /// larger than a chunk may be. + pub unusable: u64, + /// Keys that could not be copied for a reason that may clear on a later pass. + pub failed: u64, + /// Keys that had vanished from the legacy store between the scan and the copy. + pub vanished: u64, + /// Whether the pass stopped because free space reached the slack floor. + pub stopped_for_space: bool, +} + +impl CopyReport { + /// Fold another pass into this one. + pub fn merge(&mut self, other: Self) { + self.copied += other.copied; + self.bytes += other.bytes; + self.unusable += other.unusable; + self.failed += other.failed; + self.vanished += other.vanished; + self.stopped_for_space |= other.stopped_for_space; + } +} + +/// How many waves a close group is divided into, so that at most +/// [`CONCURRENT_MIGRATIONS_PER_GROUP`] of it are giving chunks up at once. +#[must_use] +pub fn migration_wave_count(close_group_size: usize) -> u64 { + let group = close_group_size.max(1) as u64; + let per_wave = CONCURRENT_MIGRATIONS_PER_GROUP.max(1) as u64; + group.div_ceil(per_wave).max(1) +} + +/// Which wave this node belongs to, derived from its own ID. +/// +/// Deterministic and needs no coordination, which is the point: a node cannot ask its +/// close group "are you migrating?" without a protocol change, and the answer would be +/// stale by the time it arrived. Hashing the peer ID spreads the members of any group +/// across the waves without anybody agreeing on anything. +/// +/// It is a stagger, not a guarantee. Seven IDs hashed into four waves will not always land +/// two, two, two, one. What makes it safe rather than merely tidy is that it composes with +/// the possession gate: a node whose turn has come still cannot give a chunk up until its +/// neighbours have proven they hold it, so an unlucky wave waits instead of over-shedding. +#[must_use] +pub fn migration_wave_for(self_id: Option<&PeerId>, close_group_size: usize) -> u64 { + let waves = migration_wave_count(close_group_size); + let Some(peer) = self_id else { + return 0; + }; + let digest = blake3::hash(&[MIGRATION_WAVE_DOMAIN, peer.as_bytes().as_slice()].concat()); + let mut head = [0u8; 8]; + head.copy_from_slice(digest.as_bytes().get(..8).unwrap_or(&[0u8; 8])); + u64::from_le_bytes(head) % waves +} + +/// Domain separator so the wave assignment cannot be confused with any other use of a +/// hashed peer ID. +const MIGRATION_WAVE_DOMAIN: &[u8] = b"ant-node/storage-migration-wave/v1"; + +/// Whether this node's wave has opened yet. +/// +/// Wave `w` opens `w * wave_hours` after this build first started on this node. A node +/// that has room for everything never consults this: it copies and retires without ever +/// being unable to serve, so it is not part of the problem the waves exist to solve. +#[must_use] +pub fn wave_has_opened(state: &MigrationState, config: &MigrationConfig, wave: u64) -> bool { + now_unix() >= wave_opens_at(state, config, wave) +} + +/// When a given wave opens, in Unix seconds. +/// +/// Measured from the END of the shed hold, not from first start. Measured from the start +/// the two settings cancel each other out: with a 72 hour hold and 24 hour waves, waves +/// would open at 0, 24, 48 and 72 hours while nothing at all may shed until hour 72, so +/// every wave would be open the moment the first one could act and the whole close group +/// would migrate together. That is the pile-up the waves exist to prevent. +#[must_use] +pub fn wave_opens_at(state: &MigrationState, config: &MigrationConfig, wave: u64) -> u64 { + state + .first_start_unix + .saturating_add(config.shed_hold_hours.saturating_mul(3600)) + .saturating_add(wave.saturating_mul(config.wave_hours.saturating_mul(3600))) +} + +/// Order keys closest-first by XOR distance from this node. +/// +/// Shedding walks this list from the far end, so the keys a node gives up are the ones it +/// is furthest from, and therefore the ones its close group covers best. +#[must_use] +pub fn rank_closest_first(mut keys: Vec, self_xor: Option) -> Vec { + let Some(me) = self_xor else { + // No identity available (devnet, unit tests). Ascending key order is stable and + // deterministic, which is all the copier needs. + keys.sort_unstable(); + return keys; + }; + keys.sort_unstable_by_key(|k| crate::client::xor_distance(k, &me)); + keys +} + +/// Structured field marking every line the fleet gate for R3 is read from. +/// +/// R3 ships when the fleet shows migrations have finished, so these lines have to be +/// queryable rather than merely readable. One field name, three values. +pub const MIGRATION_EVENT: &str = "migration_event"; + +/// Log the operator-facing summary of a completed migration. +/// +/// `freed_bytes` is what the retired environment held, which is what the deletion running +/// in the background will return. The line that says the space is actually back is +/// `migration_event = "space_returned"`, emitted by that deletion when it finishes. Two +/// lines rather than one because the deletion of a large environment takes minutes, and a +/// node that reports the disk back before it is back is a node whose operator cannot tell +/// a slow deletion from a failed one. +pub fn log_migration_complete(kept: u64, shed: u64, freed_bytes: u64) { + #[allow(clippy::cast_precision_loss)] // display only + let freed_gib = freed_bytes as f64 / (1024.0 * 1024.0 * 1024.0); + if shed == 0 { + info!( + migration_event = "complete", + kept, + shed, + freed_bytes, + "Storage migration complete: {kept} chunks now in the file store, nothing shed, \ + {freed_gib:.2} GiB being returned to the filesystem" + ); + } else { + info!( + migration_event = "complete", + kept, + shed, + freed_bytes, + "Storage migration complete: kept {kept} chunks, shed {shed} that would not fit, \ + {freed_gib:.2} GiB being returned to the filesystem. The shed keys are the ones this \ + node was furthest from; replication will refetch what still belongs here now \ + that there is room." + ); + } +} + +// ──────────────────────────────────────────────────────────────────────────── +// The driver +// ──────────────────────────────────────────────────────────────────────────── + +/// How many positions from the end of the admission group a node may give up. +/// +/// A chunk has exactly one holder at each rank, so restricting shedding to the last two +/// positions means only two of its holders ever consider dropping it, and the staged +/// rollout brings that to one. Without this rule the property is only statistical: +/// every holder could be short of space at once, each shed the same chunk, and the +/// per-volume lock would not know, because it serialises one volume and this is a +/// network-wide question. +/// +/// Measured against [`storage_admission_width`], not the close group, so the migration is +/// never more willing to drop a chunk than the pruner is. The pruner treats the wider +/// group as strictly in-range and refuses to delete inside it; shedding ranks that the +/// pruner protects would make a one-off migration weaker than the thing that runs every +/// day. +pub const SHEDDABLE_TAIL_RANKS: usize = 2; + +/// How many nodes of one close group may be giving chunks up at the same time. +/// +/// The close group is the unit that matters, not the volume and not the fleet. If every +/// holder of a chunk migrates at once, none of them can prove to the others that a copy +/// survives, and the whole group deadlocks waiting on each other. Holding it to two means +/// the other five are steady, can answer possession challenges, and are still serving the +/// chunk while the two rebuild. +pub const CONCURRENT_MIGRATIONS_PER_GROUP: usize = 2; + +/// How many keys a refusal names, so the log stays readable. +const REFUSAL_SAMPLE: usize = 4; + +/// How recently a peer must have published a commitment to be trusted as a holder. +/// +/// Commitments rotate hourly and are gossiped on the neighbour-sync cadence, so a peer +/// that has not published one for this long is not simply quiet: it has either stopped +/// speaking the protocol or retired its commitment and not yet rotated a new one. The +/// second is exactly what a node in the middle of its own migration looks like, and +/// counting it as a holder is how two migrating nodes could each conclude the other was +/// covering the chunk. +const COMMITMENT_FRESHNESS: Duration = Duration::from_secs(2 * 3600); + +/// How many keys one possession round asks about. +/// +/// The round batches by peer, so this bounds the size of a single request rather than the +/// number of requests. +const POSSESSION_BATCH_KEYS: usize = 256; + +/// How long to wait before re-evaluating a shed decision that was refused. +const SHED_REEVALUATION_INTERVAL: Duration = Duration::from_secs(600); + +/// How many copied chunks between operator-facing progress lines. +const PROGRESS_LOG_EVERY: usize = 500; + +/// How long a clean pre-retirement verification stays usable. +/// +/// Chunks written since the pass were content-checked on the way in and flushed, so the +/// only thing the window exposes is bit rot in the last half hour, which is the ordinary +/// risk of any file and is caught on read. +const VERIFICATION_REUSE_WINDOW: Duration = Duration::from_secs(1800); + +/// The network facts the driver needs, kept behind one type so the store itself stays +/// free of any knowledge of routing or commitments. +pub struct MigrationContext { + /// Routing, for close-group rank and possession checks. `None` in devnet and tests. + pub p2p: Option>, + /// This node's peer ID. + pub self_id: Option, + /// This node's address in the key space, for ordering the copy closest-first. + pub self_xor: Option, + /// The responder commitment state, which owns the retention contract. + pub commitment: Option>, + /// Replication settings, for the possession round that gates shedding. + pub replication: Option>, + /// Neighbour-sync state, which the possession challenge needs. + pub sync_state: Option>>, + /// Coordinator for the possession challenges. + pub audit_challenge_coordinator: + Option>, + /// What this node last heard each peer commit to. + /// + /// Used to require that a peer trusted to hold a chunk is currently publishing a + /// claim, rather than sitting between a retired commitment and its next rotation, + /// which is precisely the state a node in the middle of its own migration is in. + pub peer_commitments: Option< + Arc< + tokio::sync::RwLock< + HashMap, + >, + >, + >, + /// Close-group width. + pub close_group_size: usize, +} + +/// How many of the peers auditing this node now have seen its reduced commitment. +/// +/// `received` is who was sent the current root, `current` is the close group as routing +/// sees it at this moment. Only the overlap counts. A peer that received the root and has +/// since left is not going to audit this node, and a peer that has since joined has never +/// seen the root, so neither is evidence that shedding is safe. +fn enough_of_the_group_knows( + received: &HashSet, + current: &[PeerId], + needed: usize, +) -> bool { + if needed == 0 { + return false; + } + current.iter().filter(|p| received.contains(*p)).count() >= needed +} + +impl MigrationContext { + /// How many peers of the close group must have seen the reduced commitment. + /// + /// The same tolerance the pruner applies to possession proofs: all of them for a group + /// of one or two, one short of the group otherwise, so a single unreachable peer + /// cannot veto the migration forever without accepting an uninformed close group. + #[must_use] + pub fn commitment_recipients_needed(&self) -> usize { + prune_proofs_needed(self.close_group_size.saturating_sub(1)) + } + + /// Have enough of this node's close group actually received its reduced commitment? + /// + /// A rotation is not the same as neighbours knowing. Until they have seen the smaller + /// key set they keep auditing against the one this node used to hold, so giving a + /// chunk up before then turns a legitimate migration into a wave of audit failures. + pub async fn neighbours_know_the_commitment(&self) -> bool { + let needed = self.commitment_recipients_needed(); + if needed == 0 { + return false; + } + let Some(state) = self.commitment.as_ref() else { + return false; + }; + let received = state.current_delivered_peers(); + if received.is_empty() { + return false; + } + // Counted against the group as it stands now, not as it stood when the root went + // out. A peer that has since left knowing this node's reduced commitment says + // nothing about the peers that will actually audit it, and letting a departed + // peer satisfy the gate is how a node gives chunks up while its real neighbours + // still hold it to the larger key set. + let Some(current) = self.current_close_group().await else { + return false; + }; + enough_of_the_group_knows(&received, ¤t, needed) + } + + /// This node's close group as routing sees it now, or `None` if the view is too thin + /// to be evidence about a group at all. + async fn current_close_group(&self) -> Option> { + let (Some(p2p), Some(me), Some(self_xor)) = ( + self.p2p.as_ref(), + self.self_id.as_ref(), + self.self_xor.as_ref(), + ) else { + return None; + }; + // Self-inclusive, then self filtered out. The self-excluding call would return + // `close_group_size` *remote* peers, one more than the group actually has, and the + // threshold is computed from a group that includes this node. Four real + // neighbours plus one peer outside the group would then clear a bar meant to + // require five real ones. + let closest = p2p + .dht_manager() + .find_closest_nodes_local_with_self(self_xor, self.close_group_size) + .await; + let peers: Vec = closest + .iter() + .map(|n| n.peer_id) + .filter(|p| p != me) + .collect(); + if peers.len() + 1 < self.close_group_size { + return None; + } + Some(peers) + } + + /// Is this key still answerable under a retained commitment slot? + /// + /// This is the pruner's existing veto, reused verbatim: a key the node could still + /// be challenged on must not lose its last local copy. + #[must_use] + pub fn still_answerable(&self, key: &XorName) -> bool { + self.commitment + .as_ref() + .is_some_and(|state| state.is_held(key)) + } + + /// The width this node measures ranks against: the admission group, not the close + /// group. + #[must_use] + pub fn shed_width(&self) -> usize { + storage_admission_width(self.close_group_size) + } + + /// This node's position in `key`'s admission group. + pub async fn close_group_rank(&self, key: &XorName) -> GroupRank { + let (Some(p2p), Some(me)) = (self.p2p.as_ref(), self.self_id.as_ref()) else { + return GroupRank::Unknown; + }; + let closest = p2p + .dht_manager() + .find_closest_nodes_local_with_self(key, self.shed_width()) + .await; + closest + .iter() + .position(|n| n.peer_id == *me) + .map_or(GroupRank::Outside, GroupRank::Inside) + } + + /// May this node give up `key` without risking its last replica? + /// + /// Only if it is outside the admission group entirely, or sits in that group's last + /// [`SHEDDABLE_TAIL_RANKS`] positions. Never when the answer is unknown. + pub async fn may_shed(&self, key: &XorName) -> bool { + rank_is_sheddable(self.close_group_rank(key).await, self.shed_width()) + } +} + +/// Where this node sits in a key's admission group. +/// +/// `Unknown` is deliberately distinct from `Outside`. Collapsing the two would turn "this +/// node has no routing table to consult" into "no other node is closer", which is a +/// licence to give up every chunk on no evidence whatsoever. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GroupRank { + /// This node is at this position, counting from the closest. + Inside(usize), + /// This node is not among the closest for this key. + Outside, + /// Routing state is unavailable, so the question cannot be answered. + Unknown, +} + +/// Which of `keys` this node has no proof anyone else is holding. +/// +/// This is the gate on giving a chunk up at all, and it is deliberately the **same** +/// evidence the pruner demands before it deletes: cryptographic possession proofs from +/// all but one of the key's current close group, which is six of seven at +/// production width. +/// +/// Rank alone was not enough. Being far from a chunk says something about who *should* +/// hold it, not about who *does*, and during a fleet-wide migration the nodes that should +/// hold it are exactly the ones that may also be short of space. Nor is the cheap +/// `VerificationRequest` enough: it carries a self-reported `present: bool`, and a node +/// that has silently lost a chunk still answers yes. The challenge here makes a peer +/// return a digest over a nonce it has never seen, which it cannot do without the bytes. +/// +/// Returns the keys that failed, so the caller can name them. An empty result means every +/// key asked about is proven to live somewhere else. +/// +/// Without routing state, every key is unconfirmed: no view of the network is no evidence. +pub async fn unconfirmed_by_neighbours( + store: &Arc, + context: &MigrationContext, + keys: &[XorName], +) -> Vec { + let (Some(p2p), Some(self_id), Some(config), Some(sync_state), Some(coordinator)) = ( + context.p2p.as_ref(), + context.self_id.as_ref(), + context.replication.as_ref(), + context.sync_state.as_ref(), + context.audit_challenge_coordinator.as_ref(), + ) else { + return keys.to_vec(); + }; + + let local_key_count = + usize::try_from(store.current_chunks().unwrap_or(0)).unwrap_or(usize::MAX); + let dht = p2p.dht_manager(); + let mut unconfirmed = Vec::new(); + + for batch in keys.chunks(POSSESSION_BATCH_KEYS) { + // Ask only the peers that are currently closest to each key. A proof from a peer + // that has since moved out of the group is not evidence the chunk will stay there. + let mut targets_by_key: HashMap> = HashMap::new(); + let mut keys_by_peer: HashMap> = HashMap::new(); + for key in batch { + // Self-inclusive, matching the pruner, whose evidence this is. The + // self-excluding call returns one peer more than the key's group holds, and a + // proof from a peer outside it is not evidence the chunk stays in it. + let closest = dht + .find_closest_nodes_local_with_self(key, config.close_group_size) + .await; + let peers: Vec = closest + .iter() + .map(|n| n.peer_id) + .filter(|p| p != self_id) + .collect(); + for peer in &peers { + keys_by_peer.entry(*peer).or_default().push(*key); + } + targets_by_key.insert(*key, peers); + } + + let proofs = prove_peers_hold_records( + &keys_by_peer, + local_key_count, + store, + p2p, + config, + sync_state, + coordinator, + ) + .await; + + // A proof is necessary but not sufficient. The peer must also be currently + // publishing a commitment, so a node that has retired its own and not yet rotated + // a replacement, which is what a node mid-migration looks like, is not counted as + // the reason this node may give a chunk up. + let publishing = peers_publishing_a_recent_commitment(context).await; + + for key in batch { + let group = targets_by_key.get(key).map_or(&[][..], Vec::as_slice); + + // The lookup is self-inclusive, and this node is giving the chunk up, so a + // full group is `close_group_size` peers none of which is this node. Fewer + // than that is a routing view too thin to be evidence about the group at all, + // which is what a table looks like shortly after a restart. This node still + // appearing in the group means it is not outside it after all, and the + // decision to give the chunk up was taken against a view that has since + // changed. + if group.len() < config.close_group_size { + unconfirmed.push(*key); + continue; + } + + // The threshold comes from the configured group size, never from whichever + // subset happens to qualify, nor from however many peers routing returned. + // Deriving it from the filtered list is how two last holders destroy a chunk + // between them: each sees only the other publishing, so each needs exactly one + // proof, each gets it from the other, and both delete. Deriving it from the + // observed length is the same mistake more quietly: a view that has lost a + // peer lowers the bar exactly when it should not be trusted. + let needed = prune_proofs_needed(config.close_group_size); + let qualifying: Vec = group + .iter() + .filter(|p| publishing.contains(*p)) + .copied() + .collect(); + if !target_peers_reported_present(key, &qualifying, &proofs, needed) { + unconfirmed.push(*key); + } + } + } + unconfirmed +} + +/// The peers this node has heard a commitment from recently enough to trust as holders. +async fn peers_publishing_a_recent_commitment(context: &MigrationContext) -> HashSet { + let Some(records) = context.peer_commitments.as_ref() else { + return HashSet::new(); + }; + records + .read() + .await + .iter() + .filter(|(_, record)| { + record.last_commitment().is_some() + && record.received_at.elapsed() < COMMITMENT_FRESHNESS + }) + .map(|(peer, _)| *peer) + .collect() +} + +/// Whether a position in the admission group may be given up. +/// +/// Split out from the routing lookup so the rule itself is testable without a network. +/// `width` is [`storage_admission_width`], not the close-group size: a key this node is +/// outside the admission group for is one the pruner would delete anyway, and inside it +/// only the last [`SHEDDABLE_TAIL_RANKS`] positions may go. +#[must_use] +pub fn rank_is_sheddable(rank: GroupRank, width: usize) -> bool { + // A group no wider than the tail has no tail to give up. Saturating alone would set + // the threshold to zero and make every member sheddable, which is the opposite of + // what a narrow group needs. + let protected_below = if width <= SHEDDABLE_TAIL_RANKS { + width + } else { + width - SHEDDABLE_TAIL_RANKS + }; + match rank { + // No routing to ask. Never a licence: a node with no view of the network has no + // grounds at all for believing anyone else holds the chunk. + GroupRank::Unknown => false, + GroupRank::Outside => true, + GroupRank::Inside(rank) => rank >= protected_below, + } +} + +/// Whether this store needs a migration driver at all. +/// +/// The single predicate both the spawn site and its test use, so "should this node be +/// migrating" cannot be answered one way by the wiring and another way by what checks it. +#[must_use] +pub fn should_migrate(store: &Arc) -> bool { + // Or has a removal to finish, or has something at the environment's path it could not + // open. A node whose retirement was interrupted has no handle and nothing left to + // copy, but its disk has not come back. A node whose environment is a link to storage + // that was not mounted at startup has neither, and its chunks come back when the + // storage does; without a driver it would stay blind to them until a restart. + store.has_legacy() || store.has_cleanup_pending() || store.legacy_dir_is_on_disk() +} + +/// Runs the migration to completion, then returns. +/// +/// Everything it does is idempotent and derived from the filesystem, so a crash at any +/// point costs at most the work of one tick. +pub async fn run(store: Arc, context: MigrationContext, shutdown: CancellationToken) { + let config = store.migration_config().clone(); + if !worth_starting(&store, &config) { + return; + } + + let tick = Duration::from_secs(config.tick_secs.max(1)); + let mut volume_lock: Option = None; + let mut held = LockHold::default(); + let mut next_shed_evaluation = Instant::now(); + let mut next_handle_recovery = Instant::now(); + // A clean verification is a full re-read of everything both stores hold. If + // retirement is then deferred (a read still holds the legacy handle), re-hashing on + // every tick would be minutes of disk for nothing, so a recent pass is reused. + let mut verified: Option<(VerifyReport, Instant)> = None; + + loop { + tokio::select! { + () = shutdown.cancelled() => { + debug!("Storage migration stopping for shutdown"); + return; + } + () = tokio::time::sleep(tick) => {} + } + + // Never hold the volume against the rest of the machine for longer than this, + // whatever the node is waiting on. Dropping it costs a tick: if nobody else wants + // it, the branches below take it straight back. + if held.has_overstayed() { + debug!( + "Held the volume migration lock for {} hour(s); giving it back so any \ + other node on this volume gets a turn", + MAX_VOLUME_LOCK_HOLD.as_secs() / 3600 + ); + volume_lock = None; + held.give_up(); + } + + // Cleanup first. It can put an unmarked directory back under the live name, and + // recovery is what gives the node a handle to it; the other order leaves that + // until the next tick, and the completion check in between would see no handle + // and no pending cleanup and call the migration finished. + let cleanup = cleanup_state(&store); + // Before anything reads the key set: a write that nobody waited for leaves a + // note behind, and only the disk can say what became of it. + store.reconcile_pending_writes().await; + maybe_recover_lost_handle(&store, &mut next_handle_recovery).await; + if cleanup == CleanupState::Finished && !store.has_legacy() { + info!("Storage migration finished; nothing left on disk to clean up"); + return; + } + + match store.migration_phase() { + // Nothing left to migrate. Whether there is anything left to clean up is + // decided at the top of the loop, which is also where this returns from. + MigrationPhase::FilesOnly => { + volume_lock = None; + held.released(); + } + MigrationPhase::Bridging => { + // Held from the first copy through retirement, not released in between: + // a node that let go after copying would let its eleven neighbours start + // theirs before it had returned a byte, which is the exact pile-up the + // lock exists to prevent. The one exception is a node that has become + // permanently stuck (see below), which must not go on excluding the + // others for a release. + if matches!( + take_volume_lock( + &mut volume_lock, + &mut held, + store.root_dir(), + config.lock_dir.as_deref(), + ), + LockStep::WaitATick + ) { + continue; + } + if bridge_tick( + &store, + &context, + &config, + &mut next_shed_evaluation, + &shutdown, + ) + .await + { + // The copier ran. That is the volume lock being used rather than held. + held.note_disk_work(); + } else { + // Copying is blocked on something only an operator can change, so + // stop holding the volume lock against the other nodes here. + volume_lock = None; + held.released(); + } + } + MigrationPhase::Committed => { + // A node that restarted in this phase has no lock, and the work below + // (copying anything that must be kept, then re-reading the whole store to + // verify it) is exactly the disk-heavy work the lock exists to serialise. + if matches!( + take_volume_lock( + &mut volume_lock, + &mut held, + store.root_dir(), + config.lock_dir.as_deref(), + ), + LockStep::WaitATick + ) { + continue; + } + match retire_tick(&store, &context, &config, &mut verified, &shutdown).await { + // Not a return. The environment is gone from the node's point of + // view, but its directory is still being deleted in the background, + // and if that fails there has to be something left to try again. The + // loop exits at the top once nothing is pending. + RetireOutcome::Done => { + volume_lock = None; + held.released(); + } + // Time spent reading or copying is the volume lock doing its job, not + // a node sitting on it. The cap is there for a node that waits, and + // restarting a full verification because a large store took longer + // than the cap would be the cap causing the problem it prevents. + RetireOutcome::Working => held.note_disk_work(), + RetireOutcome::Waiting => {} + RetireOutcome::NoWorkToSerialise => { + // Nothing this node can do will return space, so holding the + // volume lock only stops its neighbours from trying. In R1, where + // retirement is switched off entirely, holding it would mean one + // node per volume copies and the other eleven do nothing for the + // whole release. + volume_lock = None; + held.released(); + } + } + } + } + } +} + +/// Should the driver run at all, and say why in the log if not? +fn worth_starting(store: &Arc, config: &MigrationConfig) -> bool { + if !config.enabled { + warn!( + "Storage migration is disabled. This node will keep reading both stores and \ + will never return the legacy environment's disk space." + ); + return false; + } + // The same question the spawn site asks. A different one here means a driver that is + // started and then returns immediately, which is how a node whose environment is a + // link to storage that was not mounted yet ends up never picking it up. + if !should_migrate(store) { + debug!("No legacy chunk environment; nothing to migrate"); + return false; + } + + let to_copy = store.legacy_only_keys().len(); + info!( + migration_event = "start", + to_copy, + legacy_bytes = store.legacy_bytes(), + "Storage migration starting: {to_copy} chunk(s) still only in the legacy \ + environment, {:.2} GiB to reclaim", + bytes_to_gib(store.legacy_bytes()) + ); + true +} + +/// Retry any removal that did not finish, and say whether the driver is done. +/// +/// Runs independently of the phase. A removal that could not finish leaves nothing to +/// migrate but a disk that has not come back, and the reasons it failed (a name already +/// taken, a directory that could not be flushed, a scanner holding a handle) are the kind +/// that clear on their own. +fn cleanup_state(store: &Arc) -> CleanupState { + if store.has_cleanup_pending() { + store.retry_cleanup(); + } + if store.has_cleanup_pending() || store.legacy_dir_is_on_disk() { + return CleanupState::Pending; + } + CleanupState::Finished +} + +/// Whether anything is left on disk for the driver to see through. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CleanupState { + /// Something is still there. + Pending, + /// Nothing is. + Finished, +} + +/// Put a lost legacy handle back, if there is one to put back. +/// +/// A node that lost its handle to an environment still on disk cannot read the chunks that +/// live only there. The cause is usually transient, so this runs every tick rather than +/// leaving the node half-blind until somebody restarts it. +async fn maybe_recover_lost_handle(store: &Arc, next_attempt: &mut Instant) { + if Instant::now() < *next_attempt { + return; + } + if store.recover_lost_legacy_handle().await { + info!("Reopened the legacy chunk environment; the migration continues"); + *next_attempt = Instant::now(); + return; + } + // Backed off, because each attempt scans the whole environment and a cause that has + // not cleared in half a minute is unlikely to clear in the next. + *next_attempt = Instant::now() + HANDLE_RECOVERY_INTERVAL; +} + +/// How long this node has had the volume migration lock, and when it may ask again. +/// +/// Split out because the rule is easy to get wrong in one branch and not another: a +/// branch that takes the lock without recording when, or that gives it up without a +/// cooldown, silently opts out of the cap that stops one node holding a whole machine. +#[derive(Default)] +struct LockHold { + /// When the lock was taken, or when it was last used for real disk work. + since: Option, + /// Before this, do not ask for it again. + cooldown_until: Option, +} + +impl LockHold { + /// Record that the lock has just been taken. + fn taken(&mut self) { + self.since = Some(Instant::now()); + } + + /// Record that it is no longer held. + fn released(&mut self) { + self.since = None; + } + + /// Record that this tick used the lock for what it is for. + /// + /// Copying and verifying are the exclusive disk work the lock exists to serialise, so + /// time spent on them is not time spent sitting on it. Without this a node with a + /// large store would have the cap fire in the middle of a verification pass and + /// restart it, which is the cap causing the problem it prevents. + fn note_disk_work(&mut self) { + self.since = Some(Instant::now()); + } + + /// Has this node held the lock past the cap without using it? + fn has_overstayed(&self) -> bool { + self.since + .is_some_and(|at| at.elapsed() >= MAX_VOLUME_LOCK_HOLD) + } + + /// Give the lock up and stand back so somebody else can take it. + fn give_up(&mut self) { + self.since = None; + self.cooldown_until = Some(Instant::now() + VOLUME_LOCK_COOLDOWN); + } + + /// May this node ask for the lock yet? + fn may_ask(&self) -> bool { + !self + .cooldown_until + .is_some_and(|until| Instant::now() < until) + } +} + +/// What taking the volume lock produced for the driver loop. +enum LockStep { + /// Held, or not needed because none is possible here. + Proceed, + /// Someone else has it. Try again next tick. + WaitATick, +} + +/// Take the volume lock if it is not already held, stamping when it was taken. +/// +/// One place, because the stamp is what stops a node holding the volume against every +/// other node on the machine, and a branch that acquires without stamping silently opts +/// out of that. +fn take_volume_lock( + lock: &mut Option, + held: &mut LockHold, + root_dir: &Path, + scope: Option<&Path>, +) -> LockStep { + if lock.is_some() { + return LockStep::Proceed; + } + // After giving the volume up at the cap, stand back for a moment. Reacquiring in the + // same breath would hand nobody anything. + if !held.may_ask() { + return LockStep::WaitATick; + } + match VolumeLock::try_acquire(root_dir, scope) { + LockAttempt::Acquired(taken) => { + *lock = Some(taken); + held.taken(); + LockStep::Proceed + } + LockAttempt::Busy => { + debug!("Another node on this volume is migrating; waiting"); + LockStep::WaitATick + } + // No lock is possible here, so waiting for one would strand this node + // permanently. Proceed; the slack floor is the backstop. + LockAttempt::Unavailable => LockStep::Proceed, + } +} + +/// One pass of the copier. Returns `false` when this node cannot make progress that +/// needs the volume to itself. +async fn bridge_tick( + store: &Arc, + context: &MigrationContext, + config: &MigrationConfig, + next_shed_evaluation: &mut Instant, + shutdown: &CancellationToken, +) -> bool { + let remaining = store.legacy_only_keys(); + if remaining.is_empty() { + if let Err(e) = store.commit_to_files() { + // Not progress, and not something exclusive disk access fixes. Saying it was + // would reset the hold cap every tick and let this node keep the volume from + // every other node on the machine for as long as the failure lasts. + warn!( + "Everything is copied but the migration commitment could not be recorded: \ + {e}. Retrying on the next tick." + ); + return false; + } + return true; + } + + let ordered = rank_closest_first(remaining, context.self_xor); + let batch: Vec = ordered + .into_iter() + .take(config.batch_chunks.max(1)) + .collect(); + let report = match store + .copy_batch( + &batch, + config.copier_slack_bytes(), + config.copier_throttle_mib_per_sec, + shutdown, + ) + .await + { + Ok(report) => report, + Err(e) => { + // Still retried every tick, but the volume lock goes back: if this is + // permanent, holding it would block every other node on the volume on a node + // that is getting nowhere. + warn!("Storage migration copy failed: {e}. Retrying on the next tick."); + return false; + } + }; + + if report.copied > 0 { + debug!( + "Storage migration copied {} chunk(s) ({:.2} GiB) this pass", + report.copied, + bytes_to_gib(report.bytes) + ); + // A migration runs for hours. One periodic line at info level is what an operator + // watching a node actually sees, and what says the copier has not silently stalled. + let left = store.legacy_only_keys().len(); + if left % PROGRESS_LOG_EVERY < usize::try_from(report.copied).unwrap_or(usize::MAX) { + let no_rollback = store.writes_without_a_rollback_copy(); + info!( + migration_event = "progress", + remaining = left, + no_rollback_copy = no_rollback, + "Storage migration: {left} chunk(s) left to copy out of the legacy \ + environment, {no_rollback} write(s) made without a rollback copy" + ); + } + } + if report.unusable > 0 { + warn!( + "{} chunk(s) in the legacy environment did not match their own address and \ + were dropped from the key set", + report.unusable + ); + } + + if report.stopped_for_space { + // Out of space. Whatever happens next, this node is not going to write more until + // something changes, so it stops excluding its neighbours from the volume. That + // covers the 72-hour shed hold as well as an outright refusal: holding the lock + // for three days would leave every other node on the volume unmigrated. + if Instant::now() < *next_shed_evaluation { + return false; + } + *next_shed_evaluation = Instant::now() + SHED_REEVALUATION_INTERVAL; + return evaluate_shed(store, context, config).await; + } + true +} + +/// Decide whether the node may give up what it could not copy. +async fn evaluate_shed( + store: &Arc, + context: &MigrationContext, + config: &MigrationConfig, +) -> bool { + let remaining = store.legacy_only_keys(); + let short_by = remaining.len(); + + // Read from the one switch the auditors read, not from a second copy of it. There + // used to be two constants of the same name with two environment overrides, one on + // each side of this decision, and nothing coupling them: a node could have been + // willing to shed while every peer was still applying the full penalty, which is the + // exact outcome the release ordering exists to prevent. + if !crate::replication::config::close_group_storage_penalty_suspended() { + warn!( + "This node cannot fit {short_by} chunk(s) in the file store, but this release \ + has audit penalties switched back on, so giving anything up now would be \ + penalised by every peer. Keeping both stores. Add disk, or migrate this node \ + on a build that still suspends penalties." + ); + return false; + } + + if !config.allow_shed { + warn!( + "This node cannot fit {short_by} chunk(s) in the file store and shedding is \ + turned off. Add disk, or set storage.migration.allow_shed. Until then it \ + keeps serving from both stores and the legacy environment stays." + ); + return false; + } + + let state = store.migration_state(); + + // Wait for this node's turn. A close group is split into waves so at most + // CONCURRENT_MIGRATIONS_PER_GROUP of it are giving chunks up at once; if all seven + // holders went together, none could prove to the others that a copy survived and the + // whole group would sit deadlocked waiting on each other. Only nodes that have to give + // something up wait: a node with room has already copied everything and retired. + let wave = migration_wave_for(context.self_id.as_ref(), context.close_group_size); + if !wave_has_opened(&state, config, wave) { + info!( + "This node is {short_by} chunk(s) short of disk and is in migration wave {wave} \ + of {}. Its turn opens {} hour(s) after this build first started, so the rest of \ + its close group stays steady and can keep serving what it is about to give up.", + migration_wave_count(context.close_group_size), + config + .shed_hold_hours + .saturating_add(wave.saturating_mul(config.wave_hours)) + ); + return false; + } + + // Kept as its own check even though the wave now starts after it: the hold is about + // peers on an older build still applying the penalty, the wave is about the close + // group being able to cover for whoever moves. Different reasons, both required. + if !state.shed_hold_elapsed(config) { + info!( + "This node is {short_by} chunk(s) short of disk. Holding for {} hour(s) after \ + first start before giving any up, so peers still on an older build have \ + upgraded and stopped penalising a shed.", + config.shed_hold_hours + ); + return false; + } + + // First filter, and the cheap one: a node never gives up a chunk it is near the front + // of the group for. In practice it rarely fires, by construction, because the copier + // walks closest-first, so whatever is left when the disk fills is the far end of the + // list. Finding a protected key still uncopied means the node could not fit even the + // chunks it is closest to, which is exactly when it must not shed anything. + let mut protected = Vec::new(); + for key in &remaining { + if !context.may_shed(key).await { + protected.push(*key); + if protected.len() >= REFUSAL_SAMPLE { + break; + } + } + } + if !protected.is_empty() { + let sample: Vec = protected.iter().map(hex::encode).collect(); + warn!( + "This node is {short_by} chunk(s) short of disk, and at least {} of them are \ + chunks it is near the front of the group for (for example {}). It will not \ + give those up. The legacy environment stays and its disk is not returned \ + until storage is added.", + protected.len(), + sample.join(", ") + ); + return false; + } + + // Second filter, and the one that decides it: proof that somebody else holds every + // chunk this node is about to give up. Being far from a chunk is not evidence + // that a copy exists. During a fleet-wide migration the nodes that ought to hold it + // are exactly the ones that may also be out of disk, so the question has to be asked + // rather than inferred. + info!( + "Checking that other nodes hold the {short_by} chunk(s) this node cannot fit, \ + before giving any of them up" + ); + let unconfirmed = unconfirmed_by_neighbours(store, context, &remaining).await; + if !unconfirmed.is_empty() { + let sample: Vec = unconfirmed + .iter() + .take(REFUSAL_SAMPLE) + .map(hex::encode) + .collect(); + warn!( + "{} of the {short_by} chunk(s) this node cannot fit could not be proven to \ + exist anywhere else (for example {}). Nothing is given up and the legacy \ + environment stays. Add disk, or wait for replication to place them.", + unconfirmed.len(), + sample.join(", ") + ); + return false; + } + + info!( + migration_event = "shed", + shed = short_by, + "Every one of the {short_by} chunk(s) this node cannot fit is proven to be held \ + elsewhere. Committing to what it can hold. They stay readable from the legacy \ + environment until it is removed, and replication refetches whatever still belongs \ + here once there is room." + ); + if let Err(e) = store.commit_to_files() { + warn!("Could not record the migration commitment: {e}"); + return false; + } + true +} + +/// The keys still only in the legacy store that this node is too close to give up. +async fn keys_this_node_must_not_give_up( + store: &Arc, + context: &MigrationContext, +) -> Vec { + let mut must_keep = Vec::new(); + for key in store.legacy_only_keys() { + if !context.may_shed(&key).await { + must_keep.push(key); + } + } + must_keep +} + +/// Re-ask every network gate, after verification and immediately before the deletion. +/// +/// Verification re-reads the whole store and can run for hours. A gate satisfied before it +/// started says nothing about the moment of deletion: peers leave, replicas are pruned +/// elsewhere, and a write whose file half failed adds a fresh legacy-only key that has +/// faced none of these checks. This is the last point at which the answer can still be +/// acted on, so it is the point at which it has to be true. +async fn every_gate_still_holds( + store: &Arc, + context: &MigrationContext, + candidates: &std::collections::BTreeSet, +) -> Option { + // A key that joined the legacy-only set since the snapshot was taken has been through + // none of this. Stop now rather than asking the gates about a set that has already + // moved; the next tick copies it and takes a fresh snapshot. + let live: std::collections::BTreeSet = store.legacy_only_keys().into_iter().collect(); + if live != *candidates { + debug!( + "Legacy environment not retired: the set changed while the gates were being \ + checked ({} keys then, {} now)", + candidates.len(), + live.len() + ); + return Some(RetireOutcome::Waiting); + } + if !keys_this_node_must_not_give_up(store, context) + .await + .is_empty() + { + debug!("Legacy environment not retired: the shed rule changed during verification"); + return Some(RetireOutcome::Waiting); + } + if let Some(outcome) = shedding_is_still_safe(store, context, candidates).await { + return Some(outcome); + } + // And the retention contract once more, for the same reason. + if let Some(reason) = store.retirement_blocker(|k| context.still_answerable(k)) { + debug!("Legacy environment not retired: {reason}"); + return Some(RetireOutcome::Waiting); + } + None +} + +/// The last two questions before anything is deleted, asked in this order because the +/// order is the safety argument: reduce the claim, let the group learn it, then give the +/// chunks up. +/// +/// Returns `Some` with the reason to stop, or `None` when it is safe to proceed. +async fn shedding_is_still_safe( + store: &Arc, + context: &MigrationContext, + shedding: &std::collections::BTreeSet, +) -> Option { + // Nothing below is reached until the node has reduced its commitment (the phase + // is `Committed`) and that reduction has been rebuilt and published. What remains + // is to confirm the close group has actually *received* it, and that the chunks + // being given up still exist elsewhere. Only then is anything deleted. + if !shedding.is_empty() { + // A rotation is not the same as neighbours knowing. Until they have the + // smaller key set they keep auditing this node against the one it used to + // hold, and a wave of audit failures is as damaging as losing the chunks. + // Asked only when there is a commitment to deliver. A node whose file-backed set + // is empty commits to nothing, so there is no hash for a neighbour to acknowledge + // and this gate would never open, stranding its disk for good. Nothing is lost by + // skipping it: the gate exists to stop neighbours auditing this node against a key + // set it no longer holds, and a node claiming nothing cannot fail such an audit. + // The possession check below, which proves every chunk being given up still exists + // elsewhere, is the gate that protects the data, and it still runs. + let commits_to_nothing = store + .committable_keys() + .await + .is_ok_and(|keys| keys.is_empty()); + if commits_to_nothing { + info!( + "This node's file-backed set is empty, so it commits to nothing and has \ + no reduced commitment for its close group to receive. Proceeding to the \ + possession check on the {} chunk(s) it is giving up.", + shedding.len() + ); + } else if !context.neighbours_know_the_commitment().await { + info!( + "Holding: {} of this node's close group must receive its reduced \ + commitment before it gives up {} chunk(s). {} have it so far.", + context.commitment_recipients_needed(), + shedding.len(), + context + .commitment + .as_ref() + .map_or(0, |s| s.current_delivered_peer_count()) + ); + // Give the volume back while waiting on this. It is a network condition, not + // a disk one, and it may never resolve: holding the lock through it would let + // one node stop every other node on the machine from ever starting. + return Some(RetireOutcome::NoWorkToSerialise); + } + // Asked again here, not only when the node committed. Hours pass in between, + // the group moves, and a peer that held a copy then may not now. This is the + // last moment at which the answer still matters. + let ordered: Vec = shedding.iter().copied().collect(); + let unconfirmed = unconfirmed_by_neighbours(store, context, &ordered).await; + if !unconfirmed.is_empty() { + let sample: Vec = unconfirmed + .iter() + .take(REFUSAL_SAMPLE) + .map(hex::encode) + .collect(); + warn!( + "{} of the {} chunk(s) this node is giving up can no longer be proven \ + to exist elsewhere (for example {}). The legacy environment stays.", + unconfirmed.len(), + shedding.len(), + sample.join(", ") + ); + return Some(RetireOutcome::NoWorkToSerialise); + } + } + None +} + +/// What one pass of the retirement gate concluded. +enum RetireOutcome { + /// The legacy environment is gone. The driver is finished. + Done, + /// Exclusive disk work happened this tick: copying, or re-reading the store to verify + /// it. Keep the volume, and count the time as time spent using it rather than time + /// spent holding it. + Working, + /// Still working towards it, but waiting on a clock rather than on the disk. Keep the + /// volume, because the node is about to need it, but let the hold cap run. + Waiting, + /// Blocked on something no amount of exclusive disk access will fix. + NoWorkToSerialise, +} + +/// When this node last said out loud that its migration needs a person. +static LAST_OPERATOR_WARNING: parking_lot::Mutex> = parking_lot::Mutex::new(None); + +/// How often to repeat it. Often enough to be noticed, rarely enough not to drown the log. +const OPERATOR_WARNING_INTERVAL: Duration = Duration::from_secs(3600); + +/// Should the "this needs a person" warning be repeated now? +/// +/// The condition it reports is checked on every tick and does not clear on its own, so +/// without this it would be a line every thirty seconds for as long as the node runs. +fn operator_should_hear_again() -> bool { + let mut last = LAST_OPERATOR_WARNING.lock(); + let now = Instant::now(); + if last.is_some_and(|at| now.duration_since(at) < OPERATOR_WARNING_INTERVAL) { + return false; + } + *last = Some(now); + true +} + +/// The retention contract, asked before anything else in the tick. +/// +/// `Some` with what the driver should do, or `None` when nothing is in the way. +fn blocked_before_the_gates( + store: &Arc, + context: &MigrationContext, + config: &MigrationConfig, +) -> Option { + let reason = store.retirement_blocker(|k| context.still_answerable(k))?; + // Three blockers no amount of exclusive disk access will clear: an environment this + // node cannot read, one it cannot classify at all, and one it must not delete because + // it is a link to somewhere else. Each needs a person, so give the volume back to the + // nodes that can use it and say so where an operator will see it rather than at debug. + if store.has_lost_its_legacy_handle() + || store.legacy_cannot_be_classified() + || store.legacy_is_a_link() + { + if operator_should_hear_again() { + warn!( + migration_event = "needs_an_operator", + "The legacy chunk environment will not be retired automatically: {reason}" + ); + } + return Some(RetireOutcome::NoWorkToSerialise); + } + debug!("Legacy environment not retired yet: {reason}"); + Some(if config.retire_legacy { + RetireOutcome::Waiting + } else { + // Retirement is switched off on this node, so it will never free its disk here + // however long it waits. + RetireOutcome::NoWorkToSerialise + }) +} + +/// One pass of the retirement gate. +async fn retire_tick( + store: &Arc, + context: &MigrationContext, + config: &MigrationConfig, + verified: &mut Option<(VerifyReport, Instant)>, + shutdown: &CancellationToken, +) -> RetireOutcome { + if let Some(outcome) = blocked_before_the_gates(store, context, config) { + return outcome; + } + + // Re-check the shed rule against live routing immediately before the destructive + // step, not once when the node committed hours ago. Two things put a key back into + // the legacy-only set after that decision: a file that failed verification and is now + // served from the legacy copy, and a write whose file half failed. Neither went + // through the rank check, and both would be thrown away by the removal below. + let must_keep = keys_this_node_must_not_give_up(store, context).await; + if !must_keep.is_empty() { + warn!( + "{} chunk(s) are still only in the legacy environment and this node is too \ + close to them to give them up. Copying them before anything is removed.", + must_keep.len() + ); + match store + .copy_batch( + &must_keep, + config.copier_slack_bytes(), + config.copier_throttle_mib_per_sec, + shutdown, + ) + .await + { + Ok(report) if report.stopped_for_space => { + warn!( + "Out of disk while copying {} chunk(s) this node must not give up. \ + The legacy environment stays until there is room for them.", + must_keep.len() + ); + *verified = None; + return RetireOutcome::NoWorkToSerialise; + } + Ok(_) => {} + Err(e) => { + warn!("Could not copy the chunks this node must keep: {e}"); + *verified = None; + return RetireOutcome::NoWorkToSerialise; + } + } + // Anything copied changed the file store, so a previous verification no longer + // covers it. + *verified = None; + return RetireOutcome::Working; + } + + // The real report from a recent pass, never a fabricated one. Reuse deliberately does + // NOT refresh the window: re-arming it from a reused proof would let a node that + // keeps deferring retirement run the verification exactly once and coast on it. + let reusable = verified + .filter(|(_, at)| at.elapsed() < VERIFICATION_REUSE_WINDOW) + .map(|(proof, _)| proof); + let proof = match reusable { + Some(proof) => proof, + None => match store + .verify_before_retire(config.copier_throttle_mib_per_sec, shutdown) + .await + { + Ok(proof) => { + if proof.is_clean() { + *verified = Some((proof, Instant::now())); + } + proof + } + Err(e) => { + // A pass that failed is not progress, however quickly it failed, and + // treating it as work would let a node whose store cannot be read hold + // the volume against every other node on the machine for good. + warn!("Pre-retirement verification failed: {e}. Retrying on the next tick."); + return RetireOutcome::NoWorkToSerialise; + } + }, + }; + if !proof.is_clean() { + *verified = None; + warn!( + "Pre-retirement verification found {} chunk(s) that are damaged in the file \ + store and cannot be repaired from the legacy environment. The legacy \ + environment stays.", + proof.unrepairable() + ); + return RetireOutcome::NoWorkToSerialise; + } + + // Snapshotted BEFORE the gates, not after. Every gate below is asked about exactly + // this set, and exactly this set is what the removal is permitted to destroy. Taken + // afterwards, a key that joined between the last gate and the snapshot would be + // counted as approved having passed nothing, which is the case the gates exist for. + let approved: std::collections::BTreeSet = + store.legacy_only_keys().into_iter().collect(); + + if let Some(outcome) = every_gate_still_holds(store, context, &approved).await { + return outcome; + } + + let kept = store.current_chunks().unwrap_or(0); + let shed = store.migration_state().shed_key_count; + match store + .retire_legacy( + &proof, + &|k: &XorName| context.still_answerable(k), + &approved, + ) + .await + { + Ok(freed) => { + log_migration_complete(kept, shed, freed); + RetireOutcome::Done + } + Err(e) => { + debug!("Legacy environment not retired yet: {e}"); + RetireOutcome::Waiting + } + } +} + +/// Convert a byte count to GiB for human-readable log messages. +#[allow(clippy::cast_precision_loss)] // display only +#[cfg_attr(not(feature = "logging"), allow(dead_code))] +fn bytes_to_gib(bytes: u64) -> f64 { + bytes as f64 / (1024.0 * 1024.0 * 1024.0) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + use serial_test::serial; + use tempfile::TempDir; + + fn peer_id(byte: u8) -> PeerId { + let mut bytes = [0u8; 32]; + if let Some(slot) = bytes.first_mut() { + *slot = byte; + } + PeerId::from_bytes(bytes) + } + + /// The width shedding is measured against: the admission group, not the close group. + const WIDTH: usize = storage_admission_width(7); + + #[test] + fn shedding_is_measured_against_the_width_the_pruner_protects() { + // The pruner treats the admission group (close group plus its margin) as strictly + // in-range and refuses to delete inside it. A one-off migration must not be more + // willing to drop a chunk than the thing that runs every day. + assert_eq!(WIDTH, storage_admission_width(7)); + assert!( + storage_admission_width(7) > 7, + "the admission group is wider than the close group" + ); + // A rank the close group would have called sheddable is protected here. + assert!(!rank_is_sheddable(GroupRank::Inside(5), WIDTH)); + assert!(!rank_is_sheddable(GroupRank::Inside(6), WIDTH)); + } + + #[test] + fn a_node_never_gives_up_a_chunk_it_is_among_the_closest_to() { + // This node is one of the closest for these ranks. Giving one of them up is the + // case where every short-of-space holder could drop the same chunk and take its + // last replica, so it is refused outright. + for rank in 0..WIDTH - SHEDDABLE_TAIL_RANKS { + assert!( + !rank_is_sheddable(GroupRank::Inside(rank), WIDTH), + "rank {rank} must be protected" + ); + } + // The last two positions may be given up: a chunk has exactly one holder at each, + // so it is only ever a candidate for two of its holders. + for rank in WIDTH - SHEDDABLE_TAIL_RANKS..WIDTH { + assert!( + rank_is_sheddable(GroupRank::Inside(rank), WIDTH), + "rank {rank} is in the tail and may be shed" + ); + } + // Out of range entirely: nothing to protect. + assert!(rank_is_sheddable(GroupRank::Outside, WIDTH)); + // No routing to consult is never a licence. + assert!(!rank_is_sheddable(GroupRank::Unknown, WIDTH)); + } + + #[test] + fn a_group_narrower_than_the_tail_protects_everything_in_it() { + // A group with no tail has nothing to give up. Subtracting saturatingly would put + // the threshold at zero and make every member sheddable, which is exactly + // backwards for the narrowest groups. + assert!(!rank_is_sheddable(GroupRank::Inside(0), 2)); + assert!(!rank_is_sheddable(GroupRank::Inside(0), 1)); + assert!(!rank_is_sheddable(GroupRank::Inside(1), 2)); + // Out of the group entirely is still out. + assert!(rank_is_sheddable(GroupRank::Outside, 2)); + // And a group with a tail still has one. + assert!(!rank_is_sheddable(GroupRank::Inside(0), 3)); + assert!(rank_is_sheddable(GroupRank::Inside(1), 3)); + } + + #[test] + fn the_marker_round_trips_and_a_corrupt_one_starts_over_conservatively() { + let dir = TempDir::new().expect("temp dir"); + let mut state = MigrationState::new(MigrationPhase::Bridging); + state.phase = MigrationPhase::Committed; + state.shed_key_count = 12; + state.committed_at_unix = Some(1_700_000_000); + state.save(dir.path()).expect("save"); + + let loaded = MigrationState::load_or_new(dir.path(), MigrationPhase::Bridging); + assert_eq!(loaded.phase, MigrationPhase::Committed); + assert_eq!(loaded.shed_key_count, 12); + assert_eq!(loaded.committed_at_unix, Some(1_700_000_000)); + + // An unreadable marker restarts the clocks rather than being fatal. Losing it + // delays a migration and can never rush one. + std::fs::write(state_path(dir.path()), b"not json").expect("corrupt"); + let recovered = MigrationState::load_or_new(dir.path(), MigrationPhase::Bridging); + assert_eq!(recovered.phase, MigrationPhase::Bridging); + assert_eq!(recovered.committed_at_unix, None); + } + + #[test] + fn the_shed_hold_and_retirement_clocks_run_from_recorded_times() { + let config = MigrationConfig { + shed_hold_hours: 72, + retire_delay_hours: MIN_RETIRE_DELAY_HOURS, + ..MigrationConfig::default() + }; + + let mut state = MigrationState::new(MigrationPhase::Bridging); + assert!(!state.shed_hold_elapsed(&config), "just started"); + state.first_start_unix = now_unix().saturating_sub(73 * 3600); + assert!(state.shed_hold_elapsed(&config)); + + assert!( + !state.retire_delay_elapsed(&config), + "never committed, so the clock has not started" + ); + state.committed_at_unix = Some(now_unix().saturating_sub(3600)); + assert!(!state.retire_delay_elapsed(&config), "an hour is not four"); + state.committed_at_unix = + Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); + assert!(state.retire_delay_elapsed(&config)); + } + + #[test] + fn only_one_node_on_a_volume_migrates_at_a_time() { + let volume = TempDir::new().expect("temp dir"); + let node_a = volume.path().join("node-a"); + let node_b = volume.path().join("node-b"); + std::fs::create_dir_all(&node_a).expect("mkdir"); + std::fs::create_dir_all(&node_b).expect("mkdir"); + + let LockAttempt::Acquired(held) = VolumeLock::try_acquire(&node_a, None) else { + panic!("the first node must take the lock"); + }; + assert!( + matches!(VolumeLock::try_acquire(&node_b, None), LockAttempt::Busy), + "a second node on the same volume must be told to wait, not that no lock exists" + ); + + drop(held); + assert!( + matches!( + VolumeLock::try_acquire(&node_b, None), + LockAttempt::Acquired(_) + ), + "and take it once the first is done" + ); + } + + /// Seed a real LMDB chunk store, the way a node upgrading into this build has one. + async fn seed_legacy(root: &std::path::Path, count: u32) -> Vec { + let lmdb = crate::storage::LmdbStorage::new(crate::storage::LmdbStorageConfig { + root_dir: root.to_path_buf(), + verify_on_read: true, + max_map_size: 0, + disk_reserve: 0, + }) + .await + .expect("open legacy"); + let mut keys = Vec::new(); + for i in 0..count { + let content = format!("legacy-chunk-{i}").into_bytes(); + let addr = crate::client::compute_address(&content); + lmdb.put(&addr, &content).await.expect("legacy put"); + keys.push(addr); + } + lmdb.wait_idle().await; + drop(lmdb); + keys + } + + /// The whole point, end to end: a node that starts with an LMDB chunk store and a + /// disk to hold it finishes with the chunks in files and the LMDB gone. + /// + /// Driven by `run`, the same entry point node startup calls, rather than by poking the + /// pieces. That matters: the wiring that calls it went missing once and every test + /// passed, because they all built the store directly and a node with no legacy store + /// starts no migration. + #[tokio::test] + async fn a_node_with_room_copies_everything_and_removes_the_legacy_store() { + const CHUNKS: u32 = 24; + + let tmp = TempDir::new().expect("temp dir"); + // Nested, so the volume lock this node takes lives in its own directory rather + // than one shared with every other test running in parallel. + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + let keys = seed_legacy(&root, CHUNKS).await; + + let mut config = crate::storage::ChunkStoreConfig { + root_dir: root.clone(), + ..crate::storage::ChunkStoreConfig::test_default() + }; + // Deliberately NOT setting `retire_legacy`. The shipped default has to be what + // carries this all the way to a removed environment, or the release migrates every + // node and reclaims nothing. + config.migration.tick_secs = 1; + // Scoped to this test's own directory. In production the lock is keyed by the + // filesystem, so without this every test on this machine would serialise against + // every other one that runs a migration. + config.migration.lock_dir = Some(root.clone()); + config.migration.copier_throttle_mib_per_sec = 0; + let store = Arc::new( + crate::storage::ChunkStore::new(config) + .await + .expect("open store"), + ); + + // Precondition: everything is in the legacy store and nothing is in files. + assert!(store.has_legacy(), "the node must start with an LMDB store"); + assert_eq!(store.migration_phase(), MigrationPhase::Bridging); + assert_eq!(store.legacy_only_keys().len(), CHUNKS as usize); + assert!(root.join("chunks.mdb").exists()); + + let shutdown = CancellationToken::new(); + let driver = tokio::spawn(run( + Arc::clone(&store), + MigrationContext { + p2p: None, + self_id: None, + self_xor: None, + commitment: None, + replication: None, + sync_state: None, + audit_challenge_coordinator: None, + peer_commitments: None, + close_group_size: 7, + }, + shutdown.clone(), + )); + + // The copier runs on its own and settles once nothing is left only in the legacy + // store. This node has room, so it sheds nothing and needs no network at all. + wait_for( + &store, + MigrationPhase::Committed, + "the copier should finish", + ) + .await; + assert!( + store.legacy_only_keys().is_empty(), + "every chunk should have been copied" + ); + + // Stand in for the commitment builder, which lives in the replication engine: the + // retirement gate wants the reduced commitment published and its window elapsed. + store.note_commitment_rebuilt(); + store.note_commitment_rebuilt(); + store.force_migration_state(|s| { + s.committed_at_unix = + Some(now_unix().saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); + }); + + wait_for( + &store, + MigrationPhase::FilesOnly, + "retirement should complete", + ) + .await; + shutdown.cancel(); + let _ = tokio::time::timeout(Duration::from_secs(10), driver).await; + + // The point of the whole exercise: the LMDB is gone from the filesystem. + assert!( + !root.join("chunks.mdb").exists(), + "the legacy store must be removed, which is the only moment disk comes back" + ); + assert!(!store.has_legacy()); + + // And nothing was lost: every chunk still reads, now out of a file. + assert_eq!(store.current_chunks().expect("count"), u64::from(CHUNKS)); + for (i, key) in keys.iter().enumerate() { + let expected = format!("legacy-chunk-{i}").into_bytes(); + assert_eq!( + store.get(key).await.expect("get").expect("present"), + expected, + "chunk {i} did not survive the migration" + ); + } + + // In files, under the suffix shard its address names. + let sample = keys.first().copied().expect("a key"); + let path = root + .join(crate::storage::file_store::CHUNKS_DIR_NAME) + .join(format!("{:02x}", sample.last().copied().unwrap_or(0))) + .join(hex::encode(sample)); + assert!(path.exists(), "expected a chunk file at {}", path.display()); + } + + /// The other half: a node that cannot fit its chunks and cannot prove anyone else + /// holds them keeps both stores and deletes nothing. + /// + /// This is the case that must fail safe. The node is out of disk, so it would like to + /// give chunks up, but with no view of the network it cannot show a single one exists + /// elsewhere. Refusing costs it disk. Proceeding would cost the network data. + #[tokio::test] + async fn a_node_that_cannot_prove_its_chunks_are_safe_deletes_nothing() { + const CHUNKS: u32 = 8; + + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + let keys = seed_legacy(&root, CHUNKS).await; + + let mut config = crate::storage::ChunkStoreConfig { + root_dir: root.clone(), + // Nothing will fit: the copier stops for space on its first chunk. + disk_reserve: u64::MAX / 2, + ..crate::storage::ChunkStoreConfig::test_default() + }; + config.migration.retire_legacy = true; + config.migration.tick_secs = 1; + // Scoped to this test's own directory. In production the lock is keyed by the + // filesystem, so without this every test on this machine would serialise against + // every other one that runs a migration. + config.migration.lock_dir = Some(root.clone()); + // Elapsed, so the hold is not what is doing the refusing here. + config.migration.shed_hold_hours = 0; + config.migration.wave_hours = 0; + let store = Arc::new( + crate::storage::ChunkStore::new(config) + .await + .expect("open store"), + ); + assert_eq!(store.legacy_only_keys().len(), CHUNKS as usize); + + let shutdown = CancellationToken::new(); + let driver = tokio::spawn(run( + Arc::clone(&store), + MigrationContext { + p2p: None, + self_id: None, + self_xor: None, + commitment: None, + replication: None, + sync_state: None, + audit_challenge_coordinator: None, + peer_commitments: None, + close_group_size: 7, + }, + shutdown.clone(), + )); + + // Give it long enough to have tried, re-tried, and evaluated shedding. + tokio::time::sleep(Duration::from_secs(5)).await; + shutdown.cancel(); + let _ = tokio::time::timeout(Duration::from_secs(10), driver).await; + + assert_eq!( + store.migration_phase(), + MigrationPhase::Bridging, + "a node that cannot prove its chunks are held elsewhere must not commit" + ); + assert!( + root.join("chunks.mdb").exists(), + "and must not remove the only copy of them" + ); + assert_eq!(store.legacy_only_keys().len(), CHUNKS as usize); + for (i, key) in keys.iter().enumerate() { + assert_eq!( + store.get(key).await.expect("get").expect("present"), + format!("legacy-chunk-{i}").into_bytes(), + "chunk {i} must still be served throughout" + ); + } + } + + /// Poll until the store reaches `phase`, or fail with what it reached instead. + /// + /// The deadline is generous because it is measured on the wall clock while the driver + /// it is waiting on runs on the runtime. On a saturated machine both stretch, and a + /// deadline sized for the work rather than for the contention turns a slow build into + /// a failing test. The work itself is two ticks. + async fn wait_for(store: &Arc, phase: MigrationPhase, what: &str) { + let deadline = std::time::Instant::now() + Duration::from_secs(180); + while std::time::Instant::now() < deadline { + if store.migration_phase() == phase { + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!( + "{what}: still in {:?} after the deadline, expected {phase:?}", + store.migration_phase() + ); + } + + /// A marker that cannot be used is replaced on disk, not just in memory. + /// + /// The shed hold counts from `first_start_unix`, and the whole reason this is written + /// at startup rather than at the first phase change is that a marker written later + /// would restart that clock on every reboot. A marker that is present but unusable used + /// to defeat that: it was replaced in memory and left on disk, so every start read the + /// same bad file and stamped a fresh clock. A node restarting more often than the hold + /// would then never become eligible to shed and never finish migrating, which is the + /// failure the doc comment on `load_or_create` names. + #[test] + fn an_unusable_marker_is_replaced_on_disk_so_the_hold_does_not_restart() { + for (name, bad) in [ + ("truncated", br#"{"schema":1,"phase":"brid"#.to_vec()), + ("not json at all", b"\x00\x01\x02".to_vec()), + ( + "a newer schema", + serde_json::json!({ + "schema": 9_999, + "phase": "bridging", + "first_start_unix": 1, + "committed_at_unix": null, + "rebuilds_since_commit": 0, + "shed_key_count": 0, + "kept_key_count": 0, + }) + .to_string() + .into_bytes(), + ), + // The one most worth keeping, and the one a parse into today's struct cannot + // read: a phase this build has no name for, and a field it does not know. + ( + "a newer schema this build cannot parse at all", + serde_json::json!({ + "schema": 9_999, + "phase": "some_phase_from_the_future", + "first_start_unix": 1, + "something_this_build_never_heard_of": {"a": 1}, + }) + .to_string() + .into_bytes(), + ), + ] { + let dir = TempDir::new().expect("temp dir"); + std::fs::write(state_path(dir.path()), &bad).expect("plant the bad marker"); + + let first = MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); + let written = std::fs::read(state_path(dir.path())).expect("read it back"); + assert_ne!(written, bad, "{name}: the unusable marker was left on disk"); + if name.starts_with("a newer schema") { + // Moved aside rather than destroyed: it was written on purpose by a build + // that knew more than this one. + let mut kept = state_path(dir.path()).into_os_string(); + kept.push(".schema-9999"); + let kept = PathBuf::from(kept); + assert_eq!( + std::fs::read(&kept).expect("the newer marker must be kept"), + bad, + "the newer marker was written over rather than set aside" + ); + } + + // And the clock survives the next start, which is the point of all this. + let second = MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); + assert_eq!( + second.first_start_unix, first.first_start_unix, + "{name}: the shed hold restarted on the next boot" + ); + } + } + + /// A marker whose clock is impossible is corrected on disk too. + /// + /// `with_sane_clocks` fixes a future-dated marker for the process that read it. Left + /// there, the same correction is made again on every start, from a new now each time, + /// which is the same repeating reset by another route. + #[test] + fn a_future_dated_marker_is_corrected_on_disk_not_only_in_memory() { + let dir = TempDir::new().expect("temp dir"); + let ahead = now_unix().saturating_add(60 * 60 * 24 * 365); + let planted = serde_json::json!({ + "schema": 1, + "phase": "bridging", + "first_start_unix": ahead, + "committed_at_unix": null, + "rebuilds_since_commit": 0, + "shed_key_count": 0, + "kept_key_count": 0, + }) + .to_string(); + std::fs::write(state_path(dir.path()), &planted).expect("plant it"); + + let first = MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); + assert!( + first.first_start_unix < ahead, + "the clock should have been brought back to something possible" + ); + + let reread: MigrationState = + serde_json::from_slice(&std::fs::read(state_path(dir.path())).expect("read")) + .expect("the marker on disk must now parse"); + assert_eq!( + reread.first_start_unix, first.first_start_unix, + "the correction was made in memory and not written back" + ); + } + + /// A marker that is already right is not rewritten on every start. + /// + /// The counterpart to the two above: persisting on disagreement must not turn into + /// persisting unconditionally, which would put a write on every node's start path for + /// nothing. + #[test] + fn a_usable_marker_is_left_exactly_as_it_is() { + let dir = TempDir::new().expect("temp dir"); + let first = MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); + + // Written back out by hand in a shape this build would never produce: the same + // facts, different spacing and key order. Comparing the bytes of a marker this + // build wrote against the bytes after a second start proves nothing, because an + // unconditional rewrite produces the same bytes and the test passes either way. + // Something semantically equal but textually different is the only thing that can + // tell "left alone" from "written again". + let noncanonical = format!( + "{{\"kept_key_count\":{},\"shed_key_count\":{},\"rebuilds_since_commit\":{},\ + \"committed_at_unix\":null,\"first_start_unix\":{},\"phase\":\"bridging\",\ + \"schema\":{}}}", + first.kept_key_count, + first.shed_key_count, + first.rebuilds_since_commit, + first.first_start_unix, + first.schema + ); + std::fs::write(state_path(dir.path()), &noncanonical).expect("write"); + + MigrationState::load_or_create(dir.path(), MigrationPhase::Bridging); + assert_eq!( + std::fs::read_to_string(state_path(dir.path())).expect("read"), + noncanonical, + "a marker that already said the right thing was rewritten for no reason" + ); + } + + /// Two nodes on one disk take the same lock, and the override is what makes that true + /// where they do not share a `/tmp`. + /// + /// `lock_path_for` had no coverage at all, which is how it came to be right in a way + /// that is false on our own fleet: every shipped test sets `lock_dir` and so never + /// calls it. What the lock is for is one node copying at a time on a shared disk, so + /// what has to be true is that two different node roots on one volume produce one path. + /// + /// Nothing here touches process-wide environment. An earlier version of this test set + /// `TMPDIR` to stage the private-`/tmp` case, which moved every other test's temporary + /// directory and then deleted it underneath them: sixty-three unrelated tests failed + /// with LMDB errors that pointed nowhere near the cause. + #[test] + fn nodes_on_one_volume_agree_on_a_lock_path_when_they_are_told_where_it_is() { + let volume = TempDir::new().expect("temp dir"); + let a = volume.path().join("node-0"); + let b = volume.path().join("node-1"); + std::fs::create_dir_all(&a).expect("mkdir"); + std::fs::create_dir_all(&b).expect("mkdir"); + + // With no override and one shared temporary directory, which is the case the + // default is right for: two roots on one volume, one lock. + assert_eq!( + lock_path_with(&a, None), + lock_path_with(&b, None), + "two roots on one volume must share a lock when they share a /tmp" + ); + + // Where they do not share one, the default gives each node a lock of its own and + // every one of them takes it. That is the deployment hazard, and it is why the + // override exists rather than something the code can detect. + assert_ne!( + lock_path_with(&a, Some("/tmp/private-to-node-0")), + lock_path_with(&b, Some("/tmp/private-to-node-1")), + "different lock directories must give different locks, or the override would \ + not be able to express anything" + ); + + // And told where it lives, both land on it whatever their own root is. + let told = volume.path().to_string_lossy().into_owned(); + let shared_a = lock_path_with(&a, Some(&told)); + let shared_b = lock_path_with(&b, Some(&told)); + assert_eq!( + shared_a, shared_b, + "nodes told where the lock lives must all use it" + ); + assert!(shared_a.starts_with(volume.path()), "and use the one named"); + assert_eq!( + lock_path_with(&a, Some(" ")), + lock_path_with(&a, None), + "an empty setting is not a location and must not be treated as one" + ); + + // And the lock itself then does its job across the two roots. + let LockAttempt::Acquired(held) = VolumeLock::try_acquire(&a, Some(volume.path())) else { + panic!("the first node must take it"); + }; + assert!( + matches!( + VolumeLock::try_acquire(&b, Some(volume.path())), + LockAttempt::Busy + ), + "the second must wait rather than copy alongside it" + ); + drop(held); + assert!(matches!( + VolumeLock::try_acquire(&b, Some(volume.path())), + LockAttempt::Acquired(_) + )); + } + + /// A lock directory that cannot be written is reported, not silently ignored. + /// + /// The override is a deployment fact, so a typo in it must not read as "no lock needed + /// here". `Unavailable` is the honest answer and it already warns; what this pins is + /// that a bad override does not quietly fall back to a path that would appear to work. + #[test] + fn a_lock_directory_that_does_not_exist_is_unavailable_rather_than_ignored() { + let dir = TempDir::new().expect("temp dir"); + let missing = dir.path().join("no-such-directory"); + assert!(matches!( + VolumeLock::try_acquire(dir.path(), Some(&missing)), + LockAttempt::Unavailable + )); + } + + #[tokio::test] + async fn the_driver_exits_immediately_when_there_is_nothing_to_migrate() { + // The whole feature hangs off `run` being reachable from node startup. A port that + // dropped that call once already, and nothing caught it, because a fresh node has + // no legacy environment and every test built one directly. This asserts the entry + // point is callable and terminates on its own for a node with nothing to do. + let dir = TempDir::new().expect("temp dir"); + let store = Arc::new( + crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + ..crate::storage::ChunkStoreConfig::test_default() + }) + .await + .expect("open store"), + ); + assert!(!store.has_legacy()); + + let context = MigrationContext { + p2p: None, + self_id: None, + self_xor: None, + commitment: None, + replication: None, + sync_state: None, + audit_challenge_coordinator: None, + peer_commitments: None, + close_group_size: 7, + }; + tokio::time::timeout( + Duration::from_secs(5), + run(store, context, CancellationToken::new()), + ) + .await + .expect("the driver must return rather than idle when there is nothing to migrate"); + } + + #[tokio::test] + async fn a_node_with_no_view_of_the_network_gives_up_nothing() { + // Every field is `None`, which is what a devnet or a node whose routing is not up + // yet looks like. No view of the network is no evidence, and the answer has to be + // "keep everything" rather than "nobody is closer, so give it all away". + let dir = TempDir::new().expect("temp dir"); + let store = Arc::new( + crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + ..crate::storage::ChunkStoreConfig::test_default() + }) + .await + .expect("open store"), + ); + let context = MigrationContext { + p2p: None, + self_id: None, + self_xor: None, + commitment: None, + replication: None, + sync_state: None, + audit_challenge_coordinator: None, + peer_commitments: None, + close_group_size: 7, + }; + + let keys = vec![[1u8; 32], [2u8; 32], [3u8; 32]]; + let unconfirmed = unconfirmed_by_neighbours(&store, &context, &keys).await; + assert_eq!( + unconfirmed, keys, + "with no routing state every key must count as unproven" + ); + for key in &keys { + assert!( + !context.may_shed(key).await, + "and none of them may be given up" + ); + } + } + + #[test] + fn the_possession_threshold_comes_from_the_whole_group_not_the_qualifying_subset() { + use crate::replication::pruning::{prune_proofs_needed, target_peers_reported_present}; + use std::collections::{HashMap, HashSet}; + + // Seven holders. Deriving the bar from whichever peers happen to qualify is how + // two last holders destroy a chunk between them: each sees only the other + // publishing, so each needs exactly one proof, each gets it from the other, and + // both delete. The bar must come from the group. + let key = [7u8; 32]; + let group: Vec = (0..6u8).map(peer_id).collect(); + let only_one_qualifies: Vec = group.iter().take(1).copied().collect(); + + // That one peer does answer the challenge. + let mut proofs: HashMap> = HashMap::new(); + proofs.insert(key, only_one_qualifies.iter().copied().collect()); + + // The dangerous reading: bar taken from the qualifying subset, so one is enough. + assert!( + target_peers_reported_present( + &key, + &only_one_qualifies, + &proofs, + prune_proofs_needed(only_one_qualifies.len()), + ), + "this is the mistake being guarded against, shown here to be a real risk" + ); + + // The correct reading: bar taken from the whole group, so one is nowhere near. + assert!( + !target_peers_reported_present( + &key, + &only_one_qualifies, + &proofs, + prune_proofs_needed(group.len()), + ), + "one proof must never satisfy a group of six" + ); + + // And with the whole group answering, it passes. + proofs.insert(key, group.iter().copied().collect()); + assert!(target_peers_reported_present( + &key, + &group, + &proofs, + prune_proofs_needed(group.len()), + )); + } + + #[test] + #[serial] + fn shedding_reads_the_same_switch_the_auditors_read() { + use crate::replication::config::{ + close_group_storage_penalty_suspended, set_close_group_storage_penalty_suspended, + }; + // One switch, not two. There used to be a second constant of the same name with + // its own environment override on this side of the decision, and nothing coupling + // them: a node could have been willing to shed while every peer still applied the + // full penalty, which is precisely what the release ordering exists to prevent. + set_close_group_storage_penalty_suspended(true); + assert!(close_group_storage_penalty_suspended()); + set_close_group_storage_penalty_suspended(false); + assert!(!close_group_storage_penalty_suspended()); + set_close_group_storage_penalty_suspended( + crate::replication::config::RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY, + ); + } + + #[test] + fn a_close_group_is_split_into_enough_waves_to_bound_concurrent_migrations() { + // Seven holders, two at a time, is four waves. + assert_eq!(migration_wave_count(7), 4); + assert_eq!(migration_wave_count(2), 1); + assert_eq!(migration_wave_count(1), 1); + // A degenerate width must still yield a usable wave count rather than dividing by + // zero or collapsing to "everyone at once". + assert_eq!(migration_wave_count(0), 1); + } + + #[test] + fn wave_assignment_is_stable_per_node_and_spread_across_the_group() { + use std::collections::HashMap; + let waves = migration_wave_count(7); + + // The same node always gets the same wave: a restart must not move a node into a + // turn that has already passed. + let peer = peer_id(7); + assert_eq!( + migration_wave_for(Some(&peer), 7), + migration_wave_for(Some(&peer), 7) + ); + + // And across many nodes every wave is used, so the group is genuinely staggered + // rather than all landing together. + let mut counts: HashMap = HashMap::new(); + for b in 0..=255u8 { + let w = migration_wave_for(Some(&peer_id(b)), 7); + assert!(w < waves, "wave {w} outside 0..{waves}"); + *counts.entry(w).or_default() += 1; + } + assert_eq!( + counts.len() as u64, + waves, + "every wave should be occupied, got {counts:?}" + ); + } + + #[test] + fn waves_are_actually_staggered_under_the_shipped_defaults() { + // The combination is what matters, not either setting alone. Measured from first + // start, a 72 hour hold and 24 hour waves cancel out: waves would open at 0, 24, + // 48 and 72 hours while nothing may shed until 72, so every wave is open the + // moment the first one can act and the whole close group moves together. Measured + // from the end of the hold, they stagger as intended. + let config = MigrationConfig::default(); + assert_eq!(config.shed_hold_hours, 72); + assert_eq!(config.wave_hours, 24); + + let mut state = MigrationState::new(MigrationPhase::Bridging); + let waves = migration_wave_count(7); + assert_eq!(waves, 4); + + // Nothing is open before the hold ends. + state.first_start_unix = now_unix(); + for w in 0..waves { + assert!( + !wave_has_opened(&state, &config, w), + "wave {w} opened too early" + ); + } + + // At the end of the hold, exactly the first wave is open. + state.first_start_unix = now_unix().saturating_sub(72 * 3600 + 60); + assert!(wave_has_opened(&state, &config, 0)); + for w in 1..waves { + assert!( + !wave_has_opened(&state, &config, w), + "wave {w} must wait its turn, or the group migrates together" + ); + } + + // Each later wave opens one wave_hours after the one before it. + for open in 1..waves { + state.first_start_unix = now_unix().saturating_sub((72 + open * 24) * 3600 + 60); + for w in 0..=open { + assert!(wave_has_opened(&state, &config, w)); + } + for w in open + 1..waves { + assert!(!wave_has_opened(&state, &config, w)); + } + } + } + + #[test] + fn an_implausible_clock_restarts_the_holds_rather_than_voiding_them() { + let dir = TempDir::new().expect("temp dir"); + let config = MigrationConfig::default(); + + // Zero is what a node with an unsynchronised clock writes at first boot, and it + // would make every hold vacuous. So would a time in the future. + let mut state = MigrationState::new(MigrationPhase::Committed); + state.first_start_unix = 0; + state.committed_at_unix = Some(0); + state.save(dir.path()).expect("save"); + + let loaded = MigrationState::load_or_new(dir.path(), MigrationPhase::Bridging); + assert!( + !loaded.shed_hold_elapsed(&config), + "the hold must not be void" + ); + assert!(!loaded.retire_delay_elapsed(&config)); + + let mut future = MigrationState::new(MigrationPhase::Committed); + future.first_start_unix = now_unix().saturating_add(10 * 365 * 24 * 3600); + future.committed_at_unix = Some(future.first_start_unix); + future.save(dir.path()).expect("save"); + let loaded = MigrationState::load_or_new(dir.path(), MigrationPhase::Bridging); + assert!(!loaded.shed_hold_elapsed(&config)); + assert!(!loaded.retire_delay_elapsed(&config)); + } + + #[test] + fn copy_reports_accumulate_across_passes() { + let mut total = CopyReport::default(); + total.merge(CopyReport { + copied: 3, + bytes: 300, + ..CopyReport::default() + }); + total.merge(CopyReport { + copied: 2, + bytes: 200, + unusable: 1, + stopped_for_space: true, + ..CopyReport::default() + }); + assert_eq!(total.copied, 5); + assert_eq!(total.bytes, 500); + assert_eq!(total.unusable, 1); + assert!(total.stopped_for_space); + } + /// A peer that received the commitment and then left the group is not evidence. + /// + /// It is not going to audit this node, so counting it lets a node give chunks up + /// while the neighbours who will audit it still hold it to the old, larger key set. + #[test] + fn a_departed_peer_that_knows_the_commitment_does_not_open_the_gate() { + let received: HashSet = (0..6).map(peer_id).collect(); + let still_here: Vec = (0..3).map(peer_id).collect(); + let joined_since: Vec = (100..103).map(peer_id).collect(); + let current: Vec = still_here + .iter() + .chain(joined_since.iter()) + .copied() + .collect(); + + // Six peers know it and the group is six wide, so a count that ignores who is + // actually here would sail past the threshold. + assert_eq!(received.len(), 6); + assert_eq!(current.len(), 6); + assert!(!enough_of_the_group_knows(&received, ¤t, 5)); + + // Only the three that are both here and informed count. + assert!(enough_of_the_group_knows(&received, ¤t, 3)); + assert!(!enough_of_the_group_knows(&received, ¤t, 4)); + } + + #[test] + fn a_group_that_has_all_seen_the_commitment_opens_the_gate() { + let group: Vec = (0..6).map(peer_id).collect(); + let received: HashSet = group.iter().copied().collect(); + assert!(enough_of_the_group_knows(&received, &group, 5)); + } + + #[test] + fn no_peer_ever_satisfies_a_zero_threshold() { + let group: Vec = (0..6).map(peer_id).collect(); + let received: HashSet = group.iter().copied().collect(); + // A group this node cannot reason about must not be read as unanimous consent. + assert!(!enough_of_the_group_knows(&received, &group, 0)); + } + + /// The gate stays shut when routing cannot show a full close group at all. + /// + /// Without a routing view there is no way to tell an informed neighbour from a + /// departed one, and an unanswerable question must not read as a yes. + #[tokio::test] + async fn without_a_routing_view_the_commitment_gate_stays_shut() { + let context = MigrationContext { + p2p: None, + self_id: None, + self_xor: None, + commitment: None, + replication: None, + sync_state: None, + audit_challenge_coordinator: None, + peer_commitments: None, + close_group_size: 7, + }; + assert!(!context.neighbours_know_the_commitment().await); + } +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 64f9462c..bda34ac8 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -44,11 +44,51 @@ //! listener.register_protocol(protocol).await?; //! ``` +pub(crate) mod chunk_store; +#[cfg(any(test, feature = "test-utils"))] +pub mod file_store; +#[cfg(not(any(test, feature = "test-utils")))] +pub(crate) mod file_store; mod handler; pub(crate) mod lmdb; +pub mod migration; pub use crate::ant_protocol::XorName; +pub use chunk_store::{ChunkStore, ChunkStoreConfig, VerifyReport, LEGACY_ENV_DIR}; +pub use file_store::{FileStore, FileStoreConfig, StoreLayout}; pub use handler::AntProtocol; pub(crate) use handler::ChunkRequestContext; pub(crate) use lmdb::CapacityVerdict; -pub use lmdb::{LmdbStorage, LmdbStorageConfig, StorageStats}; +pub use lmdb::{LmdbStorage, LmdbStorageConfig}; +pub use migration::{MigrationConfig, MigrationPhase, MigrationState}; + +/// Bytes in one MiB. +pub const MIB: u64 = 1024 * 1024; + +/// Bytes in one GiB. +pub const GIB: u64 = 1024 * MIB; + +/// Default free disk space to keep unused on the storage partition. +pub const DEFAULT_DISK_RESERVE: u64 = 500 * MIB; + +/// Statistics about storage operations. +/// +/// Counters other than `current_chunks` are cumulative for the lifetime of the +/// process; `current_chunks` is the live count. +#[derive(Debug, Clone, Default)] +pub struct StorageStats { + /// Total number of chunks stored. + pub chunks_stored: u64, + /// Total number of chunks retrieved. + pub chunks_retrieved: u64, + /// Total bytes stored. + pub bytes_stored: u64, + /// Total bytes retrieved. + pub bytes_retrieved: u64, + /// Number of duplicate writes (already exists). + pub duplicates: u64, + /// Number of verification failures on read. + pub verification_failures: u64, + /// Number of chunks currently persisted. + pub current_chunks: u64, +} diff --git a/tests/e2e/data_types/chunk.rs b/tests/e2e/data_types/chunk.rs index 09729b93..2c875b76 100644 --- a/tests/e2e/data_types/chunk.rs +++ b/tests/e2e/data_types/chunk.rs @@ -67,7 +67,7 @@ mod tests { EvmVerifierConfig, PaymentVerifier, PaymentVerifierConfig, PriceFloorConfig, QuoteGenerator, QuotingMetricsTracker, }; - use ant_node::storage::{AntProtocol, LmdbStorage, LmdbStorageConfig}; + use ant_node::storage::{AntProtocol, ChunkStore, ChunkStoreConfig}; use ant_node::ReplicationConfig; use evmlib::testnet::Testnet; use evmlib::RewardsAddress; @@ -355,7 +355,7 @@ mod tests { // Shut down node 0 completely (simulates node restart): // 1. Shut down the replication engine and await its background tasks - // so all Arc clones are released. + // so all Arc clones are released. // 2. Abort the protocol task that holds an Arc. // 3. Drop the node's own Arc. // This ensures the LMDB env is fully closed before reopening. @@ -433,9 +433,9 @@ mod tests { let temp_dir = std::env::temp_dir().join(format!("{test_name}_{}", rand::random::())); tokio::fs::create_dir_all(&temp_dir).await?; - let storage = LmdbStorage::new(LmdbStorageConfig { + let storage = ChunkStore::new(ChunkStoreConfig { root_dir: temp_dir.clone(), - ..LmdbStorageConfig::test_default() + ..ChunkStoreConfig::test_default() }) .await?; diff --git a/tests/e2e/fetch_local_write_guard.rs b/tests/e2e/fetch_local_write_guard.rs index 8d433542..73025ac8 100644 --- a/tests/e2e/fetch_local_write_guard.rs +++ b/tests/e2e/fetch_local_write_guard.rs @@ -16,14 +16,14 @@ //! counter; the probe scenario is observed through a sender-side count of //! verification requests, since a request that was never sent leaves no trace on //! any receiver. -//! Only `LmdbStorage::get` increments it — the replication fetch responder and +//! Only `ChunkStore::get` increments it — the replication fetch responder and //! the client GET handler; audits read through `get_raw` and leave it alone. //! It is not keyed by chunk or requester, so it is an "it served something" //! signal rather than an exact per-key one; on a freshly built testnet with no //! other traffic to the holder, a delta means it served this fetch. //! //! Two gaps this file deliberately does not close, because neither is -//! constructible without adding test-only hooks to `LmdbStorage`: +//! constructible without adding test-only hooks to `ChunkStore`: //! //! - **Ordering.** Possession is checked before capacity so a full node still //! accepts a key it already holds, matching `put`. Proving it needs a node @@ -112,7 +112,7 @@ async fn ensure_pending_verify(engine: &ReplicationEngine, key: XorName, hinter: /// /// **Phase 1, the dial.** `execute_single_fetch` refuses before the dial, so no /// holder is conscripted. Observed through the holder's `chunks_retrieved` -/// counter: only `LmdbStorage::get` moves it — the replication fetch responder +/// counter: only `ChunkStore::get` moves it — the replication fetch responder /// and the client GET handler — while audits read through `get_raw` and leave it /// alone. It is not keyed by chunk or requester, so it is an "it served /// something" signal rather than an exact per-key one; on a freshly built diff --git a/tests/e2e/replication.rs b/tests/e2e/replication.rs index 841da90a..abf5f92b 100644 --- a/tests/e2e/replication.rs +++ b/tests/e2e/replication.rs @@ -266,7 +266,7 @@ async fn test_fresh_replication_propagates_to_close_group() { /// eviction acts on), via `P2PNode::peer_trust`. #[tokio::test] #[serial] -async fn possession_check_penalises_absent_peer_only() { +async fn possession_check_penalises_absent_peer_only_and_obeys_the_release_switch() { let harness = TestHarness::setup_small().await.expect("setup"); harness.warmup_dht().await.expect("warmup"); @@ -324,6 +324,10 @@ async fn possession_check_penalises_absent_peer_only() { "precondition: C must hold the chunk" ); + // Switched on explicitly, so this half keeps testing the possession mechanism rather + // than whichever release it happens to be compiled against. + ant_node::replication::config::set_close_group_storage_penalty_suspended(false); + let trust_b_before = p2p_a.peer_trust(&peer_b); let trust_c_before = p2p_a.peer_trust(&peer_c); @@ -345,6 +349,25 @@ async fn possession_check_penalises_absent_peer_only() { "present peer C must not be penalised: {trust_c_before} -> {trust_c_after}" ); + // And the other half of the contract, on the same harness. The release that moves + // nodes off the legacy chunk store withholds exactly this penalty: a node short of + // disk cannot avoid answering "absent" while it moves its chunks out of a store that + // never returns space, and it cannot stop its peers penalising it for that, because + // the penalty is the auditor's decision. So the auditors stop one release ahead. + ant_node::replication::config::set_close_group_storage_penalty_suspended(true); + let trust_b_suspended_before = p2p_a.peer_trust(&peer_b); + engine_a + .run_possession_check_now(address, vec![peer_b, peer_c]) + .await; + let trust_b_suspended_after = p2p_a.peer_trust(&peer_b); + ant_node::replication::config::set_close_group_storage_penalty_suspended(false); + + assert!( + trust_b_suspended_after >= trust_b_suspended_before - f64::EPSILON, + "an absent peer must not be penalised while the release withholds that penalty: \ + {trust_b_suspended_before} -> {trust_b_suspended_after}" + ); + harness.teardown().await.expect("teardown"); } @@ -391,6 +414,11 @@ async fn possession_scheduler_penalises_absent_close_peer_after_delay() { .collect(); assert!(!close_group.is_empty(), "expected a non-empty close group"); + // Switched on explicitly. The release that moves nodes off the legacy chunk store + // withholds this penalty by default, so a test that asserts it must say so, or it + // silently starts asserting whichever release it is compiled against. + ant_node::replication::config::set_close_group_storage_penalty_suspended(false); + let trust_before: Vec = close_group.iter().map(|p| p2p_a.peer_trust(p)).collect(); // The checker must hold the chunk it later probes for: the possession check @@ -560,6 +588,11 @@ async fn full_close_group_node_rejects_replica_and_is_penalised_as_absent() { .await .expect("put on checker"); + // Switched on explicitly. The release that moves nodes off the legacy chunk store + // withholds this penalty by default, so a test that asserts it must say so, or it + // silently starts asserting whichever release it is compiled against. + ant_node::replication::config::set_close_group_storage_penalty_suspended(false); + let trust_before = checker_p2p.peer_trust(&full_peer); checker_engine .replicate_fresh(&address, &content, &dummy_payment_proof) @@ -2135,6 +2168,11 @@ async fn scenario_11_repeated_failures_decrease_trust() { let peer_b = *p2p_b.peer_id(); // Get initial trust score for node B (should be neutral ~0.5) + // Switched on explicitly. The release that moves nodes off the legacy chunk store + // withholds this penalty by default, so a test that asserts it must say so, or it + // silently starts asserting whichever release it is compiled against. + ant_node::replication::config::set_close_group_storage_penalty_suspended(false); + let initial_trust = p2p_a.peer_trust(&peer_b); // Report multiple application failures diff --git a/tests/e2e/testnet.rs b/tests/e2e/testnet.rs index a281f5ea..22995a3e 100644 --- a/tests/e2e/testnet.rs +++ b/tests/e2e/testnet.rs @@ -23,7 +23,7 @@ use ant_node::payment::{ QuotingMetricsTracker, }; use ant_node::replication::config::MAX_REPLICATION_MESSAGE_SIZE; -use ant_node::storage::{AntProtocol, LmdbStorage, LmdbStorageConfig}; +use ant_node::storage::{AntProtocol, ChunkStore, ChunkStoreConfig}; use ant_node::{ReplicationConfig, ReplicationEngine}; use bytes::Bytes; use evmlib::Network as EvmNetwork; @@ -448,7 +448,7 @@ impl TestNode { info!("Shutting down test node {}", self.index); // Shut down replication engine and await its background tasks so all - // Arc clones are released before we drop the engine. + // Arc clones are released before we drop the engine. if let Some(ref mut engine) = self.replication_engine { engine.shutdown().await; } @@ -1128,12 +1128,12 @@ impl TestNetwork { identity: &saorsa_core::identity::NodeIdentity, ) -> Result { // Create LMDB storage - let storage_config = LmdbStorageConfig { + let storage_config = ChunkStoreConfig { root_dir: data_dir.to_path_buf(), disk_reserve, - ..LmdbStorageConfig::test_default() + ..ChunkStoreConfig::test_default() }; - let storage = LmdbStorage::new(storage_config) + let storage = ChunkStore::new(storage_config) .await .map_err(|e| TestnetError::Core(format!("Failed to create LMDB storage: {e}")))?; diff --git a/tests/migration_crash_safety.rs b/tests/migration_crash_safety.rs new file mode 100644 index 00000000..2bf464b2 --- /dev/null +++ b/tests/migration_crash_safety.rs @@ -0,0 +1,542 @@ +//! What survives a process dying part-way through the migration. +//! +//! The design rests on being able to stop at any moment and start again: every step is +//! idempotent and re-derived from the filesystem. That is easy to assert and hard to +//! believe without trying it, so these tests kill a real child process at a real point in +//! the work and then open the store in this one and check what is there. +//! +//! **What this does and does not prove.** A killed process loses nothing the kernel has +//! already accepted, so this covers ordering and bookkeeping: a chunk is whole or absent +//! and never half-indexed, what an interrupted write leaves behind is swept, and the store +//! always opens. It does not cover losing the page cache, which is what a real power cut +//! adds and what no hosted runner can do. That remains a fleet gate, and this is the part +//! of it that can be automated. +//! +//! The children stop at a named failpoint and say so, and the parent kills them there. An +//! earlier version slept and hoped; on a quick machine the child had finished before the +//! kill arrived, so the test was checking a clean shutdown while claiming to check a +//! crash. +//! +//! Runs on every platform CI covers, which is the filesystem matrix that matters: ext4, +//! APFS and NTFS. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::missing_panics_doc, + // Test fixtures: every cast here is of a bounded loop counter into a byte, and the + // wrap is what makes the fill vary. + clippy::cast_possible_truncation +)] + +use ant_node::storage::{ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; +use tempfile::TempDir; + +/// Chunks the child writes before it is killed. +const CHUNKS: usize = 120; + +/// Deterministic content for chunk `n`. `n` goes in verbatim so no two differ only by a +/// wrap and collapse into one chunk. +fn chunk_bytes(n: usize) -> Vec { + let mut content = vec![0u8; 4096]; + content[..8].copy_from_slice(&(n as u64).to_le_bytes()); + for (i, byte) in content.iter_mut().enumerate().skip(8) { + *byte = ((i.wrapping_mul(17)).wrapping_add(n) % 251) as u8; + } + content +} + +/// Run this test binary again as a child in the mode named by `role`, wait until it has +/// reached the named point, and kill it there. +/// +/// A child process rather than a thread, because the point is to lose everything the +/// process was holding: buffers, in-memory index, locks, half-finished intentions. +/// +/// The wait is a handshake, not a sleep. An earlier version of this slept and hoped, and +/// on a quick machine the child had finished everything before the kill arrived, so the +/// test was checking a clean shutdown while claiming to check a crash. The child now stops +/// at a failpoint inside the write and says so by writing a marker; this waits for the +/// marker and then kills it, so the process always dies at the same point in the same +/// operation. +fn kill_child_at_failpoint(role: &str, root: &Path, failpoint: &str, let_through: u64) -> PathBuf { + let marker = root.join(format!("reached-{role}")); + let _ = std::fs::remove_file(&marker); + + let exe = std::env::current_exe().expect("this test binary"); + let mut child = Command::new(exe) + .arg("--exact") + .arg(role) + .arg("--nocapture") + .arg("--ignored") + .env("ANT_CRASH_TEST_ROOT", root) + .env(failpoint, &marker) + .env( + ant_node::storage::file_store::HALT_AFTER, + let_through.to_string(), + ) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .expect("spawn the child"); + + // Generous, but not unbounded. Without a deadline a failpoint that stopped working + // would hang the job rather than fail it, and a hang says nothing about the code. + let deadline = std::time::Instant::now() + Duration::from_secs(120); + while !marker.exists() { + if let Ok(Some(status)) = child.try_wait() { + panic!("the child exited before reaching the failpoint: {status}"); + } + if std::time::Instant::now() > deadline { + let _ = child.kill(); + panic!("the child never reached the failpoint"); + } + std::thread::sleep(Duration::from_millis(10)); + } + + child.kill().expect("kill the child"); + let _ = child.wait(); + marker +} + +/// Where the child was told to work. +fn child_root() -> PathBuf { + PathBuf::from(std::env::var("ANT_CRASH_TEST_ROOT").expect("the child needs a root")) +} + +/// Child mode: write chunks into a file store until killed. +#[tokio::test] +#[ignore = "child process of a crash test, not run on its own"] +async fn child_writes_until_killed() { + let root = child_root(); + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: root, + disk_reserve: 0, + ..ChunkStoreConfig::default() + }) + .await + .expect("open"); + + // Always a chunk it has not written before, so the kill lands in real work rather + // than in a re-offer of something already on disk. An earlier version cycled the same + // hundred keys and spent almost all its time confirming duplicates. + let mut n = 0usize; + loop { + let content = chunk_bytes(n); + let address = ant_node::client::compute_address(&content); + let _ = store.put(&address, &content).await; + n += 1; + } +} + +/// Child mode: copy a legacy environment into files until killed. +#[tokio::test] +#[ignore = "child process of a crash test, not run on its own"] +async fn child_migrates_until_killed() { + let root = child_root(); + let mut config = ChunkStoreConfig { + root_dir: root.clone(), + disk_reserve: 0, + ..ChunkStoreConfig::default() + }; + config.migration.tick_secs = 1; + config.migration.copier_throttle_mib_per_sec = 0; + config.migration.copier_slack_mb = 0; + config.migration.lock_dir = Some(root); + let store = ChunkStore::new(config).await.expect("open"); + + let shutdown = tokio_util::sync::CancellationToken::new(); + // One chunk at a time, so the kill lands between two of them rather than after the + // whole thing. Deliberately no sleep at the end: a child that finished and then idled + // would let this test pass having crashed nothing. + loop { + let keys = store.legacy_only_keys(); + let Some(key) = keys.first() else { + break; + }; + let _ = store.copy_batch(&[*key], 0, 0, &shutdown).await; + } + panic!("the child copied everything before it was killed, so nothing was interrupted"); +} + +/// Plant a legacy environment holding chunks numbered from `first`, and close it. +async fn seed_legacy_from(root: &Path, first: usize) -> Vec<(usize, [u8; 32])> { + let lmdb = LmdbStorage::new(LmdbStorageConfig { + root_dir: root.to_path_buf(), + verify_on_read: true, + max_map_size: 0, + disk_reserve: 0, + }) + .await + .expect("open legacy"); + let mut keys = Vec::new(); + for n in first..first + CHUNKS { + let content = chunk_bytes(n); + let address = ant_node::client::compute_address(&content); + lmdb.put(&address, &content).await.expect("seed"); + // Paired with its chunk number, because half of these are about to be deleted and + // a bare position in the surviving list no longer says which chunk it is. + keys.push((n, address)); + } + + // Then delete some, which is what makes this look like a real node rather than a + // fresh file. The environment is pinned to its current size for the whole migration, + // so a write during the bridge lands only if there are free pages to land in. On a + // production node there are plenty: this migration exists precisely because deleting + // millions of chunks filled the free list and returned nothing to the filesystem. + // Seeded and never deleted from, the environment would have no room and the bridge's + // second write would never happen, which is not the case worth testing. + let discarded: Vec<(usize, [u8; 32])> = keys.drain(..CHUNKS / 2).collect(); + for (_, address) in &discarded { + lmdb.delete(address).await.expect("make room"); + } + lmdb.wait_idle().await; + keys +} + +/// Child mode: retire the legacy environment, and be killed once it is marked. +/// +/// Everything before the mark is done here rather than in the parent, because the whole +/// point is that the process that wrote the mark is the one that dies. +#[tokio::test] +#[ignore = "child process of a crash test, not run on its own"] +async fn child_retires_until_killed() { + let root = child_root(); + let store = reopen(&root).await; + + let shutdown = tokio_util::sync::CancellationToken::new(); + let keys = store.legacy_only_keys(); + store + .copy_batch(&keys, 0, 0, &shutdown) + .await + .expect("copy every chunk"); + store.wait_idle().await; + store.commit_to_files().expect("commit to the file set"); + store.note_commitment_rebuilt(); + store.note_commitment_rebuilt(); + store.force_migration_state(|state| { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + state.committed_at_unix = Some( + now.saturating_sub(ant_node::storage::migration::MIN_RETIRE_DELAY_HOURS * 3600 + 60), + ); + }); + + let proof = store + .verify_before_retire(0, &shutdown) + .await + .expect("verify before retiring"); + // Parks inside this call, once the environment is renamed aside and marked. + let _ = store + .retire_legacy( + &proof, + &|_: &[u8; 32]| false, + &std::collections::BTreeSet::new(), + ) + .await; + panic!("the child finished retiring without being killed, so nothing was interrupted"); +} + +/// A process killed inside a publish leaves no chunk it cannot serve. +/// +/// The child is stopped at the last moment before the chunk's name exists on disk: on Unix +/// the bytes written to a temporary file with the rename not yet made, off Unix the point +/// before the file is created at all, since that platform writes under the final name +/// because a rename there carries no durability guarantee. The failure this guards against +/// is the same on both: a name outliving its bytes. The index is built from filenames at +/// startup, so a partial file wearing a real chunk name would be advertised, committed to, +/// and unservable. +#[tokio::test] +async fn a_process_killed_mid_publish_leaves_no_chunk_it_cannot_serve() { + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + // Twenty chunks land before the crash, so the store this reopens has real content in + // it. Stopping the very first write would leave nothing indexed and the loop below + // would pass by iterating over nothing. + let marker = kill_child_at_failpoint( + "child_writes_until_killed", + &root, + ant_node::storage::file_store::HALT_BEFORE_PUBLISH, + 20, + ); + assert!( + marker.exists(), + "the child must have reached the failpoint before it was killed" + ); + + // Reopening is itself part of the assertion: a store that cannot start after a crash + // is a node that cannot start. + let store = reopen(&root).await; + for key in store.all_keys().await.expect("all_keys") { + let served = store.get(&key).await; + assert!( + matches!(served, Ok(Some(_))), + "chunk {} is claimed after a crash but cannot be served: {served:?}", + hex::encode(key) + ); + } +} + +/// The temporary file a killed publish left behind is swept, not indexed. +/// +/// It carries no chunk name, so it can never be served, and leaving it would cost disk +/// for the life of the node. +/// +/// Unix only, because the leftover only exists on Unix. Off Unix the store creates the +/// file under its final name and flushes it, deliberately, since a rename there is not +/// documented to be durable. So there is no temporary file to sweep and the equivalent +/// hazard is different: a real chunk name over bytes that are short or wrong. That one is +/// covered by the store's own tests, which run on every platform, and by the +/// re-hash-everything pass the retirement does before it deletes anything. +#[cfg(unix)] +#[tokio::test] +async fn the_leftovers_of_a_killed_publish_are_swept() { + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + kill_child_at_failpoint( + "child_writes_until_killed", + &root, + ant_node::storage::file_store::HALT_BEFORE_PUBLISH, + 5, + ); + + let before = temp_files(&root.join("chunks")); + assert!( + before > 0, + "the child should have left a temporary file behind when it was killed" + ); + + let store = reopen(&root).await; + store.wait_idle().await; + assert_eq!( + temp_files(&root.join("chunks")), + 0, + "the store should sweep what an interrupted write left" + ); + drop(store); +} + +/// How many partly-written files are under `chunks_dir`. +#[cfg(unix)] +fn temp_files(chunks_dir: &Path) -> usize { + let Ok(shards) = std::fs::read_dir(chunks_dir) else { + return 0; + }; + shards + .flatten() + .filter_map(|shard| std::fs::read_dir(shard.path()).ok()) + .flat_map(std::iter::IntoIterator::into_iter) + .flatten() + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| !name.chars().all(|c| c.is_ascii_hexdigit())) + }) + .count() +} + +/// Open the store the way a restart would. +async fn reopen(root: &Path) -> ChunkStore { + let mut config = ChunkStoreConfig { + root_dir: root.to_path_buf(), + disk_reserve: 0, + ..ChunkStoreConfig::default() + }; + config.migration.lock_dir = Some(root.to_path_buf()); + ChunkStore::new(config) + .await + .expect("the store must open after a crash") +} + +/// A crash part-way through copying loses nothing: the environment still has everything. +/// +/// The copier is only allowed to drop a key from its list once the file is durably +/// published, so a crash mid-copy costs the work of one chunk, never the chunk. +#[tokio::test] +async fn a_killed_migration_still_has_every_chunk_somewhere() { + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + let keys = seed_legacy_from(&root, 0).await; + + // Ten chunks copied, the eleventh interrupted. An earlier version killed the child + // after a fixed delay, which on a fast runner meant it had copied everything and on a + // slow one meant it had copied nothing; both make this test say something other than + // what it claims. + kill_child_at_failpoint( + "child_migrates_until_killed", + &root, + ant_node::storage::file_store::HALT_BEFORE_PUBLISH, + 10, + ); + + // Interrupted, which is two claims and not one: some chunks copied, and some not. + // Only the upper bound was checked before, so a copier that did nothing at all passed + // as long as everything was still readable from the environment. + let store = reopen(&root).await; + let left = store.legacy_only_keys().len(); + assert!( + left > 0, + "the child was supposed to be killed part-way through, not after finishing" + ); + assert!( + left < keys.len(), + "the child copied nothing, so nothing was interrupted: {left} of {} left", + keys.len() + ); + + for (n, key) in &keys { + let served = store + .get(key) + .await + .expect("read after a crash") + .expect("every seeded chunk must still be readable from one store or the other"); + assert_eq!(served, chunk_bytes(*n), "chunk {n} came back wrong"); + } +} + +/// A crash between the two halves of a dual write does not leave a chunk unprotected. +/// +/// The environment's copy is written first and the file second. A crash in between leaves +/// a chunk only the environment has, and it must be on the copier's list, because a key +/// in neither view is what retirement destroys. +/// +/// The chunks the child writes are deliberately ones the environment does not already +/// hold. An earlier version seeded the same addresses the child then wrote, and the write +/// path skips the environment half for a key that is already legacy-only, so no dual +/// write happened at all and the test proved nothing. +#[tokio::test] +async fn a_crash_between_the_two_halves_leaves_the_chunk_on_the_list() { + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + // Seeded with chunks the child will not write: the child starts at 0 and these are + // far above anything it reaches in the time it has. + seed_legacy_from(&root, 1_000_000).await; + + // The crash lands inside the write of chunk 20, so chunks 0 to 19 completed and 20 + // is the one caught between the two halves. + kill_child_at_failpoint( + "child_writes_until_killed", + &root, + ant_node::storage::file_store::HALT_BEFORE_PUBLISH, + 20, + ); + + let store = reopen(&root).await; + + // Whatever the environment holds and the file store does not is on the list. It is + // derived at open from the two key sets, which is the property that makes a crash + // survivable: re-read from disk, never carried across. + // The specific key, not merely a non-empty list. The environment was seeded with + // unrelated keys, and an earlier version asserted only that something was on the + // list, which those seeds satisfied whether or not a dual write had happened at all. + let interrupted = ant_node::client::compute_address(&chunk_bytes(20)); + let legacy_only = store.legacy_only_keys(); + assert!( + legacy_only.contains(&interrupted), + "the chunk whose file half never landed must be on the copier's list: it reached \ + the environment and nothing else knows about it" + ); + for key in &legacy_only { + let served = store + .get(key) + .await + .expect("read") + .expect("a key on the copier's list must be readable from the environment"); + assert_eq!(ant_node::client::compute_address(&served), *key); + } + + // And nothing the store claims is unservable, from either side of the union. + for key in store.all_keys().await.expect("all_keys") { + assert!( + matches!(store.get(&key).await, Ok(Some(_))), + "chunk {} is claimed after a crash but cannot be served", + hex::encode(key) + ); + } +} + +/// A retirement killed after the mark is finished on the next start, never reopened. +/// +/// The most destructive moment in the whole migration. By the time the mark is written the +/// environment has been renamed aside and the node has already told the network it serves +/// those chunks from the file store. A start that put the directory back would leave the +/// node running two stores again with the disk it came here to free still spent; a start +/// that deleted an *unmarked* directory would destroy a live environment. The mark is what +/// separates the two, and it is written by the process that then dies. +/// +/// Its recovery has unit tests that plant the mark by hand. What those cannot show is that +/// the production path really writes the mark at that moment, before anything is deleted +/// and by the process that then dies. That is what this settles. It does not settle +/// durability: killing a process keeps the kernel page cache, so surviving a kill is not +/// surviving a power cut, which stays a fleet gate. +#[tokio::test] +async fn a_retirement_killed_after_the_mark_is_finished_not_reopened() { + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + let seeded = seed_legacy_from(&root, 0).await; + + kill_child_at_failpoint( + "child_retires_until_killed", + &root, + ant_node::storage::file_store::HALT_AFTER_RETIRE_MARK, + 0, + ); + + // The child died with the directory renamed aside and marked. Nothing had been + // deleted, so this is the state a power cut would leave behind. + let store = reopen(&root).await; + for _ in 0..600 { + if !store.legacy_dir_is_on_disk() && tombstones(&root) == 0 { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + assert!( + !store.legacy_dir_is_on_disk(), + "the marked environment was put back rather than finished" + ); + assert_eq!( + tombstones(&root), + 0, + "the marked directory is still on disk, so its space was never returned" + ); + + // And every chunk the environment held is still served, from the file store. + for (n, key) in &seeded { + let served = store + .get(key) + .await + .expect("read") + .expect("a chunk must survive an interrupted retirement"); + assert_eq!(served, chunk_bytes(*n), "chunk {n} came back wrong"); + } +} + +/// Directories beside the live environment that a retirement left behind. +fn tombstones(root: &Path) -> usize { + let Ok(entries) = std::fs::read_dir(root) else { + return 0; + }; + entries + .flatten() + .filter(|entry| { + entry.file_type().is_ok_and(|kind| kind.is_dir()) + && entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with(ant_node::storage::LEGACY_ENV_DIR)) + }) + .count() +} diff --git a/tests/migration_reclaims_disk.rs b/tests/migration_reclaims_disk.rs new file mode 100644 index 00000000..009e7334 --- /dev/null +++ b/tests/migration_reclaims_disk.rs @@ -0,0 +1,390 @@ +//! Proof that the migration actually returns disk to the filesystem. +//! +//! This is the claim the whole change exists to make good, and until now it was the one +//! thing the test suite did not check. The unit tests prove the environment is *removed*; +//! that is not the same as the space coming back, which is exactly the mistake that +//! started this work. The fleet deleted 2.29 million chunks, every counter said the +//! chunks were gone, and not one byte returned to the filesystem, because LMDB moves +//! freed pages to its own free list and never shortens the file. +//! +//! So these tests measure the filesystem, not the store's opinion of itself: the size of +//! the data on disk before and after, and the free space the operating system reports. +//! +//! They run on every platform CI covers, which is also the filesystem matrix that matters +//! here: ext4 on Linux, APFS on macOS, NTFS on Windows. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::missing_panics_doc, + // Test fixtures: every cast here is of a bounded loop counter into a byte, and the + // wrap is what makes the fill vary. + clippy::cast_possible_truncation +)] + +use ant_node::storage::migration::{MigrationPhase, MIN_RETIRE_DELAY_HOURS}; +use ant_node::storage::{ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; +use std::path::Path; +use std::sync::Arc; +use tempfile::TempDir; +use tokio_util::sync::CancellationToken; + +/// Chunks to plant. Enough that the environment is meaningfully larger than its own +/// overhead, so "the file shrank" cannot be an accounting artefact. +const CHUNKS: usize = 400; + +/// Bytes per chunk. +/// +/// Sized against the noise, not against the chunk. This test reads what the *filesystem* +/// says is free, which on a shared runner moves for reasons that have nothing to do with +/// it: another job's build, a package cache, an indexer. At 16 KiB a chunk the whole +/// environment came to about 14 MB and the recovery threshold to about 7 MB, which +/// ordinary runner activity can swallow. At 128 KiB it is an order of magnitude clear of +/// that, and 400 chunks still write in a few seconds. +const CHUNK_BYTES: usize = 128 * 1024; + +/// Blocks actually allocated under `path`, in bytes, following no links. +/// +/// Allocated blocks rather than file lengths. A length is what the file claims; blocks +/// are what the filesystem has handed out, and the two part company exactly where this +/// test needs to be careful: a sparse file, a file whose last block is mostly padding, or +/// a file that has been unlinked while something still holds it open. +#[cfg(unix)] +fn allocated_bytes(path: &Path) -> u64 { + use std::os::unix::fs::MetadataExt; + walk(path, &|meta| meta.blocks() * 512) +} + +/// Off Unix, the length is the best the standard library offers. +#[cfg(not(unix))] +fn allocated_bytes(path: &Path) -> u64 { + walk(path, &|meta| meta.len()) +} + +/// Sum `size` over everything under `path`. +fn walk(path: &Path, size: &dyn Fn(&std::fs::Metadata) -> u64) -> u64 { + let Ok(entries) = std::fs::read_dir(path) else { + return std::fs::symlink_metadata(path).map_or(0, |m| size(&m)); + }; + entries + .flatten() + .map(|entry| { + let path = entry.path(); + match std::fs::symlink_metadata(&path) { + Ok(meta) if meta.is_dir() => walk(&path, size), + Ok(meta) => size(&meta), + Err(_) => 0, + } + }) + .sum() +} + +/// What the filesystem says is free, right now. +/// +/// The measurement that cannot be argued with, and the one this test exists for. A path +/// disappearing proves nothing: unlink a file that something still holds open and every +/// name is gone while every block is still spoken for, which is a fair description of the +/// bug that started all this. +fn free_space(path: &Path) -> u64 { + fs2::available_space(path).expect("the filesystem should report its free space") +} + +/// Deterministic content for chunk `n`, filled so it does not compress to nothing. +/// +/// `n` goes in verbatim at the front rather than being folded into the fill, because a +/// fill that wraps makes two different `n` produce the same bytes, and content-addressed +/// storage would then hold one chunk where the test believed it held two. The first +/// version of this test did exactly that and undercounted by a third. +fn chunk_bytes(n: usize) -> Vec { + let mut content = vec![0u8; CHUNK_BYTES]; + content[..8].copy_from_slice(&(n as u64).to_le_bytes()); + for (i, byte) in content.iter_mut().enumerate().skip(8) { + *byte = ((i.wrapping_mul(31)).wrapping_add(n) % 251) as u8; + } + content +} + +/// Plant a legacy environment holding `CHUNKS` chunks and close it. +async fn seed_legacy_environment(root: &Path) -> Vec<[u8; 32]> { + let lmdb = LmdbStorage::new(LmdbStorageConfig { + root_dir: root.to_path_buf(), + verify_on_read: true, + max_map_size: 0, + disk_reserve: 0, + }) + .await + .expect("open the legacy environment"); + + let mut keys = Vec::with_capacity(CHUNKS); + for n in 0..CHUNKS { + let content = chunk_bytes(n); + let address = ant_node::client::compute_address(&content); + lmdb.put(&address, &content).await.expect("seed a chunk"); + keys.push(address); + } + lmdb.wait_idle().await; + keys +} + +/// A store configured to migrate promptly, so a test does not wait out real delays. +fn migrating_config(root: &Path) -> ChunkStoreConfig { + let mut config = ChunkStoreConfig { + root_dir: root.to_path_buf(), + disk_reserve: 0, + ..ChunkStoreConfig::default() + }; + config.migration.tick_secs = 1; + config.migration.copier_throttle_mib_per_sec = 0; + config.migration.copier_slack_mb = 0; + config.migration.lock_dir = Some(root.to_path_buf()); + config +} + +/// Take a settled store all the way through retirement. +/// +/// Drives the steps the driver would, rather than running the driver, so the test does +/// not depend on wall-clock gates it has no business waiting for. The gates themselves +/// are covered by their own tests; what this one is about is the disk. +async fn copy_everything(store: &Arc, keys: &[[u8; 32]]) { + store + .copy_batch(keys, 0, 0, &CancellationToken::new()) + .await + .expect("copy every chunk into the file store"); + assert!( + store.legacy_only_keys().is_empty(), + "every chunk should have been copied" + ); + store.wait_idle().await; +} + +/// Take an already-copied store through retirement, returning the bytes it freed. +/// +/// Separate from the copying so a caller can measure the disk in between, at the peak +/// where both stores hold everything. That is the moment a node is most at risk of +/// filling up, and measuring only the ends would miss it. +async fn retire(store: &Arc) -> u64 { + let shutdown = CancellationToken::new(); + + store + .commit_to_files() + .expect("commit to the file-backed set"); + store.note_commitment_rebuilt(); + store.note_commitment_rebuilt(); + store.force_migration_state(|state| { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + state.committed_at_unix = Some(now.saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); + }); + + let proof = store + .verify_before_retire(0, &shutdown) + .await + .expect("verify before retiring"); + assert!(proof.is_clean(), "verification must pass: {proof:?}"); + + store + .retire_legacy( + &proof, + &|_: &[u8; 32]| false, + &std::collections::BTreeSet::new(), + ) + .await + .expect("retire the legacy environment") +} + +/// The bytes the legacy environment occupied come back to the filesystem. +/// +/// Three measurements, because only the third one settles it: what the filesystem says is +/// free before anything is written, at the peak when both stores hold everything, and +/// after the environment is gone. A test that only watched paths disappear would pass +/// while every block stayed allocated, which is a fair description of the bug that +/// started all this. +/// +/// The numbers are noisy on a shared machine, so the assertion is about the shape: the +/// peak is materially below the start, and the end recovers most of the way back to it. +#[tokio::test] +async fn retiring_the_legacy_environment_returns_its_bytes_to_the_filesystem() { + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + let payload = (CHUNKS * CHUNK_BYTES) as u64; + // What the reading drifts by here, with nothing of ours happening. Printed rather + // than asserted on: it is what tells whoever reads a failure whether the space did not + // come back or the machine was simply busy. + let quiet = free_space(&root); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + let drift = quiet.abs_diff(free_space(&root)); + + let free_at_start = free_space(&root); + + let keys = seed_legacy_environment(&root).await; + let environment = root.join("chunks.mdb"); + let environment_blocks = allocated_bytes(&environment); + assert!( + environment_blocks >= payload, + "the seeded environment should have at least the chunk bytes allocated, has \ + {environment_blocks}" + ); + + let store = Arc::new( + ChunkStore::new(migrating_config(&root)) + .await + .expect("open the store"), + ); + assert!(store.has_legacy()); + + // Both stores hold everything: the peak, and the moment a node is most at risk of + // filling its disk. + copy_everything(&store, &keys).await; + let free_at_peak = free_space(&root); + assert!( + free_at_peak < free_at_start, + "holding both copies should have consumed disk" + ); + + let freed = retire(&store).await; + store.wait_idle().await; + assert_eq!(store.migration_phase(), MigrationPhase::FilesOnly); + assert!(freed > 0, "retirement reported no bytes freed"); + + // The deletion runs on a detached thread so the node can serve while it happens. + for _ in 0..400 { + if !environment.exists() && allocated_bytes(&root) < environment_blocks + payload { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + !environment.exists(), + "the environment directory is still on disk" + ); + + // Dropping the store closes every handle. A file that is unlinked while something + // still holds it open keeps its blocks and shows in no directory, so measuring before + // this point would be measuring the wrong thing. + drop(store); + + // Polled rather than sampled once after a fixed pause. Not every filesystem updates + // its accounting the instant a file goes: btrfs in particular defers it, and a single + // reading taken too early says the space never came back when it is on its way. The + // deadline is what makes this a test rather than a wait. + let free_at_end = wait_for_space(&root, free_at_peak + environment_blocks / 2).await; + + // Only the file store's copy should be left. + let left_on_disk = allocated_bytes(&root); + let file_store_blocks = allocated_bytes(&root.join("chunks")); + let recovered = free_at_end.saturating_sub(free_at_peak); + assert!( + left_on_disk <= file_store_blocks + (payload / 10), + "something other than the file store is still using disk: {left_on_disk} total \ + against {file_store_blocks} in the file store" + ); + + // And the filesystem agrees, measured against the peak rather than against a guess. + // Retiring should hand back most of what the environment was occupying, which makes + // this a statement about the environment's own size rather than about the payload. + assert!( + recovered > environment_blocks / 2, + "retiring recovered {recovered} bytes of an environment occupying \ + {environment_blocks}" + ); + + // And what is left costs roughly one copy rather than two, measured against what the + // file store actually occupies rather than against what the environment did. The two + // are not interchangeable: how much a filesystem spends on four hundred small files + // against one large one is its own business, and btrfs in particular charges very + // differently for the two. Printed as well as asserted, so a number that is drifting + // shows up in the log before it trips anything. + let consumed = free_at_start.saturating_sub(free_at_end); + println!( + "reclaim: environment {environment_blocks} bytes, file store {file_store_blocks}, \ + peak cost {}, end cost {consumed}, recovered {recovered}, ambient drift {drift} \ + in half a second", + free_at_start.saturating_sub(free_at_peak) + ); + assert!( + consumed < file_store_blocks + environment_blocks / 2, + "the filesystem is still down {consumed} bytes with only {file_store_blocks} of \ + file store to account for it, so the environment's space did not come back" + ); + + // Every chunk is still served, read back through a store opened from scratch, which + // is what a restart does. Space recovered by losing data would be no achievement, and + // it is the failure this whole change exists to avoid. + let fresh = store_reopened(&root).await; + for (n, key) in keys.iter().enumerate() { + let served = fresh + .get(key) + .await + .expect("read a migrated chunk") + .expect("a migrated chunk should still be there"); + assert_eq!(served, chunk_bytes(n), "chunk {n} came back wrong"); + } +} + +/// Wait for the filesystem to report at least `wanted` bytes free, and return what it +/// reports at the end. +/// +/// Returns whatever it last saw when the deadline passes, so the caller's assertion is +/// what fails rather than this helper, and the number in the failure is a real reading. +async fn wait_for_space(path: &Path, wanted: u64) -> u64 { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); + loop { + let free = free_space(path); + if free >= wanted || std::time::Instant::now() > deadline { + return free; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } +} + +/// Open the store again from scratch, which is what a restart does. +async fn store_reopened(root: &Path) -> ChunkStore { + ChunkStore::new(ChunkStoreConfig { + root_dir: root.to_path_buf(), + disk_reserve: 0, + ..ChunkStoreConfig::default() + }) + .await + .expect("the store must reopen after retirement") +} + +/// The file store holds the same payload in less space than the environment did. +/// +/// Not a compression claim: it is that one file per chunk carries no free list and no +/// map overhead, which is the whole reason the space can be returned at all. +#[tokio::test] +async fn the_file_store_holds_the_same_chunks_in_less_space() { + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + let keys = seed_legacy_environment(&root).await; + let environment_bytes = allocated_bytes(&root.join("chunks.mdb")); + + let store = Arc::new( + ChunkStore::new(migrating_config(&root)) + .await + .expect("open the store"), + ); + store + .copy_batch(&keys, 0, 0, &CancellationToken::new()) + .await + .expect("copy"); + store.wait_idle().await; + + let payload = (CHUNKS * CHUNK_BYTES) as u64; + let file_store_bytes = allocated_bytes(&root.join("chunks")); + assert!( + file_store_bytes >= payload, + "the file store should hold at least the payload: {file_store_bytes} < {payload}" + ); + assert!( + file_store_bytes <= environment_bytes, + "one file per chunk should not cost more than the environment did: \ + {file_store_bytes} > {environment_bytes}" + ); +} diff --git a/tests/migration_shared_volume.rs b/tests/migration_shared_volume.rs new file mode 100644 index 00000000..8ff79791 --- /dev/null +++ b/tests/migration_shared_volume.rs @@ -0,0 +1,512 @@ +//! Several nodes migrating on one disk. +//! +//! Operators run many nodes per machine, and during the bridge each one briefly holds two +//! copies of everything it stores. If they all did that at once the disk would fill, which +//! is the failure this migration exists to prevent rather than cause. A lock keyed by the +//! filesystem lets one node at a time do the copying. +//! +//! The lock has a cap on how long a single node may hold it, so one node stuck waiting on +//! its neighbours cannot keep the rest of the machine from ever starting. That cap is +//! hours long by design, so what is checked here is the exclusion itself and the +//! accounting around it, not the cap expiring. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::missing_panics_doc, + // Test fixtures: every cast here is of a bounded loop counter into a byte, and the + // wrap is what makes the fill vary. + clippy::cast_possible_truncation +)] + +use ant_node::storage::migration::{self, LockAttempt, VolumeLock, MIN_RETIRE_DELAY_HOURS}; +use ant_node::storage::{ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; +use tempfile::TempDir; +use tokio_util::sync::CancellationToken; + +/// Chunks each node holds. +const CHUNKS: usize = 60; + +fn chunk_bytes(node: usize, n: usize) -> Vec { + let mut content = vec![0u8; 8192]; + content[..8].copy_from_slice(&(n as u64).to_le_bytes()); + content[8..16].copy_from_slice(&(node as u64).to_le_bytes()); + for (i, byte) in content.iter_mut().enumerate().skip(16) { + *byte = ((i.wrapping_mul(29)).wrapping_add(n).wrapping_add(node) % 251) as u8; + } + content +} + +async fn seed_legacy(root: &Path, node: usize) -> Vec<[u8; 32]> { + let lmdb = LmdbStorage::new(LmdbStorageConfig { + root_dir: root.to_path_buf(), + verify_on_read: true, + max_map_size: 0, + disk_reserve: 0, + }) + .await + .expect("open legacy"); + let mut keys = Vec::new(); + for n in 0..CHUNKS { + let content = chunk_bytes(node, n); + let address = ant_node::client::compute_address(&content); + lmdb.put(&address, &content).await.expect("seed"); + keys.push(address); + } + lmdb.wait_idle().await; + keys +} + +/// One node at a time copies; the others wait rather than piling on. +/// +/// The lock is taken per filesystem, not per node directory. That distinction is the +/// whole point: two nodes are configured with different roots by definition, so a lock +/// beside each root would serialise neither against the other. +#[test] +fn only_one_node_on_a_volume_holds_the_lock() { + let volume = TempDir::new().expect("temp dir"); + let node_a = volume.path().join("node-a"); + let node_b = volume.path().join("node-b"); + let node_c = volume.path().join("node-c"); + for root in [&node_a, &node_b, &node_c] { + std::fs::create_dir_all(root).expect("mkdir"); + } + + // Scoped to this volume directory so the test does not contend with anything else on + // the machine's real filesystem. + let scope = Some(volume.path()); + + let LockAttempt::Acquired(held) = VolumeLock::try_acquire(&node_a, scope) else { + panic!("the first node must take the lock"); + }; + assert!( + matches!(VolumeLock::try_acquire(&node_b, scope), LockAttempt::Busy), + "a second node on the same volume must wait" + ); + assert!( + matches!(VolumeLock::try_acquire(&node_c, scope), LockAttempt::Busy), + "and so must a third" + ); + + drop(held); + assert!( + matches!( + VolumeLock::try_acquire(&node_b, scope), + LockAttempt::Acquired(_) + ), + "the lock must pass on once the first node lets go" + ); +} + +/// A migration context with no network, which is all these tests need. +/// +/// The gates that consult routing have their own tests; what is under test here is the +/// lock, and a node with no view of the network still copies. +fn offline_context() -> migration::MigrationContext { + migration::MigrationContext { + p2p: None, + self_id: None, + self_xor: None, + commitment: None, + replication: None, + sync_state: None, + audit_challenge_coordinator: None, + peer_commitments: None, + close_group_size: 7, + } +} + +/// Two migration drivers on one disk: only one copies at a time. +/// +/// This drives `migration::run`, not `copy_batch`. The copier does not take the volume +/// lock; the driver does, and an earlier version of this test called the copier directly +/// and would have passed with the lock removed from the driver entirely. +#[tokio::test] +async fn two_drivers_on_one_volume_do_not_copy_at_the_same_time() { + let volume = TempDir::new().expect("temp dir"); + let mut stores = Vec::new(); + let mut all_keys = Vec::new(); + + for node in 0..2 { + let root = volume.path().join(format!("node-{node}")); + std::fs::create_dir_all(&root).expect("mkdir"); + let keys = seed_legacy(&root, node).await; + + stores.push(driven_node(volume.path(), &root).await); + all_keys.push(keys); + } + + // Hold the volume before either driver starts, so both are shut out and neither can + // be observed making progress. + let LockAttempt::Acquired(held) = + VolumeLock::try_acquire(&volume.path().join("an-outsider"), Some(volume.path())) + else { + panic!("the outsider must take the lock"); + }; + + let shutdown = CancellationToken::new(); + let drivers: Vec<_> = stores + .iter() + .map(|store| { + tokio::spawn(migration::run( + Arc::clone(store), + offline_context(), + shutdown.clone(), + )) + }) + .collect(); + + // Long enough for several ticks. Neither driver may copy anything while the lock is + // held by somebody else. + tokio::time::sleep(Duration::from_secs(4)).await; + for (node, store) in stores.iter().enumerate() { + assert_eq!( + store.legacy_only_keys().len(), + CHUNKS, + "node {node} copied while another holder had the volume" + ); + } + + // Released: one of them takes it and copies. The other must not, because the holder + // keeps the volume from its first copy through to retiring, rather than handing it + // back between chunks. That is the point of the lock: two nodes copying at once each + // hold two copies of everything, and the disk this migration exists to free is the + // one that fills. + drop(held); + let mut copier = None; + for _ in 0..300 { + if let Some(node) = stores + .iter() + .position(|s| s.legacy_only_keys().len() < CHUNKS) + { + copier = Some(node); + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + let copier = copier.expect("one driver should have taken the volume and started"); + + // Give the other one many ticks to misbehave in. + tokio::time::sleep(Duration::from_secs(3)).await; + let waiting = 1 - copier; + assert_eq!( + stores[waiting].legacy_only_keys().len(), + CHUNKS, + "node {waiting} copied while node {copier} held the volume" + ); + + // And the one that has it finishes copying, checking on every tick that the other has + // still not started. The window being watched is the whole of the first node's copy + // rather than its two ends. + // + // Copying is as far as this one goes. Retirement is gated behind hours of wall clock + // that a test has no business waiting out, so whether the lock spans that half too has + // its own test below. + for _ in 0..600 { + assert_eq!( + stores[waiting].legacy_only_keys().len(), + CHUNKS, + "node {waiting} copied while node {copier} still held the volume" + ); + if stores[copier].legacy_only_keys().is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + shutdown.cancel(); + for driver in drivers { + let _ = driver.await; + } + + assert!( + stores[copier].legacy_only_keys().is_empty(), + "the node holding the volume did not finish copying" + ); + // Both of them, not just the one that went first. Counting only the copier would pass + // for a node that had picked up its neighbour's chunks as well as its own. + for (node, store) in stores.iter().enumerate() { + holds_exactly_its_own(store, node, &all_keys[node]).await; + } +} + +/// A node set up to be driven by `migration::run` on a shared volume. +async fn driven_node(volume: &Path, root: &Path) -> Arc { + let mut config = ChunkStoreConfig { + root_dir: root.to_path_buf(), + disk_reserve: 0, + ..ChunkStoreConfig::default() + }; + config.migration.tick_secs = 1; + config.migration.copier_throttle_mib_per_sec = 0; + config.migration.copier_slack_mb = 0; + // Small enough that copying takes several ticks, so there is a window in which the + // other node could misbehave and be caught, and large enough that the whole thing + // finishes in seconds rather than one chunk per tick. + config.migration.batch_chunks = 8; + config.migration.lock_dir = Some(volume.to_path_buf()); + Arc::new(ChunkStore::new(config).await.expect("open a node")) +} + +/// Every chunk this node seeded is still served, and nothing else is. +async fn holds_exactly_its_own(store: &ChunkStore, node: usize, keys: &[[u8; 32]]) { + assert_eq!( + store.current_chunks().expect("count") as usize, + keys.len(), + "node {node} must hold its own chunks and only its own" + ); + for (n, key) in keys.iter().enumerate() { + let served = store + .get(key) + .await + .expect("read") + .expect("every chunk this node seeded must still be here"); + assert_eq!( + served, + chunk_bytes(node, n), + "node {node} chunk {n} is wrong" + ); + } +} + +/// A node waits for the volume before it retires, not only before it copies. +/// +/// The driver is documented as holding the volume from the first copy through retirement +/// and not handing it back in between. The test above covers the copying half. This one +/// covers the other, which is the half that matters most: retiring means re-reading every +/// chunk in the store to verify it and then deleting an environment, so it is the heaviest +/// the disk gets. A driver that took the lock only for copying would run that pass while +/// eleven neighbours ran theirs. +/// +/// Shaped the same way as the copying test, and for the same reason: an outsider holds the +/// volume first, so the answer does not depend on catching a short window. The node is put +/// in the phase where retirement is the next thing it would do, and then watched for not +/// doing it. +#[tokio::test] +async fn a_node_waits_for_the_volume_before_it_retires() { + let volume = TempDir::new().expect("temp dir"); + let root = volume.path().join("node-0"); + std::fs::create_dir_all(&root).expect("mkdir"); + let keys = seed_legacy(&root, 0).await; + + let store = ready_to_retire(volume.path(), &root, &keys).await; + let shutdown = CancellationToken::new(); + + let LockAttempt::Acquired(held) = + VolumeLock::try_acquire(&volume.path().join("an-outsider"), Some(volume.path())) + else { + panic!("the outsider must take the lock"); + }; + + let driver = tokio::spawn(migration::run( + Arc::clone(&store), + offline_context(), + shutdown.clone(), + )); + + // Several ticks with everything else in place. The environment must still be there. + for _ in 0..40 { + assert!( + store.legacy_dir_is_on_disk(), + "the node retired while an outsider held the volume" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + + // Released: now it retires. Without this half the test would pass against a node that + // never retires at all, which is the failure the whole migration exists to avoid. + drop(held); + let mut retired = false; + for _ in 0..600 { + if !store.legacy_dir_is_on_disk() && !store.has_legacy() { + retired = true; + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + shutdown.cancel(); + let _ = driver.await; + assert!( + retired, + "the node never retired once the volume was free, so it was not waiting for it" + ); + + // And it still holds everything it had. Retiring is deleting the old copy, not the + // chunks. + for (n, key) in keys.iter().enumerate() { + let served = store + .get(key) + .await + .expect("read") + .expect("every chunk must survive retirement"); + assert_eq!( + served, + chunk_bytes(0, n), + "chunk {n} is wrong after retiring" + ); + } +} + +/// Open a node whose only remaining migration work is to retire. +/// +/// Copied and committed by hand rather than by waiting for the driver, because the driver +/// gets here by waiting out the shed hold, which is days. Those gates have their own +/// tests; what the caller is about to watch is the volume lock. +async fn ready_to_retire(volume: &Path, root: &Path, keys: &[[u8; 32]]) -> Arc { + let mut config = ChunkStoreConfig { + root_dir: root.to_path_buf(), + disk_reserve: 0, + ..ChunkStoreConfig::default() + }; + config.migration.tick_secs = 1; + config.migration.copier_throttle_mib_per_sec = 0; + config.migration.copier_slack_mb = 0; + config.migration.lock_dir = Some(volume.to_path_buf()); + let store = Arc::new(ChunkStore::new(config).await.expect("open a node")); + + store + .copy_batch(keys, 0, 0, &CancellationToken::new()) + .await + .expect("copy every chunk"); + store.wait_idle().await; + store.commit_to_files().expect("commit to the file set"); + store.note_commitment_rebuilt(); + store.note_commitment_rebuilt(); + store.force_migration_state(|state| { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + // Past the delay that buys the rollback window, so the only thing left between + // this node and deleting its environment is the volume. + state.committed_at_unix = Some(now.saturating_sub(MIN_RETIRE_DELAY_HOURS * 3600 + 60)); + }); + assert!( + store.legacy_dir_is_on_disk(), + "the environment should still be here before the driver runs" + ); + store +} + +/// Two nodes sharing a disk both finish, and neither loses a chunk to the other. +/// +/// Run one after the other, which is what the lock produces. What is checked is that the +/// second node's copy is unaffected by the first having already run on the same +/// filesystem: no shared state, no name collisions, no lock left behind. +/// +/// Copying only. It drives `copy_batch` rather than the driver, so it says nothing about +/// retirement and is not named as if it did: disabling retirement altogether would leave +/// it green. Retirement on a shared volume is +/// [`a_node_waits_for_the_volume_before_it_retires`]. +#[tokio::test] +async fn nodes_sharing_a_volume_do_not_take_each_others_chunks() { + let volume = TempDir::new().expect("temp dir"); + let shutdown = CancellationToken::new(); + + let mut nodes = Vec::new(); + for node in 0..2 { + let root = volume.path().join(format!("node-{node}")); + std::fs::create_dir_all(&root).expect("mkdir"); + let keys = seed_legacy(&root, node).await; + nodes.push((root, keys)); + } + + for (node, (root, keys)) in nodes.iter().enumerate() { + let mut config = ChunkStoreConfig { + root_dir: root.clone(), + disk_reserve: 0, + ..ChunkStoreConfig::default() + }; + config.migration.lock_dir = Some(volume.path().to_path_buf()); + let store = ChunkStore::new(config).await.expect("open"); + + store.copy_batch(keys, 0, 0, &shutdown).await.expect("copy"); + store.wait_idle().await; + + assert!( + store.legacy_only_keys().is_empty(), + "node {node} should have copied everything" + ); + for (n, key) in keys.iter().enumerate() { + let served = store + .get(key) + .await + .expect("read") + .expect("every chunk this node seeded must still be here"); + assert_eq!( + served, + chunk_bytes(node, n), + "node {node} chunk {n} came back wrong" + ); + } + } + + // Neither node picked up the other's chunks, which sharing a filesystem must not + // cause: the stores are separate, only the lock is shared. + let (root_a, keys_a) = &nodes[0]; + let store_a = ChunkStore::new(ChunkStoreConfig { + root_dir: root_a.clone(), + disk_reserve: 0, + ..ChunkStoreConfig::default() + }) + .await + .expect("reopen node 0"); + assert_eq!( + store_a.current_chunks().expect("count") as usize, + keys_a.len(), + "a node must hold its own chunks and only its own" + ); +} + +/// A node that cannot take the lock does not migrate, and does not lose anything either. +/// +/// Waiting is the correct answer: the chunks stay where they are, served from both stores, +/// until the volume is free. +#[tokio::test] +async fn a_node_that_cannot_take_the_lock_keeps_serving() { + let volume = TempDir::new().expect("temp dir"); + let root = volume.path().join("waiting-node"); + std::fs::create_dir_all(&root).expect("mkdir"); + let keys = seed_legacy(&root, 0).await; + + let LockAttempt::Acquired(_held) = + VolumeLock::try_acquire(&volume.path().join("busy-node"), Some(volume.path())) + else { + panic!("the other node must take the lock"); + }; + + let store = driven_node(volume.path(), &root).await; + + // A real driver, running the whole time. Without one this would say only that a store + // opens, and would still pass against a driver that ignored the lock entirely. + let shutdown = CancellationToken::new(); + let driver = tokio::spawn(migration::run( + Arc::clone(&store), + offline_context(), + shutdown.clone(), + )); + + // The store opens and serves regardless of the lock: only the copier waits for it. + assert!(store.has_legacy()); + for _ in 0..30 { + assert_eq!( + store.legacy_only_keys().len(), + CHUNKS, + "the node copied while another held the volume" + ); + for (n, key) in keys.iter().enumerate() { + let served = store + .get(key) + .await + .expect("read") + .expect("a node waiting for the volume still serves everything it holds"); + assert_eq!(served, chunk_bytes(0, n)); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + + shutdown.cancel(); + let _ = driver.await; +} diff --git a/tests/poc_audit_handler_live.rs b/tests/poc_audit_handler_live.rs index 03e865b7..5f970a08 100644 --- a/tests/poc_audit_handler_live.rs +++ b/tests/poc_audit_handler_live.rs @@ -6,7 +6,7 @@ //! `poc_commitment_audit_attacks`. This file fills the remaining gap: the //! *live* responder control-flow branches in //! [`ant_node::replication::storage_commitment_audit::handle_subtree_challenge`] — the function the -//! network actually calls — driven against a real `LmdbStorage` and a real +//! network actually calls — driven against a real `ChunkStore` and a real //! `ResponderCommitmentState`, asserting on the exact `SubtreeAuditResponse` //! variant produced. //! @@ -40,7 +40,7 @@ use ant_node::replication::storage_commitment_audit::{ handle_subtree_challenge, handle_subtree_challenge_measured, handle_subtree_slice_challenge, }; use ant_node::replication::subtree::{verify_subtree_proof, StructureVerdict}; -use ant_node::storage::{LmdbStorage, LmdbStorageConfig}; +use ant_node::storage::{ChunkStore, ChunkStoreConfig}; use saorsa_core::identity::PeerId; use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey}; use tempfile::TempDir; @@ -49,13 +49,13 @@ use tempfile::TempDir; // Fixtures // --------------------------------------------------------------------------- -async fn test_storage() -> (LmdbStorage, TempDir) { +async fn test_storage() -> (ChunkStore, TempDir) { let temp_dir = TempDir::new().expect("create temp dir"); - let config = LmdbStorageConfig { + let config = ChunkStoreConfig { root_dir: temp_dir.path().to_path_buf(), - ..LmdbStorageConfig::test_default() + ..ChunkStoreConfig::test_default() }; - let storage = LmdbStorage::new(config).await.expect("create storage"); + let storage = ChunkStore::new(config).await.expect("create storage"); (storage, temp_dir) } @@ -81,7 +81,7 @@ impl Responder { /// Build a responder that has stored `indices` and committed to them. /// The committed leaf binds `(address, BLAKE3(content))`; the responder /// reads bytes by address at audit time and rehashes them. - async fn new(storage: &LmdbStorage, indices: &[u8]) -> Self { + async fn new(storage: &ChunkStore, indices: &[u8]) -> Self { let (pk, sk) = keypair(); // Production identity derivation: peer_id == BLAKE3(pubkey_bytes). let peer_id_bytes = *blake3::hash(&pk.to_bytes()).as_bytes(); @@ -90,7 +90,7 @@ impl Responder { let mut entries = Vec::new(); for &i in indices { let content = chunk_content(i); - let addr = LmdbStorage::compute_address(&content); + let addr = ChunkStore::compute_address(&content); storage.put(&addr, &content).await.expect("put chunk"); let bytes_hash = *blake3::hash(&content).as_bytes(); entries.push((addr, bytes_hash)); @@ -112,7 +112,7 @@ impl Responder { } fn address(i: u8) -> [u8; 32] { - LmdbStorage::compute_address(&chunk_content(i)) + ChunkStore::compute_address(&chunk_content(i)) } } @@ -739,7 +739,7 @@ async fn slice_challenge_opens_a_deep_block_of_a_large_chunk() { let content: Vec = (0..100_000u32) .map(|n| (n.wrapping_mul(2_654_435_761) >> 13) as u8) .collect(); - let addr = LmdbStorage::compute_address(&content); + let addr = ChunkStore::compute_address(&content); storage.put(&addr, &content).await.expect("put chunk"); let bytes_hash = *blake3::hash(&content).as_bytes(); diff --git a/tests/poc_shutdown_lmdb_drain.rs b/tests/poc_shutdown_lmdb_drain.rs index 699765fb..135e8001 100644 --- a/tests/poc_shutdown_lmdb_drain.rs +++ b/tests/poc_shutdown_lmdb_drain.rs @@ -15,7 +15,7 @@ //! //! ## The fix //! -//! `LmdbStorage` and `PaidList` track their blocking tasks in a +//! `ChunkStore` (via its file store) and `PaidList` track their blocking tasks in a //! `TaskTracker`; `shutdown()` awaits `wait_idle()` on both after draining //! its own tasks. This test parks a chunk-store write inside its blocking //! closure, drops the awaiter (the exact leak shape), and asserts that @@ -33,7 +33,7 @@ use ant_node::payment::{ EvmVerifierConfig, PaymentVerifier, PaymentVerifierConfig, PriceFloorConfig, }; use ant_node::replication::paid_list::PaidList; -use ant_node::storage::{LmdbStorage, LmdbStorageConfig}; +use ant_node::storage::{ChunkStore, ChunkStoreConfig}; use ant_node::{ReplicationConfig, ReplicationEngine}; use evmlib::{Network as EvmNetwork, RewardsAddress}; use rand::Rng; @@ -95,9 +95,9 @@ async fn shutdown_waits_for_detached_lmdb_op_and_envs_reopen() { // The chunk store the engine will hold (and whose env we reopen below). let storage = Arc::new( - LmdbStorage::new(LmdbStorageConfig { + ChunkStore::new(ChunkStoreConfig { root_dir: root_dir.clone(), - ..LmdbStorageConfig::test_default() + ..ChunkStoreConfig::test_default() }) .await .expect("create storage"), @@ -136,7 +136,7 @@ async fn shutdown_waits_for_detached_lmdb_op_and_envs_reopen() { // mid-flight — the exact shape of a select! losing to the shutdown token // while `storage.put()` awaits `spawn_blocking`. let content = b"held-open write must block engine shutdown"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); let gate = storage.test_put_gate(); let parked = gate.write(); tokio::select! { @@ -155,7 +155,7 @@ async fn shutdown_waits_for_detached_lmdb_op_and_envs_reopen() { let blocked = tokio::time::timeout(SHUTDOWN_BLOCKED_PROBE, shutdown_fut.as_mut()).await; assert!( blocked.is_err(), - "shutdown() returned while an LMDB blocking op was in flight" + "shutdown() returned while a store write was in flight" ); // Release the write; shutdown must now run to completion. @@ -178,9 +178,9 @@ async fn shutdown_waits_for_detached_lmdb_op_and_envs_reopen() { drop(storage); // Both LMDB environments reopen cleanly from the same directory. - let reopened = LmdbStorage::new(LmdbStorageConfig { + let reopened = ChunkStore::new(ChunkStoreConfig { root_dir: root_dir.clone(), - ..LmdbStorageConfig::test_default() + ..ChunkStoreConfig::test_default() }) .await .expect("reopen chunk store"); diff --git a/tests/storage_scale.rs b/tests/storage_scale.rs new file mode 100644 index 00000000..656eedf1 --- /dev/null +++ b/tests/storage_scale.rs @@ -0,0 +1,500 @@ +//! What one file per chunk costs at scale. +//! +//! The design accepted two costs on paper and never measured either: the startup scan +//! reads every filename in the store before the node serves anything, and every chunk +//! takes an inode and a directory entry. Both grow with the store, and a node that takes +//! minutes to start, or runs a filesystem out of inodes, is a node that is down. +//! +//! These are regression gates, not benchmarks. The ceilings are generous enough that a +//! loaded shared runner does not fail them and tight enough that an order-of-magnitude +//! regression does. What they measure precisely is printed, so a number that is drifting +//! is visible in the log before it ever trips the gate. +//! +//! `ANT_SCALE_KEYS` raises the count for a deliberate larger run. The default is what a +//! hosted runner can do in reasonable time; the fleet-scale figures the ADR wants still +//! need a machine with the disk for them. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::missing_panics_doc, + // Test fixtures: every cast here is of a bounded loop counter into a byte, and the + // wrap is what makes the fill vary. + clippy::cast_possible_truncation +)] + +use ant_node::storage::{FileStore, FileStoreConfig}; +use std::path::Path; +use std::time::{Duration, Instant}; +use tempfile::TempDir; + +/// Keys to plant unless told otherwise. +const DEFAULT_KEYS: usize = 100_000; + +/// The longest a cold scan may take per chunk before this is a regression. +/// +/// Measured at 1,019 ns per key for 100,000 keys on a hosted CI runner. Fifty times that +/// leaves a slow, loaded, shared runner room to be slow while still catching a scan that +/// has gone from linear to something worse: the whole 100,000-key budget is five seconds +/// against a hundred milliseconds measured. +/// +/// Per key rather than a flat number, so that raising `ANT_SCALE_KEYS` for a larger run +/// raises the allowance with it instead of turning the gate into a coin toss. +const SCAN_CEILING_PER_KEY: Duration = Duration::from_micros(50); + +/// The most a startup scan may read, whatever the store holds. +/// +/// Fixed, deliberately, and not a fraction of the payload. A fraction grows with the +/// store, so it would keep permitting a per-chunk read as long as the chunks were big +/// enough: at the sizes below, a hundredth of the payload allowed 655 bytes per chunk, +/// which is a header read of every file in the store passing a test named for not doing +/// that. +/// +/// A scan that reads names reads the same handful of bytes whatever the store holds. +/// Measured at 125 bytes for 3,000 chunks on a hosted runner, which is the layout marker +/// and nothing else. 64 KiB is five hundred times that and still under 22 bytes per chunk +/// there, so any read that is per-chunk at all fails, and fails harder the larger the run. +#[cfg(target_os = "linux")] +const SCAN_READ_CEILING: u64 = 64 * 1024; + +/// How many keys this run should plant. +fn key_count() -> usize { + std::env::var("ANT_SCALE_KEYS") + .ok() + .and_then(|raw| raw.parse().ok()) + .unwrap_or(DEFAULT_KEYS) +} + +/// Plant `count` chunk files directly, without going through the store. +/// +/// Writing them by hand rather than through `put` is the point: this measures opening a +/// store that already holds them, which is what a restart does, not the cost of filling +/// one. +fn plant_chunks(chunks_dir: &Path, count: usize) { + for shard in 0u16..256 { + std::fs::create_dir_all(chunks_dir.join(format!("{shard:02x}"))).expect("mkdir"); + } + // One byte each. The scan reads names, never contents, so the payload would only cost + // the test disk it does not need. + for n in 0..count { + let mut address = [0u8; 32]; + address[..8].copy_from_slice(&(n as u64).to_le_bytes()); + // The shard is the last byte, so spread across all 256 rather than piling into one. + address[31] = (n % 256) as u8; + let path = chunks_dir + .join(format!("{:02x}", address[31])) + .join(hex::encode(address)); + std::fs::write(path, b"x").expect("plant a chunk"); + } +} + +/// Walk every shard under `chunks_dir`, optionally calling `metadata` on each entry. +/// +/// The control for the assertion above: the same directory, the same process, the same +/// moment, with and without the one syscall the design says the scan does not make. +fn walk(chunks_dir: &Path, stat_each: bool) -> Duration { + let started = Instant::now(); + let mut seen = 0usize; + if let Ok(shards) = std::fs::read_dir(chunks_dir) { + for shard in shards.flatten() { + let Ok(entries) = std::fs::read_dir(shard.path()) else { + continue; + }; + for entry in entries.flatten() { + // Touched so the name is not optimised away, exactly as the scan uses it. + seen += entry.file_name().as_encoded_bytes().len(); + if stat_each { + seen += usize::from(entry.metadata().is_ok()); + } + } + } + } + assert!(seen > 0, "the control walk found nothing to walk"); + started.elapsed() +} + +/// Resident memory of this process, in bytes, where the platform will say. +#[cfg(target_os = "linux")] +fn resident_bytes() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + status + .lines() + .find_map(|line| line.strip_prefix("VmRSS:")) + .and_then(|value| value.split_whitespace().next()?.parse::().ok()) + .map(|kb| kb * 1024) +} + +/// Not every platform makes this cheap to ask, and the gate below is the scan time. +#[cfg(not(target_os = "linux"))] +fn resident_bytes() -> Option { + None +} + +/// Opening a store that already holds a large number of chunks stays quick. +/// +/// This is the first thing a restarted node does and nothing is served until it finishes, +/// so it is the cost that decides whether a big node can be restarted at all. +#[tokio::test] +async fn opening_a_large_store_stays_quick() { + let keys = key_count(); + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + let chunks_dir = root.join("chunks"); + std::fs::create_dir_all(&chunks_dir).expect("mkdir"); + + let planting = Instant::now(); + plant_chunks(&chunks_dir, keys); + let planted = planting.elapsed(); + + let before = resident_bytes(); + let opening = Instant::now(); + let store = FileStore::new(FileStoreConfig { + root_dir: root.clone(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("open a store holding a large number of chunks"); + let scan = opening.elapsed(); + let after = resident_bytes(); + + let indexed = store.current_chunks().expect("count"); + assert_eq!( + indexed as usize, keys, + "the scan must find every planted chunk" + ); + + let per_key_ns = scan.as_nanos() / keys.max(1) as u128; + let growth = match (before, after) { + (Some(before), Some(after)) => format!("{} KiB", after.saturating_sub(before) / 1024), + _ => "not measured on this platform".to_string(), + }; + println!( + "scale: {keys} chunks planted in {planted:?}, scanned in {scan:?} \ + ({per_key_ns} ns/key), resident growth {growth}" + ); + + let ceiling = SCAN_CEILING_PER_KEY * u32::try_from(keys).unwrap_or(u32::MAX); + assert!( + scan < ceiling, + "scanning {keys} chunks took {scan:?} ({per_key_ns} ns/key), over the {ceiling:?} \ + ceiling" + ); + + drop(store); + + // And the scan reads names only, with no `stat` behind each one. That claim is what + // the cost above rests on, and a flat ceiling cannot settle it: one `stat` per entry + // costs about three times a bare walk, which is still far inside any ceiling loose + // enough not to flake on a shared runner. + // + // Measured against this machine instead of against a number. Two walks of the same + // directory, one reading names and one calling `metadata` on each, bracket what a scan + // of this store on this filesystem under this load costs. A scan that stats every entry + // lands at the far bracket. + // + // Three rounds, interleaved, and the median of each. One round of each would let a + // scheduling pause that happened to land on the scan and not on the walks decide the + // result: runner speed only cancels out when it moves all three together, and a + // preemption does not. Interleaving puts the three measurements next to each other in + // time and the median throws away the round that was interrupted. + let mut scans = Vec::new(); + let mut bare = Vec::new(); + let mut stats = Vec::new(); + for _ in 0..3 { + scans.push(time_a_scan(&root).await); + bare.push(walk(&chunks_dir, false)); + stats.push(walk(&chunks_dir, true)); + } + let scan = median(&mut scans); + let names_only = median(&mut bare); + let with_stat = median(&mut stats); + // Saturating, because a filesystem where a stat costs nothing would otherwise + // underflow here. On one of those the midpoint collapses onto the bare walk and this + // says little, which is the honest answer for such a filesystem. + let midpoint = names_only + with_stat.saturating_sub(names_only) / 2; + println!( + "scale: medians of three, bare walk {names_only:?} names only, {with_stat:?} with a \ + stat each, store scan {scan:?}, midpoint {midpoint:?}" + ); + assert!( + scan < midpoint, + "the scan took {scan:?}, past the {midpoint:?} midpoint between a names-only walk \ + ({names_only:?}) and one that stats every entry ({with_stat:?}), so it is doing \ + more per entry than reading a name" + ); +} + +/// Open a store at `root`, time the scan, and close it again. +async fn time_a_scan(root: &Path) -> Duration { + let started = Instant::now(); + let store = FileStore::new(FileStoreConfig { + root_dir: root.to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("reopen the store"); + let elapsed = started.elapsed(); + drop(store); + elapsed +} + +/// The middle of three, so one interrupted round does not decide anything. +fn median(samples: &mut [Duration]) -> Duration { + samples.sort_unstable(); + samples.get(samples.len() / 2).copied().unwrap_or_default() +} + +/// The index costs a bounded amount of memory per chunk. +/// +/// One inode and one directory entry per chunk is the filesystem's share, and the ADR +/// accepts it. What it did not measure is the node's own share: an in-memory set of every +/// address, which is the part that could quietly make a large node unrunnable. +/// +/// Measured in a process of its own, which is the only way this measurement means +/// anything. `VmRSS` is process-wide and the allocator hands back what earlier work freed, +/// so opening a store in a process that has already opened and dropped one grows the +/// resident set by nothing at all. That is exactly what happened here: the test read zero +/// bytes per chunk and passed, having measured the allocator rather than the index. A child +/// that has done nothing else has no freed heap to reuse. +#[cfg(target_os = "linux")] +#[tokio::test] +async fn the_in_memory_index_costs_a_bounded_amount_per_chunk() { + let keys = key_count(); + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + let chunks_dir = root.join("chunks"); + std::fs::create_dir_all(&chunks_dir).expect("mkdir"); + plant_chunks(&chunks_dir, keys); + + let per_key = index_cost_in_a_fresh_process(&root, keys); + println!("scale: index costs {per_key} bytes per chunk, measured in its own process"); + + // A 32-byte address in a sorted set, plus allocator and node overhead. Measured at 52 + // bytes per chunk on a hosted runner; 128 is comfortably above that and no longer five + // times it, which was loose enough to let an extra 128 bytes a key through unnoticed. + assert!( + per_key < 128, + "the index costs {per_key} bytes per chunk, which does not scale" + ); + // Zero is not a pass. It is what this test reported when it shared a process with one + // that had already opened and dropped a store of the same size, and it would report it + // again if the child ever stopped opening the store at all. + assert!( + per_key > 0, + "the index reported no cost at all, so nothing was measured" + ); +} + +/// Open a store of `keys` chunks in a child process and report its resident growth per key. +#[cfg(target_os = "linux")] +fn index_cost_in_a_fresh_process(root: &Path, keys: usize) -> u64 { + let exe = std::env::current_exe().expect("this test binary"); + let output = std::process::Command::new(exe) + .arg("--exact") + .arg("child_reports_index_memory") + .arg("--nocapture") + .arg("--ignored") + .env("ANT_SCALE_ROOT", root) + .env("ANT_SCALE_KEYS", keys.to_string()) + .output() + .expect("spawn the child"); + let said = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success(), + "the child failed: {said}{}", + String::from_utf8_lossy(&output.stderr) + ); + said.lines() + .find_map(|line| line.strip_prefix(INDEX_BYTES_PER_KEY)) + .and_then(|value| value.trim().parse().ok()) + .unwrap_or_else(|| panic!("the child reported no measurement: {said}")) +} + +/// What the child prints its answer behind. +#[cfg(target_os = "linux")] +const INDEX_BYTES_PER_KEY: &str = "INDEX_BYTES_PER_KEY="; + +/// Child mode: open the store named by the environment and report what it cost. +#[cfg(target_os = "linux")] +#[tokio::test] +#[ignore = "child process of the index memory measurement, not run on its own"] +async fn child_reports_index_memory() { + let root = std::path::PathBuf::from( + std::env::var("ANT_SCALE_ROOT").expect("the child needs a store to open"), + ); + let keys = key_count(); + + let before = resident_bytes().expect("linux reports this"); + let store = FileStore::new(FileStoreConfig { + root_dir: root, + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("open"); + let after = resident_bytes().expect("linux reports this"); + + assert_eq!(store.current_chunks().expect("count") as usize, keys); + let grew = after.saturating_sub(before); + println!("{INDEX_BYTES_PER_KEY}{}", grew / keys.max(1) as u64); + // Held until after the measurement is printed, so the index is still resident when it + // is read rather than freed by an early drop. + drop(store); +} + +/// Every chunk the store writes takes exactly one directory entry. +/// +/// Through `put`, not through the fixture. An earlier version planted the files itself +/// and then counted them, which proves the test can count and nothing about the store: a +/// store that wrote a sidecar beside every chunk would have passed it. +/// +/// It matters because a filesystem runs out of inodes independently of bytes, and a node +/// that fills the inode table stops accepting writes while `df` still shows free space. +#[tokio::test] +async fn each_chunk_the_store_writes_costs_one_directory_entry() { + let keys = 2_000; + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + let store = FileStore::new(FileStoreConfig { + root_dir: root.clone(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("open"); + + for n in 0..keys { + // Real content through the real path, so anything `put` writes is counted. + let mut content = vec![0u8; 512]; + content[..8].copy_from_slice(&(n as u64).to_le_bytes()); + let address = ant_node::client::compute_address(&content); + store.put(&address, &content).await.expect("put"); + } + store.wait_idle().await; + + let entries = count_entries(&root.join("chunks")); + assert_eq!( + entries.files, keys, + "the store wrote {} files for {keys} chunks", + entries.files + ); + // 256 shards and the layout marker are the fixed overhead; nothing should be + // proportional to the chunk count but the chunks themselves. + assert!( + entries.dirs <= 256, + "the store made {} directories, which grows with the store", + entries.dirs + ); +} + +/// Files and directories under a path, counted rather than summed. +struct Entries { + files: usize, + dirs: usize, +} + +fn count_entries(path: &Path) -> Entries { + let mut counted = Entries { files: 0, dirs: 0 }; + let Ok(entries) = std::fs::read_dir(path) else { + return counted; + }; + for entry in entries.flatten() { + match entry.file_type() { + Ok(kind) if kind.is_dir() => { + counted.dirs += 1; + let nested = count_entries(&entry.path()); + counted.files += nested.files; + counted.dirs += nested.dirs; + } + // The store's own two files sit beside the shards and are not chunks: the + // layout marker, and the lock that keeps a second process out. Both are + // fixed, so neither grows with the store. + Ok(_) + if entry.file_name() == ant_node::storage::file_store::LAYOUT_FILE_NAME + || entry.file_name() == ".lock" => {} + Ok(_) => counted.files += 1, + Err(_) => {} + } + } + counted +} + +/// The startup scan does not read chunk contents. +/// +/// The claim the scan's cost rests on: a store of 4 MiB chunks would be unopenable if +/// starting meant reading them. +/// +/// Measured in bytes read, not in elapsed time. Timing cannot settle this: the files were +/// written moments earlier, so reading them back comes from the page cache and costs +/// almost nothing. A version of this test that compared durations passed with a +/// deliberate `read` of every file added to the scan. `rchar` counts what the process +/// asked the kernel for whether or not the answer was cached, which is the question. +/// +/// Linux only, for `/proc/self/io`. Nothing about the scan is platform-specific, and this +/// is the platform where the answer can be had exactly. +#[cfg(target_os = "linux")] +#[tokio::test] +async fn the_startup_scan_does_not_read_chunk_contents() { + let keys = 3_000; + let chunk = 64 * 1024; + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + plant_sized(&root.join("chunks"), keys, chunk); + + let before = bytes_read().expect("linux reports this"); + let store = FileStore::new(FileStoreConfig { + root_dir: root.clone(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("open"); + let read = bytes_read() + .expect("linux reports this") + .saturating_sub(before); + + let payload = (keys * chunk) as u64; + println!( + "scale: opening a store of {keys} chunks read {read} bytes, against {payload} \ + bytes of chunk" + ); + assert_eq!(store.current_chunks().expect("count") as usize, keys); + + assert!( + read < SCAN_READ_CEILING, + "the scan read {read} bytes of a {payload} byte store, over the \ + {SCAN_READ_CEILING} byte ceiling, so it is reading contents" + ); +} + +/// Bytes this process has asked the kernel to read, cached or not. +#[cfg(target_os = "linux")] +fn bytes_read() -> Option { + let io = std::fs::read_to_string("/proc/self/io").ok()?; + io.lines() + .find_map(|line| line.strip_prefix("rchar:")) + .and_then(|value| value.trim().parse().ok()) +} + +/// Plant `count` chunk files of `bytes` each. +#[cfg(target_os = "linux")] +fn plant_sized(chunks_dir: &Path, count: usize, bytes: usize) { + for shard in 0u16..256 { + std::fs::create_dir_all(chunks_dir.join(format!("{shard:02x}"))).expect("mkdir"); + } + let payload = vec![7u8; bytes]; + for n in 0..count { + let mut address = [0u8; 32]; + address[..8].copy_from_slice(&(n as u64).to_le_bytes()); + address[31] = (n % 256) as u8; + let path = chunks_dir + .join(format!("{:02x}", address[31])) + .join(hex::encode(address)); + std::fs::write(path, &payload).expect("plant a chunk"); + } +}