From beee9487fecc01f2903a0465b043b04a8636b929 Mon Sep 17 00:00:00 2001 From: Christian Date: Mon, 13 Jul 2026 14:25:17 -0500 Subject: [PATCH 1/2] Make EC withdrawal tombstones idempotent --- crates/trusted-server-core/src/ec/finalize.rs | 210 ++++- crates/trusted-server-core/src/ec/kv.rs | 794 +++++++++++++++++- ...ue-881-idempotent-withdrawal-tombstones.md | 319 +++++++ 3 files changed, 1278 insertions(+), 45 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-13-issue-881-idempotent-withdrawal-tombstones.md diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 349df70cc..e6e8dc97b 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -6,19 +6,15 @@ use std::collections::HashSet; use edgezero_core::body::Body as EdgeBody; -use error_stack::Report; use http::Response; use super::consent::{ec_consent_granted, ec_consent_withdrawn}; -use crate::error::TrustedServerError; use crate::settings::Settings; use super::EcContext; use super::cookies::{expire_ec_cookie, set_ec_cookie}; use super::generation::{generate_ec_id, is_valid_ec_id}; -use super::kv::{ - CreateIfAbsentOutcome, KvIdentityGraph, TombstoneOutcome, apply_partner_id_updates, -}; +use super::kv::{CreateIfAbsentOutcome, KvIdentityGraph, apply_partner_id_updates}; use super::kv_types::KvEntry; use super::prebid_eids::collect_eid_cookie_updates; use super::pull_sync_marker::{expire_marker, reconcile_marker}; @@ -349,49 +345,30 @@ fn finalize_unusable_consent( // the context — pull sync discloses the raw EC ID to partners — // sees the tombstone that was just written. Only the active ID has // a snapshot in the context to correct. - let outcome = graph.write_withdrawal_tombstone(ec_id, |snapshot| { - if ec_context.ec_value() == Some(ec_id) { - ec_context.set_kv_snapshot(snapshot); - } - }); - log_tombstone_outcome(ec_id, outcome); + let initial = if ec_context.kv_snapshot().belongs_to(ec_id) { + ec_context.kv_snapshot().clone() + } else { + EcKvSnapshot::NotRead + }; + let outcome = graph.tombstone_existing_from_snapshot(ec_id, initial); + // The browser cookie is already cleared, so a failed tombstone + // leaves a live row that server-side consumers still read as + // consented. Report every failure, including the non-active cookie + // ID whose outcome is not retained on the request context. + if matches!(outcome, EcKvSnapshot::Failed { .. }) { + log::warn!( + "EC withdrawal tombstone failed for '{}': the identity-graph row may \ + still be live with consent granted", + log_id(ec_id) + ); + } + if ec_context.ec_value() == Some(ec_id) { + ec_context.set_kv_snapshot(outcome); + } }); } } -/// Records what happened to one withdrawal tombstone. -/// -/// An unknown identity is expected traffic rather than a fault: the identifier -/// comes from a client-supplied cookie, so it may name something this -/// deployment never issued. An error is different: nothing was recorded, so a -/// real row may have gone unmarked, and that is logged as a fault. The browser -/// cookie is expired in every case, and that is the primary enforcement. -fn log_tombstone_outcome( - ec_id: &str, - outcome: Result>, -) { - match outcome { - Ok(TombstoneOutcome::Written) => {} - Ok(TombstoneOutcome::UnknownIdentity) => { - log::debug!( - "Skipping withdrawal tombstone for unknown EC ID '{}'", - log_id(ec_id), - ); - } - Err(err) => { - // Covers both a failed write and a check that could not determine - // whether the identity exists. Either way no marker was recorded, - // so a withdrawal may go unrecorded for the batch-sync window; the - // browser cookie is expired regardless. - log::error!( - "Could not record the withdrawal of EC ID '{}', so it may go unrecorded \ - for the batch-sync window; the browser cookie is still expired: {err:?}", - log_id(ec_id), - ); - } - } -} - fn withdrawal_ec_ids(ec_context: &EcContext) -> HashSet { let mut hashes = HashSet::new(); @@ -1554,6 +1531,151 @@ mod tests { ); } + #[test] + fn finalize_withdrawal_tombstones_both_present_ids_once() { + let settings = create_test_settings(); + let active_ec = sample_ec_id("activ3"); + let cookie_ec = sample_ec_id("cook3e"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = + make_context_with_consent(Some(&active_ec), Some(&cookie_ec), true, false, consent); + let graph = KvIdentityGraph::in_memory("test_store"); + graph + .create( + &active_ec, + &KvEntry::minimal("active.example.com", "active-uid", 1_000), + ) + .expect("should seed active row"); + graph + .create( + &cookie_ec, + &KvEntry::minimal("cookie.example.com", "cookie-uid", 1_000), + ) + .expect("should seed cookie row"); + ec_context.set_kv_snapshot(graph.load_snapshot(&active_ec)); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let (active_tombstone, active_generation) = graph + .get(&active_ec) + .expect("should read active row") + .expect("should retain active tombstone"); + let (cookie_tombstone, cookie_generation) = graph + .get(&cookie_ec) + .expect("should read cookie row") + .expect("should retain cookie tombstone"); + assert!( + !active_tombstone.consent.ok, + "active row should be withdrawn" + ); + assert!( + active_tombstone.ids.is_empty(), + "active IDs should be cleared" + ); + assert!( + !cookie_tombstone.consent.ok, + "cookie row should be withdrawn" + ); + assert!( + cookie_tombstone.ids.is_empty(), + "cookie IDs should be cleared" + ); + + let mut repeated_response = empty_response(); + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut repeated_response, + ); + + assert_eq!( + graph + .get(&active_ec) + .expect("should read active row") + .expect("should retain active tombstone") + .1, + active_generation, + "repeated finalization should not rewrite active tombstone" + ); + assert_eq!( + graph + .get(&cookie_ec) + .expect("should read cookie row") + .expect("should retain cookie tombstone") + .1, + cookie_generation, + "repeated finalization should not rewrite cookie tombstone" + ); + } + + #[test] + fn finalize_withdrawal_keeps_cookie_deletion_on_kv_failure() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("failw1"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = + make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); + let graph = KvIdentityGraph::failing("unavailable-store"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let cookies = response + .headers() + .get_all(http::header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect::>(); + assert_eq!( + response.status(), + 200, + "KV failure should not change response status" + ); + assert!( + cookies + .iter() + .any(|cookie| { cookie.starts_with("ts-ec=;") && cookie.contains("Max-Age=0") }), + "KV failure should not prevent EC cookie deletion" + ); + assert!( + cookies.iter().any(|cookie| { + cookie.starts_with("ts-ec-pull-complete=;") && cookie.contains("Max-Age=0") + }), + "KV failure should not prevent marker deletion" + ); + } + #[test] fn finalize_sets_marker_for_complete_pull_partner_snapshot() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index fd9aa5773..ba81eff14 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -975,6 +975,169 @@ impl KvIdentityGraph { }) } + /// Resolves a tombstone attempt whose point read reported the row absent. + /// + /// A proven-absent key is a no-op: there is nothing to withdraw, and a + /// forged cookie must not mint a row. A key that provably exists is + /// tombstoned unconditionally — no CAS generation is available after a + /// missed read, and a withdrawal must win over any concurrent write. An + /// existence check that itself fails leaves the withdrawal unresolved + /// rather than silently dropped. + fn tombstone_unproven_missing(&self, ec_id: &str, missing: EcKvSnapshot) -> EcKvSnapshot { + match self.key_exists_confirmed(ec_id) { + Ok(false) => missing, + Ok(true) => { + log::warn!( + "withdrawal tombstone for '{}': point read missed a row the store still \ + lists; writing an unconditional tombstone", + log_id(ec_id) + ); + let mut outcome_snapshot = EcKvSnapshot::NotRead; + match self.write_withdrawal_tombstone(ec_id, |snapshot| { + outcome_snapshot = snapshot; + }) { + Ok(TombstoneOutcome::Written) => outcome_snapshot, + Ok(TombstoneOutcome::UnknownIdentity) => EcKvSnapshot::Missing { + ec_id: ec_id.to_owned(), + }, + Err(err) => { + log::warn!( + "unconditional withdrawal tombstone failed for '{}': {err:?}", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + } + } + Err(err) => { + log::warn!( + "withdrawal tombstone for '{}': existence check failed, cannot confirm \ + absence: {err:?}", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + } + } + + /// Writes a tombstone only when an existing row can be confirmed. + /// + /// Existing-key-only behavior is deliberate: a forged or expired `ts-ec` + /// cookie must not mint a row. But a *point read* cannot prove absence on + /// an eventually-consistent store, and dropping a withdrawal is worse than + /// a redundant read, so absence is established in two stages: + /// + /// 1. Any snapshot that is not a usable `Present` for this EC ID — a + /// publisher preload that read `Missing`, a read that `Failed`, or one + /// lacking a CAS generation — is re-read. On the publisher path that + /// re-read is separated from the preload by the full origin round trip, + /// which gives replication time to converge. + /// 2. A re-read that still reports the row absent is checked against + /// [`key_exists_confirmed`](Self::key_exists_confirmed), which reads + /// the primary data source. + /// + /// Resolving the initial snapshot happens outside the retry counter, so all + /// [`MAX_CAS_RETRIES`] iterations stay available for the tombstone write. + pub(crate) fn tombstone_existing_from_snapshot( + &self, + ec_id: &str, + snapshot: EcKvSnapshot, + ) -> EcKvSnapshot { + let mut current = match snapshot { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + ref entry, + .. + } if snapshot_id == ec_id && !entry.consent.ok => return snapshot, + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + generation: Some(_), + .. + } if snapshot_id == ec_id => snapshot, + _ => self.load_snapshot(ec_id), + }; + + for _attempt in 0..MAX_CAS_RETRIES { + let generation = match current { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + ref entry, + .. + } if snapshot_id == ec_id && !entry.consent.ok => return current, + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + generation: Some(generation), + .. + } if snapshot_id == ec_id => generation, + // A missing row (including one that disappeared mid-retry) is + // only a no-op once absence is proven against the primary data + // source. + EcKvSnapshot::Missing { + ec_id: ref snapshot_id, + } if snapshot_id == ec_id => { + return self.tombstone_unproven_missing(ec_id, current); + } + // A refreshed read that failed (or any other unusable state) + // fails closed rather than silently dropping the withdrawal. + _ => { + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } + }; + + let tombstone = KvEntry::tombstone(current_timestamp()); + let Ok((body, meta_str)) = Self::serialize_entry(&tombstone, self.store_name()) else { + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + }; + match self.write_entry( + ec_id, + &body, + &meta_str, + TOMBSTONE_TTL, + EcKvWriteMode::IfGenerationMatch(generation), + ) { + Ok(EcKvWriteOutcome::Written) => { + return EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(tombstone), + generation: None, + }; + } + Ok(EcKvWriteOutcome::PreconditionFailed) => { + current = self.load_snapshot(ec_id); + } + Err(err) => { + log::warn!( + "conditional withdrawal tombstone failed for '{}': {err:?}", + log_id(ec_id) + ); + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } + } + } + + // Withdrawal enforcement lost every CAS race, so the row can still be + // live with consent granted while the browser cookie is cleared. That + // divergence is only visible to operators if it is logged here. + log::warn!( + "withdrawal tombstone for '{}': CAS conflict after {MAX_CAS_RETRIES} retries; the \ + identity-graph row may still be live with consent granted", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + /// Counts the number of keys sharing the same EC hash prefix. /// /// Uses the platform KV list API with a prefix filter, limited to @@ -1147,6 +1310,85 @@ mod tests { use super::*; use crate::ec::kv_backend::test_support::InMemoryEcKv; + /// [`EcKvStore`] wrapper whose first CAS write both fails the precondition + /// and deletes the key, simulating a concurrent withdrawal that removes the + /// row between this writer's read and its write. + struct DisappearOnConflictEcKv { + inner: InMemoryEcKv, + conflicts_remaining: std::sync::Mutex, + } + + impl DisappearOnConflictEcKv { + fn new(conflicts: u32) -> Self { + Self { + inner: InMemoryEcKv::new("disappear-store"), + conflicts_remaining: std::sync::Mutex::new(conflicts), + } + } + + fn seed_live(&self, ec_id: &str) { + let (body, meta) = + KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) + .expect("should serialize seeded entry"); + self.inner + .insert( + ec_id, + EcKvWrite { + body: &body, + metadata: &meta, + ttl: ENTRY_TTL, + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed live entry"); + } + } + + impl EcKvStore for DisappearOnConflictEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { + let mut remaining = self + .conflicts_remaining + .lock() + .expect("should lock conflict counter"); + if *remaining > 0 { + *remaining -= 1; + self.inner.delete(key).expect("should delete on conflict"); + return Ok(EcKvWriteOutcome::PreconditionFailed); + } + } + self.inner.insert(key, write) + } + + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + self.inner.count_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + fn snapshot_ec_id() -> String { format!("{}.ABC123", "a".repeat(64)) } @@ -1240,6 +1482,24 @@ mod tests { entry } + fn concurrent_live_entry() -> KvEntry { + let mut entry = live_entry(); + entry.ids.insert( + "concurrent.example.com".to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: "concurrent-uid".to_owned(), + }, + ); + entry + } + + // ----------------------------------------------------------------------- + // CAS-conflict injection tests + // ----------------------------------------------------------------------- + + use crate::ec::kv_backend::EcKvLookup; + use crate::ec::kv_backend::test_support::InMemoryEcKv; + /// [`EcKvStore`] wrapper that injects generation conflicts: the first /// `conflicts_remaining` `IfGenerationMatch` inserts return /// [`EcKvWriteOutcome::PreconditionFailed`] without writing, optionally @@ -1248,6 +1508,7 @@ mod tests { inner: InMemoryEcKv, conflicts_remaining: std::sync::Mutex, revive_on_conflict: bool, + partner_update_on_conflict: bool, } impl ConflictInjectingEcKv { @@ -1256,6 +1517,16 @@ mod tests { inner: InMemoryEcKv::new("conflict-store"), conflicts_remaining: std::sync::Mutex::new(conflicts), revive_on_conflict, + partner_update_on_conflict: false, + } + } + + fn with_partner_update_on_conflict(conflicts: u32) -> Self { + Self { + inner: InMemoryEcKv::new("partner-conflict-store"), + conflicts_remaining: std::sync::Mutex::new(conflicts), + revive_on_conflict: true, + partner_update_on_conflict: true, } } @@ -1324,8 +1595,13 @@ mod tests { if self.revive_on_conflict { // Simulate a concurrent writer reviving the entry // between this writer's read and its CAS write. + let concurrent_entry = if self.partner_update_on_conflict { + concurrent_live_entry() + } else { + live_entry() + }; let (body, meta) = KvIdentityGraph::serialize_entry( - &live_entry(), + &concurrent_entry, self.inner.store_name(), ) .expect("should serialize concurrent live entry"); @@ -1697,6 +1973,45 @@ mod tests { assert!(kv.get(&ec_id).expect("should read store").is_none()); } + #[test] + fn tombstone_existing_from_snapshot_never_creates_missing_key() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let snapshot = EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }; + + let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!(matches!(outcome, EcKvSnapshot::Missing { .. })); + assert!( + kv.get(&ec_id).expect("should read store").is_none(), + "withdrawal must not create a tombstone for an absent key" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_uses_existing_generation() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()).expect("should create"); + let snapshot = kv.load_snapshot(&ec_id); + + let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "should return the persisted tombstone" + ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read store") + .expect("should preserve existing key"); + assert!(!stored.consent.ok, "should persist withdrawal state"); + } + #[test] fn write_withdrawal_tombstone_overwrites_live_entry() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -1717,6 +2032,103 @@ mod tests { assert!(!loaded.consent.ok, "should be withdrawn after tombstone"); } + // ----------------------------------------------------------------------- + // Snapshot-aware mutation stores and tests + // ----------------------------------------------------------------------- + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + struct RecordedEcKvInsert { + mode: EcKvWriteMode, + ttl: Duration, + } + + #[derive(Default)] + struct RecordedEcKvOperations { + lookups: std::sync::atomic::AtomicUsize, + inserts: std::sync::Mutex>, + } + + impl RecordedEcKvOperations { + fn reset(&self) { + self.lookups.store(0, std::sync::atomic::Ordering::Relaxed); + self.inserts + .lock() + .expect("should lock recorded inserts") + .clear(); + } + + fn lookup_count(&self) -> usize { + self.lookups.load(std::sync::atomic::Ordering::Relaxed) + } + + fn inserts(&self) -> Vec { + self.inserts + .lock() + .expect("should lock recorded inserts") + .clone() + } + } + + /// In-memory store that records every backend operation before delegation. + struct RecordingEcKv { + inner: InMemoryEcKv, + operations: Arc, + } + + impl RecordingEcKv { + fn new(operations: Arc) -> Self { + Self { + inner: InMemoryEcKv::new("recording-store"), + operations, + } + } + } + + impl EcKvStore for RecordingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + self.operations + .lookups + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + self.operations + .inserts + .lock() + .expect("should lock recorded inserts") + .push(RecordedEcKvInsert { + mode: write.mode, + ttl: write.ttl, + }); + self.inner.insert(key, write) + } + + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + self.inner.count_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + /// [`EcKvStore`] whose reads succeed but every write fails, simulating a /// store that becomes unwritable mid-request. struct WriteFailingEcKv { @@ -2023,6 +2435,386 @@ mod tests { ); } + #[test] + fn tombstone_existing_from_snapshot_skips_backend_for_authoritative_tombstone() { + for generation in [Some(7), None] { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + let ec_id = snapshot_ec_id(); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(KvEntry::tombstone(1_000)), + generation, + }; + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot.clone()); + + assert_eq!( + outcome, snapshot, + "should preserve authoritative tombstone state" + ); + assert_eq!( + operations.lookup_count(), + 0, + "should not reread a tombstone" + ); + assert!( + operations.inserts().is_empty(), + "should not attempt to rewrite a tombstone" + ); + } + } + + #[test] + fn tombstone_existing_from_snapshot_repeated_request_preserves_first_write() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + let live_snapshot = graph.load_snapshot(&ec_id); + operations.reset(); + + graph.tombstone_existing_from_snapshot(&ec_id, live_snapshot); + + assert_eq!( + operations.lookup_count(), + 0, + "usable generation should avoid a read" + ); + assert_eq!( + operations.inserts(), + vec![RecordedEcKvInsert { + mode: EcKvWriteMode::IfGenerationMatch(1), + ttl: TOMBSTONE_TTL, + }], + "first withdrawal should perform one conditional tombstone write" + ); + let first_snapshot = graph.load_snapshot(&ec_id); + let (first_entry, first_generation) = match &first_snapshot { + EcKvSnapshot::Present { + entry, generation, .. + } => (entry.as_ref().clone(), *generation), + other => panic!("should load first tombstone, got {other:?}"), + }; + operations.reset(); + + let second_outcome = graph.tombstone_existing_from_snapshot(&ec_id, first_snapshot); + + assert_eq!( + operations.lookup_count(), + 0, + "repeated withdrawal should not reread" + ); + assert!( + operations.inserts().is_empty(), + "repeated withdrawal should not refresh the tombstone TTL" + ); + assert_eq!( + second_outcome.generation_for(&ec_id), + first_generation, + "repeated withdrawal should preserve the stored generation" + ); + assert_eq!( + second_outcome + .entry_for(&ec_id) + .map(|entry| entry.consent.updated), + Some(first_entry.consent.updated), + "repeated withdrawal should preserve the first tombstone timestamp" + ); + } + + #[test] + fn tombstone_existing_from_stale_parallel_snapshot_stops_after_conflict() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + let stale_snapshot = graph.load_snapshot(&ec_id); + operations.reset(); + + let first_outcome = graph.tombstone_existing_from_snapshot(&ec_id, stale_snapshot.clone()); + let second_outcome = graph.tombstone_existing_from_snapshot(&ec_id, stale_snapshot); + + assert_eq!( + operations.inserts(), + vec![ + RecordedEcKvInsert { + mode: EcKvWriteMode::IfGenerationMatch(1), + ttl: TOMBSTONE_TTL, + }, + RecordedEcKvInsert { + mode: EcKvWriteMode::IfGenerationMatch(1), + ttl: TOMBSTONE_TTL, + }, + ], + "parallel loser should attempt stale CAS once and never replace the winner" + ); + assert_eq!( + operations.lookup_count(), + 1, + "parallel loser should reread exactly once after its conflict" + ); + assert_eq!( + second_outcome.generation_for(&ec_id), + Some(2), + "parallel loser should return the winner's stored generation" + ); + assert_eq!( + second_outcome + .entry_for(&ec_id) + .map(|entry| entry.consent.updated), + first_outcome + .entry_for(&ec_id) + .map(|entry| entry.consent.updated), + "parallel loser should preserve the winner's tombstone" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_succeeds_without_backend_for_tombstone() { + let graph = KvIdentityGraph::failing("unavailable-store"); + let ec_id = snapshot_ec_id(); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(KvEntry::tombstone(1_000)), + generation: Some(3), + }; + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot.clone()); + + assert_eq!( + outcome, snapshot, + "authoritative tombstone should not touch unavailable backend" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_non_authoritative_states_reread_live_row() { + let ec_id = snapshot_ec_id(); + let states = [ + EcKvSnapshot::NotRead, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }, + EcKvSnapshot::Present { + ec_id: "different-ec-id".to_owned(), + entry: Box::new(KvEntry::tombstone(1_000)), + generation: Some(9), + }, + ]; + + for state in states { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + operations.reset(); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, state); + + assert_eq!( + operations.lookup_count(), + 1, + "state should force one reread" + ); + assert_eq!( + operations.inserts().len(), + 1, + "live reread should write once" + ); + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "reread live row should be tombstoned" + ); + } + } + + #[test] + fn tombstone_existing_from_snapshot_retries_cas_conflict() { + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, false)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "should retry the conflict and persist the tombstone" + ); + } + + #[test] + fn tombstone_gen_unavailable_survives_four_conflicts_then_writes() { + // A generation-unavailable snapshot refreshes once before its CAS. That + // refresh must not spend a CAS attempt, so a withdrawal tombstone still + // persists after four conflicts and a successful fifth write. + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(4, false)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }; + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "the fifth CAS attempt must persist the tombstone after a refresh and four conflicts" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_returns_failed_after_cas_exhaustion() { + // Every CAS attempt loses its race, so the row stays live with consent + // granted while the browser cookie is already cleared. The caller must + // see a failure it can report rather than a silent no-op. + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(MAX_CAS_RETRIES, false)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + matches!(outcome, EcKvSnapshot::Failed { .. }), + "CAS exhaustion must report a failed withdrawal" + ); + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("row should remain"); + assert!( + stored.consent.ok, + "the row is still live, which is exactly why the failure must be reported" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_overrides_concurrent_live_update() { + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::with_partner_update_on_conflict(1)); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + let entry = outcome + .entry_for(&ec_id) + .expect("should return persisted tombstone"); + assert!(!entry.consent.ok, "withdrawal should win after retry"); + assert!( + entry.ids.is_empty(), + "withdrawal should clear concurrent partner IDs" + ); + } + + #[test] + fn upsert_partner_id_rejects_tombstone() { + let graph = KvIdentityGraph::in_memory("test-store"); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &KvEntry::tombstone(1_000)) + .expect("should seed tombstone"); + + let result = graph.upsert_partner_id(&ec_id, "ssp.example.com", "uid-1"); + + assert!(result.is_err(), "public upsert should reject a tombstone"); + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("should preserve tombstone"); + assert!(!stored.consent.ok, "entry should remain withdrawn"); + assert!( + stored.ids.is_empty(), + "upsert should not repopulate partner IDs" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_store_failure_returns_failed() { + let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); + let ec_id = snapshot_ec_id(); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: Some(1), + }; + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!(matches!(outcome, EcKvSnapshot::Failed { .. })); + } + + #[test] + fn tombstone_existing_from_snapshot_noop_when_row_disappears_on_retry() { + let store = DisappearOnConflictEcKv::new(1); + store.seed_live(&snapshot_ec_id()); + let graph = KvIdentityGraph::new(store); + let ec_id = snapshot_ec_id(); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + matches!(outcome, EcKvSnapshot::Missing { .. }), + "a row that disappears during retry becomes a no-op" + ); + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "must not recreate the disappeared key" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_reretries_failed_snapshot_read() { + // A prior request-scoped read failed, so the snapshot is `Failed`. A + // withdrawal must not silently drop consent removal: re-read the store + // and tombstone the row if it is authoritatively present. + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = snapshot_ec_id(); + kv.create(&ec_id, &live_entry()).expect("should seed live"); + + let outcome = kv.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + ); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "a failed snapshot must re-read and persist the tombstone" + ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read store") + .expect("should preserve existing key"); + assert!(!stored.consent.ok, "withdrawal must reach the store"); + } + #[test] fn key_exists_confirmed_distinguishes_absence_from_a_stale_point_read() { let graph = KvIdentityGraph::stale_lookup("stale-store", 1); diff --git a/docs/superpowers/plans/2026-07-13-issue-881-idempotent-withdrawal-tombstones.md b/docs/superpowers/plans/2026-07-13-issue-881-idempotent-withdrawal-tombstones.md new file mode 100644 index 000000000..9dfc6db88 --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-issue-881-idempotent-withdrawal-tombstones.md @@ -0,0 +1,319 @@ +# Issue #881: Idempotent EC Withdrawal Tombstones Plan + +- **Date:** 2026-07-13 +- **Status:** Implemented and verified +- **Issue:** [#881 — Make EC withdrawal tombstoning idempotent across request bursts](https://github.com/IABTechLab/trusted-server/issues/881) +- **Stack base:** [Draft PR #900 — Avoid no-op EC KV reads in post-send pull sync](https://github.com/IABTechLab/trusted-server/pull/900) +- **Underlying dependency:** [PR #885 — Request-scoped EC KV snapshot and orphan recovery](https://github.com/IABTechLab/trusted-server/pull/885) + +## Goal + +Make explicit EC withdrawal idempotent across repeated and concurrent requests +without weakening the existing-key-only privacy invariant introduced by PR +#885. The first successful withdrawal of a live row writes a CAS-protected +24-hour tombstone. A request that already has authoritative tombstone state +returns without reading or writing KV, so it cannot refresh the tombstone's +entry timestamp or TTL. + +Browser-cookie deletion remains synchronous and best-effort KV failure must +never block the response. + +## Clarified Semantics + +- An authoritative missing row is a no-op. A valid-looking but unverified + browser cookie must never create a KV root. +- A matching authoritative tombstone snapshot is returned unchanged before any + lookup, serialization, or write, regardless of whether its generation is + available. +- A matching live snapshot with a generation uses one conditional write. +- A live snapshot without a generation, a failed/not-read snapshot, or a + snapshot for another EC ID rereads the requested row before deciding. +- CAS conflicts reread and retry. If another withdrawal has already written a + tombstone, retry ends without another write. If a concurrent live update or + re-consent changed the generation first, withdrawal retries against that live + row and tombstones it. +- A later re-consent may legitimately win when it linearizes after a completed + or no-op withdrawal. Idempotency does not impose global withdrawal priority. +- Repeated withdrawal preserves the original tombstone expiration because it + performs no second write. +- When cookie and active EC IDs differ, every valid existing row is withdrawn + independently; missing or malformed IDs are never created. +- `ts-ec` and the pull-completeness marker are expired before best-effort KV + work. Store failure is logged/swallowed by finalization. + +## Non-Goals + +- Do not recreate missing roots from browser cookies. +- Do not add a cross-request lock, deduplication store, or withdrawal cookie. +- Do not restrict withdrawal handling to document navigations. +- Do not change the 24-hour tombstone duration or KV schema. +- Do not change EC generation, marker behavior, batch sync, pull sync, or + partner-upsert semantics beyond preserving tombstone rejection. +- Do not make withdrawal dominate a re-consent that occurs after withdrawal's + linearization point. +- Do not rewrite archival specs that describe the superseded unconditional + helper. + +## Current Behavior + +`KvIdentityGraph::tombstone_existing_from_snapshot` already preserves PR #885's +existing-key-only and CAS behavior, but it writes a fresh tombstone whenever the +snapshot is `Present`, including when that entry is already a tombstone. Parallel +withdrawals therefore converge safely but still perform a redundant CAS write, +and repeated requests reset the tombstone's 24-hour TTL. + +The older public `write_withdrawal_tombstone` unconditional-overwrite helper is +now dead in production but remains available and could bypass the conditional +path in future code. + +Finalization already: + +- expires browser state before KV work; +- collects both valid cookie and active EC IDs; +- uses the carried snapshot only for its matching ID; +- independently resolves the other ID; +- logs/swallows KV failures. + +The implementation should therefore remain concentrated in the core KV method, +with finalization changes limited to acceptance-level integration tests unless a +test exposes a defect. + +## Proposed Design + +### 1. Add an authoritative tombstone fast path + +At the beginning of each `tombstone_existing_from_snapshot` retry iteration, +match a `Present` snapshot only when its `ec_id` equals the requested ID. + +- If `entry.consent.ok == false`, return the exact snapshot immediately. +- Perform this check before requiring a generation or constructing a new + `KvEntry::tombstone`. +- Preserve the snapshot's entry timestamp and generation exactly. + +Then retain the existing state machine: + +| Initial/refreshed state | Action | +| ----------------------------------------- | ----------------------------------- | +| Matching tombstone | Return unchanged; zero backend work | +| Matching live entry + generation | CAS-write a 24-hour tombstone | +| Matching live entry without generation | Reread | +| Matching `Missing` | Return unchanged; never create | +| `Failed`, `NotRead`, or wrong-ID snapshot | Reread requested ID | +| CAS precondition failure | Reread and retry | +| Row disappears during retry | Return `Missing` | +| Store failure or retry exhaustion | Return ID-bound `Failed` | + +A successful tombstone remains `consent.ok = false`, has empty partner IDs, and +uses `TOMBSTONE_TTL`. + +### 2. Remove the unconditional bypass + +Delete the now-unused public `write_withdrawal_tombstone` method and its obsolete +overwrite test. Update `KvIdentityGraph::delete` documentation so it describes +the snapshot-aware conditional withdrawal path without linking to the removed +API. + +Repository search must confirm there is no live Rust caller before removal. +Historical design documents may remain unchanged. + +### 3. Prove operation-level idempotency + +Use a focused recording backend around the existing in-memory store. It must +count every lookup and insert attempt, including inserts that return a CAS +precondition failure, and record each write's mode and TTL. Tests must show: + +- a supplied matching tombstone returns with zero lookups and zero insert + attempts; +- generation-unavailable tombstone state also performs no backend operation; +- the first live withdrawal performs exactly one `IfGenerationMatch` insert + with `TOMBSTONE_TTL`, while the second causes zero additional insert attempts + and leaves stored generation and `consent.updated` unchanged; +- two stale live snapshots model parallel requests: the first writes the + tombstone; the second conflicts, rereads the tombstone, and performs no + replacement write; +- a supplied tombstone succeeds even against an always-failing backend, proving + no hidden operation; +- live state with generation avoids an initial read and uses one CAS write; +- missing state remains a no-op; +- `NotRead`, failed, generation-unavailable, and wrong-ID states reread the + requested row before applying the documented write/no-create behavior. + +The recording backend's first-write TTL assertion plus zero additional insert +attempts on repetition is the authoritative proof that the original tombstone +TTL was not refreshed. + +### 4. Preserve race ordering and tombstone authority + +Extend conflict tests to model a concurrent live update/re-consent that changes +the generation before withdrawal's first CAS. Withdrawal must reread the live +row and eventually write the tombstone, clearing any partner IDs. + +Retain the existing batch-sync conditional-upsert and snapshot bulk-upsert +tombstone tests, and add focused coverage for the public single-partner upsert +path so all live enrichment APIs are proven unable to repopulate tombstones: + +- `upsert_partner_id_if_exists` rejects tombstones; +- `upsert_partner_id` returns an error and leaves tombstone IDs empty; +- snapshot bulk upsert cannot repopulate a tombstone; +- a disappeared row is not recreated; +- store errors return failed state rather than claiming persistence. + +### 5. Verify finalization behavior + +Retain the existing finalization coverage for malformed/absent IDs and a +present active ID plus missing secondary ID. Add only the missing integration +cases: + +- differing valid cookie and active EC IDs are both tombstoned when both rows + exist; +- repeated finalization preserves existing tombstone generations; +- a failing KV graph still returns the response and emits both applicable + browser-cookie expiration headers. + +Production finalization should not change unless these tests expose a defect. + +## File Map + +### Modify + +- `crates/trusted-server-core/src/ec/kv.rs` + - Add the matching-tombstone no-op branch. + - Remove the unconditional overwrite helper. + - Update withdrawal documentation. + - Add operation-count, repetition, and concurrency tests. +- `crates/trusted-server-core/src/ec/finalize.rs` + - Add two-ID, repeated-withdrawal, and KV-failure integration coverage. + +### Add + +- `docs/superpowers/plans/2026-07-13-issue-881-idempotent-withdrawal-tombstones.md` + - Record the reviewed design and verification contract. + +No dependency, configuration, adapter, JavaScript, or public wire-format change +is expected. + +## Implementation Tasks + +### Task 1 — Establish failing idempotency tests + +- [x] Add a recording withdrawal backend that counts lookups and every insert + attempt and captures write mode/TTL. +- [x] Add a supplied-tombstone test proving zero lookups and zero inserts. +- [x] Add repeated and stale-parallel snapshot tests proving the first insert is + `IfGenerationMatch` with `TOMBSTONE_TTL`, then no further insert occurs and + generation/`consent.updated` remain unchanged. +- [x] Add live-generation, unavailable-generation, `NotRead`, failed, wrong-ID, + and missing state tests. +- [x] Run `cargo test-fastly tombstone_existing_from_snapshot` and confirm the + new repeated/no-backend tests fail before implementation. + +### Task 2 — Implement the no-op branch and remove the bypass + +- [x] Return a matching tombstone snapshot before generation lookup, + serialization, or write. +- [x] Keep live/missing/failed/mismatched/CAS behavior unchanged. +- [x] Remove `write_withdrawal_tombstone` and update the `delete` documentation. +- [x] Search for remaining Rust references to the removed helper. +- [x] Run focused KV tests until green. + +### Task 3 — Cover concurrent state changes + +- [x] Model another withdrawal winning between read and CAS; prove the loser + rereads and stops without replacing the tombstone. +- [x] Model a concurrent live update/re-consent winning before CAS; prove + withdrawal retries and tombstones the refreshed row. +- [x] Assert final tombstones contain no partner IDs. +- [x] Add direct `upsert_partner_id` tombstone rejection coverage and re-run the + existing conditional and snapshot-bulk rejection tests. + +### Task 4 — Verify finalization + +- [x] Add a test with both differing valid IDs present and assert both become + tombstones. +- [x] Retain the existing missing and invalid ID tests unchanged as regression + coverage. +- [x] Add repeated-finalization generation-stability coverage. +- [x] Add a failing-store test proving EC and marker cookie deletion survives KV + failure. +- [x] Run focused withdrawal/finalization tests. + +### Task 5 — Review and full verification + +- [x] Run independent correctness/concurrency and test-quality reviews. +- [x] Apply only fixes required by issue scope. +- [x] Mark this plan implemented only after all checks below pass. + +## Acceptance Mapping + +| Issue requirement | Planned evidence | +| ------------------------------------------------ | ----------------------------------------------------------------------------- | +| First withdrawal establishes a 24-hour tombstone | Live-entry CAS test and existing `TOMBSTONE_TTL` assertion | +| Repeated requests avoid overwrite writes | Operation counts plus unchanged generation and `consent.updated` | +| Concurrent withdrawal cannot restore IDs | Stale-snapshot and concurrent-live-update conflict tests | +| Both differing valid IDs are withdrawn | Finalization test with both rows seeded | +| Missing/unverified IDs create no root | Existing-key-only and invalid-ID tests | +| KV failure remains best-effort | Finalization response/cookie test with failing graph | +| Late partner updates cannot repopulate | Conditional, public single, and snapshot-bulk upsert rejection tests | +| Original TTL is not refreshed | First insert records `TOMBSTONE_TTL`; repetition records zero further inserts | + +## Verification Contract + +Run focused checks during implementation: + +```bash +cargo test-fastly tombstone_existing_from_snapshot +cargo test-fastly withdrawal +cargo test-fastly upsert_partner_id_if_exists_rejects_tombstone +cargo test-fastly upsert_partner_id_rejects_tombstone +cargo test-fastly snapshot_upsert_rejects_tombstone +``` + +Before committing and opening the draft PR, run: + +```bash +cargo fmt --all -- --check +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cd crates/trusted-server-js/lib && npx vitest run +cd crates/trusted-server-js/lib && npm run format +cd docs && npm run format +cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 +git diff --check +``` + +## Definition of Done + +- The first live-row withdrawal writes one CAS-protected 24-hour tombstone. +- Repeated and concurrent withdrawals observing that tombstone perform no + replacement write and do not refresh its expiration. +- Missing IDs remain absent; the unconditional overwrite API no longer exists. +- Concurrent live updates before successful withdrawal are tombstoned on retry. +- Later partner writes cannot repopulate tombstones. +- Both valid differing IDs are handled independently. +- Browser-cookie deletion remains independent of KV success. +- Focused tests, independent review, and every applicable repository gate pass. + +## Risks and Mitigations + +- **Snapshot binding:** The no-op branch must require the snapshot ID to match + the requested EC ID; a tombstone for another ID cannot suppress withdrawal. +- **Linearization:** Returning an observed tombstone linearizes withdrawal at + that observation. A later re-consent may legitimately win. +- **TTL visibility:** Record the first insert's TTL and every subsequent insert + attempt at the wrapper boundary; stable generation/timestamp alone is not + sufficient evidence. +- **Dead API removal:** Compile and repository-search after deletion to catch any + hidden caller. +- **Conflict-test realism:** Inject actual generation changes and persisted + state, not endless synthetic precondition failures. +- **Stack dependency:** Reconcile changes if PR #885 or draft PR #900 modifies + snapshot/finalization contracts before this stack lands. From 1cc335b3c7ad14c943039170bf4013b39e0f1355 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 3 Sep 2026 12:46:50 -0500 Subject: [PATCH 2/2] Prevent stale misses from refreshing EC tombstones --- crates/trusted-server-core/src/ec/kv.rs | 333 +++++++++++++++++- docs/guide/edge-cookies.md | 2 +- ...ue-881-idempotent-withdrawal-tombstones.md | 117 +++--- 3 files changed, 397 insertions(+), 55 deletions(-) diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index ba81eff14..b3ae46fae 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -39,6 +39,9 @@ const ENTRY_TTL: Duration = Duration::from_secs(365 * 24 * 60 * 60); /// TTL for withdrawal tombstones (24 hours). const TOMBSTONE_TTL: Duration = Duration::from_secs(24 * 60 * 60); +/// Namespace for completion markers written after a withdrawal tombstone. +const WITHDRAWAL_MARKER_PREFIX: &str = "__ts_ec_withdrawal_complete__:"; + /// Outcome of an [`KvIdentityGraph::upsert_partner_id_if_exists`] call. /// /// Like [`KvIdentityGraph::upsert_partner_id`], this method fails closed when @@ -380,6 +383,10 @@ impl KvIdentityGraph { // Serialize once and reuse across the fast path and CAS loop. let (body, meta_str) = Self::serialize_entry(entry, self.store_name())?; + // Completion markers belong to withdrawn generations. Remove any + // marker before this key can become live again. + self.clear_withdrawal_marker(ec_id)?; + // Try create first — fast path for new entries. if self.write_entry(ec_id, &body, &meta_str, ENTRY_TTL, EcKvWriteMode::Add)? == EcKvWriteOutcome::Written @@ -411,6 +418,11 @@ impl KvIdentityGraph { let mut current_gen = generation; for attempt in 0..MAX_CAS_RETRIES { + // A completion marker belongs to the tombstone generation. Remove + // it before making this key live so a later withdrawal cannot be + // suppressed by stale fallback state. + self.clear_withdrawal_marker(ec_id)?; + match self.write_entry( ec_id, &body, @@ -862,12 +874,66 @@ impl KvIdentityGraph { self.store.key_exists(ec_id) } + fn withdrawal_marker_key(ec_id: &str) -> String { + format!("{WITHDRAWAL_MARKER_PREFIX}{ec_id}") + } + + fn withdrawal_marker_exists(&self, ec_id: &str) -> Result> { + let marker_key = Self::withdrawal_marker_key(ec_id); + Ok(self.store.count_keys_with_prefix(&marker_key, 1)? > 0) + } + + fn write_withdrawal_marker(&self, ec_id: &str) -> Result<(), Report> { + let marker_key = Self::withdrawal_marker_key(ec_id); + match self.store.insert( + &marker_key, + EcKvWrite { + body: "1", + metadata: "{}", + ttl: TOMBSTONE_TTL, + mode: EcKvWriteMode::Add, + }, + )? { + EcKvWriteOutcome::Written | EcKvWriteOutcome::PreconditionFailed => Ok(()), + } + } + + fn clear_withdrawal_marker(&self, ec_id: &str) -> Result<(), Report> { + if !self.withdrawal_marker_exists(ec_id)? { + return Ok(()); + } + + let marker_key = Self::withdrawal_marker_key(ec_id); + match self.store.delete(&marker_key) { + Ok(()) => Ok(()), + Err(delete_err) => match self.withdrawal_marker_exists(ec_id) { + // Another request removed the marker first. + Ok(false) => Ok(()), + Ok(true) | Err(_) => Err(delete_err), + }, + } + } + + fn record_withdrawal_completion(&self, ec_id: &str) { + if let Err(err) = self.write_withdrawal_marker(ec_id) { + // The root is already tombstoned. Preserve that successful privacy + // write even if the cost-control marker cannot be recorded. + log::warn!( + "withdrawal completion marker failed for '{}': {err:?}", + log_id(ec_id) + ); + } + } + /// Writes a withdrawal tombstone for consent enforcement. /// /// Overwrites the entry with `consent.ok = false`, empty partner IDs, /// and a 24-hour TTL. Uses unconditional overwrite (no CAS) since the /// entry is being withdrawn regardless of concurrent state. /// + /// A successful write records a same-TTL completion marker so repeated + /// stale misses do not overwrite the root or refresh its tombstone TTL. + /// /// The tombstone preserves consent enforcement for batch sync clients /// (`POST /_ts/api/v1/batch-sync`) during the 24-hour revocation window. /// @@ -931,13 +997,17 @@ impl KvIdentityGraph { }, }); - written.map(|entry| { + let outcome = written.map(|entry| { if entry.is_some() { TombstoneOutcome::Written } else { TombstoneOutcome::UnknownIdentity } - }) + }); + if matches!(outcome, Ok(TombstoneOutcome::Written)) { + self.record_withdrawal_completion(ec_id); + } + outcome } /// Tombstones a held identity, returning the entry written. @@ -987,6 +1057,24 @@ impl KvIdentityGraph { match self.key_exists_confirmed(ec_id) { Ok(false) => missing, Ok(true) => { + match self.withdrawal_marker_exists(ec_id) { + Ok(true) => { + log::debug!( + "withdrawal tombstone for '{}': completion marker already exists", + log_id(ec_id) + ); + return missing; + } + Ok(false) => {} + Err(err) => { + // Marker failure must not weaken withdrawal. Fall back + // to the existing unconditional privacy write. + log::warn!( + "withdrawal completion marker lookup failed for '{}': {err:?}", + log_id(ec_id) + ); + } + } log::warn!( "withdrawal tombstone for '{}': point read missed a row the store still \ lists; writing an unconditional tombstone", @@ -1104,6 +1192,7 @@ impl KvIdentityGraph { EcKvWriteMode::IfGenerationMatch(generation), ) { Ok(EcKvWriteOutcome::Written) => { + self.record_withdrawal_completion(ec_id); return EcKvSnapshot::Present { ec_id: ec_id.to_owned(), entry: Box::new(tombstone), @@ -1239,7 +1328,7 @@ impl KvIdentityGraph { Ok(Some(cluster_size)) } - /// Hard-deletes the entry. + /// Hard-deletes the entry and any withdrawal completion marker. /// /// Reserved for the IAB data deletion framework (deferred). For consent /// withdrawal, use [`write_withdrawal_tombstone`](Self::write_withdrawal_tombstone). @@ -1250,7 +1339,8 @@ impl KvIdentityGraph { pub fn delete(&self, ec_id: &str) -> Result<(), Report> { // The backend's delete already attaches store context, so propagate // without re-wrapping the same message. - self.store.delete(ec_id) + self.store.delete(ec_id)?; + self.clear_withdrawal_marker(ec_id) } } @@ -1890,6 +1980,54 @@ mod tests { assert!(loaded.consent.ok, "should be live after revive"); } + #[test] + fn create_or_revive_clears_withdrawal_marker() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + let snapshot = kv.load_snapshot(&ec_id); + kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + assert!( + kv.withdrawal_marker_exists(&ec_id) + .expect("should read withdrawal marker"), + "withdrawal should record completion" + ); + + kv.create_or_revive(&ec_id, &live_entry()) + .expect("should revive tombstone"); + + let (loaded, _) = kv + .get(&ec_id) + .expect("should read revived entry") + .expect("should find revived entry"); + assert!(loaded.consent.ok, "should be live after revive"); + assert!( + !kv.withdrawal_marker_exists(&ec_id) + .expect("should read withdrawal marker"), + "revival should clear stale withdrawal completion" + ); + } + + #[test] + fn delete_removes_withdrawal_marker() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + let snapshot = kv.load_snapshot(&ec_id); + kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + + kv.delete(&ec_id).expect("should delete entry and marker"); + + assert!(kv.get(&ec_id).expect("should read store").is_none()); + assert!( + !kv.withdrawal_marker_exists(&ec_id) + .expect("should read withdrawal marker"), + "hard delete should remove withdrawal completion" + ); + } + #[test] fn upsert_partner_id_if_exists_reports_missing_key() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -2073,13 +2211,19 @@ mod tests { struct RecordingEcKv { inner: InMemoryEcKv, operations: Arc, + stale_lookups_remaining: std::sync::Mutex, } impl RecordingEcKv { fn new(operations: Arc) -> Self { + Self::with_stale_lookups(operations, 0) + } + + fn with_stale_lookups(operations: Arc, stale_lookups: u32) -> Self { Self { inner: InMemoryEcKv::new("recording-store"), operations, + stale_lookups_remaining: std::sync::Mutex::new(stale_lookups), } } } @@ -2093,6 +2237,14 @@ mod tests { self.operations .lookups .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let mut stale_lookups = self + .stale_lookups_remaining + .lock() + .expect("should lock stale lookup counter"); + if *stale_lookups > 0 { + *stale_lookups -= 1; + return Ok(None); + } self.inner.lookup(key) } @@ -2129,6 +2281,76 @@ mod tests { } } + /// Store whose completion-marker operations fail while root operations work. + struct MarkerFailingEcKv { + inner: InMemoryEcKv, + stale_lookups_remaining: std::sync::Mutex, + } + + impl MarkerFailingEcKv { + fn new(stale_lookups: u32) -> Self { + Self { + inner: InMemoryEcKv::new("marker-failing-store"), + stale_lookups_remaining: std::sync::Mutex::new(stale_lookups), + } + } + + fn marker_error(&self, operation: &str) -> Report { + Report::new(TrustedServerError::KvStore { + store_name: self.inner.store_name().to_owned(), + message: format!("completion marker {operation} failed"), + }) + } + } + + impl EcKvStore for MarkerFailingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + let mut stale_lookups = self + .stale_lookups_remaining + .lock() + .expect("should lock stale lookup counter"); + if *stale_lookups > 0 { + *stale_lookups -= 1; + return Ok(None); + } + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + if key.starts_with(WITHDRAWAL_MARKER_PREFIX) { + return Err(self.marker_error("write")); + } + self.inner.insert(key, write) + } + + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + if prefix.starts_with(WITHDRAWAL_MARKER_PREFIX) { + return Err(self.marker_error("lookup")); + } + self.inner.count_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + /// [`EcKvStore`] whose reads succeed but every write fails, simulating a /// store that becomes unwritable mid-request. struct WriteFailingEcKv { @@ -2485,11 +2707,17 @@ mod tests { ); assert_eq!( operations.inserts(), - vec![RecordedEcKvInsert { - mode: EcKvWriteMode::IfGenerationMatch(1), - ttl: TOMBSTONE_TTL, - }], - "first withdrawal should perform one conditional tombstone write" + vec![ + RecordedEcKvInsert { + mode: EcKvWriteMode::IfGenerationMatch(1), + ttl: TOMBSTONE_TTL, + }, + RecordedEcKvInsert { + mode: EcKvWriteMode::Add, + ttl: TOMBSTONE_TTL, + }, + ], + "first withdrawal should write the root and its completion marker" ); let first_snapshot = graph.load_snapshot(&ec_id); let (first_entry, first_generation) = match &first_snapshot { @@ -2525,6 +2753,57 @@ mod tests { ); } + #[test] + fn tombstone_existing_from_repeated_stale_miss_preserves_first_write() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( + Arc::clone(&operations), + 2, + )); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + operations.reset(); + + let first_outcome = graph.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + let first_updated = first_outcome + .entry_for(&ec_id) + .expect("should return first tombstone") + .consent + .updated; + operations.reset(); + + graph.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + + assert!( + operations.inserts().is_empty(), + "a repeated stale miss should not rewrite the completed tombstone" + ); + let (stored, generation) = graph + .get(&ec_id) + .expect("should read stored tombstone") + .expect("should preserve tombstone"); + assert_eq!( + generation, 2, + "only the first withdrawal should advance the root generation" + ); + assert_eq!( + stored.consent.updated, first_updated, + "repeated withdrawal should preserve the first tombstone timestamp" + ); + } + #[test] fn tombstone_existing_from_stale_parallel_snapshot_stops_after_conflict() { let operations = Arc::new(RecordedEcKvOperations::default()); @@ -2546,6 +2825,10 @@ mod tests { mode: EcKvWriteMode::IfGenerationMatch(1), ttl: TOMBSTONE_TTL, }, + RecordedEcKvInsert { + mode: EcKvWriteMode::Add, + ttl: TOMBSTONE_TTL, + }, RecordedEcKvInsert { mode: EcKvWriteMode::IfGenerationMatch(1), ttl: TOMBSTONE_TTL, @@ -2629,8 +2912,8 @@ mod tests { ); assert_eq!( operations.inserts().len(), - 1, - "live reread should write once" + 2, + "live reread should write the root and completion marker" ); assert!( outcome @@ -2641,6 +2924,34 @@ mod tests { } } + #[test] + fn tombstone_stale_miss_still_writes_when_marker_operations_fail() { + let graph = KvIdentityGraph::new(MarkerFailingEcKv::new(1)); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + + let outcome = graph.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "marker failures must not suppress the withdrawal write" + ); + let (stored, _) = graph + .get(&ec_id) + .expect("should read stored row") + .expect("should preserve the root"); + assert!(!stored.consent.ok, "root should remain tombstoned"); + } + #[test] fn tombstone_existing_from_snapshot_retries_cas_conflict() { let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, false)); diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 1137b9e9d..feca67018 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -124,7 +124,7 @@ flowchart TD - **Non-regulated**: EC always allowed. - **Unknown**: Fail-closed when jurisdiction cannot be determined. -The `ec_identity_store` KV store is the only EC lifecycle store. It holds identity graph state, source-domain keyed partner UIDs, a minimal consent snapshot used for EC entry metadata, and withdrawal tombstones. Consent interpretation for each request remains based on the live request signals listed above. +The `ec_identity_store` KV store is the only EC lifecycle store. It holds identity graph state, source-domain keyed partner UIDs, a minimal consent snapshot used for EC entry metadata, withdrawal tombstones, and same-TTL completion markers that prevent stale point-read misses from rewriting completed tombstones. Consent interpretation for each request remains based on the live request signals listed above. ## Partner Sync Channels diff --git a/docs/superpowers/plans/2026-07-13-issue-881-idempotent-withdrawal-tombstones.md b/docs/superpowers/plans/2026-07-13-issue-881-idempotent-withdrawal-tombstones.md index 9dfc6db88..ec424e0f7 100644 --- a/docs/superpowers/plans/2026-07-13-issue-881-idempotent-withdrawal-tombstones.md +++ b/docs/superpowers/plans/2026-07-13-issue-881-idempotent-withdrawal-tombstones.md @@ -11,9 +11,11 @@ Make explicit EC withdrawal idempotent across repeated and concurrent requests without weakening the existing-key-only privacy invariant introduced by PR #885. The first successful withdrawal of a live row writes a CAS-protected -24-hour tombstone. A request that already has authoritative tombstone state -returns without reading or writing KV, so it cannot refresh the tombstone's -entry timestamp or TTL. +24-hour tombstone and a same-TTL completion marker. A request that already has +authoritative tombstone state returns without reading or writing KV. When an +eventually consistent point read instead misses the existing tombstone, the +strongly read completion marker prevents an unconditional replacement write, +so neither path refreshes the tombstone's entry timestamp or TTL. Browser-cookie deletion remains synchronous and best-effort KV failure must never block the response. @@ -35,7 +37,12 @@ never block the response. - A later re-consent may legitimately win when it linearizes after a completed or no-op withdrawal. Idempotency does not impose global withdrawal priority. - Repeated withdrawal preserves the original tombstone expiration because it - performs no second write. + performs no second root write. A completion-marker insert is attempted only + after the first successful tombstone write. +- A repeated stale point-read miss uses the strongly consistent completion + marker to avoid another unconditional root write. +- Completion-marker failure never suppresses the privacy write, and revival or + hard deletion clears the marker before the key can become live again. - When cookie and active EC IDs differ, every valid existing row is withdrawn independently; missing or malformed IDs are never created. - `ts-ec` and the pull-completeness marker are expired before best-effort KV @@ -44,10 +51,11 @@ never block the response. ## Non-Goals - Do not recreate missing roots from browser cookies. -- Do not add a cross-request lock, deduplication store, or withdrawal cookie. +- Do not add a separate deduplication store, cross-request lock, or withdrawal + cookie. The stale-miss completion marker lives in the existing EC store. - Do not restrict withdrawal handling to document navigations. -- Do not change the 24-hour tombstone duration or KV schema. -- Do not change EC generation, marker behavior, batch sync, pull sync, or +- Do not change the 24-hour tombstone duration or root-entry KV schema. +- Do not change EC generation, browser pull-marker behavior, batch sync, pull sync, or partner-upsert semantics beyond preserving tombstone rejection. - Do not make withdrawal dominate a re-consent that occurs after withdrawal's linearization point. @@ -62,9 +70,10 @@ snapshot is `Present`, including when that entry is already a tombstone. Paralle withdrawals therefore converge safely but still perform a redundant CAS write, and repeated requests reset the tombstone's 24-hour TTL. -The older public `write_withdrawal_tombstone` unconditional-overwrite helper is -now dead in production but remains available and could bypass the conditional -path in future code. +The unconditional `write_withdrawal_tombstone` helper remains necessary when +point reads miss a root that the strong list still sees. Without durable +completion state, repeated stale misses bypass the `Present` fast path and keep +rewriting that root. Finalization already: @@ -92,29 +101,38 @@ match a `Present` snapshot only when its `ec_id` equals the requested ID. Then retain the existing state machine: -| Initial/refreshed state | Action | -| ----------------------------------------- | ----------------------------------- | -| Matching tombstone | Return unchanged; zero backend work | -| Matching live entry + generation | CAS-write a 24-hour tombstone | -| Matching live entry without generation | Reread | -| Matching `Missing` | Return unchanged; never create | -| `Failed`, `NotRead`, or wrong-ID snapshot | Reread requested ID | -| CAS precondition failure | Reread and retry | -| Row disappears during retry | Return `Missing` | -| Store failure or retry exhaustion | Return ID-bound `Failed` | +| Initial/refreshed state | Action | +| ----------------------------------------- | ------------------------------------- | +| Matching tombstone | Return unchanged; zero backend work | +| Matching live entry + generation | CAS-write a 24-hour tombstone | +| Matching live entry without generation | Reread | +| Refreshed `Missing` | Prove absence or use guarded fallback | +| `Failed`, `NotRead`, or wrong-ID snapshot | Reread requested ID | +| CAS precondition failure | Reread and retry | +| Row disappears during retry | Return `Missing` | +| Store failure or retry exhaustion | Return ID-bound `Failed` | A successful tombstone remains `consent.ok = false`, has empty partner IDs, and uses `TOMBSTONE_TTL`. -### 2. Remove the unconditional bypass +Each successful root tombstone also creates an add-only completion marker in the +same EC store with `TOMBSTONE_TTL`. The marker namespace cannot collide with EC +IDs and is excluded from hash-prefix cluster counts. If a later point read +misses but the strong root-existence check and marker check both succeed, +withdrawal returns without rewriting the root. Marker read or write failures +fall back to the root privacy write. Same-key revival and hard deletion remove +the marker. -Delete the now-unused public `write_withdrawal_tombstone` method and its obsolete -overwrite test. Update `KvIdentityGraph::delete` documentation so it describes -the snapshot-aware conditional withdrawal path without linking to the removed -API. +### 2. Contain the unconditional fallback -Repository search must confirm there is no live Rust caller before removal. -Historical design documents may remain unchanged. +Keep `write_withdrawal_tombstone` only for the case where eventually consistent +point reads miss a root that the strong list still sees. Record completion after +that write so another stale miss cannot refresh the root. The marker is +best-effort after a successful root write; marker failure is logged and cannot +turn a completed privacy write into a reported failure. + +Update hard deletion and same-key revival to remove completion state. Historical +design documents may remain unchanged. ### 3. Prove operation-level idempotency @@ -125,9 +143,13 @@ precondition failure, and record each write's mode and TTL. Tests must show: - a supplied matching tombstone returns with zero lookups and zero insert attempts; - generation-unavailable tombstone state also performs no backend operation; -- the first live withdrawal performs exactly one `IfGenerationMatch` insert - with `TOMBSTONE_TTL`, while the second causes zero additional insert attempts - and leaves stored generation and `consent.updated` unchanged; +- the first live withdrawal performs one `IfGenerationMatch` root insert and + one add-only completion-marker insert with `TOMBSTONE_TTL`, while a repeated + authoritative tombstone causes zero additional insert attempts and leaves + stored root generation and `consent.updated` unchanged; +- two consecutive stale point-read misses cause one unconditional root write; + the second request observes the strong completion marker and leaves the root + generation and `consent.updated` unchanged; - two stale live snapshots model parallel requests: the first writes the tombstone; the second conflicts, rereads the tombstone, and performs no replacement write; @@ -178,9 +200,10 @@ Production finalization should not change unless these tests expose a defect. - `crates/trusted-server-core/src/ec/kv.rs` - Add the matching-tombstone no-op branch. - - Remove the unconditional overwrite helper. + - Gate the stale-miss fallback with a same-store completion marker. + - Clear completion markers on same-key revival and hard deletion. - Update withdrawal documentation. - - Add operation-count, repetition, and concurrency tests. + - Add operation-count, repetition, stale-read, and concurrency tests. - `crates/trusted-server-core/src/ec/finalize.rs` - Add two-ID, repeated-withdrawal, and KV-failure integration coverage. @@ -190,7 +213,7 @@ Production finalization should not change unless these tests expose a defect. - Record the reviewed design and verification contract. No dependency, configuration, adapter, JavaScript, or public wire-format change -is expected. +is expected. The EC store gains an internal completion-marker key namespace. ## Implementation Tasks @@ -207,13 +230,14 @@ is expected. - [x] Run `cargo test-fastly tombstone_existing_from_snapshot` and confirm the new repeated/no-backend tests fail before implementation. -### Task 2 — Implement the no-op branch and remove the bypass +### Task 2 — Implement the no-op branch and guard the fallback - [x] Return a matching tombstone snapshot before generation lookup, serialization, or write. - [x] Keep live/missing/failed/mismatched/CAS behavior unchanged. -- [x] Remove `write_withdrawal_tombstone` and update the `delete` documentation. -- [x] Search for remaining Rust references to the removed helper. +- [x] Record completion after successful tombstone writes. +- [x] Suppress repeated stale-miss overwrites when the completion marker exists. +- [x] Clear completion state on same-key revival and hard deletion. - [x] Run focused KV tests until green. ### Task 3 — Cover concurrent state changes @@ -248,7 +272,7 @@ is expected. | Issue requirement | Planned evidence | | ------------------------------------------------ | ----------------------------------------------------------------------------- | | First withdrawal establishes a 24-hour tombstone | Live-entry CAS test and existing `TOMBSTONE_TTL` assertion | -| Repeated requests avoid overwrite writes | Operation counts plus unchanged generation and `consent.updated` | +| Repeated requests avoid overwrite writes | Present and stale-miss operation counts plus unchanged root generation/time | | Concurrent withdrawal cannot restore IDs | Stale-snapshot and concurrent-live-update conflict tests | | Both differing valid IDs are withdrawn | Finalization test with both rows seeded | | Missing/unverified IDs create no root | Existing-key-only and invalid-ID tests | @@ -295,7 +319,10 @@ git diff --check - The first live-row withdrawal writes one CAS-protected 24-hour tombstone. - Repeated and concurrent withdrawals observing that tombstone perform no replacement write and do not refresh its expiration. -- Missing IDs remain absent; the unconditional overwrite API no longer exists. +- Repeated stale point-read misses use the completion marker and do not rewrite + the root tombstone. +- Missing IDs remain absent; unconditional overwrite remains confined to the + strongly confirmed stale-miss fallback. - Concurrent live updates before successful withdrawal are tombstoned on retry. - Later partner writes cannot repopulate tombstones. - Both valid differing IDs are handled independently. @@ -308,11 +335,15 @@ git diff --check the requested EC ID; a tombstone for another ID cannot suppress withdrawal. - **Linearization:** Returning an observed tombstone linearizes withdrawal at that observation. A later re-consent may legitimately win. -- **TTL visibility:** Record the first insert's TTL and every subsequent insert - attempt at the wrapper boundary; stable generation/timestamp alone is not - sufficient evidence. -- **Dead API removal:** Compile and repository-search after deletion to catch any - hidden caller. +- **TTL visibility:** Record the first root and completion-marker TTLs and every + subsequent insert attempt at the wrapper boundary; stable root + generation/timestamp alone is not sufficient evidence. +- **Stale-miss completion:** Write the marker only after the root tombstone + succeeds. Marker failures must fall back to the privacy write, while revival + and hard deletion must remove stale completion state. +- **Fallback containment:** Keep unconditional overwrite behind strong root + existence and completion-marker checks so no other path can refresh a + completed tombstone. - **Conflict-test realism:** Inject actual generation changes and persisted state, not endless synthetic precondition failures. - **Stack dependency:** Reconcile changes if PR #885 or draft PR #900 modifies