From 0fc5896312365be88607a00ff4100f2d8bfe88b8 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 28 Aug 2026 05:30:17 -0600 Subject: [PATCH 01/12] Revert "Revert "Merge pull request #299 from ProjectASAP/feat/accuracy-guarantee-propagation-172"" This reverts commit 5c6489b01c83f2245cfa2869d8e3a0899a59aae4. --- crates/asap-aware-mapping/src/accuracy.rs | 1088 +++++++++++++++++ crates/asap-aware-mapping/src/cost_model.rs | 2 + crates/asap-aware-mapping/src/grouping.rs | 6 + crates/asap-aware-mapping/src/lib.rs | 26 +- crates/asap-aware-mapping/src/recurrence.rs | 6 +- crates/asap-aware-mapping/src/replacement.rs | 1000 ++++++++++++++- crates/devtools/src/bin/dag_export.rs | 45 +- crates/types/src/dag_export.rs | 187 ++- crates/types/src/post_asap/expr.rs | 13 + crates/types/src/post_asap/guarantee.rs | 450 +++++++ crates/types/src/post_asap/mod.rs | 5 + crates/types/src/post_asap/sketch.rs | 6 +- .../asap-aware-mapping/summary_properties.md | 29 + 13 files changed, 2802 insertions(+), 61 deletions(-) create mode 100644 crates/asap-aware-mapping/src/accuracy.rs create mode 100644 crates/types/src/post_asap/guarantee.rs diff --git a/crates/asap-aware-mapping/src/accuracy.rs b/crates/asap-aware-mapping/src/accuracy.rs new file mode 100644 index 0000000..9393404 --- /dev/null +++ b/crates/asap-aware-mapping/src/accuracy.rs @@ -0,0 +1,1088 @@ +//! Planning-time accuracy algebra (issue #172): the [`AccuracyModel`] +//! extension point, its conservative default, and end-to-end +//! accuracy-budget allocation. +//! +//! ## Why a second trait next to `CostModel` +//! +//! Accuracy legality and cost ranking are different responsibilities. +//! [`crate::cost_model::CostModel`] answers "which legal candidate is +//! cheapest"; this module answers "which candidates are legal at all". The +//! pipeline [`crate::replacement`] runs is, in order: +//! +//! ```text +//! candidate generation +//! -> guarantee propagation (AccuracyModel::propagate) +//! -> AccuracyTarget satisfaction (AccuracyModel::satisfies) +//! -> legal candidates only (illegal ones become MemoGroup::rejected) +//! -> cost ranking / global selection (CostModel) +//! ``` +//! +//! A `CostModel` only ever sees the survivors, so it cannot override a +//! legality decision — the same "permutation only, never prune" contract +//! `CostModel::rank_candidates` already has, applied one stage earlier. +//! +//! ## What the default model admits +//! +//! [`DefaultAccuracyModel`] is deliberately conservative and fail-closed: +//! +//! | operator | rule | result | +//! |---|---|---| +//! | any, all inputs exact | exact input | the local guarantee (or exact) | +//! | `ApproximateAggregate`, all `AbsoluteValue` | additive | `Σ B`, `δ` by union bound | +//! | `ApproximateAggregate`, all `RelativeValue`, values known non-negative | multiplicative | `ε_in + ε_out + ε_in·ε_out`, `δ` by union bound | +//! | `Lipschitz { L }`, one `AbsoluteValue` input | Lipschitz | `L·B_in + B_local`, `δ` by union bound | +//! | `ExactSum`, value-like inputs | sum | `Σ B_i` (`AbsoluteValue`), `δ` by union bound over inputs | +//! | `ExactExtremum`, same-metric inputs | max/min | `max B_i`, `δ` by union bound over inputs | +//! | anything else | — | [`AccuracyError::UnsupportedComposition`] | +//! +//! Cross-metric compositions (a `Rank` error under a value-additive rule, +//! a `Cardinality` error under a `Frequency` sketch, …) have no registered +//! rule and are rejected. The child is **never** treated as exact. Nothing +//! assumes independence: every probability combinator is the union bound. +//! A statistic the rule needs but [`PropagationStats`] does not supply +//! (an input row count, a stream's L1 norm) stays a +//! [`BoundExpr::Unknown`] leaf — the guarantee is still produced, but it +//! cannot satisfy any target until something instantiates the statistic. +//! +//! ## Precedence between root and per-node targets +//! +//! - A root `QueryRequirements.accuracy`, when supplied to +//! [`crate::replacement::search_workload_with_targets`], is the +//! end-to-end target for that query's root value. It is checked against +//! the root group's candidates *before* cost ranking; a candidate whose +//! guarantee is unknown, or misses the target, is moved to +//! `MemoGroup::rejected`. +//! - For an approximate node over an **exact** child, the node's own +//! `AggIntent.accuracy` sizes its sketch, exactly as before this module +//! existed, and the readout's guarantee is that sketch's local guarantee. +//! - For an approximate node over an **approximate** child, the outer +//! node's `AggIntent.accuracy` is the end-to-end target *for that value*. +//! The inner node's `AggIntent.accuracy` is only its declared local +//! requirement: the as-declared composition is evaluated and kept only if +//! it satisfies the outer target, and the [`AccuracyBudgetAllocator`] +//! additionally proposes re-sized splits of the outer target. A front end +//! that copied the same target onto every node has therefore *not* +//! produced a valid end-to-end allocation — the composed guarantee is +//! what decides. +//! - `AccuracyTarget::Exact` on a node admits only exact realizations +//! (unchanged), and an approximate layer can never satisfy it. + +use asap_types::post_asap::{ + AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, GuaranteeSource, ProbabilityExpr, + ResultGuarantee, SketchAlgorithm, SketchParams, SketchQuery, SummaryFamilyType, +}; +use asap_types::types::AccuracyTarget; + +/// Statistics a propagation rule may consult. Every field is optional and +/// defaults to "unknown": a rule that needs a missing statistic emits a +/// [`BoundExpr::Unknown`] leaf (or rejects) rather than guessing. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct PropagationStats { + /// Whether every input value is known to be non-negative — required by + /// the multiplicative relative-error rule, which is unsound across a + /// sign change. + pub values_non_negative: Option, + /// Number of input rows an exact aggregation consumes (e.g. the number + /// of groups a `sum` folds), for `ExactSum`/`ExactExtremum`'s union + /// bound over per-input failures. + pub input_row_count: Option, +} + +/// The deployment-extensible accuracy algebra. `asap-aware-mapping` ships +/// [`DefaultAccuracyModel`]; a deployment with a proof for a composition the +/// default rejects (a registered cross-metric conversion, say) implements +/// this trait and passes it to +/// [`crate::replacement::SketchAlgorithmStrategy::with_models`]. +pub trait AccuracyModel { + /// The guarantee of reading `query` out of a summary of family `family` + /// built over an **exact** input — derived from the family's committed + /// parameters by inverting the same sizing formulas + /// [`crate::replacement::default_size_params`] uses. `None` when this + /// model has no error model for the family (the default has none for + /// `Sample`/`Wavelet`/`StatModel`). + fn local_guarantee( + &self, + family: &SummaryFamilyType, + query: &SketchQuery, + ) -> Option; + + /// Compose `inputs`' guarantees (in the parent's child order) with the + /// parent's own `local` guarantee under `op`. `Err` is the fail-closed + /// answer: no registered rule, or a missing input guarantee. + fn propagate( + &self, + op: &CompositionOperator, + inputs: &[ResultGuarantee], + local: Option<&ResultGuarantee>, + stats: &PropagationStats, + ) -> Result; + + /// Does `guarantee` meet `target`? An unevaluable bound or probability + /// never satisfies anything. + fn satisfies(&self, guarantee: &ResultGuarantee, target: &AccuracyTarget) -> bool; +} + +/// The conservative, fail-closed default — see the module docs' table. +#[derive(Debug, Default, Clone, Copy)] +pub struct DefaultAccuracyModel; + +/// Failure probability this crate attributes to a KLL bound sized by +/// `k = ⌈2/ε⌉`: the 99%-confidence convention the `k`-to-rank-error tables +/// of the reference implementation (Apache DataSketches) quote that formula +/// at. KLL's sizing ignores δ, so this is the confidence level the sizing +/// implicitly claims, not something re-derived from `k`. +pub const KLL_FAILURE_PROBABILITY: f64 = 0.01; + +/// Failure probability this crate attributes to an HLL/Theta/KMV bound +/// sized by its relative *standard error* (`1.04/√m`, `1/√k`): a +/// one-standard-deviation bound of an approximately Gaussian estimator is +/// exceeded with probability ≈ 0.3173. Those families' sizing ignores δ, so +/// an `EpsilonDelta` target with a tighter δ is honestly not met by the +/// parameters the sizing picked; the guarantee says so instead of hiding it. +pub const STANDARD_ERROR_FAILURE_PROBABILITY: f64 = 0.3173; + +/// Small relative tolerance for comparing an evaluated bound against a +/// target, so a parameter sized by `⌈·⌉` to *exactly* meet ε is not rejected +/// by floating-point noise. +const SATISFACTION_TOLERANCE: f64 = 1e-9; + +impl DefaultAccuracyModel { + /// The local guarantee of one sketch `(algorithm, params)` for `query` + /// — each arm inverts the matching formula in + /// [`crate::replacement::default_size_params`]. + pub fn sketch_guarantee( + algorithm: &SketchAlgorithm, + params: &SketchParams, + query: &SketchQuery, + ) -> Option { + let (metric, bound, delta) = match params { + // KLL: rank error ε ≈ 2/k. + SketchParams::Kll { k } => ( + ErrorMetric::Rank, + 2.0 / f64::from(*k), + ProbabilityExpr::Constant { + value: KLL_FAILURE_PROBABILITY, + }, + ), + // DDSketch: deterministic relative value error α. + SketchParams::DDSketch { alpha } => { + (ErrorMetric::RelativeValue, *alpha, ProbabilityExpr::Zero) + } + // HLL: relative standard error 1.04/√(2^p). + SketchParams::Hll { precision } => ( + ErrorMetric::Cardinality, + 1.04 / 2f64.powi(i32::from(*precision)).sqrt(), + ProbabilityExpr::Constant { + value: STANDARD_ERROR_FAILURE_PROBABILITY, + }, + ), + // KMV / Theta: relative standard error 1/√k. + SketchParams::Kmv { k } | SketchParams::Theta { k } => ( + ErrorMetric::Cardinality, + 1.0 / f64::from(*k).sqrt(), + ProbabilityExpr::Constant { + value: STANDARD_ERROR_FAILURE_PROBABILITY, + }, + ), + // CMS family: over-count ≤ (e/w)·‖f‖₁ with probability ≥ 1 − e^{−d}. + // Count-Sketch is sized with the same placeholder formula (see + // `default_size_params`), so it gets the same placeholder bound. + SketchParams::Cms { width, depth } + | SketchParams::CountSketch { width, depth } + | SketchParams::CmsWithHeap { width, depth, .. } + | SketchParams::CountSketchWithHeap { width, depth, .. } => ( + ErrorMetric::Frequency, + std::f64::consts::E / f64::from(*width), + ProbabilityExpr::Constant { + value: (-f64::from(*depth)).exp(), + }, + ), + }; + let mut provenance = vec![GuaranteeSource::SketchReadout { + algorithm: format!("{algorithm:?}"), + params: serde_json::to_value(params).unwrap_or(serde_json::Value::Null), + query: format!("{query:?}"), + }]; + // A `TopK` readout's count for each *reported* key carries the + // sketch's frequency bound; which keys are reported is not certified + // by anything here (issue #172, PR 3) — say so in the trail rather + // than claiming `TopKMembership`. + if matches!(query, SketchQuery::TopK { .. }) { + provenance.push(GuaranteeSource::UnavailableStatistic { + statistic: "topk_membership_margin_certificate".into(), + }); + } + Some(ResultGuarantee { + metric, + bound: BoundExpr::Constant { value: bound }, + failure_probability: delta, + provenance, + }) + } + + fn additive( + op: &CompositionOperator, + inputs: &[ResultGuarantee], + local: &ResultGuarantee, + rule: &str, + ) -> ResultGuarantee { + let mut terms: Vec = inputs.iter().map(|g| g.bound.clone()).collect(); + terms.push(local.bound.clone()); + let mut deltas: Vec = inputs + .iter() + .map(|g| g.failure_probability.clone()) + .collect(); + deltas.push(local.failure_probability.clone()); + ResultGuarantee { + metric: local.metric, + bound: BoundExpr::Sum { terms }, + failure_probability: ProbabilityExpr::UnionBound { terms: deltas }, + provenance: composed_provenance(op, inputs, local, rule), + } + } + + /// `(1 + ε_total) = Π (1 + ε_i)` ⇒ for two factors + /// `ε_in + ε_out + ε_in·ε_out`; written out as the sum of all + /// cross-products so the expression tree is exact for any input count. + fn multiplicative( + op: &CompositionOperator, + inputs: &[ResultGuarantee], + local: &ResultGuarantee, + ) -> ResultGuarantee { + let factors: Vec<&BoundExpr> = inputs + .iter() + .map(|g| &g.bound) + .chain(std::iter::once(&local.bound)) + .collect(); + // Every non-empty subset's product: Π(1+ε_i) − 1 = Σ_{S≠∅} Π_{i∈S} ε_i. + let mut terms = Vec::new(); + for mask in 1..(1u32 << factors.len()) { + let subset: Vec = factors + .iter() + .enumerate() + .filter(|(i, _)| mask & (1 << i) != 0) + .map(|(_, b)| (*b).clone()) + .collect(); + terms.push(if subset.len() == 1 { + subset.into_iter().next().expect("one element") + } else { + BoundExpr::Product { factors: subset } + }); + } + let mut deltas: Vec = inputs + .iter() + .map(|g| g.failure_probability.clone()) + .collect(); + deltas.push(local.failure_probability.clone()); + ResultGuarantee { + metric: ErrorMetric::RelativeValue, + bound: BoundExpr::Sum { terms }, + failure_probability: ProbabilityExpr::UnionBound { terms: deltas }, + provenance: composed_provenance(op, inputs, local, "relative_cross_term_union_bound"), + } + } + + fn lipschitz( + op: &CompositionOperator, + constant: f64, + inputs: &[ResultGuarantee], + local: Option<&ResultGuarantee>, + ) -> ResultGuarantee { + let input = &inputs[0]; + let scaled = BoundExpr::Scaled { + factor: constant, + inner: Box::new(input.bound.clone()), + }; + let (bound, delta) = match local { + Some(local) => ( + BoundExpr::Sum { + terms: vec![scaled, local.bound.clone()], + }, + ProbabilityExpr::UnionBound { + terms: vec![ + input.failure_probability.clone(), + local.failure_probability.clone(), + ], + }, + ), + None => (scaled, input.failure_probability.clone()), + }; + let exact_local = ResultGuarantee::exact("deterministic Lipschitz transformation"); + ResultGuarantee { + metric: ErrorMetric::AbsoluteValue, + bound, + failure_probability: delta, + provenance: composed_provenance( + op, + inputs, + local.unwrap_or(&exact_local), + "lipschitz_union_bound", + ), + } + } + + /// Exact `sum` over approximate inputs: `B ≤ Σ B_i`, `δ ≤ Σ δ_i`. The + /// planner composes one *per-value* child guarantee over an unknown + /// number of input rows, so both the bound and the union bound scale by + /// `stats.input_row_count` — an [`BoundExpr::Unknown`] leaf when it is + /// not supplied. Each input's normalized bound is first converted to + /// absolute units via the statistic its metric is normalized by (also + /// unknown unless supplied); a `Rank` input has no such conversion. + fn exact_sum( + op: &CompositionOperator, + inputs: &[ResultGuarantee], + stats: &PropagationStats, + ) -> Result { + let mut terms = Vec::with_capacity(inputs.len()); + let mut deltas = Vec::with_capacity(inputs.len()); + let mut provenance = Vec::new(); + for (i, input) in inputs.iter().enumerate() { + let absolute = + absolute_bound(input).ok_or_else(|| AccuracyError::UnsupportedComposition { + operator: op.clone(), + input_metrics: inputs.iter().map(|g| g.metric).collect(), + local_metric: None, + reason: format!( + "input {i} carries a {:?} guarantee, which has no registered \ + conversion to an absolute value error", + input.metric + ), + })?; + if let BoundExpr::Product { factors } = &absolute { + for f in factors { + if let BoundExpr::Unknown { statistic } = f { + provenance.push(GuaranteeSource::UnavailableStatistic { + statistic: statistic.clone(), + }); + } + } + } + terms.push(absolute); + deltas.push(input.failure_probability.clone()); + } + let count = row_count(stats, &mut provenance); + let exact_local = ResultGuarantee::exact("ExactAggregate(Sum)"); + provenance.extend(composed_provenance( + op, + inputs, + &exact_local, + "exact_sum_union_bound", + )); + Ok(ResultGuarantee { + metric: ErrorMetric::AbsoluteValue, + bound: BoundExpr::Product { + factors: vec![count.clone(), BoundExpr::Sum { terms }], + }, + failure_probability: ProbabilityExpr::Scaled { + count, + inner: Box::new(ProbabilityExpr::UnionBound { terms: deltas }), + }, + provenance, + }) + } + + /// Exact `max`/`min` over approximate inputs of one shared metric: the + /// returned value's error is at most the largest input bound (order + /// statistics are monotone under a uniform perturbation), with + /// probability by the union bound over every input row. This bounds the + /// returned *value*; it does not identify the true winning key. + fn exact_extremum( + op: &CompositionOperator, + inputs: &[ResultGuarantee], + stats: &PropagationStats, + ) -> Result { + let metric = inputs[0].metric; + if inputs.iter().any(|g| g.metric != metric) || metric == ErrorMetric::TopKMembership { + return Err(AccuracyError::UnsupportedComposition { + operator: op.clone(), + input_metrics: inputs.iter().map(|g| g.metric).collect(), + local_metric: None, + reason: "exact max/min needs every input under one value-like metric".into(), + }); + } + let mut provenance = Vec::new(); + let count = row_count(stats, &mut provenance); + let exact_local = ResultGuarantee::exact("ExactAggregate(MinMax)"); + provenance.extend(composed_provenance( + op, + inputs, + &exact_local, + "exact_extremum_union_bound", + )); + Ok(ResultGuarantee { + metric, + bound: BoundExpr::Max { + terms: inputs.iter().map(|g| g.bound.clone()).collect(), + }, + failure_probability: ProbabilityExpr::Scaled { + count, + inner: Box::new(ProbabilityExpr::UnionBound { + terms: inputs + .iter() + .map(|g| g.failure_probability.clone()) + .collect(), + }), + }, + provenance, + }) + } +} + +/// `stats.input_row_count` as a bound factor, or an `Unknown` leaf (recorded +/// in `provenance`) when absent. +fn row_count(stats: &PropagationStats, provenance: &mut Vec) -> BoundExpr { + match stats.input_row_count { + Some(n) => BoundExpr::Constant { value: n as f64 }, + None => { + provenance.push(GuaranteeSource::UnavailableStatistic { + statistic: "input_row_count".into(), + }); + BoundExpr::Unknown { + statistic: "input_row_count".into(), + } + } + } +} + +/// `input`'s bound converted to absolute value units, multiplying a +/// normalized metric by the (unknown) statistic it is normalized by. `None` +/// for a metric with no such conversion (`Rank`, `TopKMembership`). +fn absolute_bound(input: &ResultGuarantee) -> Option { + let normalizer = match input.metric { + ErrorMetric::AbsoluteValue => return Some(input.bound.clone()), + ErrorMetric::RelativeValue => "true_value_magnitude", + ErrorMetric::Cardinality => "true_cardinality", + ErrorMetric::Frequency => "stream_l1_norm", + // `Rank` has no distribution-free conversion to a value error; a + // metric this crate does not know has no registered conversion. + ErrorMetric::Rank | ErrorMetric::TopKMembership | _ => return None, + }; + if input.bound.is_zero() { + return Some(BoundExpr::Zero); + } + Some(BoundExpr::Product { + factors: vec![ + input.bound.clone(), + BoundExpr::Unknown { + statistic: normalizer.into(), + }, + ], + }) +} + +fn composed_provenance( + op: &CompositionOperator, + inputs: &[ResultGuarantee], + local: &ResultGuarantee, + rule: &str, +) -> Vec { + let mut provenance: Vec = inputs + .iter() + .enumerate() + .map(|(input_index, g)| GuaranteeSource::ChildGuarantee { + input_index, + guarantee: Box::new(g.clone()), + }) + .collect(); + provenance.extend(local.provenance.iter().cloned()); + provenance.push(GuaranteeSource::CompositionStep { + operator: op.clone(), + rule: rule.into(), + }); + provenance +} + +impl AccuracyModel for DefaultAccuracyModel { + fn local_guarantee( + &self, + family: &SummaryFamilyType, + query: &SketchQuery, + ) -> Option { + match family { + SummaryFamilyType::Plain(_) => Some(ResultGuarantee::exact("Plain value")), + SummaryFamilyType::ExactAggregate(kind, _) => { + Some(ResultGuarantee::exact(format!("ExactAggregate({kind:?})"))) + } + SummaryFamilyType::Sketch(kind, _) => { + Self::sketch_guarantee(kind.algorithm(), kind.params(), query) + } + // No error model is registered for these families. + SummaryFamilyType::Sample(..) + | SummaryFamilyType::Wavelet(..) + | SummaryFamilyType::StatModel(..) => None, + } + } + + fn propagate( + &self, + op: &CompositionOperator, + inputs: &[ResultGuarantee], + local: Option<&ResultGuarantee>, + stats: &PropagationStats, + ) -> Result { + // Exact input: only the local guarantee remains (or the value is exact). + if inputs.iter().all(ResultGuarantee::is_exact) { + return Ok(match local { + Some(local) => { + let mut out = local.clone(); + out.provenance + .extend(inputs.iter().enumerate().map(|(input_index, g)| { + GuaranteeSource::ChildGuarantee { + input_index, + guarantee: Box::new(g.clone()), + } + })); + out.provenance.push(GuaranteeSource::CompositionStep { + operator: op.clone(), + rule: "exact_input".into(), + }); + out + } + None => { + let mut out = ResultGuarantee::exact(format!("{op:?} over exact inputs")); + out.provenance.push(GuaranteeSource::CompositionStep { + operator: op.clone(), + rule: "exact_input".into(), + }); + out + } + }); + } + + let input_metrics: Vec = inputs.iter().map(|g| g.metric).collect(); + let unsupported = |reason: String| AccuracyError::UnsupportedComposition { + operator: op.clone(), + input_metrics: input_metrics.clone(), + local_metric: local.map(|g| g.metric), + reason, + }; + // An exact input is compatible with every metric; only approximate + // inputs constrain the rule. + let approximate: Vec<&ResultGuarantee> = inputs.iter().filter(|g| !g.is_exact()).collect(); + let same_metric = |metric: ErrorMetric| approximate.iter().all(|g| g.metric == metric); + + match op { + CompositionOperator::ApproximateAggregate => { + let local = local.ok_or_else(|| { + unsupported("approximate operator has no local guarantee to compose".into()) + })?; + if !same_metric(local.metric) { + return Err(unsupported(format!( + "no registered cross-metric rule from {input_metrics:?} to {:?}", + local.metric + ))); + } + match local.metric { + ErrorMetric::AbsoluteValue => { + Ok(Self::additive(op, inputs, local, "additive_union_bound")) + } + ErrorMetric::RelativeValue => { + if stats.values_non_negative != Some(true) { + return Err(unsupported( + "relative-error composition needs values of known sign \ + (PropagationStats::values_non_negative)" + .into(), + )); + } + Ok(Self::multiplicative(op, inputs, local)) + } + ErrorMetric::Rank + | ErrorMetric::Cardinality + | ErrorMetric::Frequency + | ErrorMetric::TopKMembership + | _ => Err(unsupported(format!( + "no registered same-metric composition rule for {:?} over {:?}", + local.metric, local.metric + ))), + } + } + CompositionOperator::Lipschitz { constant } => { + if !(constant.is_finite() && *constant >= 0.0) { + return Err(unsupported(format!( + "Lipschitz constant {constant} is not a finite non-negative number" + ))); + } + if inputs.len() != 1 || !same_metric(ErrorMetric::AbsoluteValue) { + return Err(unsupported( + "Lipschitz rule is registered for exactly one AbsoluteValue input".into(), + )); + } + if local.is_some_and(|g| g.metric != ErrorMetric::AbsoluteValue) { + return Err(unsupported( + "Lipschitz rule needs an AbsoluteValue local guarantee".into(), + )); + } + Ok(Self::lipschitz(op, *constant, inputs, local)) + } + CompositionOperator::ExactSum => Self::exact_sum(op, inputs, stats), + CompositionOperator::ExactExtremum => Self::exact_extremum(op, inputs, stats), + CompositionOperator::TopKSelection => Err(unsupported( + "top-k membership over approximate inputs needs a margin certificate \ + (issue #172, PR 3)" + .into(), + )), + // An operator this crate does not know has no registered rule. + _ => Err(unsupported("no registered rule for this operator".into())), + } + } + + fn satisfies(&self, guarantee: &ResultGuarantee, target: &AccuracyTarget) -> bool { + let within = |value: Option, limit: f64| { + value.is_some_and(|v| v <= limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON) + }; + match target { + AccuracyTarget::Exact => guarantee.is_exact(), + AccuracyTarget::Epsilon(eps) => within(guarantee.bound.evaluate(), *eps), + AccuracyTarget::EpsilonDelta { epsilon, delta } => { + within(guarantee.bound.evaluate(), *epsilon) + && within(guarantee.failure_probability.evaluate(), *delta) + } + } + } +} + +// ── Budget allocation ─────────────────────────────────────────────────────── + +/// The shape of a composition an allocator splits a budget across. +#[derive(Debug, Clone, PartialEq)] +pub struct CompositionShape { + /// The metric the composed guarantee will carry — decides whether the + /// budget composes additively (`Σ ε_i ≤ ε`) or multiplicatively + /// (`Π(1+ε_i) ≤ 1+ε`). + pub metric: ErrorMetric, + /// How many approximate layers share the budget (≥ 1). + pub approximate_layer_count: usize, +} + +/// One way of splitting an end-to-end target across a composition's +/// approximate layers. `layers[0]` is the outermost layer's local target; +/// the remainder are the inner layers', outermost first. +#[derive(Debug, Clone, PartialEq)] +pub struct AccuracyAllocation { + pub allocator: &'static str, + pub layers: Vec, +} + +impl AccuracyAllocation { + /// The end-to-end budget left for everything below `layers[0]` — what + /// the inner subtree must satisfy as a whole (it re-splits internally). + /// `None` for a single-layer allocation. + pub fn inner_target(&self, shape: &CompositionShape) -> Option { + let inner = &self.layers[1..]; + if inner.is_empty() { + return None; + } + let (eps, delta): (Vec, Vec>) = inner + .iter() + .map(|t| match t { + AccuracyTarget::Exact => (0.0, Some(0.0)), + AccuracyTarget::Epsilon(e) => (*e, None), + AccuracyTarget::EpsilonDelta { epsilon, delta } => (*epsilon, Some(*delta)), + }) + .unzip(); + let epsilon = match shape.metric { + ErrorMetric::RelativeValue => eps.iter().map(|e| 1.0 + e).product::() - 1.0, + _ => eps.iter().sum(), + }; + Some(match delta.iter().copied().sum::>() { + Some(delta) => AccuracyTarget::EpsilonDelta { epsilon, delta }, + None => AccuracyTarget::Epsilon(epsilon), + }) + } +} + +/// Enumerates the finite set of budget splits the search tries for one +/// composition. Exposed as its own hook because equal splitting is rarely +/// cost-optimal; a deployment can return several candidate splits and let +/// cost ranking pick among the legal ones. +pub trait AccuracyBudgetAllocator { + fn allocations( + &self, + target: &AccuracyTarget, + composition: &CompositionShape, + ) -> Vec; +} + +/// The initial deterministic allocator: every approximate layer gets an +/// equal share — `ε_i = ε / n`, `δ_i = δ / n` for an additively composed +/// metric, and `ε_i = (1 + ε)^{1/n} − 1` for a multiplicatively composed +/// one — so the composed bound meets the target exactly with no slack. +/// `AccuracyTarget::Exact` yields no allocation: no approximate layer can +/// meet it. +#[derive(Debug, Default, Clone, Copy)] +pub struct EqualSplitAllocator; + +impl AccuracyBudgetAllocator for EqualSplitAllocator { + fn allocations( + &self, + target: &AccuracyTarget, + composition: &CompositionShape, + ) -> Vec { + let n = composition.approximate_layer_count.max(1); + let (epsilon, delta) = match target { + AccuracyTarget::Exact => return Vec::new(), + AccuracyTarget::Epsilon(e) => (*e, None), + AccuracyTarget::EpsilonDelta { epsilon, delta } => (*epsilon, Some(*delta)), + }; + if !(epsilon.is_finite() && epsilon > 0.0) { + return Vec::new(); + } + let local_epsilon = match composition.metric { + ErrorMetric::RelativeValue => (1.0 + epsilon).powf(1.0 / n as f64) - 1.0, + _ => epsilon / n as f64, + }; + let layer = match delta { + Some(delta) => AccuracyTarget::EpsilonDelta { + epsilon: local_epsilon, + delta: delta / n as f64, + }, + None => AccuracyTarget::Epsilon(local_epsilon), + }; + vec![AccuracyAllocation { + allocator: "EqualSplitAllocator", + layers: vec![layer; n], + }] + } +} + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::post_asap::{GroupingStrategy, SketchKind}; + + fn abs(bound: f64, delta: f64) -> ResultGuarantee { + ResultGuarantee { + metric: ErrorMetric::AbsoluteValue, + bound: BoundExpr::Constant { value: bound }, + failure_probability: ProbabilityExpr::Constant { value: delta }, + provenance: vec![], + } + } + + fn rel(bound: f64) -> ResultGuarantee { + ResultGuarantee { + metric: ErrorMetric::RelativeValue, + bound: BoundExpr::Constant { value: bound }, + failure_probability: ProbabilityExpr::Zero, + provenance: vec![], + } + } + + fn with_metric(metric: ErrorMetric, bound: f64) -> ResultGuarantee { + ResultGuarantee { + metric, + ..abs(bound, 0.0) + } + } + + #[test] + fn exact_child_contributes_zero_error() { + let local = abs(0.05, 0.01); + let out = DefaultAccuracyModel + .propagate( + &CompositionOperator::ApproximateAggregate, + &[ResultGuarantee::exact("sum")], + Some(&local), + &PropagationStats::default(), + ) + .unwrap(); + assert_eq!(out.bound.evaluate(), Some(0.05)); + assert_eq!(out.failure_probability.evaluate(), Some(0.01)); + assert_eq!(out.metric, ErrorMetric::AbsoluteValue); + } + + #[test] + fn additive_bounds_and_delta_union_bound_compose() { + let out = DefaultAccuracyModel + .propagate( + &CompositionOperator::ApproximateAggregate, + &[abs(0.02, 0.01)], + Some(&abs(0.03, 0.02)), + &PropagationStats::default(), + ) + .unwrap(); + assert!((out.bound.evaluate().unwrap() - 0.05).abs() < 1e-12); + // Union bound, not 1 − (1−0.01)(1−0.02) = 0.0298. + assert!((out.failure_probability.evaluate().unwrap() - 0.03).abs() < 1e-12); + assert!(out.provenance.iter().any(|s| matches!( + s, + GuaranteeSource::CompositionStep { rule, .. } if rule == "additive_union_bound" + ))); + } + + #[test] + fn relative_error_includes_the_cross_term() { + let stats = PropagationStats { + values_non_negative: Some(true), + ..Default::default() + }; + let out = DefaultAccuracyModel + .propagate( + &CompositionOperator::ApproximateAggregate, + &[rel(0.1)], + Some(&rel(0.2)), + &stats, + ) + .unwrap(); + // 0.1 + 0.2 + 0.1·0.2 = 0.32, not 0.3. + assert!((out.bound.evaluate().unwrap() - 0.32).abs() < 1e-12); + assert_eq!(out.metric, ErrorMetric::RelativeValue); + } + + #[test] + fn relative_error_without_sign_knowledge_is_rejected() { + let err = DefaultAccuracyModel + .propagate( + &CompositionOperator::ApproximateAggregate, + &[rel(0.1)], + Some(&rel(0.2)), + &PropagationStats::default(), + ) + .unwrap_err(); + assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); + } + + #[test] + fn incompatible_metrics_are_rejected_not_treated_as_exact() { + // HLL cardinality error under a CMS frequency guarantee. + let err = DefaultAccuracyModel + .propagate( + &CompositionOperator::ApproximateAggregate, + &[with_metric(ErrorMetric::Cardinality, 0.01)], + Some(&with_metric(ErrorMetric::Frequency, 0.01)), + &PropagationStats::default(), + ) + .unwrap_err(); + assert!(matches!( + err, + AccuracyError::UnsupportedComposition { + input_metrics, + local_metric: Some(ErrorMetric::Frequency), + .. + } if input_metrics == vec![ErrorMetric::Cardinality] + )); + // Quantile rank error under value-additive logic. + let err = DefaultAccuracyModel + .propagate( + &CompositionOperator::ApproximateAggregate, + &[with_metric(ErrorMetric::Rank, 0.01)], + Some(&abs(0.01, 0.0)), + &PropagationStats::default(), + ) + .unwrap_err(); + assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); + } + + #[test] + fn same_metric_rank_over_rank_has_no_registered_rule() { + let err = DefaultAccuracyModel + .propagate( + &CompositionOperator::ApproximateAggregate, + &[with_metric(ErrorMetric::Rank, 0.01)], + Some(&with_metric(ErrorMetric::Rank, 0.01)), + &PropagationStats::default(), + ) + .unwrap_err(); + assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); + } + + #[test] + fn lipschitz_scales_the_input_bound() { + let out = DefaultAccuracyModel + .propagate( + &CompositionOperator::Lipschitz { constant: 3.0 }, + &[abs(0.1, 0.01)], + Some(&abs(0.05, 0.02)), + &PropagationStats::default(), + ) + .unwrap(); + assert!((out.bound.evaluate().unwrap() - 0.35).abs() < 1e-12); + assert!((out.failure_probability.evaluate().unwrap() - 0.03).abs() < 1e-12); + } + + #[test] + fn exact_sum_over_approximate_sums_bounds_and_keeps_unknown_row_count_unknown() { + let out = DefaultAccuracyModel + .propagate( + &CompositionOperator::ExactSum, + &[abs(0.1, 0.01)], + None, + &PropagationStats::default(), + ) + .unwrap(); + assert_eq!(out.metric, ErrorMetric::AbsoluteValue); + assert_eq!( + out.bound.evaluate(), + None, + "unknown row count stays unknown" + ); + assert!(out.provenance.iter().any(|s| matches!( + s, + GuaranteeSource::UnavailableStatistic { statistic } if statistic == "input_row_count" + ))); + assert!(!DefaultAccuracyModel.satisfies(&out, &AccuracyTarget::Epsilon(1.0))); + + let known = PropagationStats { + input_row_count: Some(4), + ..Default::default() + }; + let out = DefaultAccuracyModel + .propagate( + &CompositionOperator::ExactSum, + &[abs(0.1, 0.01)], + None, + &known, + ) + .unwrap(); + assert!((out.bound.evaluate().unwrap() - 0.4).abs() < 1e-12); + assert!((out.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); + } + + #[test] + fn exact_extremum_takes_the_max_bound() { + let known = PropagationStats { + input_row_count: Some(2), + ..Default::default() + }; + let out = DefaultAccuracyModel + .propagate( + &CompositionOperator::ExactExtremum, + &[abs(0.1, 0.01), abs(0.3, 0.01)], + None, + &known, + ) + .unwrap(); + assert!((out.bound.evaluate().unwrap() - 0.3).abs() < 1e-12); + assert!((out.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); + } + + #[test] + fn topk_selection_is_unsupported() { + let err = DefaultAccuracyModel + .propagate( + &CompositionOperator::TopKSelection, + &[abs(0.1, 0.01)], + None, + &PropagationStats::default(), + ) + .unwrap_err(); + assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); + } + + #[test] + fn local_guarantee_inverts_the_sizing_formulas() { + use crate::replacement::default_size_params; + use asap_types::pre_asap::agg_intent::{default_cardinality, default_quantile}; + + let q = default_quantile(0.99); + let params = default_size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01); + let g = DefaultAccuracyModel + .local_guarantee( + &SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, params), + GroupingStrategy::default(), + ), + &SketchQuery::Quantile { q: 0.99 }, + ) + .unwrap(); + assert_eq!(g.metric, ErrorMetric::Rank); + assert!(DefaultAccuracyModel.satisfies(&g, &AccuracyTarget::Epsilon(0.01))); + assert_eq!(g.approximate_layer_count(), 1); + + let c = default_cardinality(); + let params = default_size_params(SketchAlgorithm::Hll, &c, 0.01, 0.01); + let g = DefaultAccuracyModel + .local_guarantee( + &SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Hll, params), + GroupingStrategy::default(), + ), + &SketchQuery::Cardinality, + ) + .unwrap(); + assert_eq!(g.metric, ErrorMetric::Cardinality); + assert!(DefaultAccuracyModel.satisfies(&g, &AccuracyTarget::Epsilon(0.01))); + + let params = default_size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001); + let g = DefaultAccuracyModel + .local_guarantee( + &SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Cms, params), + GroupingStrategy::default(), + ), + &SketchQuery::Cardinality, + ) + .unwrap(); + assert_eq!(g.metric, ErrorMetric::Frequency); + assert!(DefaultAccuracyModel.satisfies( + &g, + &AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001 + } + )); + } + + #[test] + fn satisfies_is_fail_closed_on_unknowns_and_exact() { + let unknown = ResultGuarantee { + bound: BoundExpr::Unknown { + statistic: "x".into(), + }, + ..abs(0.0, 0.0) + }; + assert!(!DefaultAccuracyModel.satisfies(&unknown, &AccuracyTarget::Epsilon(1.0))); + assert!(!DefaultAccuracyModel.satisfies(&abs(0.0, 0.01), &AccuracyTarget::Exact)); + assert!( + DefaultAccuracyModel.satisfies(&ResultGuarantee::exact("x"), &AccuracyTarget::Exact) + ); + } + + #[test] + fn equal_split_respects_the_root_epsilon_and_delta() { + let target = AccuracyTarget::EpsilonDelta { + epsilon: 0.1, + delta: 0.02, + }; + let shape = CompositionShape { + metric: ErrorMetric::AbsoluteValue, + approximate_layer_count: 2, + }; + let allocations = EqualSplitAllocator.allocations(&target, &shape); + assert_eq!(allocations.len(), 1); + let layers = &allocations[0].layers; + assert_eq!(layers.len(), 2); + let (eps, deltas): (Vec, Vec) = layers + .iter() + .map(|t| match t { + AccuracyTarget::EpsilonDelta { epsilon, delta } => (*epsilon, *delta), + other => panic!("unexpected {other:?}"), + }) + .unzip(); + assert!((eps.iter().sum::() - 0.1).abs() < 1e-12); + assert!((deltas.iter().sum::() - 0.02).abs() < 1e-12); + assert_eq!( + allocations[0].inner_target(&shape), + Some(AccuracyTarget::EpsilonDelta { + epsilon: 0.05, + delta: 0.01 + }) + ); + + // Multiplicative composition: (1+ε_i)^2 = 1+ε, not 2ε_i = ε. + let rel_shape = CompositionShape { + metric: ErrorMetric::RelativeValue, + approximate_layer_count: 2, + }; + let allocations = + EqualSplitAllocator.allocations(&AccuracyTarget::Epsilon(0.21), &rel_shape); + let AccuracyTarget::Epsilon(e) = allocations[0].layers[0] else { + panic!() + }; + assert!((e - 0.1).abs() < 1e-12); + + assert!(EqualSplitAllocator + .allocations(&AccuracyTarget::Exact, &shape) + .is_empty()); + } +} diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 681402e..98b4f91 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -838,6 +838,7 @@ mod tests { fields: vec![], time_index: None, }, + guarantee: None, }), family: family.clone(), col: asap_types::pre_asap::expr_ir::ColumnRef::Named("value".into()), @@ -852,6 +853,7 @@ mod tests { }], time_index: None, }, + guarantee: None, } } diff --git a/crates/asap-aware-mapping/src/grouping.rs b/crates/asap-aware-mapping/src/grouping.rs index c5485f4..0e3f294 100644 --- a/crates/asap-aware-mapping/src/grouping.rs +++ b/crates/asap-aware-mapping/src/grouping.rs @@ -275,6 +275,11 @@ fn with_grouping(node: Rc, grouping: GroupingStrategy) -> Rc, grouping: GroupingStrategy) -> Rc, + pub rejected: Vec, +} + /// A replacement strategy: given a [`TargetSubDAG`], does this strategy have /// an opinion on it at all (`matches`), and if so, every semantically valid /// replacement (`replacements`)? @@ -517,6 +554,19 @@ pub trait ReplacementStrategy { /// Reporting "every valid candidate" is this method's whole job; picking /// the best one is a [`CostModel`]'s job, out of scope here. fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec; + + /// [`replacements`](Self::replacements) plus the accuracy-illegal + /// candidates this strategy refused to propose (issue #172). Default: + /// every candidate from `replacements`, no rejections — a strategy that + /// never performs an accuracy check need not override this. + /// [`search_workload_with`] calls this (not `replacements`) so the + /// rejections land in [`MemoGroup::rejected`]. + fn propose(&self, target: &TargetSubDAG<'_>) -> Proposals { + Proposals { + candidates: self.replacements(target), + rejected: Vec::new(), + } + } } // ── Implementation: how one AggIntent may be realised ─────────────────────── @@ -1074,6 +1124,33 @@ fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { /// `DefaultCostModel` is a unit struct with no state, so one instance serves /// every caller. static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; +static DEFAULT_ACCURACY_MODEL: DefaultAccuracyModel = DefaultAccuracyModel; +static DEFAULT_ALLOCATOR: EqualSplitAllocator = EqualSplitAllocator; + +/// The three deployment-pluggable models one candidate construction +/// consults, bundled so the construction path threads one argument rather +/// than three. `cost` ranks and sizes; `accuracy` and `allocator` decide +/// legality (issue #172) — see [`crate::accuracy`]'s module docs for why +/// those are separate from `cost` and run before it. +#[derive(Clone, Copy)] +pub(crate) struct Models<'a> { + pub cost: &'a dyn CostModel, + pub accuracy: &'a dyn AccuracyModel, + pub allocator: &'a dyn AccuracyBudgetAllocator, +} + +impl<'a> Models<'a> { + /// `cost` with the built-in [`DefaultAccuracyModel`]/ + /// [`EqualSplitAllocator`] — what every entry point that only takes a + /// `CostModel` uses. + pub(crate) fn with_default_accuracy(cost: &'a dyn CostModel) -> Self { + Self { + cost, + accuracy: &DEFAULT_ACCURACY_MODEL, + allocator: &DEFAULT_ALLOCATOR, + } + } +} /// Wraps [`implementations_for_with`]'s exhaustive, ranked list directly: for /// a bindable `Aggregate`, every valid candidate summary realization as its @@ -1084,8 +1161,17 @@ static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; /// [`SketchAlgorithmStrategy::new`] — so a deployment-specific cost model's /// other hooks (`size_params`, `realize_extension`, `readout_extension`) are /// still consulted while binding each candidate. +/// +/// The one thing that *does* drop a candidate is accuracy legality (issue +/// #172), decided by the [`AccuracyModel`] — never by the cost model: a +/// sketch over an approximate child is proposed only if its composed +/// guarantee has a sound propagation rule and satisfies the node's own +/// `AccuracyTarget`; otherwise it is reported through +/// [`ReplacementStrategy::propose`] as a [`RejectedCandidate`]. See +/// [`crate::accuracy`]'s module docs for the rules and the precedence +/// between root and per-node targets. pub struct SketchAlgorithmStrategy<'a> { - cost_model: &'a dyn CostModel, + models: Models<'a>, } impl SketchAlgorithmStrategy<'static> { @@ -1093,7 +1179,7 @@ impl SketchAlgorithmStrategy<'static> { /// what a deployment gets with no custom cost model plugged in. pub fn default_cost_model() -> Self { Self { - cost_model: &DEFAULT_COST_MODEL, + models: Models::with_default_accuracy(&DEFAULT_COST_MODEL), } } } @@ -1101,39 +1187,199 @@ impl SketchAlgorithmStrategy<'static> { impl<'a> SketchAlgorithmStrategy<'a> { /// A strategy that ranks/binds via `cost_model` instead of the built-in /// static preference order — the same customization point - /// [`implementations_for_with`] already offers. + /// [`implementations_for_with`] already offers. Accuracy legality stays + /// with the built-in [`DefaultAccuracyModel`]/[`EqualSplitAllocator`]. pub fn new(cost_model: &'a dyn CostModel) -> Self { - Self { cost_model } + Self { + models: Models::with_default_accuracy(cost_model), + } } -} -impl ReplacementStrategy for SketchAlgorithmStrategy<'_> { - fn matches(&self, target: &TargetSubDAG<'_>) -> bool { - bindable_intent(target.root).is_some() + /// A strategy with every model plugged in explicitly: `cost_model` for + /// ranking/sizing, `accuracy_model` for guarantee derivation/propagation/ + /// satisfaction, `allocator` for end-to-end budget splits. One model + /// never overrides another: legality is settled by `accuracy_model` + /// before `cost_model` ranks what is left. + pub fn with_models( + cost_model: &'a dyn CostModel, + accuracy_model: &'a dyn AccuracyModel, + allocator: &'a dyn AccuracyBudgetAllocator, + ) -> Self { + Self { + models: Models { + cost: cost_model, + accuracy: accuracy_model, + allocator, + }, + } } - fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { - let Some(intent) = bindable_intent(target.root) else { - return Vec::new(); + pub(crate) fn from_models(models: Models<'a>) -> Self { + Self { models } + } + + /// The whole enumeration for one target, with `intent_override` + /// substituting the target's own intent (only ever its `AccuracyTarget` + /// differs — see [`realize_child_with`]). + fn propose_with(&self, root: &Rc, intent_override: Option<&AggIntent>) -> Proposals { + let mut proposals = Proposals::default(); + let Some(declared) = bindable_intent(root) else { + return proposals; }; + let intent = intent_override.unwrap_or(declared); + let models = self.models; + + // Is the child approximate? Probed once, up front: a candidate over + // an approximate child needs the end-to-end budget split across both + // layers, which changes which candidates exist at all. + let child_layers = aggregate_child(root) + .and_then(|child| realize_child_with(child, models, None).ok()) + .and_then(|child| { + child + .guarantee + .as_ref() + .filter(|g| !g.is_exact()) + .map(ResultGuarantee::approximate_layer_count) + }); + // `implementations_for_with` is already exhaustive and ranked — no // separate dispatch needed here. Only `Sketch` has more than one // candidate in practice (every other variant's own dispatch produces // exactly one `Implementation`), but this loop doesn't need to know // that; it just constructs whatever the list contains. - implementations_for_with(intent, self.cost_model) - .into_iter() - .filter_map(|implementation| { - let rationale = describe_implementation(intent, &implementation); - let node = construct_summary(target.root, implementation, self.cost_model).ok()?; - Some(ReplacementSubDAG { + for implementation in implementations_for_with(intent, models.cost) { + let rationale = describe_implementation(intent, &implementation); + // The as-declared composition: every layer sized to its own + // declared `AccuracyTarget`. Legal iff the composed guarantee + // satisfies this node's target — a front end copying one target + // onto every node does not make that so. + proposals.record( + rationale.clone(), + construct_summary_with(root, intent, implementation.clone(), models, None, None), + ); + + // Budget-split alternatives (issue #172, PR 2): re-size this + // layer and the approximate child under each allocation of this + // node's target across every approximate layer. + let (Some(child_layers), Implementation::Sketch(kind), Some(target)) = + (child_layers, &implementation, accuracy_target(intent)) + else { + continue; + }; + let Some(readout_query) = aggregate_child(root) + .and_then(|child| child.output_schema().ok()) + .map(|schema| readout(intent, &summarised_column(intent, &schema), models.cost)) + else { + continue; + }; + let family = SummaryFamilyType::Sketch(kind.clone(), GroupingStrategy::default()); + let Some(local) = models.accuracy.local_guarantee(&family, &readout_query) else { + continue; + }; + let shape = CompositionShape { + metric: local.metric, + approximate_layer_count: 1 + child_layers, + }; + let allocations = models.allocator.allocations(target, &shape); + if allocations.is_empty() { + proposals.rejected.push(RejectedCandidate { strategy: "SketchAlgorithmStrategy", - replacement: Replacement::Summary(node), - provenance: ReplacementProvenance::SummaryImplementation, - rationale, - }) - }) - .collect() + description: rationale.clone(), + error: AccuracyError::NoLegalAllocation { + target: target.clone(), + layer_count: shape.approximate_layer_count, + }, + }); + continue; + } + let declared_child_target = aggregate_child(root) + .and_then(|child| bindable_intent(child)) + .and_then(accuracy_target); + for allocation in allocations { + let outer_target = &allocation.layers[0]; + let inner_target = allocation.inner_target(&shape); + let (eps, delta) = accuracy_budget(outer_target); + let resized = Implementation::Sketch(SketchKind::new( + kind.algorithm().clone(), + models + .cost + .size_params(kind.algorithm().clone(), intent, eps, delta), + )); + // Identical to the as-declared composition already recorded + // above — nothing new to propose. + if resized == implementation && inner_target.as_ref() == declared_child_target { + continue; + } + let note = GuaranteeSource::BudgetAllocation { + allocator: allocation.allocator.to_string(), + layer: 0, + layer_count: shape.approximate_layer_count, + local_target: outer_target.clone(), + end_to_end_target: target.clone(), + }; + proposals.record( + format!( + "{rationale}; sized under {} budget split of {target:?} across \ + {} approximate layers (this layer {outer_target:?}, child subtree \ + {inner_target:?})", + allocation.allocator, shape.approximate_layer_count + ), + construct_summary_with( + root, + intent, + resized, + models, + inner_target.as_ref(), + Some(note), + ), + ); + } + } + proposals + } +} + +impl Proposals { + /// File one construction attempt: a legal node becomes a candidate, an + /// [`ImplementError::Accuracy`] becomes a [`RejectedCandidate`], and a + /// schema-derivation failure is skipped exactly as it always was. + fn record(&mut self, rationale: String, built: Result, ImplementError>) { + match built { + Ok(node) => self.candidates.push(ReplacementSubDAG { + strategy: "SketchAlgorithmStrategy", + replacement: Replacement::Summary(node), + provenance: ReplacementProvenance::SummaryImplementation, + rationale, + }), + Err(ImplementError::Accuracy(error)) => self.rejected.push(RejectedCandidate { + strategy: "SketchAlgorithmStrategy", + description: rationale, + error, + }), + Err(ImplementError::Schema(_)) => {} + } + } +} + +/// The `child` of a [`bindable_intent`]-shaped `Aggregate`. +fn aggregate_child(node: &QueryExpr) -> Option<&Rc> { + match node { + QueryExpr::Aggregate { child, .. } => Some(child), + _ => None, + } +} + +impl ReplacementStrategy for SketchAlgorithmStrategy<'_> { + fn matches(&self, target: &TargetSubDAG<'_>) -> bool { + bindable_intent(target.root).is_some() + } + + fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { + self.propose(target).candidates + } + + fn propose(&self, target: &TargetSubDAG<'_>) -> Proposals { + self.propose_with(target.root, None) } } @@ -1224,9 +1470,32 @@ pub(crate) fn realize_child( root: &Rc, cost_model: &dyn CostModel, ) -> Result, ImplementError> { - let target = TargetSubDAG::new(root); - match SketchAlgorithmStrategy::new(cost_model) - .replacements(&target) + realize_child_with(root, Models::with_default_accuracy(cost_model), None) +} + +/// [`realize_child`] with every model explicit, plus an optional +/// `end_to_end_target` for `root`'s own value (issue #172): when an +/// [`AccuracyBudgetAllocator`] hands an approximate child a share of its +/// parent's budget, the child is re-enumerated with that share substituted +/// for its declared `AccuracyTarget` — sizing its sketch (and, recursively, +/// re-splitting for its own approximate children) under the allocated +/// budget. A child whose declared target is `Exact` keeps it: an allocation +/// never approximates something the caller declared exact. +pub(crate) fn realize_child_with( + root: &Rc, + models: Models<'_>, + end_to_end_target: Option<&AccuracyTarget>, +) -> Result, ImplementError> { + let overridden = end_to_end_target.and_then(|target| { + let declared = bindable_intent(root)?; + match accuracy_target(declared) { + Some(AccuracyTarget::Exact) | None => None, + Some(_) => Some(override_accuracy(declared, target)), + } + }); + match SketchAlgorithmStrategy::from_models(models) + .propose_with(root, overridden.as_ref()) + .candidates .into_iter() .next() { @@ -1242,12 +1511,28 @@ pub(crate) fn realize_child( } // No candidate at all: `root` isn't `bindable_intent` shape (or its // intent has no realization `implementations_for_with` can't - // produce — never happens, that match is exhaustive) — the same - // conservative fallback `SketchAlgorithmStrategy::matches` uses. + // produce — never happens, that match is exhaustive), or every + // candidate was accuracy-illegal — either way the same conservative + // fallback `SketchAlgorithmStrategy::matches` uses: keep the + // pre-ASAP subtree, executed exactly. None => keep_pre_asap(root), } } +/// `intent` with its `AccuracyTarget` replaced by `target` — a no-op for an +/// intent that carries none (see [`accuracy_target`]). +fn override_accuracy(intent: &AggIntent, target: &AccuracyTarget) -> AggIntent { + let mut out = intent.clone(); + match &mut out { + AggIntent::Quantile { accuracy, .. } + | AggIntent::Cardinality { accuracy, .. } + | AggIntent::Count { accuracy } + | AggIntent::TopK { accuracy, .. } => *accuracy = target.clone(), + _ => {} + } + out +} + /// Wrap an unrewritten pre-ASAP subtree, lifting its schema with every column /// `SummaryFamilyType::Plain`. `pub` so a caller can fall back to this /// explicitly — e.g. when `SketchAlgorithmStrategy::replacements()` returns no @@ -1263,6 +1548,9 @@ fn keep_pre_asap_rc(expr: Rc) -> Result, ImplementErr Ok(Rc::new(SummaryNode { expr: SummaryExpr::KeepPreAsap(expr), schema: lift(&schema), + // A kept pre-ASAP subtree is executed exactly by the runtime + // (`Implementation::PassThrough`'s contract) — zero error. + guarantee: Some(ResultGuarantee::exact("KeepPreAsap")), })) } @@ -1312,21 +1600,48 @@ pub(crate) fn construct_summary( expr: &QueryExpr, implementation: Implementation, cost_model: &dyn CostModel, +) -> Result, ImplementError> { + let models = Models::with_default_accuracy(cost_model); + match bindable_intent(expr) { + Some(intent) => construct_summary_with(expr, intent, implementation, models, None, None), + None => keep_pre_asap_rc(Rc::new(expr.clone())), + } +} + +/// [`construct_summary`] with every model explicit (issue #172). `intent` +/// is `expr`'s own [`bindable_intent`], or a copy of it with an allocated +/// `AccuracyTarget` substituted (see [`realize_child_with`]). +/// `child_target`, when set, is the end-to-end budget the child subtree is +/// re-enumerated under; `allocation` is the provenance note recording the +/// split that produced both. `Err(ImplementError::Accuracy)` is the +/// fail-closed answer for a composition with no sound rule or one that +/// misses `intent`'s target. +pub(crate) fn construct_summary_with( + expr: &QueryExpr, + intent: &AggIntent, + implementation: Implementation, + models: Models<'_>, + child_target: Option<&AccuracyTarget>, + allocation: Option, ) -> Result, ImplementError> { if let QueryExpr::Aggregate { - reduction, - measures, - having, - child, - .. + reduction, child, .. } = expr { - // The bindable shape: exactly one intent, no HAVING. (Multi-intent - // nodes and HAVING stay logical — see `bindable_intent`.) - if let ([intent], None) = (measures.as_slice(), having) { + // `bindable_intent` already established the shape: exactly one + // intent, no HAVING. (Multi-intent nodes and HAVING stay logical.) + if bindable_intent(expr).is_some() { if let Some((family, estimate)) = summary_family(implementation) { return construct_summary_agg( - expr, reduction, intent, child, family, estimate, cost_model, + expr, + reduction, + intent, + child, + family, + estimate, + models, + child_target, + allocation, ); } } @@ -1371,7 +1686,9 @@ fn construct_summary_agg( child: &Rc, family: SummaryFamilyType, estimate: bool, - cost_model: &dyn CostModel, + models: Models<'_>, + child_target: Option<&AccuracyTarget>, + allocation: Option, ) -> Result, ImplementError> { let child_schema = child.output_schema()?; // The single canonical pre-ASAP derivation (per-series vs cross-series, @@ -1386,13 +1703,29 @@ fn construct_summary_agg( let state_idx = summary_col_index(&out_schema, &by, per_series); let col = summarised_column(intent, &child_schema); - let query = estimate.then(|| readout(intent, &col, cost_model)); + let query = estimate.then(|| readout(intent, &col, models.cost)); let mut state_schema = lift(&out_schema); if let Some(field) = state_schema.fields.get_mut(state_idx) { field.dtype = family.clone(); } + let bound_child = realize_child_with(child, models, child_target)?; + + // ── Guarantee (issue #172) ────────────────────────────────────────── + // Derived *before* the node exists, so an illegal composition is never + // materialized: the local guarantee of this family's readout (or exact + // accumulator) composed over the child's, under the operator this + // family applies to the child's values. + let guarantee = compose_guarantee( + &family, + query.as_ref(), + &bound_child, + intent, + models.accuracy, + allocation, + )?; + // `reduction` is carried onto `SummaryAgg` verbatim — not flattened to a // bare `Vec` — so `SummaryExecutor::find_candidates` can tell // a genuine empty-`by` reduction apart from a per-entity shape with no @@ -1400,13 +1733,16 @@ fn construct_summary_agg( // single place that decides this; nothing downstream re-derives it. let agg = Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { - child: realize_child(child, cost_model)?, + child: bound_child, family, col, reduction: reduction.clone(), grouping: GroupingStrategy::default(), }, schema: state_schema, + // Summary *state* carries no caller-visible guarantee; only a + // finalized value does. An exact accumulator's state is its value. + guarantee: if estimate { None } else { guarantee.clone() }, }); match query { // The readout: downstream of the estimate the schema is the plain @@ -1418,11 +1754,102 @@ fn construct_summary_agg( query, }, schema: lift(&out_schema), + guarantee, })), None => Ok(agg), } } +/// The guarantee of the value a `family` node produces over `child` — +/// [`AccuracyModel::propagate`] under the [`CompositionOperator`] this family +/// applies to its child's values — checked against `intent`'s own +/// `AccuracyTarget` whenever the child is approximate (an approximate +/// parent over an exact child is sized to that target by construction and +/// is not re-checked here, so single-layer behavior is unchanged; see +/// [`crate::accuracy`]'s precedence rules). `Ok(None)` is "no error model" +/// (an approximate family the model has no local guarantee for, over an +/// exact child) — unknown, never exact. +fn compose_guarantee( + family: &SummaryFamilyType, + query: Option<&PostAsapSketchQuery>, + child: &SummaryNode, + intent: &AggIntent, + accuracy: &dyn AccuracyModel, + allocation: Option, +) -> Result, AccuracyError> { + let (op, local) = match (family, query) { + (SummaryFamilyType::ExactAggregate(kind, _), _) => { + let op = match kind { + ExactKind::Sum => CompositionOperator::ExactSum, + ExactKind::MinMax => CompositionOperator::ExactExtremum, + // A row count does not depend on the rows' values: exact + // regardless of the child's own error. + ExactKind::Count => { + return Ok(Some(ResultGuarantee::exact( + "ExactAggregate(Count): row count is independent of input values", + ))) + } + // Counter-reset detection over perturbed values has no finite + // Lipschitz constant — over an approximate child this is a + // deterministic transform with no registered rule. + ExactKind::Increase | ExactKind::Rate => CompositionOperator::Lipschitz { + constant: f64::INFINITY, + }, + }; + ( + op, + Some(ResultGuarantee::exact(format!("ExactAggregate({kind:?})"))), + ) + } + (_, Some(query)) => ( + CompositionOperator::ApproximateAggregate, + accuracy.local_guarantee(family, query), + ), + (_, None) => (CompositionOperator::ApproximateAggregate, None), + }; + let Some(input) = child.guarantee.clone() else { + // A child with no guarantee at all is an unknown quantity, which + // nothing can be composed over (a `Sample` readout, say) — unless + // this node is itself the unknown family, in which case it inherits + // "unknown" rather than fabricating a guarantee for its child. + return match local { + Some(_) => Err(AccuracyError::MissingInputGuarantee { + operator: op, + input_index: 0, + }), + None => Ok(None), + }; + }; + if local.is_none() && input.is_exact() { + return Ok(None); + } + let mut guarantee = accuracy.propagate( + &op, + std::slice::from_ref(&input), + local.as_ref(), + &PropagationStats::default(), + )?; + if let Some(note) = allocation { + guarantee.provenance.push(note); + } + if let Some(target) = accuracy_target(intent) { + guarantee.provenance.push(GuaranteeSource::AccuracyTarget { + target: target.clone(), + }); + // Approximate over approximate: the composed guarantee must meet the + // target this node's value was requested at. + if !input.is_exact() && !accuracy.satisfies(&guarantee, target) { + return Err(AccuracyError::TargetNotSatisfied { + metric: guarantee.metric, + bound: guarantee.bound.evaluate(), + failure_probability: guarantee.failure_probability.evaluate(), + target: target.clone(), + }); + } + } + Ok(Some(guarantee)) +} + /// Index of the summary-state column in the aggregate's output schema: /// cross-series output is `by ++ [agg]` (the column after the keys); /// a per-series reduction keeps every label and replaces the sample value @@ -1599,6 +2026,12 @@ pub struct MemoGroup { /// order (not ranked — see [`PlanSpace::cost_sorted`] for the ranked /// view). pub candidates: Vec, + /// Every candidate a strategy considered for `target` but refused on + /// accuracy-legality grounds (issue #172), plus any `candidates` entry + /// the root-target check ([`search_workload_with_targets`]) moved here. + /// Never ranked — [`PlanSpace::cost_sorted`]/[`PlanSpace::global_selection`] + /// read only `candidates`, so a [`CostModel`] cannot resurrect one. + pub rejected: Vec, } impl MemoGroup { @@ -1607,6 +2040,7 @@ impl MemoGroup { target, consumer_count, candidates: Vec::new(), + rejected: Vec::new(), } } @@ -2900,6 +3334,93 @@ pub fn search_workload_with<'s, Id>( search_cse_workload_with(cse_workload(roots), strategies) } +/// [`search_workload_with`] plus a per-root end-to-end `AccuracyTarget` +/// (issue #172) — the workload's `QueryRequirements.accuracy`, threaded +/// alongside each root. After the search, every root that carries a target +/// has its group's bound [`Replacement::Summary`] candidates checked with +/// `accuracy_model`'s [`AccuracyModel::satisfies`]: a candidate whose +/// guarantee is absent (unknown) or misses the target is moved from +/// [`MemoGroup::candidates`] to [`MemoGroup::rejected`] *before* +/// [`PlanSpace::cost_sorted`]/[`PlanSpace::global_selection`] ever rank the +/// group, so a `CostModel` cannot pick it. A `KeepPreAsap` candidate is +/// exact and always survives — the raw/pre-ASAP alternative is what an +/// unsatisfiable root keeps. Logical [`Replacement::Rewrite`] candidates +/// are not bound values and are left alone; the targets *inside* a rewrite +/// are their own groups. +/// +/// Precedence against per-node `AggIntent.accuracy` is documented in +/// [`crate::accuracy`]'s module docs. +pub fn search_workload_with_targets<'s, Id>( + roots: Vec<(Id, Rc, Option)>, + strategies: &[Box], + accuracy_model: &dyn AccuracyModel, +) -> PlanSpace { + let mut targets = Vec::with_capacity(roots.len()); + let roots = roots + .into_iter() + .map(|(id, root, target)| { + targets.push(target); + (id, root) + }) + .collect(); + let mut space = search_workload_with(roots, strategies); + // `cse_workload` preserves root order, so targets zip by position. + let root_ptrs: Vec<(*const QueryExpr, AccuracyTarget)> = space + .roots + .iter() + .zip(targets) + .filter_map(|((_, root), target)| target.map(|t| (Rc::as_ptr(root), t))) + .collect(); + for (ptr, target) in root_ptrs { + let Some(group) = space.groups.get_mut(&ptr) else { + continue; + }; + let (legal, illegal): (Vec<_>, Vec<_>) = + group + .candidates + .drain(..) + .partition(|candidate| match &candidate.replacement { + Replacement::Summary(node) => node + .guarantee + .as_ref() + .is_some_and(|g| accuracy_model.satisfies(g, &target)), + Replacement::Rewrite(_) => true, + }); + group.candidates = legal; + group.rejected.extend(illegal.into_iter().map(|candidate| { + let (metric, bound, failure_probability) = match &candidate.replacement { + Replacement::Summary(node) => node + .guarantee + .as_ref() + .map(|g| { + ( + g.metric, + g.bound.evaluate(), + g.failure_probability.evaluate(), + ) + }) + .unwrap_or(( + asap_types::post_asap::ErrorMetric::AbsoluteValue, + None, + None, + )), + Replacement::Rewrite(_) => unreachable!("rewrites are never rejected here"), + }; + RejectedCandidate { + strategy: candidate.strategy, + description: format!("{} (root end-to-end target check)", candidate.rationale), + error: AccuracyError::TargetNotSatisfied { + metric, + bound, + failure_probability, + target: target.clone(), + }, + } + })); + } + space +} + fn cse_workload(roots: Vec<(Id, Rc)>) -> Vec<(Id, Rc)> { // `share_common_subtrees` wants owned `QueryExpr`s, not already-`Rc` // roots — the same `Rc::try_unwrap`-with-clone-fallback pattern @@ -2975,15 +3496,19 @@ fn search_cse_workload_with<'s, Id>( let target = TargetSubDAG::with_consumer_count(&root, consumer_count); let mut proposed = Vec::new(); + let mut rejected = Vec::new(); for strategy in strategies { if strategy.matches(&target) { let name = strategy.name(); - proposed.extend(strategy.replacements(&target).into_iter().map( - |mut candidate| { - candidate.strategy = name; - candidate - }, - )); + let proposals = strategy.propose(&target); + proposed.extend(proposals.candidates.into_iter().map(|mut candidate| { + candidate.strategy = name; + candidate + })); + rejected.extend(proposals.rejected.into_iter().map(|mut rejection| { + rejection.strategy = name; + rejection + })); } } if rollup_strategy.matches(&target) { @@ -3029,6 +3554,7 @@ fn search_cse_workload_with<'s, Id>( for candidate in proposed { group.add_candidate(candidate); } + group.rejected.extend(rejected); } // Any pointer `discover_new_descendant_targets` appended to `order` @@ -3956,10 +4482,20 @@ mod tests { fn enumerating_the_targets_candidates_does_not_leak_into_a_nested_aggregate() { // outer: quantile(0.99, ...) over inner: quantile(0.5, m) — both // Quantile, so both share the [Kll, DDSketch] candidate list. + // + // Rank-over-rank has no registered rule in `DefaultAccuracyModel` + // (issue #172 — see `approximate_over_approximate_is_rejected_by_default`), + // so this test injects `RankAdditiveModel` to admit the composition + // and keep exercising the per-node enumeration property it is about. let inner = agg(vec![2], default_quantile(0.5), metric_scan(&["job"])); let outer = Rc::new(agg(vec![], default_quantile(0.99), inner)); let target = TargetSubDAG::new(&outer); - let replacements = SketchAlgorithmStrategy::default_cost_model().replacements(&target); + let replacements = SketchAlgorithmStrategy::with_models( + &DefaultCostModel, + &RankAdditiveModel, + &EqualSplitAllocator, + ) + .replacements(&target); let ddsketch = replacements .iter() @@ -5818,4 +6354,374 @@ mod tests { }; assert_eq!(col, &ColumnRef::Named("bytes".into())); } + + // ── Accuracy guarantees and fail-closed composition (issue #172) ───── + + use asap_types::post_asap::{BoundExpr, ErrorMetric}; + + /// A test-only `AccuracyModel` that *registers* a rule the default + /// deliberately lacks — a sketch over rank-bounded inputs composes + /// additively, keeping the outer sketch's own metric — so the + /// composition/allocation machinery can be exercised end to end. + /// Everything else delegates to `DefaultAccuracyModel`. + struct RankAdditiveModel; + + impl AccuracyModel for RankAdditiveModel { + fn local_guarantee( + &self, + family: &SummaryFamilyType, + query: &PostAsapSketchQuery, + ) -> Option { + DefaultAccuracyModel.local_guarantee(family, query) + } + + fn propagate( + &self, + op: &CompositionOperator, + inputs: &[ResultGuarantee], + local: Option<&ResultGuarantee>, + stats: &PropagationStats, + ) -> Result { + let rank = |g: &ResultGuarantee| g.is_exact() || g.metric == ErrorMetric::Rank; + if let (CompositionOperator::ApproximateAggregate, true, Some(local)) = + (op, inputs.iter().all(rank), local) + { + let relabel = |g: &ResultGuarantee| ResultGuarantee { + metric: ErrorMetric::AbsoluteValue, + ..g.clone() + }; + let inputs: Vec<_> = inputs.iter().map(relabel).collect(); + let mut out = + DefaultAccuracyModel.propagate(op, &inputs, Some(&relabel(local)), stats)?; + out.metric = local.metric; + return Ok(out); + } + DefaultAccuracyModel.propagate(op, inputs, local, stats) + } + + fn satisfies(&self, guarantee: &ResultGuarantee, target: &AccuracyTarget) -> bool { + DefaultAccuracyModel.satisfies(guarantee, target) + } + } + + fn quantile_eps(q: f64, eps: f64) -> AggIntent { + AggIntent::Quantile { + col: None, + q, + accuracy: AccuracyTarget::Epsilon(eps), + } + } + + /// The `SketchParams::Kll { k }` of the top `SummaryAgg` under `node`. + fn kll_k_of(node: &SummaryNode) -> u32 { + match &node.expr { + SummaryExpr::SummaryEstimate { summary_input, .. } => kll_k_of(summary_input), + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::Sketch(kind, _), + .. + } => match kind.params() { + SketchParams::Kll { k } => *k, + other => panic!("expected KLL params, got {other:?}"), + }, + other => panic!("expected a sketch SummaryAgg, got {other:?}"), + } + } + + fn summary_child(node: &SummaryNode) -> &Rc { + match &node.expr { + SummaryExpr::SummaryEstimate { summary_input, .. } => summary_child(summary_input), + SummaryExpr::SummaryAgg { child, .. } => child, + other => panic!("expected a SummaryAgg, got {other:?}"), + } + } + + #[test] + fn approximate_over_approximate_is_rejected_by_default_not_treated_as_exact() { + // quantile(0.99, quantile by (job) (0.5, m)): rank over rank — no + // registered rule, so every outer sketch candidate is refused with a + // typed reason and the raw/pre-ASAP alternative is what remains. + let inner = agg(vec![2], default_quantile(0.5), metric_scan(&["job"])); + let outer = Rc::new(agg(vec![], default_quantile(0.99), inner)); + let proposals = + SketchAlgorithmStrategy::default_cost_model().propose(&TargetSubDAG::new(&outer)); + assert!( + proposals.candidates.is_empty(), + "no outer sketch may be proposed over an approximate child without a rule: {:?}", + proposals.candidates + ); + // Every attempt — the as-declared composition and the equal-split + // re-sizing, for each of KLL/DDSketch — is refused for the same + // typed reason: no rule, whatever the budget. + assert_eq!(proposals.rejected.len(), 4, "{:?}", proposals.rejected); + for rejection in &proposals.rejected { + assert!( + matches!( + &rejection.error, + AccuracyError::UnsupportedComposition { input_metrics, .. } + if input_metrics == &vec![ErrorMetric::Rank] + ), + "{:?}", + rejection.error + ); + } + // Fallback keeps the whole subtree pre-ASAP — executed exactly. + let realized = realize_child(&outer, &DefaultCostModel).unwrap(); + assert!(matches!(realized.expr, SummaryExpr::KeepPreAsap(_))); + assert!(realized + .guarantee + .as_ref() + .is_some_and(ResultGuarantee::is_exact)); + + // Cross-metric: a quantile over a cardinality estimate. + let inner = agg(vec![2], default_cardinality(), metric_scan(&["job"])); + let outer = Rc::new(agg(vec![], default_quantile(0.99), inner)); + let proposals = + SketchAlgorithmStrategy::default_cost_model().propose(&TargetSubDAG::new(&outer)); + assert!(proposals.candidates.is_empty()); + assert!(proposals.rejected.iter().all(|r| matches!( + &r.error, + AccuracyError::UnsupportedComposition { input_metrics, .. } + if input_metrics == &vec![ErrorMetric::Cardinality] + ))); + } + + #[test] + fn exact_child_contributes_zero_error() { + // quantile(0.9, sum by (job) (m)): KLL over an exact Sum accumulator + // — the readout's guarantee is exactly KLL's own local guarantee. + let inner = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); + let outer = agg(vec![], default_quantile(0.9), inner); + let root = realize(&outer).unwrap(); + let guarantee = root + .guarantee + .as_ref() + .expect("a readout carries a guarantee"); + assert_eq!(guarantee.metric, ErrorMetric::Rank); + assert_eq!(guarantee.bound.evaluate(), Some(2.0 / 200.0)); + assert_eq!(guarantee.approximate_layer_count(), 1); + assert!(guarantee.provenance.iter().any(|s| matches!( + s, + GuaranteeSource::ChildGuarantee { guarantee, .. } if guarantee.is_exact() + ))); + assert!(guarantee.provenance.iter().any(|s| matches!( + s, + GuaranteeSource::CompositionStep { rule, .. } if rule == "exact_input" + ))); + // The sketch *state* node carries no guarantee; the exact + // accumulator's state is its value and does. + let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { + panic!() + }; + assert!(summary_input.guarantee.is_none()); + assert!(summary_child(&root) + .guarantee + .as_ref() + .is_some_and(ResultGuarantee::is_exact)); + } + + #[test] + fn exact_sum_over_approximate_child_keeps_the_row_count_unknown() { + // sum(count_distinct by (job) (m)): an exact sum over HLL estimates + // is representable (Σ B_i) but its bound depends on the group count + // and the true cardinalities — unknown at planning time, so the + // guarantee exists, says what it needs, and satisfies nothing. + let inner = agg(vec![2], default_cardinality(), metric_scan(&["job"])); + let outer = agg(vec![], AggIntent::Sum { col: None }, inner); + let root = realize(&outer).unwrap(); + let guarantee = root + .guarantee + .as_ref() + .expect("an exact sum carries a guarantee"); + assert_eq!(guarantee.metric, ErrorMetric::AbsoluteValue); + assert_eq!(guarantee.bound.evaluate(), None); + assert!(guarantee.provenance.iter().any(|s| matches!( + s, + GuaranteeSource::UnavailableStatistic { statistic } if statistic == "input_row_count" + ))); + assert!(!DefaultAccuracyModel.satisfies(guarantee, &AccuracyTarget::Epsilon(1e9))); + + // count(...) over the same child is exact: a row count does not + // depend on the rows' values. + let inner = agg(vec![2], default_cardinality(), metric_scan(&["job"])); + let outer = agg( + vec![], + AggIntent::Count { + accuracy: AccuracyTarget::Exact, + }, + inner, + ); + let root = realize(&outer).unwrap(); + assert!(root + .guarantee + .as_ref() + .is_some_and(ResultGuarantee::is_exact)); + } + + #[test] + fn equal_split_allocation_makes_a_legal_tighter_candidate_and_rejects_the_declared_one() { + // With a registered rank-additive rule: outer ε=0.1 over inner ε=0.1 + // composes to 0.2 > 0.1 as declared (cheap: k=20 each) — illegal. + // The equal split (0.05 + 0.05) re-sizes both layers to k=40 — + // pricier, and the only legal way to meet the outer target. + let inner = agg(vec![2], quantile_eps(0.5, 0.1), metric_scan(&["job"])); + let outer = Rc::new(agg(vec![], quantile_eps(0.99, 0.1), inner)); + let strategy = SketchAlgorithmStrategy::with_models( + &DefaultCostModel, + &RankAdditiveModel, + &EqualSplitAllocator, + ); + let proposals = strategy.propose(&TargetSubDAG::new(&outer)); + + let declared = proposals + .rejected + .iter() + .filter(|r| { + matches!( + &r.error, + AccuracyError::TargetNotSatisfied { + metric: ErrorMetric::Rank, + bound: Some(b), + target: AccuracyTarget::Epsilon(e), + .. + } if (b - 0.2).abs() < 1e-12 && *e == 0.1 + ) + }) + .count(); + assert_eq!( + declared, 1, + "the as-declared KLL composition is rejected: {:?}", + proposals.rejected + ); + + let kll: Vec<_> = proposals + .candidates + .iter() + .filter_map(|c| match &c.replacement { + Replacement::Summary(node) + if summary_family_algorithm(node) == SketchAlgorithm::Kll => + { + Some(node) + } + _ => None, + }) + .collect(); + assert_eq!( + kll.len(), + 1, + "exactly one legal KLL candidate (the allocated one)" + ); + let node = kll[0]; + assert_eq!(kll_k_of(node), 40, "outer re-sized to ε/2"); + assert_eq!(kll_k_of(summary_child(node)), 40, "inner re-sized to ε/2"); + let guarantee = node.guarantee.as_ref().unwrap(); + assert_eq!(guarantee.metric, ErrorMetric::Rank); + assert!((guarantee.bound.evaluate().unwrap() - 0.1).abs() < 1e-12); + assert_eq!(guarantee.approximate_layer_count(), 2); + assert!(guarantee.provenance.iter().any(|s| matches!( + s, + GuaranteeSource::BudgetAllocation { allocator, layer_count: 2, .. } + if allocator == "EqualSplitAllocator" + ))); + assert!(matches!(guarantee.bound, BoundExpr::Sum { .. })); + // No candidate with the cheaper illegal sizing exists anywhere. + assert!(proposals.candidates.iter().all(|c| match &c.replacement { + Replacement::Summary(node) + if summary_family_algorithm(node) == SketchAlgorithm::Kll => + kll_k_of(node) != 20, + _ => true, + })); + } + + #[test] + fn legality_precedes_cost_in_search_and_global_selection() { + // Same fixture through the workload search: the illegal cheaper + // candidate is absent from the group *before* any cost ranking, the + // rejection is recorded on the group, and global selection commits + // to the legal, more expensive one. + let inner = agg(vec![2], quantile_eps(0.5, 0.1), metric_scan(&["job"])); + let outer = Rc::new(agg(vec![], quantile_eps(0.99, 0.1), inner)); + let strategies: Vec> = + vec![Box::new(SketchAlgorithmStrategy::with_models( + &DefaultCostModel, + &RankAdditiveModel, + &EqualSplitAllocator, + ))]; + let space = search_workload_with(vec![("q", Rc::clone(&outer))], &strategies); + let root = &space.roots[0].1; + let group = space.group_for(root).unwrap(); + assert!(!group.rejected.is_empty()); + assert!(group.candidates.iter().all(|c| match &c.replacement { + Replacement::Summary(node) => node.guarantee.as_ref().is_some_and(|g| { + DefaultAccuracyModel.satisfies(g, &AccuracyTarget::Epsilon(0.1)) + }), + Replacement::Rewrite(_) => false, + })); + let ranked = space.cost_sorted(&DefaultCostModel); + let root_ranked = ranked.iter().find(|g| Rc::ptr_eq(g.target, root)).unwrap(); + assert_eq!(root_ranked.candidates.len(), group.candidates.len()); + + let selection = space.global_selection(&DefaultCostModel); + let chosen = selection + .for_target(root) + .unwrap() + .chosen + .expect("a legal candidate wins"); + let Replacement::Summary(node) = &chosen.replacement else { + panic!() + }; + assert_eq!(kll_k_of(node), 40); + } + + #[test] + fn root_target_check_removes_candidates_before_cost_ranking() { + let q = Rc::new(agg(vec![2], default_quantile(0.99), metric_scan(&["job"]))); + // A root target tighter than the node's own ε=0.01: every sketch + // candidate misses it and is moved to `rejected`; nothing is left + // for the cost model to rank. + let space = search_workload_with_targets( + vec![("q", Rc::clone(&q), Some(AccuracyTarget::Epsilon(0.001)))], + &default_strategies(), + &DefaultAccuracyModel, + ); + let root = &space.roots[0].1; + let group = space.group_for(root).unwrap(); + assert!(group + .candidates + .iter() + .all(|c| matches!(c.replacement, Replacement::Rewrite(_)))); + assert!(group.rejected.iter().all(|r| matches!( + r.error, + AccuracyError::TargetNotSatisfied { target: AccuracyTarget::Epsilon(e), .. } if e == 0.001 + ))); + assert!(group.rejected.len() >= 2); + let selection = space.global_selection(&DefaultCostModel); + assert!(selection.for_target(root).unwrap().chosen.is_none()); + + // A root target the node's own sizing meets keeps every candidate. + let space = search_workload_with_targets( + vec![("q", Rc::clone(&q), Some(AccuracyTarget::Epsilon(0.01)))], + &default_strategies(), + &DefaultAccuracyModel, + ); + let group = space.group_for(&space.roots[0].1).unwrap(); + assert!(group + .candidates + .iter() + .any(|c| matches!(c.replacement, Replacement::Summary(_)))); + + // An `Exact` root target admits only exact candidates. + let space = search_workload_with_targets( + vec![("q", Rc::clone(&q), Some(AccuracyTarget::Exact))], + &default_strategies(), + &DefaultAccuracyModel, + ); + let group = space.group_for(&space.roots[0].1).unwrap(); + assert!(group.candidates.iter().all(|c| match &c.replacement { + Replacement::Summary(node) => node + .guarantee + .as_ref() + .is_some_and(ResultGuarantee::is_exact), + Replacement::Rewrite(_) => true, + })); + } } diff --git a/crates/devtools/src/bin/dag_export.rs b/crates/devtools/src/bin/dag_export.rs index 3dd7dde..071c4a4 100644 --- a/crates/devtools/src/bin/dag_export.rs +++ b/crates/devtools/src/bin/dag_export.rs @@ -56,8 +56,8 @@ use std::time::Instant; use asap_aware_mapping::cost_model::DefaultCostModel; use asap_aware_mapping::replacement::{search_workload, Replacement, ReplacementSubDAG}; use asap_types::dag_export::{ - self, DagDecision, DagGraph, DagNote, NamedGraph, PostAsapSubstitution, TargetReplacement, - TargetReplacementAfter, WorkloadGraph, + self, DagDecision, DagGraph, DagNote, NamedGraph, PostAsapSubstitution, TargetRejection, + TargetReplacement, TargetReplacementAfter, WorkloadGraph, }; use asap_types::post_asap::SummaryExpr; use asap_types::pre_asap::cse::{structural_hash, HashCache}; @@ -358,6 +358,10 @@ struct PostAsapResults { /// [`dag_export::export_post_asap`] — every winning candidate spliced /// directly into that query's own pre-ASAP shape in place. post_graphs: Vec<(String, DagGraph)>, + /// One `(query_name, TargetRejection)` per accuracy-illegal candidate + /// the search refused (`MemoGroup::rejected`, issue #172) whose target + /// node is found in that query's own exported graph. + rejections: Vec<(String, TargetRejection)>, } /// Assign collision-free, explicit identities to structurally equal nodes @@ -545,7 +549,19 @@ fn run_post_asap_with_progress( // subtree and inside `post_graph` as a whole. let mut lookup_cache = HashCache::new(); let mut replacements = Vec::new(); + let mut rejections = Vec::new(); let mut matched = vec![false; winners.len()]; + // Groups with accuracy-refused candidates (issue #172): matched to a + // query's graph nodes the same hash-then-structural-equality way. + let rejected_groups: Vec<_> = space + .groups() + .filter(|group| !group.rejected.is_empty()) + .collect(); + let mut rejected_by_hash: HashMap> = HashMap::new(); + for (i, group) in rejected_groups.iter().enumerate() { + let hash = structural_hash(&group.target, &mut by_hash_cache); + rejected_by_hash.entry(hash).or_default().push(i); + } for (name, _, qe) in lowered_queries { let graph = dag_export::export(qe); for node in &graph.nodes { @@ -559,6 +575,24 @@ fn run_post_asap_with_progress( )); matched[i] = true; } + let hash = structural_hash(source_expr, &mut lookup_cache); + for &i in rejected_by_hash.get(&hash).into_iter().flatten() { + let group = rejected_groups[i]; + if *source_expr != *group.target { + continue; + } + rejections.extend(group.rejected.iter().map(|rejected| { + ( + name.clone(), + TargetRejection { + target_pre_id: node.id, + strategy: rejected.strategy.to_string(), + description: rejected.description.clone(), + error: rejected.error.clone(), + }, + ) + })); + } } } @@ -602,6 +636,7 @@ fn run_post_asap_with_progress( PostAsapResults { replacements, post_graphs, + rejections, } } @@ -675,6 +710,7 @@ async fn main() { graph, replacements: Vec::new(), post_graph: None, + rejections: Vec::new(), }); } for (explanation, matched) in explanations.iter().zip(matched) { @@ -704,6 +740,11 @@ async fn main() { named.post_graph = Some(post_graph); } } + for (query_name, rejection) in results.rejections { + if let Some(named) = queries.iter_mut().find(|q| q.name == query_name) { + named.rejections.push(rejection); + } + } } { diff --git a/crates/types/src/dag_export.rs b/crates/types/src/dag_export.rs index 252d319..5df2ed8 100644 --- a/crates/types/src/dag_export.rs +++ b/crates/types/src/dag_export.rs @@ -46,7 +46,7 @@ use std::rc::Rc; use serde::Serialize; -use crate::post_asap::{SummaryExpr, SummaryNode}; +use crate::post_asap::{AccuracyError, ResultGuarantee, SummaryExpr, SummaryNode}; use crate::pre_asap::cse::{structural_hash, HashCache}; use crate::pre_asap::query_expr::{QueryExpr, Source}; @@ -199,6 +199,12 @@ pub struct NamedGraph { /// `NamedGraph` is unaffected. #[serde(default, skip_serializing_if = "Option::is_none")] pub post_graph: Option, + /// Accuracy-illegal candidates a higher layer's search refused for + /// targets in this query (issue #172) — see [`TargetRejection`]. Always + /// empty coming out of this module; omitted from the JSON when empty, + /// same additive rule as `replacements`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub rejections: Vec, } /// A batch of named queries — the shape the viewer's multi-query / compare @@ -277,6 +283,33 @@ pub struct SummaryDagNode { /// Child node ids, in the variant's field order (e.g. `SummaryJoin` is /// `[outer, inner]`). pub children: Vec, + /// The value's machine-readable accuracy guarantee (issue #172) — + /// [`SummaryNode::guarantee`] serialized structurally (metric, symbolic + /// bound, failure probability, provenance including any budget + /// allocation), not as prose. Omitted when the node carries none (raw + /// summary state, or a family with no error model), so every consumer + /// predating this field parses the same shape it always has. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub guarantee: Option, +} + +/// One accuracy-illegal candidate a higher layer's search refused for a +/// target (issue #172) — `asap_aware_mapping::replacement::RejectedCandidate` +/// re-shaped into this crate's own crate-agnostic vocabulary, the same +/// layering rule as [`TargetReplacement`]. Carried on +/// [`NamedGraph::rejections`] so a renderer can explain *why* a target kept +/// its raw/pre-ASAP form, not only what won elsewhere. +#[derive(Debug, Clone, Serialize)] +pub struct TargetRejection { + /// Id of the [`DagNode`] in this query's own `graph.nodes` the refused + /// candidate targeted. + pub target_pre_id: u32, + /// Which strategy considered the candidate. + pub strategy: String, + /// What the candidate would have been. + pub description: String, + /// The typed reason it was refused. + pub error: AccuracyError, } /// One post-ASAP `SummaryNode` tree, flattened the same way [`DagGraph`] @@ -315,6 +348,7 @@ fn push_summary_node( label: String, detail: serde_json::Value, children: Vec, + guarantee: Option, ) -> u32 { let id = nodes.len() as u32; nodes.push(SummaryDagNode { @@ -323,6 +357,7 @@ fn push_summary_node( label, detail, children, + guarantee, }); id } @@ -433,14 +468,21 @@ fn build_summary(node: &SummaryNode, nodes: &mut Vec) -> u32 { let inner_kind = pre_asap_subgraph.nodes[pre_asap_subgraph.root as usize].kind; let label = format!("KeepPreAsap({inner_kind})"); let detail = serde_json::json!({ "pre_asap_subgraph": pre_asap_subgraph }); - return push_summary_node(nodes, "KeepPreAsap", label, detail, vec![]); + return push_summary_node( + nodes, + "KeepPreAsap", + label, + detail, + vec![], + node.guarantee.clone(), + ); } let children: Vec = summary_children(&node.expr) .into_iter() .map(|child| build_summary(child, nodes)) .collect(); let (kind, label, detail) = summary_shape(&node.expr); - push_summary_node(nodes, kind, label, detail, children) + push_summary_node(nodes, kind, label, detail, children, node.guarantee.clone()) } /// One replacement site a higher layer (the `dag_export` binary) found by @@ -725,7 +767,17 @@ fn build_summary_hybrid( .into_iter() .map(|child| build_summary_hybrid(child, nodes, cache, find_winner)) .collect(); - let (kind, label, detail) = summary_shape(&node.expr); + let (kind, label, mut detail) = summary_shape(&node.expr); + // The merged graph's `DagNode` has no dedicated guarantee field (it is + // the pre-ASAP node shape); the guarantee rides in `detail` under the + // same key/shape `SummaryDagNode::guarantee` uses, additively. + if let Some(guarantee) = &node.guarantee { + if let (serde_json::Value::Object(map), Ok(value)) = + (&mut detail, serde_json::to_value(guarantee)) + { + map.insert("guarantee".into(), value); + } + } let id = push_summary_originated_node(nodes, kind, label, detail, children); nodes[id as usize].schema = Some(summary_schema_json(&node.schema)); id @@ -1415,4 +1467,131 @@ mod tests { on the Aggregate subtree it represents, not just the root" ); } + + /// Issue #172: a readout's guarantee is exported structurally — metric, + /// symbolic bound, failure probability, provenance (allocation + /// included) — and a rejection carries its typed reason. + #[test] + fn export_carries_guarantee_allocation_and_rejection_reason() { + use crate::post_asap::{ + BoundExpr, CompositionOperator, ErrorMetric, GroupingStrategy, GuaranteeSource, + ProbabilityExpr, SketchAlgorithm, SketchKind, SketchParams, SketchQuery, + SummaryFamilyType, SummarySchema, + }; + let leaf = Rc::new(scan("t", vec![Column::new("v", DataType::Float64, false)])); + let kept = Rc::new(SummaryNode { + expr: SummaryExpr::KeepPreAsap(Rc::clone(&leaf)), + schema: SummarySchema { + fields: vec![], + time_index: None, + }, + guarantee: Some(ResultGuarantee::exact("KeepPreAsap")), + }); + let agg = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child: kept, + family: SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 40 }), + GroupingStrategy::default(), + ), + col: crate::pre_asap::expr_ir::ColumnRef::Named("v".into()), + reduction: Reduction::by(vec![]), + grouping: GroupingStrategy::default(), + }, + schema: SummarySchema { + fields: vec![], + time_index: None, + }, + guarantee: None, + }); + let guarantee = ResultGuarantee { + metric: ErrorMetric::Rank, + bound: BoundExpr::Sum { + terms: vec![ + BoundExpr::Constant { value: 0.05 }, + BoundExpr::Constant { value: 0.05 }, + ], + }, + failure_probability: ProbabilityExpr::UnionBound { + terms: vec![ProbabilityExpr::Constant { value: 0.01 }], + }, + provenance: vec![ + GuaranteeSource::CompositionStep { + operator: CompositionOperator::ApproximateAggregate, + rule: "additive_union_bound".into(), + }, + GuaranteeSource::BudgetAllocation { + allocator: "EqualSplitAllocator".into(), + layer: 0, + layer_count: 2, + local_target: AccuracyTarget::Epsilon(0.05), + end_to_end_target: AccuracyTarget::Epsilon(0.1), + }, + ], + }; + let root = SummaryNode { + expr: SummaryExpr::SummaryEstimate { + summary_input: agg, + query: SketchQuery::Quantile { q: 0.99 }, + }, + schema: SummarySchema { + fields: vec![], + time_index: None, + }, + guarantee: Some(guarantee), + }; + let graph = export_summary(&root); + let json = serde_json::to_value(&graph).unwrap(); + let root_json = &json["nodes"][graph.root as usize]; + assert_eq!(root_json["guarantee"]["metric"], "rank"); + assert_eq!(root_json["guarantee"]["bound"]["op"], "sum"); + assert_eq!( + root_json["guarantee"]["failure_probability"]["op"], + "union_bound" + ); + let provenance = root_json["guarantee"]["provenance"].as_array().unwrap(); + assert!(provenance + .iter() + .any(|s| s["kind"] == "budget_allocation" && s["layer_count"] == 2)); + assert!(provenance.iter().any(|s| s["kind"] == "composition_step")); + // Raw sketch state carries none; the exact leaf carries zero error. + let state = &json["nodes"][1]; + assert_eq!(state["kind"], "SummaryAgg"); + assert!(state.get("guarantee").is_none()); + assert_eq!(json["nodes"][0]["guarantee"]["bound"]["op"], "zero"); + + let named = NamedGraph { + name: "q".into(), + source: None, + graph: export(&leaf), + replacements: vec![], + post_graph: None, + rejections: vec![TargetRejection { + target_pre_id: 0, + strategy: "SketchAlgorithmStrategy".into(), + description: "quantile over quantile".into(), + error: AccuracyError::UnsupportedComposition { + operator: CompositionOperator::ApproximateAggregate, + input_metrics: vec![ErrorMetric::Rank], + local_metric: Some(ErrorMetric::Rank), + reason: "no registered rule".into(), + }, + }], + }; + let json = serde_json::to_value(&named).unwrap(); + assert_eq!( + json["rejections"][0]["error"]["kind"], + "unsupported_composition" + ); + assert_eq!(json["rejections"][0]["error"]["input_metrics"][0], "rank"); + // Additive: a graph with no rejections omits the key entirely. + let plain = NamedGraph { + rejections: vec![], + ..named + }; + assert!(serde_json::to_value(&plain) + .unwrap() + .get("rejections") + .is_none()); + } } diff --git a/crates/types/src/post_asap/expr.rs b/crates/types/src/post_asap/expr.rs index 2402b22..b93aa90 100644 --- a/crates/types/src/post_asap/expr.rs +++ b/crates/types/src/post_asap/expr.rs @@ -1,5 +1,6 @@ use std::rc::Rc; +use super::guarantee::ResultGuarantee; use super::schema::{SummaryFamilyType, SummarySchema}; use super::sketch::{GroupingStrategy, SketchQuery}; use crate::pre_asap::{ColumnRef, QueryExpr, Reduction}; @@ -16,6 +17,18 @@ pub struct SummaryNode { /// Output schema of `expr` — the schema of the data flowing on the edge /// leading *from* this node to its parent(s). pub schema: SummarySchema, + /// The machine-readable accuracy guarantee of the *value* this node + /// produces (issue #172) — `Some` on every finalized, caller-visible + /// value: a `SummaryEstimate` readout, an `ExactAggregate`-family + /// `SummaryAgg` (its state *is* the value), or a `KeepPreAsap` subtree + /// (executed exactly). `None` on raw summary state — a sketch-family + /// `SummaryAgg`, `SummaryMerge`, `SummarySubtract`, `SummaryDelete`, + /// `SummaryJoin` — whose guarantee only exists once something reads it + /// out; and `None` on a readout of a family the plugged-in + /// `AccuracyModel` has no local guarantee for (`Sample`/`Wavelet`/ + /// `StatModel`), which a fail-closed consumer must treat as "unknown", + /// never as exact. + pub guarantee: Option, } // ── Post-ASAP sketch-bound IR ──────────────────────────────────────────────── diff --git a/crates/types/src/post_asap/guarantee.rs b/crates/types/src/post_asap/guarantee.rs new file mode 100644 index 0000000..04471a7 --- /dev/null +++ b/crates/types/src/post_asap/guarantee.rs @@ -0,0 +1,450 @@ +//! Machine-readable accuracy guarantees for finalized post-ASAP values +//! (issue #172). +//! +//! A selected post-ASAP plan used to carry no statement about the error of +//! the value it produces: every approximate layer was sized from its own +//! [`AccuracyTarget`] as if its input were exact, so an approximate parent +//! could silently consume an approximate child. This module is the +//! *vocabulary* that fixes that — the typed metric, the symbolic bound and +//! failure-probability expressions, the provenance trail, and the typed +//! rejection reasons. The *algebra* that composes these (the `AccuracyModel` +//! trait, its default conservative rules, and budget allocation) lives one +//! layer up in `asap_aware_mapping::accuracy`, the same layering +//! [`crate::dag_export`] keeps for cost decisions: this crate defines the +//! shapes, the planning crate decides. +//! +//! ## What a guarantee says +//! +//! [`ResultGuarantee`] is attached to a finalized, caller-visible value — +//! [`super::SummaryNode::guarantee`] on a `SummaryEstimate` readout, an +//! exact accumulator, or a kept pre-ASAP subtree — never to raw summary +//! state (a `SummaryAgg` sketch node carries `None`; its readout carries the +//! guarantee). Its statement is: +//! +//! ```text +//! Pr[ err_metric(estimate, truth) > bound ] <= failure_probability +//! ``` +//! +//! where `err_metric` is fixed by [`ErrorMetric`] and each metric has its +//! own normalization (documented per variant). Metrics are **not** +//! interchangeable: a cardinality error and a frequency error are different +//! quantities, and composing them needs an explicit rule, never an implicit +//! "add the epsilons". +//! +//! ## Why expressions, not numbers +//! +//! [`BoundExpr`]/[`ProbabilityExpr`] are tiny serializable expression trees +//! rather than bare `f64`s so a planning-time guarantee can reference a +//! statistic it does not have (a group count, a stream's L1 norm) and stay +//! honestly *unknown* until something instantiates it — a deployment's own +//! cardinality estimate, or a runtime posterior observation (issue #239). +//! [`BoundExpr::evaluate`] returns `None`, never `0`, for such a bound; +//! "unknown" and "zero" are different answers and a fail-closed planner +//! treats them differently. +//! +//! ## What is deliberately *not* here +//! +//! No `CorrectnessPolicy`-style enum: [`AccuracyTarget`] remains the one +//! authoritative requirement type, and [`AccuracyError`] is the typed reason +//! a candidate failed against it. No independence assumptions: the only +//! probability combinator is the union bound. + +use serde::{Deserialize, Serialize}; + +use crate::types::AccuracyTarget; + +/// Which error quantity a [`ResultGuarantee`] bounds. `#[non_exhaustive]`: +/// a deployment's own `AccuracyModel` may need a metric this crate does not +/// enumerate yet, and downstream matches must not assume the list is closed. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ErrorMetric { + /// `|estimate − truth| ≤ bound`, in the value's own units. + AbsoluteValue, + /// `|estimate − truth| ≤ bound · |truth|` — a multiplicative guarantee, + /// only meaningful for values of known sign (DDSketch's α). + RelativeValue, + /// The returned value's *rank* in the input multiset is within + /// `bound · n` of the requested rank (KLL's ε). Says nothing about how + /// far the returned *value* is from the true quantile value. + Rank, + /// `|estimate − truth| ≤ bound · truth` for a distinct count (HLL/Theta/ + /// KMV's relative standard error). + Cardinality, + /// `|estimate − truth| ≤ bound · ‖f‖₁` for a point-frequency query + /// (CMS's ε, normalized by the stream's L1 norm). + Frequency, + /// The returned key set equals the true top-k set. No shipped model + /// produces this yet: it needs a per-key interval margin certificate + /// (issue #172, PR 3). Present so the vocabulary can name the metric a + /// `TopK` readout would need without pretending a frequency bound is + /// one. + TopKMembership, +} + +/// A symbolic, serializable error-bound expression. Non-negative real +/// arithmetic only — there is no subtraction, so a bound can never be +/// tightened by construction, only by evaluating a known statistic. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum BoundExpr { + /// Exactly zero error (deterministic exact computation). + Zero, + /// A resolved numeric bound in the metric's own normalization. + Constant { value: f64 }, + /// `Σ terms` — the additive composition rule. + Sum { terms: Vec }, + /// `Π factors` — e.g. the relative-error cross term, or a normalized + /// bound times the (possibly unknown) statistic it is normalized by. + Product { factors: Vec }, + /// `factor · inner` — an explicitly registered Lipschitz constant. + Scaled { factor: f64, inner: Box }, + /// `max(terms)` — exact max/min over bounded inputs. + Max { terms: Vec }, + /// A statistic this bound needs but nothing has supplied yet. Evaluates + /// to `None`, never `0`: an unknown quantity is not a small one. + Unknown { statistic: String }, +} + +impl BoundExpr { + /// Numeric value of this bound, or `None` if any [`BoundExpr::Unknown`] + /// leaf is reachable. + pub fn evaluate(&self) -> Option { + match self { + BoundExpr::Zero => Some(0.0), + BoundExpr::Constant { value } => Some(*value), + BoundExpr::Sum { terms } => terms.iter().map(BoundExpr::evaluate).sum(), + BoundExpr::Product { factors } => factors.iter().map(BoundExpr::evaluate).product(), + BoundExpr::Scaled { factor, inner } => inner.evaluate().map(|b| factor * b), + BoundExpr::Max { terms } => terms + .iter() + .map(BoundExpr::evaluate) + .try_fold(0.0_f64, |acc, t| t.map(|t| acc.max(t))), + BoundExpr::Unknown { .. } => None, + } + } + + /// `true` iff this bound is structurally zero (every leaf is + /// [`BoundExpr::Zero`], or a `Product`/`Scaled` contains a zero factor). + /// Distinct from `evaluate() == Some(0.0)` only in that it never + /// depends on floating-point evaluation. + pub fn is_zero(&self) -> bool { + match self { + BoundExpr::Zero => true, + BoundExpr::Constant { value } => *value == 0.0, + BoundExpr::Sum { terms } | BoundExpr::Max { terms } => { + terms.iter().all(BoundExpr::is_zero) + } + BoundExpr::Product { factors } => factors.iter().any(BoundExpr::is_zero), + BoundExpr::Scaled { factor, inner } => *factor == 0.0 || inner.is_zero(), + BoundExpr::Unknown { .. } => false, + } + } +} + +/// A symbolic, serializable failure-probability expression. The only +/// combinator over several events is the union bound — the default model +/// never assumes independence between sketch errors. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum ProbabilityExpr { + /// The guarantee is deterministic. + Zero, + /// A resolved probability in `[0, 1]`. + Constant { value: f64 }, + /// `min(1, Σ terms)` — Boole's inequality over the listed events. + UnionBound { terms: Vec }, + /// `min(1, count · inner)` — the union bound over `count` events that + /// each fail with probability at most `inner` (e.g. one per input row of + /// an exact aggregation). `count` is a [`BoundExpr`] so it may be an + /// [`BoundExpr::Unknown`] statistic. + Scaled { + count: BoundExpr, + inner: Box, + }, + /// A probability nothing has supplied yet — same stance as + /// [`BoundExpr::Unknown`]. + Unknown { statistic: String }, +} + +impl ProbabilityExpr { + /// Numeric value clamped to `[0, 1]`, or `None` if any unknown leaf is + /// reachable. + pub fn evaluate(&self) -> Option { + let raw = match self { + ProbabilityExpr::Zero => 0.0, + ProbabilityExpr::Constant { value } => *value, + ProbabilityExpr::UnionBound { terms } => terms + .iter() + .map(ProbabilityExpr::evaluate) + .sum::>()?, + ProbabilityExpr::Scaled { count, inner } => count.evaluate()? * inner.evaluate()?, + ProbabilityExpr::Unknown { .. } => return None, + }; + Some(raw.clamp(0.0, 1.0)) + } + + /// `true` iff this probability is structurally zero. + pub fn is_zero(&self) -> bool { + match self { + ProbabilityExpr::Zero => true, + ProbabilityExpr::Constant { value } => *value == 0.0, + ProbabilityExpr::UnionBound { terms } => terms.iter().all(ProbabilityExpr::is_zero), + ProbabilityExpr::Scaled { count, inner } => count.is_zero() || inner.is_zero(), + ProbabilityExpr::Unknown { .. } => false, + } + } +} + +/// How a parent operator consumes its inputs' values — the shape an +/// `AccuracyModel::propagate` rule is registered against. `#[non_exhaustive]` +/// for the same reason [`ErrorMetric`] is. +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum CompositionOperator { + /// An approximate summary built over its inputs' (approximate) values + /// — the sketch-over-sketch case. Its own `local` guarantee composes + /// with the inputs' under a same-metric rule. + ApproximateAggregate, + /// A deterministic transformation with an explicitly registered global + /// Lipschitz constant: `B_out ≤ constant · B_in + B_local`. The planner + /// never derives `constant` itself; only a caller that has proved it + /// may construct this operator. + Lipschitz { constant: f64 }, + /// An exact sum over approximate inputs: `B ≤ Σ B_i`, `δ ≤ Σ δ_i`. + ExactSum, + /// An exact max/min over approximate inputs — bounds the returned + /// *value* (`max` of the input bounds) but does not identify which key + /// is the true winner. + ExactExtremum, + /// A top-k selection over approximate inputs. Unsupported by the default + /// model until the margin certificate of issue #172 PR 3 exists. + TopKSelection, +} + +/// One entry in a [`ResultGuarantee`]'s provenance trail — enough for a +/// reader to reconstruct *why* the bound is what it is without re-running +/// the planner. `#[non_exhaustive]` so runtime evidence (issue #239) and +/// deployment-specific sources can be appended later. +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum GuaranteeSource { + /// Deterministic exact computation — zero error by construction. + Exact { + /// What made it exact (e.g. `"ExactAggregate(Sum)"`, + /// `"KeepPreAsap"`). + reason: String, + }, + /// The target this readout's sketch was sized against. + AccuracyTarget { target: AccuracyTarget }, + /// The concrete sketch a readout's local guarantee was derived from. + SketchReadout { + algorithm: String, + params: serde_json::Value, + query: String, + }, + /// A composed input's own guarantee, carried verbatim so the trail is + /// self-contained. `input_index` is the input's position in the + /// composition (0-based, in the parent's child order). + ChildGuarantee { + input_index: usize, + guarantee: Box, + }, + /// The propagation rule that produced this guarantee from its inputs. + CompositionStep { + operator: CompositionOperator, + /// Stable rule name (e.g. `"additive_union_bound"`). + rule: String, + }, + /// The budget split that produced this layer's local target — present + /// only when an `AccuracyBudgetAllocator` re-sized a layer. + BudgetAllocation { + allocator: String, + layer: usize, + layer_count: usize, + local_target: AccuracyTarget, + end_to_end_target: AccuracyTarget, + }, + /// A statistic the bound needs but nothing supplied — the reason a + /// [`BoundExpr::Unknown`]/[`ProbabilityExpr::Unknown`] leaf exists. + UnavailableStatistic { statistic: String }, + /// Query-time evidence (issue #239's posterior bounds). Never produced + /// at planning time; reserved so a runtime can append its observation + /// to the same trail instead of inventing a parallel one. + RuntimeObservation { + source: String, + detail: serde_json::Value, + }, +} + +/// The machine-readable accuracy statement attached to a finalized +/// post-ASAP value — see the module docs for its semantics. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ResultGuarantee { + pub metric: ErrorMetric, + pub bound: BoundExpr, + pub failure_probability: ProbabilityExpr, + pub provenance: Vec, +} + +impl ResultGuarantee { + /// The zero-error, zero-failure guarantee of a deterministic exact + /// computation. `metric` is [`ErrorMetric::AbsoluteValue`]: an exact + /// value is exact under every metric, and absolute error is the one + /// every same-metric rule accepts as a zero input. + pub fn exact(reason: impl Into) -> Self { + Self { + metric: ErrorMetric::AbsoluteValue, + bound: BoundExpr::Zero, + failure_probability: ProbabilityExpr::Zero, + provenance: vec![GuaranteeSource::Exact { + reason: reason.into(), + }], + } + } + + /// `true` iff this guarantee promises zero error with certainty. + pub fn is_exact(&self) -> bool { + self.bound.is_zero() && self.failure_probability.is_zero() + } + + /// How many approximate sketch readouts contributed to this value — + /// `1` for a plain readout, `0` for an exact value, and the transitive + /// count through every [`GuaranteeSource::ChildGuarantee`] for a + /// composition. An `AccuracyBudgetAllocator` uses this as the number + /// of layers a budget must be split across. + pub fn approximate_layer_count(&self) -> usize { + self.provenance + .iter() + .map(|source| match source { + GuaranteeSource::SketchReadout { .. } => 1, + GuaranteeSource::ChildGuarantee { guarantee, .. } => { + guarantee.approximate_layer_count() + } + _ => 0, + }) + .sum() + } +} + +/// Why an accuracy check rejected a candidate. Typed, serializable, and +/// carried through to DAG export so a rejection is as inspectable as a +/// selection. Never a reason to "treat the child as exact". +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, thiserror::Error)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum AccuracyError { + /// No registered propagation rule covers this operator over these + /// input metrics (or this local metric). + #[error( + "unsupported accuracy composition: {operator:?} over inputs {input_metrics:?} \ + with local {local_metric:?} — {reason}" + )] + UnsupportedComposition { + operator: CompositionOperator, + input_metrics: Vec, + local_metric: Option, + reason: String, + }, + /// An approximate input carries no guarantee at all, so nothing can be + /// composed over it. + #[error("input {input_index} of {operator:?} carries no accuracy guarantee")] + MissingInputGuarantee { + operator: CompositionOperator, + input_index: usize, + }, + /// The composed guarantee does not satisfy the applicable + /// [`AccuracyTarget`]. `bound`/`failure_probability` are the evaluated + /// values when known. + #[error( + "composed guarantee ({metric:?}, bound {bound:?}, failure probability \ + {failure_probability:?}) does not satisfy {target:?}" + )] + TargetNotSatisfied { + metric: ErrorMetric, + bound: Option, + failure_probability: Option, + target: AccuracyTarget, + }, + /// No budget allocation could make the composition legal under the + /// end-to-end target. + #[error("no legal accuracy-budget allocation for {target:?} across {layer_count} layers")] + NoLegalAllocation { + target: AccuracyTarget, + layer_count: usize, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unknown_statistic_never_evaluates_to_zero() { + let b = BoundExpr::Product { + factors: vec![ + BoundExpr::Constant { value: 0.01 }, + BoundExpr::Unknown { + statistic: "input_row_count".into(), + }, + ], + }; + assert_eq!(b.evaluate(), None); + assert!(!b.is_zero()); + let p = ProbabilityExpr::Scaled { + count: BoundExpr::Unknown { + statistic: "input_row_count".into(), + }, + inner: Box::new(ProbabilityExpr::Constant { value: 0.01 }), + }; + assert_eq!(p.evaluate(), None); + } + + #[test] + fn union_bound_sums_and_clamps() { + let p = ProbabilityExpr::UnionBound { + terms: vec![ + ProbabilityExpr::Constant { value: 0.7 }, + ProbabilityExpr::Constant { value: 0.6 }, + ], + }; + assert_eq!(p.evaluate(), Some(1.0)); + } + + #[test] + fn exact_guarantee_is_zero_layers() { + let g = ResultGuarantee::exact("test"); + assert!(g.is_exact()); + assert_eq!(g.approximate_layer_count(), 0); + } + + #[test] + fn guarantee_round_trips_through_json() { + let g = ResultGuarantee { + metric: ErrorMetric::Frequency, + bound: BoundExpr::Sum { + terms: vec![ + BoundExpr::Constant { value: 0.01 }, + BoundExpr::Scaled { + factor: 2.0, + inner: Box::new(BoundExpr::Constant { value: 0.005 }), + }, + ], + }, + failure_probability: ProbabilityExpr::UnionBound { + terms: vec![ProbabilityExpr::Constant { value: 0.01 }], + }, + provenance: vec![GuaranteeSource::CompositionStep { + operator: CompositionOperator::Lipschitz { constant: 2.0 }, + rule: "lipschitz".into(), + }], + }; + let json = serde_json::to_value(&g).unwrap(); + assert_eq!(json["metric"], "frequency"); + assert_eq!(json["bound"]["op"], "sum"); + let back: ResultGuarantee = serde_json::from_value(json).unwrap(); + assert_eq!(back, g); + } +} diff --git a/crates/types/src/post_asap/mod.rs b/crates/types/src/post_asap/mod.rs index 5809eb7..5ce4538 100644 --- a/crates/types/src/post_asap/mod.rs +++ b/crates/types/src/post_asap/mod.rs @@ -28,11 +28,16 @@ //! — see `asap_aware_mapping::grouping`'s module docs for why. pub mod expr; +pub mod guarantee; pub mod query_time; pub mod schema; pub mod sketch; pub use expr::{SummaryExpr, SummaryNode}; +pub use guarantee::{ + AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, GuaranteeSource, ProbabilityExpr, + ResultGuarantee, +}; pub use query_time::{ classic_cms_sizing, cms_posterior_error_bound, count_sketch_posterior_error_bound, cu_sketch_posterior_error_bound, traditional_a_priori_bound, diff --git a/crates/types/src/post_asap/sketch.rs b/crates/types/src/post_asap/sketch.rs index e3a4a8b..3e40aac 100644 --- a/crates/types/src/post_asap/sketch.rs +++ b/crates/types/src/post_asap/sketch.rs @@ -1,3 +1,5 @@ +use serde::{Deserialize, Serialize}; + use crate::pre_asap::ColumnRef; // ── Exact accumulators ────────────────────────────────────────────────────── @@ -38,7 +40,7 @@ pub enum ExactParams { /// [`SketchParams`]. Each algorithm belongs to exactly one [`SketchKind`] /// category (e.g. `Kll` and `DDSketch` both realize quantile sketches); /// [`SketchKind::new`] is where that classification is made. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] pub enum SketchAlgorithm { /// KLL quantile sketch (mergeable, ε-accurate rank queries). Kll, @@ -67,7 +69,7 @@ pub enum SketchAlgorithm { /// instance. The variant must correspond to the associated `SketchAlgorithm`; /// mismatches are caught at post-ASAP bind time, before any later, /// deployment-specific stage ever sees the plan. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum SketchParams { Kll { k: u32, diff --git a/docs/design_docs/asap-aware-mapping/summary_properties.md b/docs/design_docs/asap-aware-mapping/summary_properties.md index f7a7cff..5602572 100644 --- a/docs/design_docs/asap-aware-mapping/summary_properties.md +++ b/docs/design_docs/asap-aware-mapping/summary_properties.md @@ -95,3 +95,32 @@ Does the summary's error behavior change as more items are inserted? Some summaries provide guarantees largely independent of stream length, while others may degrade or require resizing. The planner needs this information when selecting long-lived summaries. + +## End-to-end guarantees for nested summaries + +A single summary is sized against its own `AccuracyTarget`, but a summary +over another summary's readout is legal only if the composed error still +meets the requirement on the outer value. Legality is established before +costing: + +```text +candidate generation + -> guarantee propagation (AccuracyModel::propagate) + -> AccuracyTarget satisfaction (AccuracyModel::satisfies) + -> legal candidates only (illegal ones -> MemoGroup::rejected) + -> cost ranking / global selection (CostModel) +``` + +Every finalized post-ASAP value carries a machine-readable +`ResultGuarantee`: a typed error metric, symbolic bound and probability +expressions, and provenance. Exact values have zero error; a sketch +readout's guarantee is derived from the sizing formula that produced its +parameters. + +The default `AccuracyModel` is deliberately conservative. It supports +registered same-metric additive, relative, and Lipschitz rules and exact +sum/max/min over approximate inputs. It uses union-bound probabilities +without assuming independence, preserves unknown statistics as unknown, +and rejects unsupported composition instead of treating the child as exact. +An `AccuracyBudgetAllocator` can propose resized layers; the `CostModel` +ranks only candidates that satisfy the accuracy requirement. From 34e5b34ffd9a118f0ebf50764d5677e60b707a6f Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 28 Aug 2026 05:39:13 -0600 Subject: [PATCH 02/12] fix(accuracy): fail closed for unmodeled sketch bounds --- crates/asap-aware-mapping/src/accuracy.rs | 118 ++++++++++++------- crates/asap-aware-mapping/src/grouping.rs | 16 ++- crates/asap-aware-mapping/src/replacement.rs | 28 +++++ 3 files changed, 112 insertions(+), 50 deletions(-) diff --git a/crates/asap-aware-mapping/src/accuracy.rs b/crates/asap-aware-mapping/src/accuracy.rs index 9393404..eb083ca 100644 --- a/crates/asap-aware-mapping/src/accuracy.rs +++ b/crates/asap-aware-mapping/src/accuracy.rs @@ -126,21 +126,6 @@ pub trait AccuracyModel { #[derive(Debug, Default, Clone, Copy)] pub struct DefaultAccuracyModel; -/// Failure probability this crate attributes to a KLL bound sized by -/// `k = ⌈2/ε⌉`: the 99%-confidence convention the `k`-to-rank-error tables -/// of the reference implementation (Apache DataSketches) quote that formula -/// at. KLL's sizing ignores δ, so this is the confidence level the sizing -/// implicitly claims, not something re-derived from `k`. -pub const KLL_FAILURE_PROBABILITY: f64 = 0.01; - -/// Failure probability this crate attributes to an HLL/Theta/KMV bound -/// sized by its relative *standard error* (`1.04/√m`, `1/√k`): a -/// one-standard-deviation bound of an approximately Gaussian estimator is -/// exceeded with probability ≈ 0.3173. Those families' sizing ignores δ, so -/// an `EpsilonDelta` target with a tighter δ is honestly not met by the -/// parameters the sizing picked; the guarantee says so instead of hiding it. -pub const STANDARD_ERROR_FAILURE_PROBABILITY: f64 = 0.3173; - /// Small relative tolerance for comparing an evaluated bound against a /// target, so a parameter sized by `⌈·⌉` to *exactly* meet ε is not rejected /// by floating-point noise. @@ -155,13 +140,19 @@ impl DefaultAccuracyModel { params: &SketchParams, query: &SketchQuery, ) -> Option { + // A frequency bound for each reported key does not certify that the + // reported key set is the true top-k set. Until a margin certificate + // is modeled, TopK has no end-to-end guarantee. + if matches!(query, SketchQuery::TopK { .. }) { + return None; + } let (metric, bound, delta) = match params { // KLL: rank error ε ≈ 2/k. SketchParams::Kll { k } => ( ErrorMetric::Rank, 2.0 / f64::from(*k), - ProbabilityExpr::Constant { - value: KLL_FAILURE_PROBABILITY, + ProbabilityExpr::Unknown { + statistic: "kll_failure_probability".into(), }, ), // DDSketch: deterministic relative value error α. @@ -172,46 +163,40 @@ impl DefaultAccuracyModel { SketchParams::Hll { precision } => ( ErrorMetric::Cardinality, 1.04 / 2f64.powi(i32::from(*precision)).sqrt(), - ProbabilityExpr::Constant { - value: STANDARD_ERROR_FAILURE_PROBABILITY, + ProbabilityExpr::Unknown { + statistic: "hll_failure_probability".into(), }, ), // KMV / Theta: relative standard error 1/√k. SketchParams::Kmv { k } | SketchParams::Theta { k } => ( ErrorMetric::Cardinality, 1.0 / f64::from(*k).sqrt(), - ProbabilityExpr::Constant { - value: STANDARD_ERROR_FAILURE_PROBABILITY, - }, - ), - // CMS family: over-count ≤ (e/w)·‖f‖₁ with probability ≥ 1 − e^{−d}. - // Count-Sketch is sized with the same placeholder formula (see - // `default_size_params`), so it gets the same placeholder bound. - SketchParams::Cms { width, depth } - | SketchParams::CountSketch { width, depth } - | SketchParams::CmsWithHeap { width, depth, .. } - | SketchParams::CountSketchWithHeap { width, depth, .. } => ( - ErrorMetric::Frequency, - std::f64::consts::E / f64::from(*width), - ProbabilityExpr::Constant { - value: (-f64::from(*depth)).exp(), + ProbabilityExpr::Unknown { + statistic: "kmv_theta_failure_probability".into(), }, ), + // CMS: over-count ≤ (e/w)·‖f‖₁ with probability ≥ 1 − e^{−d}. + SketchParams::Cms { width, depth } | SketchParams::CmsWithHeap { width, depth, .. } => { + ( + ErrorMetric::Frequency, + std::f64::consts::E / f64::from(*width), + ProbabilityExpr::Constant { + value: (-f64::from(*depth)).exp(), + }, + ) + } + // CountSketch has an L2-based error bound and a different sizing + // relationship. Its current sizing is explicitly a placeholder, + // so it cannot support a formal guarantee. + SketchParams::CountSketch { .. } | SketchParams::CountSketchWithHeap { .. } => { + return None + } }; - let mut provenance = vec![GuaranteeSource::SketchReadout { + let provenance = vec![GuaranteeSource::SketchReadout { algorithm: format!("{algorithm:?}"), params: serde_json::to_value(params).unwrap_or(serde_json::Value::Null), query: format!("{query:?}"), }]; - // A `TopK` readout's count for each *reported* key carries the - // sketch's frequency bound; which keys are reported is not certified - // by anything here (issue #172, PR 3) — say so in the trail rather - // than claiming `TopKMembership`. - if matches!(query, SketchQuery::TopK { .. }) { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: "topk_membership_margin_certificate".into(), - }); - } Some(ResultGuarantee { metric, bound: BoundExpr::Constant { value: bound }, @@ -987,6 +972,14 @@ mod tests { .unwrap(); assert_eq!(g.metric, ErrorMetric::Rank); assert!(DefaultAccuracyModel.satisfies(&g, &AccuracyTarget::Epsilon(0.01))); + assert_eq!(g.failure_probability.evaluate(), None); + assert!(!DefaultAccuracyModel.satisfies( + &g, + &AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.01, + } + )); assert_eq!(g.approximate_layer_count(), 1); let c = default_cardinality(); @@ -1002,6 +995,7 @@ mod tests { .unwrap(); assert_eq!(g.metric, ErrorMetric::Cardinality); assert!(DefaultAccuracyModel.satisfies(&g, &AccuracyTarget::Epsilon(0.01))); + assert_eq!(g.failure_probability.evaluate(), None); let params = default_size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001); let g = DefaultAccuracyModel @@ -1023,6 +1017,42 @@ mod tests { )); } + #[test] + fn unsupported_sketch_guarantees_fail_closed() { + use crate::replacement::default_size_params; + use asap_types::pre_asap::agg_intent::default_cardinality; + + let intent = default_cardinality(); + let count_sketch = default_size_params(SketchAlgorithm::CountSketch, &intent, 0.01, 0.01); + assert!(DefaultAccuracyModel + .local_guarantee( + &SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::CountSketch, count_sketch), + GroupingStrategy::default(), + ), + &SketchQuery::PointCount { + key: asap_types::pre_asap::expr_ir::ColumnRef::SampleValue, + value: None, + }, + ) + .is_none()); + + let cms_heap = SketchParams::CmsWithHeap { + width: 272, + depth: 5, + heap_size: 10, + }; + assert!(DefaultAccuracyModel + .local_guarantee( + &SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::CmsWithHeap, cms_heap), + GroupingStrategy::default(), + ), + &SketchQuery::TopK { k: 10 }, + ) + .is_none()); + } + #[test] fn satisfies_is_fail_closed_on_unknowns_and_exact() { let unknown = ResultGuarantee { diff --git a/crates/asap-aware-mapping/src/grouping.rs b/crates/asap-aware-mapping/src/grouping.rs index 0e3f294..d82afc3 100644 --- a/crates/asap-aware-mapping/src/grouping.rs +++ b/crates/asap-aware-mapping/src/grouping.rs @@ -275,11 +275,10 @@ fn with_grouping(node: Rc, grouping: GroupingStrategy) -> Rc, grouping: GroupingStrategy) -> Rc true, })); } + + #[test] + fn topk_accuracy_target_rejects_uncertified_membership() { + let q = Rc::new(agg( + vec![2], + AggIntent::TopK { + k: 10, + accuracy: AccuracyTarget::Epsilon(0.01), + }, + metric_scan(&["job"]), + )); + let space = search_workload_with_targets( + vec![("q", Rc::clone(&q), Some(AccuracyTarget::Epsilon(0.01)))], + &default_strategies(), + &DefaultAccuracyModel, + ); + let group = space.group_for(&space.roots[0].1).unwrap(); + + assert!(group + .candidates + .iter() + .all(|candidate| matches!(candidate.replacement, Replacement::Rewrite(_)))); + assert!(!group.rejected.is_empty()); + assert!(group + .rejected + .iter() + .all(|rejected| matches!(rejected.error, AccuracyError::TargetNotSatisfied { .. }))); + } } From 500105954288299d1b750dbd0b766e1aa5fae0b5 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 28 Aug 2026 06:35:00 -0600 Subject: [PATCH 03/12] fix(accuracy): reject unmodeled Hydra guarantees --- crates/asap-aware-mapping/src/grouping.rs | 103 ++++------------------ 1 file changed, 18 insertions(+), 85 deletions(-) diff --git a/crates/asap-aware-mapping/src/grouping.rs b/crates/asap-aware-mapping/src/grouping.rs index d82afc3..19c17a2 100644 --- a/crates/asap-aware-mapping/src/grouping.rs +++ b/crates/asap-aware-mapping/src/grouping.rs @@ -26,10 +26,12 @@ //! with no grouping concept at all) has nothing for a //! shared-multi-subpopulation structure to multiplex across. //! - **The family has a Hydra variant** -//! ([`asap_types::post_asap::hydra_kind_for`]): only `Cms` and -//! `CountSketch` are selectable today because their error guarantees are -//! modeled. `HydraKll` remains an explicit experimental IR value, but the -//! paper excludes quantiles and search therefore never emits it. +//! ([`asap_types::post_asap::hydra_kind_for`]): `Cms` and `CountSketch` +//! have structural Hydra mappings. `HydraKll` remains an explicit +//! experimental IR value, but the paper excludes quantiles and search +//! therefore never emits it. Accuracy-targeted candidates are also +//! withheld until the shared grid's collision error is composed with the +//! inner sketch guarantee. //! //! Whether Hydra is *worth it* for a given estimated subpopulation //! cardinality is a cost-model question, deliberately out of scope here — @@ -79,7 +81,7 @@ use asap_types::pre_asap::query_expr::{QueryExpr, Reduction}; use crate::cost_model::{CostModel, DefaultCostModel}; use crate::replacement::{ - bindable_intent, construct_summary, describe_intent, implementations_for_with, + accuracy_target, bindable_intent, construct_summary, describe_intent, implementations_for_with, summary_candidates, Implementation, Replacement, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; @@ -158,6 +160,13 @@ impl<'a> HydraGroupingStrategy<'a> { let Some(intent) = bindable_intent(target.root) else { return Vec::new(); }; + // Hydra adds shared-grid collision error that is not modeled yet. + // `with_grouping` clears the per-subpopulation guarantee so exports + // remain honest, but an unknown guarantee must also keep the + // candidate out of cost ranking when this node has a requirement. + if accuracy_target(intent).is_some() { + return Vec::new(); + } summary_candidates(intent) .iter() .filter_map(|kind| hydra_kind_for(kind).map(|hydra_kind| (kind.clone(), hydra_kind))) @@ -324,7 +333,6 @@ fn with_grouping(node: Rc, grouping: GroupingStrategy) -> Rc { - let SketchParams::Cms { width, depth } = kind.params() else { - panic!("expected Cms params, got {:?}", kind.params()); - }; - assert_eq!( - grouping, - &GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width: *width, - depth: *depth, - shared_rows: *depth, - shared_columns: *width, - }, - } - ); - } - SketchAlgorithm::CountSketch => { - let SketchParams::CountSketch { width, depth } = kind.params() else { - panic!("expected CountSketch params, got {:?}", kind.params()); - }; - assert_eq!( - grouping, - &GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCountSketch, - params: HydraParams::HydraCountSketch { - width: *width, - depth: *depth, - shared_rows: *depth, - shared_columns: *width, - }, - } - ); - } - other => panic!("unexpected Hydra candidate algorithm: {other:?}"), - } - assert!(!replacement.rationale.is_empty()); - } + assert!(replacements.is_empty(), "{replacements:?}"); } #[test] From c83a90eb26830cafc42539ad69e83b2c8fcfcc18 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 28 Aug 2026 06:53:21 -0600 Subject: [PATCH 04/12] fix(accuracy): validate symbolic guarantee values --- crates/asap-aware-mapping/src/replacement.rs | 18 +++++------ crates/types/src/post_asap/guarantee.rs | 33 ++++++++++++++++---- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 74403b6..99f810d 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -4718,7 +4718,7 @@ mod tests { // ── discovery + MEMO shape ─────────────────────────────────────────── #[test] - fn single_bindable_aggregate_gets_every_sketch_and_grouping_candidate() { + fn single_bindable_aggregate_excludes_unprovable_hydra_candidates() { let intent = AggIntent::Count { accuracy: AccuracyTarget::EpsilonDelta { epsilon: 0.01, @@ -4738,8 +4738,8 @@ mod tests { assert_eq!(agg_group.consumer_count, 1); assert_eq!( agg_group.candidates.len(), - 4, - "grouped approximate count has independent and Hydra CMS/CountSketch candidates: {:?}", + 2, + "only independent CMS/CountSketch candidates have modeled guarantees: {:?}", agg_group.candidates ); assert!(agg_group @@ -4766,8 +4766,8 @@ mod tests { ) }) .count(), - 2, - "the default workload search must register the Hydra grouping strategy" + 0, + "Hydra shared-grid error is unmodeled, so accuracy-targeted candidates must be absent" ); let scan_group = space @@ -5094,7 +5094,7 @@ mod tests { } #[test] - fn grouping_cost_prefers_hydra_only_for_high_subpopulation_cardinality() { + fn grouping_cost_cannot_resurrect_unprovable_hydra_candidates() { struct EstimatedSubpopulations(usize); impl CostModel for EstimatedSubpopulations { @@ -5139,10 +5139,10 @@ mod tests { .clone() } - assert!(matches!( + assert_eq!( first_grouping(10_000), - GroupingStrategy::SharedMultiSubpopulation { .. } - )); + GroupingStrategy::PerSubpopulationInstance + ); assert_eq!( first_grouping(10), GroupingStrategy::PerSubpopulationInstance diff --git a/crates/types/src/post_asap/guarantee.rs b/crates/types/src/post_asap/guarantee.rs index 04471a7..812dcb2 100644 --- a/crates/types/src/post_asap/guarantee.rs +++ b/crates/types/src/post_asap/guarantee.rs @@ -111,18 +111,23 @@ impl BoundExpr { /// Numeric value of this bound, or `None` if any [`BoundExpr::Unknown`] /// leaf is reachable. pub fn evaluate(&self) -> Option { - match self { + let value = match self { BoundExpr::Zero => Some(0.0), - BoundExpr::Constant { value } => Some(*value), + BoundExpr::Constant { value } => value.is_finite().then_some(*value), BoundExpr::Sum { terms } => terms.iter().map(BoundExpr::evaluate).sum(), BoundExpr::Product { factors } => factors.iter().map(BoundExpr::evaluate).product(), - BoundExpr::Scaled { factor, inner } => inner.evaluate().map(|b| factor * b), + BoundExpr::Scaled { factor, inner } => factor + .is_finite() + .then_some(*factor) + .zip(inner.evaluate()) + .map(|(factor, bound)| factor * bound), BoundExpr::Max { terms } => terms .iter() .map(BoundExpr::evaluate) .try_fold(0.0_f64, |acc, t| t.map(|t| acc.max(t))), BoundExpr::Unknown { .. } => None, - } + }?; + (value.is_finite() && value >= 0.0).then_some(value) } /// `true` iff this bound is structurally zero (every leaf is @@ -174,7 +179,8 @@ impl ProbabilityExpr { pub fn evaluate(&self) -> Option { let raw = match self { ProbabilityExpr::Zero => 0.0, - ProbabilityExpr::Constant { value } => *value, + ProbabilityExpr::Constant { value } if (0.0..=1.0).contains(value) => *value, + ProbabilityExpr::Constant { .. } => return None, ProbabilityExpr::UnionBound { terms } => terms .iter() .map(ProbabilityExpr::evaluate) @@ -182,7 +188,7 @@ impl ProbabilityExpr { ProbabilityExpr::Scaled { count, inner } => count.evaluate()? * inner.evaluate()?, ProbabilityExpr::Unknown { .. } => return None, }; - Some(raw.clamp(0.0, 1.0)) + raw.is_finite().then(|| raw.clamp(0.0, 1.0)) } /// `true` iff this probability is structurally zero. @@ -402,6 +408,21 @@ mod tests { assert_eq!(p.evaluate(), None); } + #[test] + fn invalid_numeric_leaves_fail_closed() { + assert_eq!(BoundExpr::Constant { value: -0.1 }.evaluate(), None); + assert_eq!( + BoundExpr::Scaled { + factor: -1.0, + inner: Box::new(BoundExpr::Constant { value: 0.1 }), + } + .evaluate(), + None + ); + assert_eq!(ProbabilityExpr::Constant { value: -0.1 }.evaluate(), None); + assert_eq!(ProbabilityExpr::Constant { value: 1.1 }.evaluate(), None); + } + #[test] fn union_bound_sums_and_clamps() { let p = ProbabilityExpr::UnionBound { From 063e235278abce1c852312c81bd6d90e68e15102 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 28 Aug 2026 09:40:41 -0600 Subject: [PATCH 05/12] model deferred sketch accuracy guarantees --- crates/asap-aware-mapping/src/accuracy.rs | 180 ++++++++++++++---- crates/asap-aware-mapping/src/grouping.rs | 95 +++++++-- crates/asap-aware-mapping/src/replacement.rs | 144 +++++++------- .../tests/sql_to_post_asap.rs | 4 +- crates/types/src/post_asap/guarantee.rs | 4 + .../asap-aware-mapping/summary_properties.md | 17 ++ 6 files changed, 315 insertions(+), 129 deletions(-) diff --git a/crates/asap-aware-mapping/src/accuracy.rs b/crates/asap-aware-mapping/src/accuracy.rs index eb083ca..e14e044 100644 --- a/crates/asap-aware-mapping/src/accuracy.rs +++ b/crates/asap-aware-mapping/src/accuracy.rs @@ -86,6 +86,15 @@ pub struct PropagationStats { /// of groups a `sum` folds), for `ExactSum`/`ExactExtremum`'s union /// bound over per-input failures. pub input_row_count: Option, + /// Lower confidence bound of the kth selected TopK item, after widening + /// the interval by the sketch's own estimation error. + pub topk_selected_lower_bound: Option, + /// Greatest upper confidence bound among excluded TopK items, after + /// widening the interval by the sketch's own estimation error. + pub topk_excluded_upper_bound: Option, + /// Union-bound failure probability of all intervals used by the margin + /// certificate. + pub topk_interval_failure_probability: Option, } /// The deployment-extensible accuracy algebra. `asap-aware-mapping` ships @@ -131,6 +140,13 @@ pub struct DefaultAccuracyModel; /// by floating-point noise. const SATISFACTION_TOLERANCE: f64 = 1e-9; +fn count_sketch_failure_probability(depth: u32) -> Option { + if depth == 0 || depth.is_multiple_of(2) { + return None; + } + Some((-f64::from(depth) / 18.0).exp()) +} + impl DefaultAccuracyModel { /// The local guarantee of one sketch `(algorithm, params)` for `query` /// — each arm inverts the matching formula in @@ -140,40 +156,32 @@ impl DefaultAccuracyModel { params: &SketchParams, query: &SketchQuery, ) -> Option { - // A frequency bound for each reported key does not certify that the - // reported key set is the true top-k set. Until a margin certificate - // is modeled, TopK has no end-to-end guarantee. - if matches!(query, SketchQuery::TopK { .. }) { - return None; - } let (metric, bound, delta) = match params { - // KLL: rank error ε ≈ 2/k. + // The committed KLL contract is its 99%-confidence normalized + // rank bound. Tighter confidence needs runtime-supported + // amplification and therefore remains unavailable. SketchParams::Kll { k } => ( ErrorMetric::Rank, 2.0 / f64::from(*k), - ProbabilityExpr::Unknown { - statistic: "kll_failure_probability".into(), - }, + ProbabilityExpr::Constant { value: 0.01 }, ), // DDSketch: deterministic relative value error α. SketchParams::DDSketch { alpha } => { (ErrorMetric::RelativeValue, *alpha, ProbabilityExpr::Zero) } - // HLL: relative standard error 1.04/√(2^p). + // HLL: RSE = 1.04/√(2^p). Chebyshev turns ten standard + // deviations into a distribution-free 99% confidence bound. SketchParams::Hll { precision } => ( ErrorMetric::Cardinality, - 1.04 / 2f64.powi(i32::from(*precision)).sqrt(), - ProbabilityExpr::Unknown { - statistic: "hll_failure_probability".into(), - }, + 10.0 * 1.04 / 2f64.powi(i32::from(*precision)).sqrt(), + ProbabilityExpr::Constant { value: 0.01 }, ), - // KMV / Theta: relative standard error 1/√k. + // KMV / Theta: RSE <= 1/√(k-2); the same Chebyshev conversion + // gives a conservative parameter-derived 99% confidence bound. SketchParams::Kmv { k } | SketchParams::Theta { k } => ( ErrorMetric::Cardinality, - 1.0 / f64::from(*k).sqrt(), - ProbabilityExpr::Unknown { - statistic: "kmv_theta_failure_probability".into(), - }, + 10.0 / f64::from(k.saturating_sub(2).max(1)).sqrt(), + ProbabilityExpr::Constant { value: 0.01 }, ), // CMS: over-count ≤ (e/w)·‖f‖₁ with probability ≥ 1 − e^{−d}. SketchParams::Cms { width, depth } | SketchParams::CmsWithHeap { width, depth, .. } => { @@ -185,12 +193,18 @@ impl DefaultAccuracyModel { }, ) } - // CountSketch has an L2-based error bound and a different sizing - // relationship. Its current sizing is explicitly a placeholder, - // so it cannot support a formal guarantee. - SketchParams::CountSketch { .. } | SketchParams::CountSketchWithHeap { .. } => { - return None - } + // CountSketch: one row has variance at most ‖f‖₂²/w. With + // ε=√(3/w), Chebyshev makes a row bad with probability <=1/3; + // the median across independent odd-depth rows has the binomial + // tail bounded by Hoeffding below. + SketchParams::CountSketch { width, depth } + | SketchParams::CountSketchWithHeap { width, depth, .. } => ( + ErrorMetric::L2Frequency, + (3.0 / f64::from(*width)).sqrt(), + ProbabilityExpr::Constant { + value: count_sketch_failure_probability(*depth)?, + }, + ), }; let provenance = vec![GuaranteeSource::SketchReadout { algorithm: format!("{algorithm:?}"), @@ -438,6 +452,7 @@ fn absolute_bound(input: &ResultGuarantee) -> Option { ErrorMetric::RelativeValue => "true_value_magnitude", ErrorMetric::Cardinality => "true_cardinality", ErrorMetric::Frequency => "stream_l1_norm", + ErrorMetric::L2Frequency => "stream_l2_norm", // `Rank` has no distribution-free conversion to a value error; a // metric this crate does not know has no registered conversion. ErrorMetric::Rank | ErrorMetric::TopKMembership | _ => return None, @@ -506,7 +521,9 @@ impl AccuracyModel for DefaultAccuracyModel { stats: &PropagationStats, ) -> Result { // Exact input: only the local guarantee remains (or the value is exact). - if inputs.iter().all(ResultGuarantee::is_exact) { + if inputs.iter().all(ResultGuarantee::is_exact) + && !matches!(op, CompositionOperator::TopKSelection) + { return Ok(match local { Some(local) => { let mut out = local.clone(); @@ -574,6 +591,7 @@ impl AccuracyModel for DefaultAccuracyModel { ErrorMetric::Rank | ErrorMetric::Cardinality | ErrorMetric::Frequency + | ErrorMetric::L2Frequency | ErrorMetric::TopKMembership | _ => Err(unsupported(format!( "no registered same-metric composition rule for {:?} over {:?}", @@ -601,11 +619,50 @@ impl AccuracyModel for DefaultAccuracyModel { } CompositionOperator::ExactSum => Self::exact_sum(op, inputs, stats), CompositionOperator::ExactExtremum => Self::exact_extremum(op, inputs, stats), - CompositionOperator::TopKSelection => Err(unsupported( - "top-k membership over approximate inputs needs a margin certificate \ - (issue #172, PR 3)" - .into(), - )), + CompositionOperator::TopKSelection => { + let (Some(selected_lower), Some(excluded_upper), Some(delta)) = ( + stats.topk_selected_lower_bound, + stats.topk_excluded_upper_bound, + stats.topk_interval_failure_probability, + ) else { + return Err(unsupported( + "top-k membership needs selected-lower, excluded-upper, and interval \ + failure-probability evidence" + .into(), + )); + }; + if !(selected_lower.is_finite() + && excluded_upper.is_finite() + && delta.is_finite() + && (0.0..=1.0).contains(&delta) + && selected_lower > excluded_upper) + { + return Err(unsupported( + "top-k confidence intervals overlap or contain invalid evidence".into(), + )); + } + let mut provenance = inputs + .iter() + .enumerate() + .map(|(input_index, guarantee)| GuaranteeSource::ChildGuarantee { + input_index, + guarantee: Box::new(guarantee.clone()), + }) + .collect::>(); + if let Some(local) = local { + provenance.extend(local.provenance.clone()); + } + provenance.push(GuaranteeSource::CompositionStep { + operator: op.clone(), + rule: "topk_membership_margin_certificate".into(), + }); + Ok(ResultGuarantee { + metric: ErrorMetric::TopKMembership, + bound: BoundExpr::Zero, + failure_probability: ProbabilityExpr::Constant { value: delta }, + provenance, + }) + } // An operator this crate does not know has no registered rule. _ => Err(unsupported("no registered rule for this operator".into())), } @@ -942,7 +999,7 @@ mod tests { } #[test] - fn topk_selection_is_unsupported() { + fn topk_selection_requires_a_separated_margin_certificate() { let err = DefaultAccuracyModel .propagate( &CompositionOperator::TopKSelection, @@ -952,6 +1009,36 @@ mod tests { ) .unwrap_err(); assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); + + let certified = DefaultAccuracyModel + .propagate( + &CompositionOperator::TopKSelection, + &[abs(0.1, 0.01)], + None, + &PropagationStats { + topk_selected_lower_bound: Some(101.0), + topk_excluded_upper_bound: Some(100.0), + topk_interval_failure_probability: Some(0.005), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(certified.metric, ErrorMetric::TopKMembership); + assert_eq!(certified.bound.evaluate(), Some(0.0)); + assert_eq!(certified.failure_probability.evaluate(), Some(0.005)); + + let overlapping = DefaultAccuracyModel.propagate( + &CompositionOperator::TopKSelection, + &[abs(0.1, 0.01)], + None, + &PropagationStats { + topk_selected_lower_bound: Some(100.0), + topk_excluded_upper_bound: Some(100.0), + topk_interval_failure_probability: Some(0.005), + ..Default::default() + }, + ); + assert!(overlapping.is_err()); } #[test] @@ -972,8 +1059,8 @@ mod tests { .unwrap(); assert_eq!(g.metric, ErrorMetric::Rank); assert!(DefaultAccuracyModel.satisfies(&g, &AccuracyTarget::Epsilon(0.01))); - assert_eq!(g.failure_probability.evaluate(), None); - assert!(!DefaultAccuracyModel.satisfies( + assert_eq!(g.failure_probability.evaluate(), Some(0.01)); + assert!(DefaultAccuracyModel.satisfies( &g, &AccuracyTarget::EpsilonDelta { epsilon: 0.01, @@ -994,8 +1081,8 @@ mod tests { ) .unwrap(); assert_eq!(g.metric, ErrorMetric::Cardinality); - assert!(DefaultAccuracyModel.satisfies(&g, &AccuracyTarget::Epsilon(0.01))); - assert_eq!(g.failure_probability.evaluate(), None); + assert_eq!(g.failure_probability.evaluate(), Some(0.01)); + assert!(!DefaultAccuracyModel.satisfies(&g, &AccuracyTarget::Epsilon(0.01))); let params = default_size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001); let g = DefaultAccuracyModel @@ -1018,13 +1105,13 @@ mod tests { } #[test] - fn unsupported_sketch_guarantees_fail_closed() { + fn count_sketch_uses_an_l2_guarantee() { use crate::replacement::default_size_params; use asap_types::pre_asap::agg_intent::default_cardinality; let intent = default_cardinality(); let count_sketch = default_size_params(SketchAlgorithm::CountSketch, &intent, 0.01, 0.01); - assert!(DefaultAccuracyModel + let guarantee = DefaultAccuracyModel .local_guarantee( &SummaryFamilyType::Sketch( SketchKind::new(SketchAlgorithm::CountSketch, count_sketch), @@ -1035,14 +1122,22 @@ mod tests { value: None, }, ) - .is_none()); + .expect("CountSketch has a parameter-derived L2 guarantee"); + assert_eq!(guarantee.metric, ErrorMetric::L2Frequency); + assert!(DefaultAccuracyModel.satisfies( + &guarantee, + &AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.01, + } + )); let cms_heap = SketchParams::CmsWithHeap { width: 272, depth: 5, heap_size: 10, }; - assert!(DefaultAccuracyModel + let topk_frequency = DefaultAccuracyModel .local_guarantee( &SummaryFamilyType::Sketch( SketchKind::new(SketchAlgorithm::CmsWithHeap, cms_heap), @@ -1050,7 +1145,8 @@ mod tests { ), &SketchQuery::TopK { k: 10 }, ) - .is_none()); + .expect("heap sketch still provides per-key frequency intervals"); + assert_eq!(topk_frequency.metric, ErrorMetric::Frequency); } #[test] diff --git a/crates/asap-aware-mapping/src/grouping.rs b/crates/asap-aware-mapping/src/grouping.rs index 19c17a2..863dea7 100644 --- a/crates/asap-aware-mapping/src/grouping.rs +++ b/crates/asap-aware-mapping/src/grouping.rs @@ -29,9 +29,9 @@ //! ([`asap_types::post_asap::hydra_kind_for`]): `Cms` and `CountSketch` //! have structural Hydra mappings. `HydraKll` remains an explicit //! experimental IR value, but the paper excludes quantiles and search -//! therefore never emits it. Accuracy-targeted candidates are also -//! withheld until the shared grid's collision error is composed with the -//! inner sketch guarantee. +//! therefore never emits it. The shared-grid term is represented +//! symbolically and accuracy-targeted candidates are withheld until its +//! required statistics are supplied. //! //! Whether Hydra is *worth it* for a given estimated subpopulation //! cardinality is a cost-model question, deliberately out of scope here — @@ -73,8 +73,9 @@ use std::rc::Rc; use asap_types::post_asap::{ - default_hydra_params, hydra_kind_for, GroupingStrategy, HydraKind, SketchAlgorithm, - SketchParams, SummaryExpr, SummaryFamilyType, SummaryNode, + default_hydra_params, hydra_kind_for, BoundExpr, CompositionOperator, GroupingStrategy, + GuaranteeSource, HydraKind, ProbabilityExpr, ResultGuarantee, SketchAlgorithm, SketchParams, + SummaryExpr, SummaryFamilyType, SummaryNode, }; use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::query_expr::{QueryExpr, Reduction}; @@ -160,10 +161,9 @@ impl<'a> HydraGroupingStrategy<'a> { let Some(intent) = bindable_intent(target.root) else { return Vec::new(); }; - // Hydra adds shared-grid collision error that is not modeled yet. - // `with_grouping` clears the per-subpopulation guarantee so exports - // remain honest, but an unknown guarantee must also keep the - // candidate out of cost ranking when this node has a requirement. + // Hydra's shared-grid term is symbolic until deployment/data + // statistics instantiate it. Keep that unevaluable guarantee out of + // cost ranking when this node has an accuracy requirement. if accuracy_target(intent).is_some() { return Vec::new(); } @@ -284,10 +284,7 @@ fn with_grouping(node: Rc, grouping: GroupingStrategy) -> Rc, grouping: GroupingStrategy) -> Rc ResultGuarantee { + let mut provenance = inner.provenance.clone(); + provenance.push(GuaranteeSource::ChildGuarantee { + input_index: 0, + guarantee: Box::new(inner.clone()), + }); + provenance.push(GuaranteeSource::UnavailableStatistic { + statistic: "hydra_shared_grid_collision_bound".into(), + }); + provenance.push(GuaranteeSource::UnavailableStatistic { + statistic: "hydra_shared_grid_failure_probability".into(), + }); + provenance.push(GuaranteeSource::CompositionStep { + operator: CompositionOperator::ApproximateAggregate, + rule: "hydra_shared_grid_union_bound".into(), + }); + ResultGuarantee { + metric: inner.metric, + bound: BoundExpr::Sum { + terms: vec![ + inner.bound.clone(), + BoundExpr::Unknown { + statistic: "hydra_shared_grid_collision_bound".into(), + }, + ], + }, + failure_probability: ProbabilityExpr::UnionBound { + terms: vec![ + inner.failure_probability.clone(), + ProbabilityExpr::Unknown { + statistic: "hydra_shared_grid_failure_probability".into(), + }, + ], + }, + provenance, + } +} + #[cfg(test)] mod tests { use super::*; + use asap_types::post_asap::ErrorMetric; use asap_types::pre_asap::agg_intent::{default_cardinality, default_quantile}; use asap_types::pre_asap::query_expr::Source; use asap_types::pre_asap::schema::{Column, DataType, Schema}; @@ -398,6 +438,35 @@ mod tests { )))); } + #[test] + fn hydra_composes_inner_and_shared_grid_error_symbolically() { + let inner = ResultGuarantee { + metric: ErrorMetric::Frequency, + bound: BoundExpr::Constant { value: 0.01 }, + failure_probability: ProbabilityExpr::Constant { value: 0.02 }, + provenance: vec![], + }; + let composed = hydra_guarantee(&inner); + + assert_eq!(composed.metric, ErrorMetric::Frequency); + assert!(matches!( + composed.bound, + BoundExpr::Sum { ref terms } + if matches!(terms.as_slice(), [ + BoundExpr::Constant { value }, + BoundExpr::Unknown { statistic }, + ] if *value == 0.01 && statistic == "hydra_shared_grid_collision_bound") + )); + assert!(matches!( + composed.failure_probability, + ProbabilityExpr::UnionBound { ref terms } + if matches!(terms.as_slice(), [ + ProbabilityExpr::Constant { value }, + ProbabilityExpr::Unknown { statistic }, + ] if *value == 0.02 && statistic == "hydra_shared_grid_failure_probability") + )); + } + // ── HydraGroupingStrategy ───────────────────────────────────────────── #[test] diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 99f810d..176716c 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -907,7 +907,7 @@ pub fn default_size_params( depth: cms_depth(delta), }, SketchAlgorithm::Hll => SketchParams::Hll { - precision: hll_precision(eps), + precision: hll_precision_99(eps), }, SketchAlgorithm::CmsWithHeap => { let k = match intent { @@ -924,17 +924,11 @@ pub fn default_size_params( // CountSketchWithHeap) are only reachable once a cost model picks // them; sized here so that wiring is local. SketchAlgorithm::DDSketch => SketchParams::DDSketch { alpha: eps }, - SketchAlgorithm::Theta => SketchParams::Theta { k: kmv_k(eps) }, - SketchAlgorithm::Kmv => SketchParams::Kmv { k: kmv_k(eps) }, - // Count-Sketch is CMS's balanced/zero-mean-error alternative — - // same (width, depth) shape, sized the same way for now (a - // Count-Sketch-specific bound uses an L2-norm error guarantee - // rather than CMS's L1-norm one; this is a placeholder pending - // that refinement, same status as the other non-preferred - // candidates above). + SketchAlgorithm::Theta => SketchParams::Theta { k: kmv_k_99(eps) }, + SketchAlgorithm::Kmv => SketchParams::Kmv { k: kmv_k_99(eps) }, SketchAlgorithm::CountSketch => SketchParams::CountSketch { - width: cms_width(eps), - depth: cms_depth(delta), + width: count_sketch_width(eps), + depth: count_sketch_depth(delta), }, SketchAlgorithm::CountSketchWithHeap => { let k = match intent { @@ -942,8 +936,8 @@ pub fn default_size_params( _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), }; SketchParams::CountSketchWithHeap { - width: cms_width(eps), - depth: cms_depth(delta), + width: count_sketch_width(eps), + depth: count_sketch_depth(delta), heap_size: k as u32, } } @@ -1045,20 +1039,14 @@ pub fn posterior_aware_size_params( heap_size: k as u32, } } - SketchAlgorithm::CountSketch => SketchParams::CountSketch { - width: relaxed_width(eps), - depth: cms_depth(delta), - }, + SketchAlgorithm::CountSketch => { + // CMS's expected-L1 collision relaxation is not a CountSketch + // L2 theorem; retain the formal CountSketch sizing unchanged. + default_size_params(kind, intent, eps, delta) + } SketchAlgorithm::CountSketchWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), - }; - SketchParams::CountSketchWithHeap { - width: relaxed_width(eps), - depth: cms_depth(delta), - heap_size: k as u32, - } + // As above, do not apply CMS's L1 relaxation to CountSketch. + default_size_params(kind, intent, eps, delta) } // Every other kind is untouched by this issue's CMS-specific // relaxation — defer to the existing formula verbatim. Spelled out @@ -1086,12 +1074,17 @@ fn kll_k(eps: f64) -> u32 { saturating_ceil(2.0 / eps, 8, 65_535) } -/// HLL: standard error ≈ 1.04/√(2^p) ⇒ `p = ⌈log2((1.04/ε)²)⌉`. The default -/// `Cardinality` target (`asap-ir::default_cardinality`) inverts to p = 14. +/// HLL standard-error inversion. The committed guarantee applies the +/// 99%-confidence conversion below before calling this helper. fn hll_precision(eps: f64) -> u8 { saturating_ceil((1.04 / eps).powi(2).log2(), 4, 18) as u8 } +/// 99%-confidence HLL relative bound via Chebyshev: ten times the RSE. +fn hll_precision_99(eps: f64) -> u8 { + hll_precision(eps / 10.0) +} + /// CMS: over-count ≤ ε·N with width `w = ⌈e/ε⌉` columns. fn cms_width(eps: f64) -> u32 { saturating_ceil(std::f64::consts::E / eps, 2, 1 << 26) @@ -1103,9 +1096,29 @@ fn cms_depth(delta: f64) -> u32 { saturating_ceil((1.0 / delta).ln(), 1, 32) } -/// KMV / theta: relative error ≈ 1/√k ⇒ `k = ⌈1/ε²⌉`. -fn kmv_k(eps: f64) -> u32 { - saturating_ceil(1.0 / (eps * eps), 16, 1 << 26) +/// 99%-confidence KMV/Theta relative bound via Chebyshev, using +/// `RSE <= 1/sqrt(k-2)` and a ten-standard-deviation interval. +fn kmv_k_99(eps: f64) -> u32 { + saturating_ceil(100.0 / (eps * eps) + 2.0, 16, 1 << 26) +} + +/// CountSketch `L2` point-query width: ε = sqrt(3/w). +fn count_sketch_width(eps: f64) -> u32 { + saturating_ceil(3.0 / (eps * eps), 2, 1 << 26) +} + +/// Positive odd depth satisfying Hoeffding's median failure bound +/// `exp(-depth/18) <= delta` for per-row failure at most 1/3. +fn count_sketch_depth(delta: f64) -> u32 { + if !(delta.is_finite() && delta > 0.0 && delta < 1.0) { + return 255; + } + let depth = saturating_ceil(18.0 * (1.0 / delta).ln(), 1, 255); + if depth.is_multiple_of(2) { + (depth + 1).min(255) + } else { + depth + } } /// `⌈x⌉` clamped to `[lo, hi]`; NaN / non-positive x saturate to `hi` @@ -1802,7 +1815,11 @@ fn compose_guarantee( ) } (_, Some(query)) => ( - CompositionOperator::ApproximateAggregate, + if matches!(query, PostAsapSketchQuery::TopK { .. }) { + CompositionOperator::TopKSelection + } else { + CompositionOperator::ApproximateAggregate + }, accuracy.local_guarantee(family, query), ), (_, None) => (CompositionOperator::ApproximateAggregate, None), @@ -1836,9 +1853,10 @@ fn compose_guarantee( guarantee.provenance.push(GuaranteeSource::AccuracyTarget { target: target.clone(), }); - // Approximate over approximate: the composed guarantee must meet the - // target this node's value was requested at. - if !input.is_exact() && !accuracy.satisfies(&guarantee, target) { + // Check even over an exact input: parameter clamps or a conservative + // confidence conversion can make the tightest available sketch miss + // its requested target. + if !accuracy.satisfies(&guarantee, target) { return Err(AccuracyError::TargetNotSatisfied { metric: guarantee.metric, bound: guarantee.bound.evaluate(), @@ -3982,14 +4000,12 @@ mod tests { } #[test] - fn default_cardinality_inverts_to_hll_precision_14() { - // `default_cardinality` encodes HLL's standard error at p=14; the - // sizing must invert it back exactly. + fn default_cardinality_uses_the_tightest_supported_hll_precision() { assert_eq!( preferred(&default_cardinality()), Implementation::Sketch(SketchKind::new( SketchAlgorithm::Hll, - SketchParams::Hll { precision: 14 }, + SketchParams::Hll { precision: 18 }, )) ); } @@ -4197,7 +4213,7 @@ mod tests { } #[test] - fn posterior_aware_sizing_applies_to_every_cms_family_kind() { + fn posterior_aware_sizing_does_not_apply_cms_l1_relaxation_to_count_sketch() { let cms_heap_intent = AggIntent::TopK { k: 7, accuracy: eps(0.01), @@ -4214,10 +4230,12 @@ mod tests { 0.01, assumption ), - SketchParams::CountSketch { - width: 68, - depth: 5 - }, // ceil(272 * 0.25) + default_size_params( + SketchAlgorithm::CountSketch, + &count_intent(0.01), + 0.01, + 0.01 + ), ); // CmsWithHeap / CountSketchWithHeap carry k through untouched. match posterior_aware_size_params( @@ -4369,7 +4387,7 @@ mod tests { } #[test] - fn cardinality_enumerates_all_three_summary_candidates() { + fn cardinality_rejects_hll_when_its_parameter_cap_misses_the_target() { let q = Rc::new(agg(vec![2], default_cardinality(), metric_scan(&["job"]))); let target = TargetSubDAG::new(&q); let replacements = SketchAlgorithmStrategy::default_cost_model().replacements(&target); @@ -4382,12 +4400,8 @@ mod tests { .collect(); assert_eq!( kinds, - vec![ - SketchAlgorithm::Hll, - SketchAlgorithm::Theta, - SketchAlgorithm::Kmv - ], - "expected every summary_candidates entry for Cardinality" + vec![SketchAlgorithm::Theta, SketchAlgorithm::Kmv], + "the conservative 99% HLL bound cannot meet the target at precision <= 18" ); } @@ -4789,7 +4803,7 @@ mod tests { .groups() .find(|g| matches!(g.target.as_ref(), QueryExpr::Aggregate { .. })) .unwrap(); - assert_eq!(agg_group.candidates.len(), 3); + assert_eq!(agg_group.candidates.len(), 2); } #[test] @@ -6301,7 +6315,7 @@ mod tests { } #[test] - fn topk_realizes_cms_with_heap_and_topk_readout() { + fn topk_without_margin_evidence_falls_back_to_pre_asap() { let q = agg( vec![2], AggIntent::TopK { @@ -6311,21 +6325,7 @@ mod tests { metric_scan(&["job"]), ); let root = realize(&q).unwrap(); - let SummaryExpr::SummaryEstimate { - summary_input, - query, - } = &root.expr - else { - panic!("expected estimate root, got {:?}", root.expr); - }; - assert!(matches!(query, PostAsapSketchQuery::TopK { k: 5 })); - assert!(matches!( - &summary_input.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::Sketch(kind, _), - .. - } if kind.algorithm() == &SketchAlgorithm::CmsWithHeap - )); + assert!(matches!(root.expr, SummaryExpr::KeepPreAsap(_))); } #[test] @@ -6747,9 +6747,9 @@ mod tests { .iter() .all(|candidate| matches!(candidate.replacement, Replacement::Rewrite(_)))); assert!(!group.rejected.is_empty()); - assert!(group - .rejected - .iter() - .all(|rejected| matches!(rejected.error, AccuracyError::TargetNotSatisfied { .. }))); + assert!(group.rejected.iter().all(|rejected| matches!( + rejected.error, + AccuracyError::TargetNotSatisfied { .. } | AccuracyError::UnsupportedComposition { .. } + ))); } } diff --git a/crates/integration-tests/tests/sql_to_post_asap.rs b/crates/integration-tests/tests/sql_to_post_asap.rs index 42564b0..78b289e 100644 --- a/crates/integration-tests/tests/sql_to_post_asap.rs +++ b/crates/integration-tests/tests/sql_to_post_asap.rs @@ -240,7 +240,7 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { /// distinct branch of the sketch-vs-exact decision than the quantile test /// above. #[tokio::test] -async fn sql_count_distinct_binds_hll_sketch_over_named_column() { +async fn sql_count_distinct_binds_a_confidence_certified_sketch_over_named_column() { let pre_asap = lower( "SELECT COUNT(DISTINCT service) FROM metrics", AccuracyTarget::Epsilon(0.01), @@ -275,7 +275,7 @@ async fn sql_count_distinct_binds_hll_sketch_over_named_column() { assert_eq!( family, &SummaryFamilyType::Sketch( - SketchKind::new(SketchAlgorithm::Hll, SketchParams::Hll { precision: 14 }), + SketchKind::new(SketchAlgorithm::Theta, SketchParams::Theta { k: 1_000_002 }), GroupingStrategy::default() ) ); diff --git a/crates/types/src/post_asap/guarantee.rs b/crates/types/src/post_asap/guarantee.rs index 812dcb2..016e82e 100644 --- a/crates/types/src/post_asap/guarantee.rs +++ b/crates/types/src/post_asap/guarantee.rs @@ -75,6 +75,10 @@ pub enum ErrorMetric { /// `|estimate − truth| ≤ bound · ‖f‖₁` for a point-frequency query /// (CMS's ε, normalized by the stream's L1 norm). Frequency, + /// `|estimate − truth| ≤ bound · ‖f‖₂` for a point-frequency query. + /// CountSketch uses this normalization; it is intentionally distinct + /// from CMS's [`ErrorMetric::Frequency`] (`L1`) guarantee. + L2Frequency, /// The returned key set equals the true top-k set. No shipped model /// produces this yet: it needs a per-key interval margin certificate /// (issue #172, PR 3). Present so the vocabulary can name the metric a diff --git a/docs/design_docs/asap-aware-mapping/summary_properties.md b/docs/design_docs/asap-aware-mapping/summary_properties.md index 5602572..63bfa37 100644 --- a/docs/design_docs/asap-aware-mapping/summary_properties.md +++ b/docs/design_docs/asap-aware-mapping/summary_properties.md @@ -117,6 +117,23 @@ expressions, and provenance. Exact values have zero error; a sketch readout's guarantee is derived from the sizing formula that produced its parameters. +The built-in sketch contracts distinguish their error norms and confidence +semantics: + +- CMS uses an L1-frequency bound, while CountSketch uses an L2-frequency + bound and is sized from both width and odd median depth. The two are not + interchangeable. +- KLL, HLL, KMV, and Theta expose conservative 99%-confidence guarantees + derived from their committed parameters. A tighter requested `delta` is + rejected unless another implementation supplies a stronger contract. +- TopK membership is certified only when the widened confidence interval of + the kth selected item is strictly above every excluded item's widened + interval. Missing or overlapping interval evidence fails closed. +- Hydra adds its shared-grid collision error to the inner sketch error and + union-bounds their failure probabilities. Until deployment/data statistics + instantiate the shared-grid terms, the expressions remain symbolic and an + accuracy-targeted Hydra candidate is not admitted. + The default `AccuracyModel` is deliberately conservative. It supports registered same-metric additive, relative, and Lipschitz rules and exact sum/max/min over approximate inputs. It uses union-bound probabilities From 089324ad1f9e924d9a3bed65bcecef633451dee2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 28 Aug 2026 10:53:19 -0600 Subject: [PATCH 06/12] docs: specify end-to-end accuracy guarantees --- docs/design_docs/asap-aware-mapping/README.md | 2 + .../end-to-end-accuracy-guarantees.md | 462 ++++++++++++++++++ 2 files changed, 464 insertions(+) create mode 100644 docs/design_docs/end-to-end-accuracy-guarantees.md diff --git a/docs/design_docs/asap-aware-mapping/README.md b/docs/design_docs/asap-aware-mapping/README.md index b8ea19a..eb0b045 100644 --- a/docs/design_docs/asap-aware-mapping/README.md +++ b/docs/design_docs/asap-aware-mapping/README.md @@ -88,6 +88,8 @@ The design is split into focused documents: subpopulation and time organization, roll-ups, sharing, semantic rewrites, and hybrid execution. - [Summary properties](summary_properties.md) lists the capabilities used to determine whether summaries and optimizations can be composed safely. +- [End-to-end accuracy guarantees](../end-to-end-accuracy-guarantees.md) specifies the typed + guarantee IR, sketch contracts, composition rules, target checking, and fail-closed boundaries. - [Explainability](explainability.md) describes how the planner reports available replacements using the same candidate space it optimizes. diff --git a/docs/design_docs/end-to-end-accuracy-guarantees.md b/docs/design_docs/end-to-end-accuracy-guarantees.md new file mode 100644 index 0000000..1296f8e --- /dev/null +++ b/docs/design_docs/end-to-end-accuracy-guarantees.md @@ -0,0 +1,462 @@ +# End-to-End Accuracy Guarantees + +## Status and scope + +This document specifies the accuracy design implemented by PR #303 and tracked +by issue #172. It covers how ASAPPlanner represents, composes, checks, and +explains approximation guarantees for post-ASAP plans, including nested +summaries. + +ASAPPlanner is a mathematical planner. It does not execute sketches or import a +sketch runtime. The planner derives guarantees from committed parameters and +keeps data- or runtime-dependent quantities symbolic. A serving system may +later provide observations that instantiate those symbols, but unavailable +evidence must never be replaced with an optimistic value. + +The design has one governing rule: + +> Accuracy legality is decided before cost ranking. A cheaper candidate cannot +> override a missing or insufficient guarantee. + +## Problem + +A post-ASAP plan can contain more than one approximate layer: + +```text +raw input + -> inner summary + -> inner estimate + -> outer summary or transformation + -> caller-visible result +``` + +Sizing every layer independently against the caller's full error target is not +an end-to-end proof. The layers may use different error metrics, their bounds +may compose, and their failure probabilities consume a shared budget. Some +operations, such as TopK selection, need evidence that cannot be expressed by +adding point-estimation epsilons. + +ASAPPlanner therefore uses this pipeline: + +```text +candidate generation + -> accuracy-budget allocation + -> local guarantee derivation + -> end-to-end guarantee propagation + -> AccuracyTarget satisfaction + -> legal candidates only + -> cost ranking and global selection +``` + +Rejected candidates remain available in explanatory output with a structured +reason. The planner retains an exact or pre-ASAP fallback when no approximate +candidate can be proved legal. + +## Requirements + +The design must: + +1. Use the existing `AccuracyTarget::{Exact, Epsilon, EpsilonDelta}` as the + correctness requirement. +2. Preserve the meaning of each error metric instead of treating every bound + as an interchangeable epsilon. +3. Derive local guarantees from the same committed parameters used to size the + sketch. +4. Propagate guarantees through supported compositions without assuming + independence. +5. Check the final guarantee against the target before invoking the cost model. +6. Preserve symbolic statistics, provenance, allocations, and rejection + reasons in the post-ASAP IR and DAG export. +7. Fail closed for invalid numeric values, unknown required statistics, + incompatible metrics, and unsupported compositions. + +## Guarantee IR + +Caller-visible post-ASAP values may carry a `ResultGuarantee`: + +```rust +struct ResultGuarantee { + metric: ErrorMetric, + bound: BoundExpr, + failure_probability: ProbabilityExpr, + provenance: Vec, +} +``` + +Summary state does not itself claim a caller-visible result guarantee. The +guarantee is attached to a finalized value, such as a `SummaryEstimate`. + +### Error metrics + +The built-in model distinguishes: + +| Metric | Meaning | +| --- | --- | +| `AbsoluteValue` | Absolute error in the returned value | +| `RelativeValue` | Error relative to the true value magnitude | +| `Rank` | Normalized rank error | +| `Cardinality` | Relative cardinality error | +| `Frequency` | Frequency error normalized by the stream L1 norm | +| `L2Frequency` | Frequency error normalized by the stream L2 norm | +| `TopKMembership` | Correctness of the selected TopK membership set | + +These metrics are not implicitly convertible. In particular, Count-Min Sketch +and CountSketch use different frequency norms, and a point-frequency guarantee +does not prove TopK membership. + +### Symbolic expressions + +`BoundExpr` represents constants, sums, products, maxima, and unavailable +statistics. `ProbabilityExpr` represents zero, constants, union bounds, and +unavailable probabilities. Evaluation returns no value when a required leaf +is unknown or malformed. + +Numeric leaves must be finite. Bounds must be non-negative, and probabilities +must lie in `[0, 1]`. Invalid values fail closed rather than passing a target +comparison through floating-point behavior. + +### Provenance + +`GuaranteeSource` records why a guarantee is believed: + +- sketch algorithm and committed parameters; +- child guarantees; +- composition rules; +- accuracy-budget allocations; +- runtime observations, when supplied; and +- required statistics that are currently unavailable. + +This information is exported with candidate and rejection data so that a plan +can be audited without reconstructing the proof from planner internals. + +## Extension points + +Accuracy reasoning and allocation are separate from cost modeling: + +```rust +trait AccuracyModel { + fn local_guarantee(/* family, query, parameters */) + -> Option; + + fn propagate( + &self, + op: &CompositionOperator, + inputs: &[ResultGuarantee], + local: Option<&ResultGuarantee>, + stats: &PropagationStats, + ) -> Result; + + fn satisfies( + &self, + guarantee: &ResultGuarantee, + target: &AccuracyTarget, + ) -> bool; +} +``` + +`AccuracyBudgetAllocator` proposes finite parameter allocations for nested +approximate layers. `CostModel` may rank only the candidates that survive +guarantee propagation and target checking. Deployments may provide stronger +accuracy models, but the default model remains conservative. + +## Core composition rules + +### Exact values + +An exact value contributes zero error and zero failure probability: + +```text +B_exact = 0 +delta_exact = 0 +``` + +An exact input does not exempt a local approximate sketch from target checking. +This matters when parameter ranges are clamped and the tightest representable +configuration still misses the requested target. + +### Additive and Lipschitz composition + +Compatible absolute bounds compose without an independence assumption: + +```text +B_total <= B_input + B_local +delta_total <= delta_input + delta_local +``` + +For a registered `L`-Lipschitz transformation: + +```text +B_output <= L * B_input + B_local +delta_output <= delta_input + delta_local +``` + +Failure probabilities use a union bound. + +### Relative composition + +For a registered multiplicative rule whose values are known to be +non-negative: + +```text +epsilon_total = + epsilon_input + + epsilon_local + + epsilon_input * epsilon_local +``` + +The rule is rejected when sign information is missing or values may cross +zero. + +### Exact aggregation over approximate values + +For an exact sum, input bounds are summed and input failures are union-bounded. +When the number of folded rows is required but unknown, the resulting bound +stays symbolic. + +For exact minimum or maximum, the value-error bound is the maximum input bound +and failures are union-bounded. This bounds the returned value; it does not by +itself prove the identity of a winning key. + +### Unsupported composition + +Approximate-over-approximate composition is accepted only when the model has a +rule for the operator and metric. Cross-metric composition and unregistered +same-metric composition return `UnsupportedComposition`. They never treat an +approximate child as exact. + +## Sketch contracts + +The formulas below are planner contracts derived from committed sketch +parameters. Parameter clamps are followed by a target check. + +### KLL + +KLL uses normalized rank error: + +```text +epsilon_rank = 2 / k +k = ceil(2 / requested_epsilon) +``` + +The built-in contract has failure probability `0.01` (99% confidence). A target +with a tighter `delta` requires a stronger deployment-specific model or an +amplification contract and otherwise fails closed. + +### DDSketch + +DDSketch uses its committed deterministic relative-error parameter: + +```text +epsilon_relative = alpha +delta = 0 +``` + +### HLL + +HLL starts from relative standard error: + +```text +RSE = 1.04 / sqrt(2^p) +``` + +The default model uses a conservative Chebyshev 99% interval: + +```text +epsilon_cardinality = 10 * 1.04 / sqrt(2^p) +delta = 0.01 +``` + +Sizing inverts this bound. If the supported precision cap cannot satisfy the +target, HLL is rejected even when it would be the cheaper or preferred family. + +### KMV and Theta + +The default model uses: + +```text +RSE <= 1 / sqrt(k - 2) +epsilon_cardinality = 10 / sqrt(max(k - 2, 1)) +delta = 0.01 +k = ceil(100 / requested_epsilon^2 + 2) +``` + +This is also a conservative Chebyshev 99% contract. Tighter confidence fails +closed unless another model proves it. + +### Count-Min Sketch + +CMS uses an L1-normalized one-sided frequency bound: + +```text +epsilon_l1 = e / width +delta <= exp(-depth) +absolute_error <= epsilon_l1 * ||f||_1 +``` + +Sizing uses: + +```text +width = ceil(e / requested_epsilon) +depth = ceil(ln(1 / requested_delta)) +``` + +Posterior CMS relaxation may reduce its width only under its documented L1 +assumptions. + +### CountSketch + +CountSketch has a separate L2-normalized point-frequency contract. It does not +reuse CMS sizing or posterior relaxation: + +```text +epsilon_l2 = sqrt(3 / width) +absolute_error <= epsilon_l2 * ||f||_2 +``` + +One row is bad with probability at most `1/3`. For an independent odd number of +rows, the median is assigned the conservative Hoeffding bound: + +```text +delta <= exp(-depth / 18) +width = ceil(3 / requested_epsilon^2) +depth = an odd integer >= ceil(18 * ln(1 / requested_delta)) +``` + +Zero or even depth has no modeled guarantee. Width/depth caps are checked +against the target after sizing. + +## TopK membership certificate + +A heap attached to CMS or CountSketch can provide point-frequency estimates, +but those estimates do not prove that the returned keys are the true TopK set. + +`PropagationStats` therefore accepts three pieces of evidence: + +- the lower confidence bound of the kth selected item; +- the greatest upper confidence bound among excluded items; and +- the union-bound failure probability of all intervals used by the + certificate. + +The intervals must already be widened by the underlying sketch error. Exact +membership is certified only when: + +```text +selected_kth_lower_bound > excluded_max_upper_bound +``` + +The result then has metric `TopKMembership`, zero membership error, and the +provided interval failure probability. Missing values, non-finite values, +invalid probabilities, equality, or overlap reject the candidate. + +Static planning currently has no ordinary source for these boundary intervals, +so TopK sketch candidates remain fail-closed until planning-time or runtime +evidence is supplied. + +## Hydra shared-grid composition + +A Hydra result includes both the inner per-subpopulation sketch error and an +outer shared-grid collision term: + +```text +B_hydra = B_inner + B_shared_grid +delta_hydra <= delta_inner + delta_shared_grid +``` + +The second expression is a union bound. The shared-grid collision bound and +failure probability depend on deployment/data statistics, so the default +planner records them as: + +```text +hydra_shared_grid_collision_bound +hydra_shared_grid_failure_probability +``` + +The resulting expression and provenance are visible in the IR. Because these +leaves cannot yet be evaluated, an accuracy-targeted Hydra candidate is not +admitted. Copying the inner guarantee onto Hydra without the shared term would +be unsound. + +## Accuracy targets and allocation + +`AccuracyTarget::Exact` accepts only a zero bound and zero failure probability. +`AccuracyTarget::Epsilon` checks the evaluated magnitude. An +`AccuracyTarget::EpsilonDelta` requires both an evaluated magnitude no greater +than epsilon and an evaluated failure probability no greater than delta. + +The initial allocator uses conservative finite choices, including equal splits +for additive nested layers: + +```text +epsilon_i = epsilon_total / approximate_layer_count +delta_i = delta_total / approximate_layer_count +``` + +Every allocated candidate is resized, propagated, and checked. Allocation does +not constitute proof by itself, and recording a requested delta in metadata is +not evidence that an algorithm achieves it. + +## Planner and runtime boundary + +PR #303 implements the guarantee algebra and the parameter-derived contracts in +ASAPPlanner. It does not require ASAPPlanner to link to `asap_sketchlib`. + +Runtime or planning-time observations are still useful for quantities that are +not fixed by static parameters: + +- TopK boundary intervals; +- a concrete stream L2 norm when an absolute CountSketch bound is needed; +- Hydra shared-grid collision statistics; and +- stronger implementation-specific confidence or amplification contracts. + +Such evidence must enter through explicit observation/statistics fields and be +recorded in provenance. Its absence leaves expressions symbolic and candidates +unprovable. + +## Explainability and export + +DAG export includes: + +- the selected guarantee and metric; +- symbolic bound and probability expressions; +- guarantee provenance; +- accuracy allocations; and +- rejected candidates with their rejection reasons. + +This makes the correctness decision inspectable and ensures the explanation +uses the same candidate space and legality checks as optimization. + +## Validation strategy + +Tests cover: + +- exact, additive, relative, Lipschitz, sum, and extremum propagation; +- incompatible metrics and unsupported composition; +- malformed and unknown symbolic values; +- target checking after parameter clamps; +- CountSketch L2 sizing distinct from CMS L1 sizing; +- accepted separated and rejected overlapping TopK intervals; +- Hydra inner-plus-shared-grid symbolic composition; +- KLL/HLL/KMV/Theta confidence and `EpsilonDelta` behavior; +- rejection before cost ranking and global selection; and +- DAG export and frontend-to-post-ASAP integration. + +The repository validation commands are: + +```text +cargo fmt --all +cargo test --workspace +cargo clippy --workspace --all-targets -- -D warnings +git diff --check +``` + +## Non-goals + +- Adding another correctness-policy enum alongside `AccuracyTarget`. +- Assuming statistical independence by default. +- Treating all sketch errors as a common epsilon. +- Claiming arbitrary nonlinear or cross-metric composition. +- Executing sketches or importing a sketch runtime into ASAPPlanner. +- Treating unavailable runtime evidence as zero. + +Advanced nonlinear, induced-norm, interval/Jacobian, and correlation-aware +propagation belongs in separate follow-up work. From 79d1d02293230ae91aa0bdfee69441d9a373a747 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 28 Aug 2026 11:18:50 -0600 Subject: [PATCH 07/12] fix(accuracy): align KLL sizing with confidence contract --- crates/asap-aware-mapping/src/accuracy.rs | 15 +- crates/asap-aware-mapping/src/replacement.rs | 45 ++-- .../tests/promql_to_post_asap.rs | 8 +- .../tests/sql_to_post_asap.rs | 8 +- .../end-to-end-accuracy-guarantees.md | 218 ++++++++++++++---- 5 files changed, 216 insertions(+), 78 deletions(-) diff --git a/crates/asap-aware-mapping/src/accuracy.rs b/crates/asap-aware-mapping/src/accuracy.rs index e14e044..99685ba 100644 --- a/crates/asap-aware-mapping/src/accuracy.rs +++ b/crates/asap-aware-mapping/src/accuracy.rs @@ -140,6 +140,13 @@ pub struct DefaultAccuracyModel; /// by floating-point noise. const SATISFACTION_TOLERANCE: f64 = 1e-9; +pub(crate) const KLL_RANK_ERROR_COEFFICIENT_99: f64 = 2.296; +pub(crate) const KLL_RANK_ERROR_EXPONENT_99: f64 = 0.9723; + +pub(crate) fn kll_rank_error_99(k: u32) -> f64 { + KLL_RANK_ERROR_COEFFICIENT_99 / f64::from(k).powf(KLL_RANK_ERROR_EXPONENT_99) +} + fn count_sketch_failure_probability(depth: u32) -> Option { if depth == 0 || depth.is_multiple_of(2) { return None; @@ -157,12 +164,12 @@ impl DefaultAccuracyModel { query: &SketchQuery, ) -> Option { let (metric, bound, delta) = match params { - // The committed KLL contract is its 99%-confidence normalized - // rank bound. Tighter confidence needs runtime-supported - // amplification and therefore remains unavailable. + // Apache DataSketches' single-sided KLL fit is the empirical 99th + // percentile normalized rank error for quantile/rank queries. + // Tighter confidence needs an amplification contract. SketchParams::Kll { k } => ( ErrorMetric::Rank, - 2.0 / f64::from(*k), + kll_rank_error_99(*k), ProbabilityExpr::Constant { value: 0.01 }, ), // DDSketch: deterministic relative value error α. diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 176716c..60b3bed 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -366,7 +366,8 @@ use thiserror::Error; use crate::accuracy::{ AccuracyBudgetAllocator, AccuracyModel, CompositionShape, DefaultAccuracyModel, - EqualSplitAllocator, PropagationStats, + EqualSplitAllocator, PropagationStats, KLL_RANK_ERROR_COEFFICIENT_99, + KLL_RANK_ERROR_EXPONENT_99, }; use crate::accuracy_reconciliation::AccuracyReconciliationStrategy; use crate::cost_model::{CostModel, CseCandidate, DefaultCostModel, ShareDecision}; @@ -1068,10 +1069,14 @@ pub fn posterior_aware_size_params( // smallest parameter satisfying the target, clamped to the family's sane // range. A non-positive ε saturates to the clamp maximum (tightest allowed). -/// KLL: rank error ε ≈ 2/k ⇒ `k = ⌈2/ε⌉`. ε = 0.01 → k = 200, matching the -/// design doc's worked example (`KLL{k=200}` satisfies ε=0.01). +/// Invert Apache DataSketches' empirical 99th-percentile, single-sided KLL +/// normalized rank-error fit: `epsilon = 2.296 / k^0.9723`. fn kll_k(eps: f64) -> u32 { - saturating_ceil(2.0 / eps, 8, 65_535) + saturating_ceil( + (KLL_RANK_ERROR_COEFFICIENT_99 / eps).powf(1.0 / KLL_RANK_ERROR_EXPONENT_99), + 8, + 65_535, + ) } /// HLL standard-error inversion. The committed guarantee applies the @@ -3981,7 +3986,7 @@ mod tests { preferred(&approx), Implementation::Sketch(SketchKind::new( SketchAlgorithm::Kll, - SketchParams::Kll { k: 200 }, // design.md worked example + SketchParams::Kll { k: 269 }, )) ); @@ -3994,7 +3999,7 @@ mod tests { preferred(&looser), Implementation::Sketch(SketchKind::new( SketchAlgorithm::Kll, - SketchParams::Kll { k: 40 }, // ⌈2/0.05⌉ + SketchParams::Kll { k: 52 }, )) ); } @@ -5860,7 +5865,7 @@ mod tests { #[test] fn quantile_realizes_kll_wrapped_in_estimate() { // quantile by (job) (m) at ε=0.01 → Estimate(Quantile) over - // SummaryAgg(Kll{k:200}) over KeepPreAsap(Scan). job = col 2. + // SummaryAgg(Kll{k:269}) over KeepPreAsap(Scan). job = col 2. let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); let root = realize(&q).unwrap(); @@ -5895,7 +5900,7 @@ mod tests { assert_eq!( family, &SummaryFamilyType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 269 }), GroupingStrategy::default() ) ); @@ -5905,7 +5910,7 @@ mod tests { assert_eq!( field(&summary_input.schema, "quantile_0_99").dtype, SummaryFamilyType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 269 }), GroupingStrategy::default() ) ); @@ -6497,7 +6502,10 @@ mod tests { .as_ref() .expect("a readout carries a guarantee"); assert_eq!(guarantee.metric, ErrorMetric::Rank); - assert_eq!(guarantee.bound.evaluate(), Some(2.0 / 200.0)); + assert_eq!( + guarantee.bound.evaluate(), + Some(crate::accuracy::kll_rank_error_99(269)) + ); assert_eq!(guarantee.approximate_layer_count(), 1); assert!(guarantee.provenance.iter().any(|s| matches!( s, @@ -6560,8 +6568,8 @@ mod tests { #[test] fn equal_split_allocation_makes_a_legal_tighter_candidate_and_rejects_the_declared_one() { // With a registered rank-additive rule: outer ε=0.1 over inner ε=0.1 - // composes to 0.2 > 0.1 as declared (cheap: k=20 each) — illegal. - // The equal split (0.05 + 0.05) re-sizes both layers to k=40 — + // composes above 0.1 as declared (cheap: k=26 each) — illegal. + // The equal split re-sizes both layers to k=52 — // pricier, and the only legal way to meet the outer target. let inner = agg(vec![2], quantile_eps(0.5, 0.1), metric_scan(&["job"])); let outer = Rc::new(agg(vec![], quantile_eps(0.99, 0.1), inner)); @@ -6583,7 +6591,8 @@ mod tests { bound: Some(b), target: AccuracyTarget::Epsilon(e), .. - } if (b - 0.2).abs() < 1e-12 && *e == 0.1 + } if (b - 2.0 * crate::accuracy::kll_rank_error_99(26)).abs() < 1e-12 + && *e == 0.1 ) }) .count(); @@ -6611,11 +6620,13 @@ mod tests { "exactly one legal KLL candidate (the allocated one)" ); let node = kll[0]; - assert_eq!(kll_k_of(node), 40, "outer re-sized to ε/2"); - assert_eq!(kll_k_of(summary_child(node)), 40, "inner re-sized to ε/2"); + assert_eq!(kll_k_of(node), 52, "outer re-sized to ε/2"); + assert_eq!(kll_k_of(summary_child(node)), 52, "inner re-sized to ε/2"); let guarantee = node.guarantee.as_ref().unwrap(); assert_eq!(guarantee.metric, ErrorMetric::Rank); - assert!((guarantee.bound.evaluate().unwrap() - 0.1).abs() < 1e-12); + let composed_bound = guarantee.bound.evaluate().unwrap(); + assert!(composed_bound <= 0.1); + assert!((composed_bound - 2.0 * crate::accuracy::kll_rank_error_99(52)).abs() < 1e-12); assert_eq!(guarantee.approximate_layer_count(), 2); assert!(guarantee.provenance.iter().any(|s| matches!( s, @@ -6669,7 +6680,7 @@ mod tests { let Replacement::Summary(node) = &chosen.replacement else { panic!() }; - assert_eq!(kll_k_of(node), 40); + assert_eq!(kll_k_of(node), 52); } #[test] diff --git a/crates/integration-tests/tests/promql_to_post_asap.rs b/crates/integration-tests/tests/promql_to_post_asap.rs index d0f48da..124c544 100644 --- a/crates/integration-tests/tests/promql_to_post_asap.rs +++ b/crates/integration-tests/tests/promql_to_post_asap.rs @@ -56,7 +56,7 @@ fn dtype<'a>(schema: &'a SummarySchema, name: &str) -> &'a SummaryFamilyType { /// /// ```text /// SummaryEstimate { query: Quantile{0.99} } → {quantile_0_99: Float64} -/// └─ SummaryAgg { Kll{k:200}, col: SampleValue } → {quantile_0_99: Sketch(Kll, {k:200})} +/// └─ SummaryAgg { Kll{k:269}, col: SampleValue } → {quantile_0_99: Sketch(Kll, {k:269})} /// └─ SummaryAgg { Rate, col: SampleValue } → {ts, value: ExactAggregate(Rate), …} /// └─ KeepPreAsap(TimeRange{5m} → Scan) → {ts, value} /// ``` @@ -88,7 +88,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { "the summary-state type must not propagate past the estimate" ); - // The quantile: KLL committed, k=200 sized from ε=0.01. `quantile(...)` + // The quantile: KLL committed, k=269 sized for ε=0.01 at 99% confidence. // is an aggregation operator with no `by(...)`: a genuine full // reduction, one output row — not to be confused with the inner rate's // per-entity grouping below, even though both once collapsed to the @@ -106,7 +106,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { assert_eq!( family, &SummaryFamilyType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 269 }), GroupingStrategy::default() ) ); @@ -119,7 +119,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { assert_eq!( dtype(&summary_input.schema, "quantile_0_99"), &SummaryFamilyType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 269 }), GroupingStrategy::default() ) ); diff --git a/crates/integration-tests/tests/sql_to_post_asap.rs b/crates/integration-tests/tests/sql_to_post_asap.rs index 78b289e..f869cbb 100644 --- a/crates/integration-tests/tests/sql_to_post_asap.rs +++ b/crates/integration-tests/tests/sql_to_post_asap.rs @@ -143,13 +143,13 @@ async fn sql_full_query_root_stays_logical_under_the_identity_projection() { /// /// ```text /// SummaryEstimate { query: Quantile{0.99} } → {…: Float64} -/// └─ SummaryAgg { Kll{k:200}, col: metrics.latency } → {…: Sketch(Kll, {k:200})} +/// └─ SummaryAgg { Kll{k:269}, col: metrics.latency } → {…: Sketch(Kll, {k:269})} /// └─ KeepPreAsap(Scan) → {ts, service, latency, bytes} /// ``` /// /// The SQL counterpart of `promql_to_post_asap.rs`'s /// `promql_quantile_of_rate_binds_kll_over_rate_accumulator`: same intent -/// (`Quantile`), same KLL sizing (k=200 from ε=0.01), but the summarised +/// (`Quantile`), same 99%-confidence KLL sizing (k=269 from ε=0.01), but the summarised /// column is the intent's own *named* SQL column rather than PromQL's /// synthetic sample value. #[tokio::test] @@ -194,7 +194,7 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { assert_eq!( family, &SummaryFamilyType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 269 }), GroupingStrategy::default() ) ); @@ -214,7 +214,7 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { assert_eq!( summary_input.schema.fields[0].dtype, SummaryFamilyType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 269 }), GroupingStrategy::default() ) ); diff --git a/docs/design_docs/end-to-end-accuracy-guarantees.md b/docs/design_docs/end-to-end-accuracy-guarantees.md index 1296f8e..72a3a6d 100644 --- a/docs/design_docs/end-to-end-accuracy-guarantees.md +++ b/docs/design_docs/end-to-end-accuracy-guarantees.md @@ -1,11 +1,10 @@ -# End-to-End Accuracy Guarantees +# Design: End-to-End Accuracy Guarantees -## Status and scope +## Audience and context -This document specifies the accuracy design implemented by PR #303 and tracked -by issue #172. It covers how ASAPPlanner represents, composes, checks, and -explains approximation guarantees for post-ASAP plans, including nested -summaries. +This document is for ASAPPlanner developers, architects, and researchers. It +defines how the planner represents, composes, checks, and explains approximation +guarantees for post-ASAP plans, including nested summaries. ASAPPlanner is a mathematical planner. It does not execute sketches or import a sketch runtime. The planner derives guarantees from committed parameters and @@ -18,7 +17,7 @@ The design has one governing rule: > Accuracy legality is decided before cost ranking. A cheaper candidate cannot > override a missing or insufficient guarantee. -## Problem +## Problem and why now A post-ASAP plan can contain more than one approximate layer: @@ -36,6 +35,18 @@ may compose, and their failure probabilities consume a shared budget. Some operations, such as TopK selection, need evidence that cannot be expressed by adding point-estimation epsilons. +Without an end-to-end model, the planner can select a locally well-sized sketch +whose caller-visible result violates the requested accuracy. Nested summaries, +shared grouping, and cross-query reuse make this a planning concern rather than +an isolated sketch-implementation detail. + +## Inputs, outputs, and end-to-end behavior + +The observable input is a pre-ASAP query plan whose aggregate intents carry an +`AccuracyTarget`, plus optional statistics and runtime observations. The output +is a candidate space of post-ASAP plans. Each caller-visible approximate value +has a typed `ResultGuarantee`, while rejected candidates carry a reason. + ASAPPlanner therefore uses this pipeline: ```text @@ -52,7 +63,41 @@ Rejected candidates remain available in explanatory output with a structured reason. The planner retains an exact or pre-ASAP fallback when no approximate candidate can be proved legal. -## Requirements +## Goals and non-goals + +The minimum successful outcome is that no selected approximate result lacks a +machine-readable guarantee that satisfies its accuracy target. The model must +also distinguish incompatible error metrics and preserve the evidence used to +reach its decision. + +This design does not execute sketches, import a sketch runtime, assume +statistical independence, or prove arbitrary nonlinear and cross-metric +composition. It does not introduce another correctness policy alongside +`AccuracyTarget`. + +## Heilmeier questions + +- **What are we trying to do?** Prevent the planner from selecting an + approximate plan unless it can prove the result meets the caller's accuracy + requirement. +- **How is it done without this design?** Sketches can be sized locally, but a + nested plan has no common representation or rule for its combined error. +- **What is new?** Typed guarantee expressions, explicit composition rules, + budget allocation, and legality filtering before cost ranking. +- **Who cares?** Query authors need accuracy requirements to be meaningful; + planner and runtime developers need an auditable contract between selected + parameters and observable results. +- **What are the risks and costs?** Conservative rules can reject useful plans; + incorrect estimator assumptions can admit unsound plans; symbolic evidence + increases IR and explanation size. +- **How long will it take?** The core algebra and built-in contracts are one + planner change. Supplying runtime-dependent TopK and Hydra evidence is a + separate integration increment. +- **How is success checked?** Unit tests exercise every registered rule and + rejection boundary; end-to-end tests confirm illegal candidates cannot reach + cost selection; exported plans expose the proof and rejection reason. + +## Required behavior The design must: @@ -70,7 +115,9 @@ The design must: 7. Fail closed for invalid numeric values, unknown required statistics, incompatible metrics, and unsupported compositions. -## Guarantee IR +## Proposed design + +### Guarantee IR Caller-visible post-ASAP values may carry a `ResultGuarantee`: @@ -86,7 +133,7 @@ struct ResultGuarantee { Summary state does not itself claim a caller-visible result guarantee. The guarantee is attached to a finalized value, such as a `SummaryEstimate`. -### Error metrics +#### Error metrics The built-in model distinguishes: @@ -104,7 +151,7 @@ These metrics are not implicitly convertible. In particular, Count-Min Sketch and CountSketch use different frequency norms, and a point-frequency guarantee does not prove TopK membership. -### Symbolic expressions +#### Symbolic expressions `BoundExpr` represents constants, sums, products, maxima, and unavailable statistics. `ProbabilityExpr` represents zero, constants, union bounds, and @@ -115,7 +162,7 @@ Numeric leaves must be finite. Bounds must be non-negative, and probabilities must lie in `[0, 1]`. Invalid values fail closed rather than passing a target comparison through floating-point behavior. -### Provenance +#### Provenance `GuaranteeSource` records why a guarantee is believed: @@ -129,7 +176,7 @@ comparison through floating-point behavior. This information is exported with candidate and rejection data so that a plan can be audited without reconstructing the proof from planner internals. -## Extension points +### Extension points Accuracy reasoning and allocation are separate from cost modeling: @@ -159,9 +206,9 @@ approximate layers. `CostModel` may rank only the candidates that survive guarantee propagation and target checking. Deployments may provide stronger accuracy models, but the default model remains conservative. -## Core composition rules +### Core composition rules -### Exact values +#### Exact values An exact value contributes zero error and zero failure probability: @@ -174,7 +221,7 @@ An exact input does not exempt a local approximate sketch from target checking. This matters when parameter ranges are clamped and the tightest representable configuration still misses the requested target. -### Additive and Lipschitz composition +#### Additive and Lipschitz composition Compatible absolute bounds compose without an independence assumption: @@ -192,7 +239,7 @@ delta_output <= delta_input + delta_local Failure probabilities use a union bound. -### Relative composition +#### Relative composition For a registered multiplicative rule whose values are known to be non-negative: @@ -207,7 +254,7 @@ epsilon_total = The rule is rejected when sign information is missing or values may cross zero. -### Exact aggregation over approximate values +#### Exact aggregation over approximate values For an exact sum, input bounds are summed and input failures are union-bounded. When the number of folded rows is required but unknown, the resulting bound @@ -217,32 +264,37 @@ For exact minimum or maximum, the value-error bound is the maximum input bound and failures are union-bounded. This bounds the returned value; it does not by itself prove the identity of a winning key. -### Unsupported composition +#### Unsupported composition Approximate-over-approximate composition is accepted only when the model has a rule for the operator and metric. Cross-metric composition and unregistered same-metric composition return `UnsupportedComposition`. They never treat an approximate child as exact. -## Sketch contracts +### Sketch contracts The formulas below are planner contracts derived from committed sketch parameters. Parameter clamps are followed by a target check. -### KLL +#### KLL -KLL uses normalized rank error: +For quantile and rank queries, the built-in contract follows Apache +DataSketches' empirical 99th-percentile single-sided normalized rank-error fit: ```text -epsilon_rank = 2 / k -k = ceil(2 / requested_epsilon) +epsilon_rank = 2.296 / k^0.9723 +k = ceil((2.296 / requested_epsilon)^(1 / 0.9723)) ``` -The built-in contract has failure probability `0.01` (99% confidence). A target -with a tighter `delta` requires a stronger deployment-specific model or an -amplification contract and otherwise fails closed. +The contract has failure probability `0.01`. A target with a tighter `delta` +requires a stronger deployment-specific model or an amplification contract and +otherwise fails closed. The coefficients are specific to the cited KLL contract +and must change if the selected implementation uses different semantics. See +[Apache DataSketches KLL accuracy](https://datasketches.apache.org/docs/KLL/KLLAccuracyAndSize.html) +and its +[C++ contract](https://github.com/apache/datasketches-cpp/blob/master/kll/include/kll_sketch.hpp). -### DDSketch +#### DDSketch DDSketch uses its committed deterministic relative-error parameter: @@ -251,7 +303,7 @@ epsilon_relative = alpha delta = 0 ``` -### HLL +#### HLL HLL starts from relative standard error: @@ -268,8 +320,10 @@ delta = 0.01 Sizing inverts this bound. If the supported precision cap cannot satisfy the target, HLL is rejected even when it would be the cheaper or preferred family. +This contract requires an estimator whose variance is bounded by the stated RSE; +changing estimator semantics requires changing the accuracy model. -### KMV and Theta +#### KMV and Theta The default model uses: @@ -281,9 +335,10 @@ k = ceil(100 / requested_epsilon^2 + 2) ``` This is also a conservative Chebyshev 99% contract. Tighter confidence fails -closed unless another model proves it. +closed unless another model proves it. The variance premise follows the +[Theta/KMV equations](https://datasketches.apache.org/docs/pdf/ThetaSketchEquations.pdf). -### Count-Min Sketch +#### Count-Min Sketch CMS uses an L1-normalized one-sided frequency bound: @@ -303,7 +358,7 @@ depth = ceil(ln(1 / requested_delta)) Posterior CMS relaxation may reduce its width only under its documented L1 assumptions. -### CountSketch +#### CountSketch CountSketch has a separate L2-normalized point-frequency contract. It does not reuse CMS sizing or posterior relaxation: @@ -325,7 +380,7 @@ depth = an odd integer >= ceil(18 * ln(1 / requested_delta)) Zero or even depth has no modeled guarantee. Width/depth caps are checked against the target after sizing. -## TopK membership certificate +### TopK membership certificate A heap attached to CMS or CountSketch can provide point-frequency estimates, but those estimates do not prove that the returned keys are the true TopK set. @@ -352,7 +407,7 @@ Static planning currently has no ordinary source for these boundary intervals, so TopK sketch candidates remain fail-closed until planning-time or runtime evidence is supplied. -## Hydra shared-grid composition +### Hydra shared-grid composition A Hydra result includes both the inner per-subpopulation sketch error and an outer shared-grid collision term: @@ -376,7 +431,7 @@ leaves cannot yet be evaluated, an accuracy-targeted Hydra candidate is not admitted. Copying the inner guarantee onto Hydra without the shared term would be unsound. -## Accuracy targets and allocation +### Accuracy targets and allocation `AccuracyTarget::Exact` accepts only a zero bound and zero failure probability. `AccuracyTarget::Epsilon` checks the evaluated magnitude. An @@ -395,10 +450,10 @@ Every allocated candidate is resized, propagated, and checked. Allocation does not constitute proof by itself, and recording a requested delta in metadata is not evidence that an algorithm achieves it. -## Planner and runtime boundary +### Planner and runtime boundary -PR #303 implements the guarantee algebra and the parameter-derived contracts in -ASAPPlanner. It does not require ASAPPlanner to link to `asap_sketchlib`. +The guarantee algebra and parameter-derived contracts live in ASAPPlanner. They +do not require the planner to link to `asap_sketchlib`. Runtime or planning-time observations are still useful for quantities that are not fixed by static parameters: @@ -412,7 +467,7 @@ Such evidence must enter through explicit observation/statistics fields and be recorded in provenance. Its absence leaves expressions symbolic and candidates unprovable. -## Explainability and export +### Explainability and export DAG export includes: @@ -425,7 +480,63 @@ DAG export includes: This makes the correctness decision inspectable and ensures the explanation uses the same candidate space and legality checks as optimization. -## Validation strategy +## Minimal complexity + +Three concepts are necessary: + +- `ResultGuarantee` is the authoritative description of caller-visible error; + reusing `AccuracyTarget` would conflate a request with evidence that the + request was met. +- `AccuracyModel` separates correctness rules from `CostModel`; embedding + legality in cost values would let ranking accidentally override correctness. +- `AccuracyBudgetAllocator` separates a proposed per-layer budget from the + propagated proof; assigning the full target to every layer is unsound. + +Symbolic expressions are used instead of a general theorem prover or free-form +text. They are the smallest representation that can preserve unknown +statistics, evaluate supported formulas, serialize the result, and explain why +a candidate was rejected. + +## Alternatives and decisions + +- **One untyped epsilon:** rejected because rank, cardinality, L1 frequency, L2 + frequency, value, and membership errors are not interchangeable. +- **Put correctness in `CostModel`:** rejected because legality must not depend + on ranking policy. +- **Treat missing evidence as zero:** rejected because it silently converts an + unproved candidate into a valid one. +- **Assume independent errors:** rejected; the default uses union bounds. +- **Require a runtime library dependency:** rejected because parameter-derived + planning contracts and runtime observations have different lifecycles. +- **Copy the inner guarantee onto Hydra:** rejected because it omits shared-grid + collisions. +- **Use a point-frequency guarantee for TopK:** rejected because it does not + establish membership at the selection boundary. + +## Quality attributes and evidence + +- **Maintainability and extensibility:** each new sketch or composition adds one + local contract and focused tests; unknown variants remain fail-closed through + non-exhaustive enums. +- **Debuggability and understandability:** DAG export exposes expressions, + provenance, allocations, and rejection reasons. The observable proxy is that + a rejected candidate can be diagnosed from exported data without replaying + cost ranking. +- **Performance and scalability:** expressions are small trees evaluated during + candidate construction. No numerical performance claim is made; candidate + count and planning latency should be measured before adding richer allocation + enumeration. +- **Operability:** runtime-dependent values have named symbolic leaves and an + explicit observation provenance path. +- **Security and robustness:** malformed non-finite or out-of-range numeric + leaves fail closed. + +## Acceptance and test design + +Acceptance behavior was derived from the required formulas and fail-closed +boundaries. The current tests were written alongside or after implementation, +so they are not an independent test design; final review is likewise not +independent unless performed by another reviewer. Tests cover: @@ -449,14 +560,23 @@ cargo clippy --workspace --all-targets -- -D warnings git diff --check ``` -## Non-goals +## Risks, rollout, and exit criteria + +The main correctness risk is a mismatch between a planner contract and the +estimator actually selected by a serving implementation. Each parameter-derived +contract must therefore state its estimator premise, and deployments with +different semantics must replace the model rather than reuse the guarantee. + +The main availability risk is conservative rejection. TopK and Hydra stay on +the exact/pre-ASAP path until their required evidence is available. This is the +intended rollback behavior: removing or disabling a questionable rule reduces +optimization opportunities without weakening correctness. -- Adding another correctness-policy enum alongside `AccuracyTarget`. -- Assuming statistical independence by default. -- Treating all sketch errors as a common epsilon. -- Claiming arbitrary nonlinear or cross-metric composition. -- Executing sketches or importing a sketch runtime into ASAPPlanner. -- Treating unavailable runtime evidence as zero. +The design is ready for use when all workspace tests and warnings-as-errors +checks pass, every selected approximate result has an evaluable satisfying +guarantee, and exported rejection data identifies unavailable evidence. Runtime +observation integration exits its follow-up phase when TopK boundary and Hydra +shared-grid fixtures can be supplied end to end without implicit assumptions. Advanced nonlinear, induced-norm, interval/Jacobian, and correlation-aware -propagation belongs in separate follow-up work. +propagation remains separate work. From 86328428dd892967d5ef7660bd33e120f7982811 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 28 Aug 2026 13:13:30 -0600 Subject: [PATCH 08/12] docs: allow summary-specific accuracy targets --- .../end-to-end-accuracy-guarantees.md | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/design_docs/end-to-end-accuracy-guarantees.md b/docs/design_docs/end-to-end-accuracy-guarantees.md index 72a3a6d..e6d56d3 100644 --- a/docs/design_docs/end-to-end-accuracy-guarantees.md +++ b/docs/design_docs/end-to-end-accuracy-guarantees.md @@ -101,8 +101,11 @@ composition. It does not introduce another correctness policy alongside The design must: -1. Use the existing `AccuracyTarget::{Exact, Epsilon, EpsilonDelta}` as the - correctness requirement. +1. Represent each summary's caller-visible correctness requirement with an + `AccuracyTarget`. The existing `Exact`, `Epsilon`, and `EpsilonDelta` + variants remain valid where their semantics match, but they are not a + closed set: add or refine target variants when a new summary requires a + correctness contract that those variants cannot express faithfully. 2. Preserve the meaning of each error metric instead of treating every bound as an interchangeable epsilon. 3. Derive local guarantees from the same committed parameters used to size the @@ -438,6 +441,21 @@ be unsound. `AccuracyTarget::EpsilonDelta` requires both an evaluated magnitude no greater than epsilon and an evaluated failure probability no greater than delta. +These variants are the currently supported target vocabulary, not a permanent +restriction on summary semantics. A new summary may introduce or motivate a +change to `AccuracyTarget` when its caller-visible requirement is not an +epsilon-style numeric error contract. Any new or revised target must define: + +- the result semantics being constrained, including its error metric; +- the evidence and parameters required to prove satisfaction; +- the satisfaction rule and its unknown/unsupported behavior; and +- its serialization, explainability, allocation, and compatibility behavior. + +The planner must fail closed until the corresponding guarantee model, +propagation rules, and final satisfaction check exist. It must not force a new +summary into `Exact`, `Epsilon`, or `EpsilonDelta` merely to reuse the existing +API. + The initial allocator uses conservative finite choices, including equal splits for additive nested layers: From 1c03ed09c65a315dd0b8a3700e11f6a0e79bb1fc Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 28 Aug 2026 13:20:03 -0600 Subject: [PATCH 09/12] docs: split accuracy design and developer guidance --- .../end-to-end-accuracy-guarantees.md | 307 ++---------------- .../end-to-end-accuracy-guarantees.md | 283 ++++++++++++++++ 2 files changed, 313 insertions(+), 277 deletions(-) create mode 100644 docs/developer_docs/end-to-end-accuracy-guarantees.md diff --git a/docs/design_docs/end-to-end-accuracy-guarantees.md b/docs/design_docs/end-to-end-accuracy-guarantees.md index e6d56d3..372ebfb 100644 --- a/docs/design_docs/end-to-end-accuracy-guarantees.md +++ b/docs/design_docs/end-to-end-accuracy-guarantees.md @@ -6,6 +6,12 @@ This document is for ASAPPlanner developers, architects, and researchers. It defines how the planner represents, composes, checks, and explains approximation guarantees for post-ASAP plans, including nested summaries. +Implementation contracts, sketch formulas, extension steps, and validation +commands live in the +[developer guide](../developer_docs/end-to-end-accuracy-guarantees.md). This +document is the authority for architectural decisions and correctness +invariants; the developer guide is the authority for implementing them. + ASAPPlanner is a mathematical planner. It does not execute sketches or import a sketch runtime. The planner derives guarantees from committed parameters and keeps data- or runtime-dependent quantities symbolic. A serving system may @@ -179,260 +185,28 @@ comparison through floating-point behavior. This information is exported with candidate and rejection data so that a plan can be audited without reconstructing the proof from planner internals. -### Extension points - -Accuracy reasoning and allocation are separate from cost modeling: - -```rust -trait AccuracyModel { - fn local_guarantee(/* family, query, parameters */) - -> Option; - - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[ResultGuarantee], - local: Option<&ResultGuarantee>, - stats: &PropagationStats, - ) -> Result; - - fn satisfies( - &self, - guarantee: &ResultGuarantee, - target: &AccuracyTarget, - ) -> bool; -} -``` - -`AccuracyBudgetAllocator` proposes finite parameter allocations for nested -approximate layers. `CostModel` may rank only the candidates that survive -guarantee propagation and target checking. Deployments may provide stronger -accuracy models, but the default model remains conservative. - -### Core composition rules - -#### Exact values - -An exact value contributes zero error and zero failure probability: - -```text -B_exact = 0 -delta_exact = 0 -``` - -An exact input does not exempt a local approximate sketch from target checking. -This matters when parameter ranges are clamped and the tightest representable -configuration still misses the requested target. - -#### Additive and Lipschitz composition - -Compatible absolute bounds compose without an independence assumption: - -```text -B_total <= B_input + B_local -delta_total <= delta_input + delta_local -``` - -For a registered `L`-Lipschitz transformation: - -```text -B_output <= L * B_input + B_local -delta_output <= delta_input + delta_local -``` - -Failure probabilities use a union bound. - -#### Relative composition - -For a registered multiplicative rule whose values are known to be -non-negative: - -```text -epsilon_total = - epsilon_input - + epsilon_local - + epsilon_input * epsilon_local -``` - -The rule is rejected when sign information is missing or values may cross -zero. - -#### Exact aggregation over approximate values - -For an exact sum, input bounds are summed and input failures are union-bounded. -When the number of folded rows is required but unknown, the resulting bound -stays symbolic. - -For exact minimum or maximum, the value-error bound is the maximum input bound -and failures are union-bounded. This bounds the returned value; it does not by -itself prove the identity of a winning key. - -#### Unsupported composition - -Approximate-over-approximate composition is accepted only when the model has a -rule for the operator and metric. Cross-metric composition and unregistered -same-metric composition return `UnsupportedComposition`. They never treat an -approximate child as exact. - -### Sketch contracts - -The formulas below are planner contracts derived from committed sketch -parameters. Parameter clamps are followed by a target check. - -#### KLL - -For quantile and rank queries, the built-in contract follows Apache -DataSketches' empirical 99th-percentile single-sided normalized rank-error fit: - -```text -epsilon_rank = 2.296 / k^0.9723 -k = ceil((2.296 / requested_epsilon)^(1 / 0.9723)) -``` - -The contract has failure probability `0.01`. A target with a tighter `delta` -requires a stronger deployment-specific model or an amplification contract and -otherwise fails closed. The coefficients are specific to the cited KLL contract -and must change if the selected implementation uses different semantics. See -[Apache DataSketches KLL accuracy](https://datasketches.apache.org/docs/KLL/KLLAccuracyAndSize.html) -and its -[C++ contract](https://github.com/apache/datasketches-cpp/blob/master/kll/include/kll_sketch.hpp). - -#### DDSketch - -DDSketch uses its committed deterministic relative-error parameter: - -```text -epsilon_relative = alpha -delta = 0 -``` - -#### HLL - -HLL starts from relative standard error: - -```text -RSE = 1.04 / sqrt(2^p) -``` - -The default model uses a conservative Chebyshev 99% interval: - -```text -epsilon_cardinality = 10 * 1.04 / sqrt(2^p) -delta = 0.01 -``` - -Sizing inverts this bound. If the supported precision cap cannot satisfy the -target, HLL is rejected even when it would be the cheaper or preferred family. -This contract requires an estimator whose variance is bounded by the stated RSE; -changing estimator semantics requires changing the accuracy model. - -#### KMV and Theta - -The default model uses: +### Accuracy-model boundary -```text -RSE <= 1 / sqrt(k - 2) -epsilon_cardinality = 10 / sqrt(max(k - 2, 1)) -delta = 0.01 -k = ceil(100 / requested_epsilon^2 + 2) -``` - -This is also a conservative Chebyshev 99% contract. Tighter confidence fails -closed unless another model proves it. The variance premise follows the -[Theta/KMV equations](https://datasketches.apache.org/docs/pdf/ThetaSketchEquations.pdf). - -#### Count-Min Sketch +Accuracy reasoning and budget allocation are separate from cost modeling. +The accuracy model derives local guarantees, propagates compatible guarantees, +and checks the caller-visible result against its target. The budget allocator +only proposes allocations; it does not prove them. `CostModel` may rank only +candidates that survive the complete accuracy check. -CMS uses an L1-normalized one-sided frequency bound: - -```text -epsilon_l1 = e / width -delta <= exp(-depth) -absolute_error <= epsilon_l1 * ||f||_1 -``` +Composition rules are explicit and metric-aware. The built-in model supports +only registered exact, additive, Lipschitz, relative, and exact-aggregation +rules. It uses union bounds rather than assuming independence. Cross-metric and +unregistered approximate compositions are unsupported and fail closed. -Sizing uses: - -```text -width = ceil(e / requested_epsilon) -depth = ceil(ln(1 / requested_delta)) -``` - -Posterior CMS relaxation may reduce its width only under its documented L1 -assumptions. - -#### CountSketch - -CountSketch has a separate L2-normalized point-frequency contract. It does not -reuse CMS sizing or posterior relaxation: - -```text -epsilon_l2 = sqrt(3 / width) -absolute_error <= epsilon_l2 * ||f||_2 -``` - -One row is bad with probability at most `1/3`. For an independent odd number of -rows, the median is assigned the conservative Hoeffding bound: - -```text -delta <= exp(-depth / 18) -width = ceil(3 / requested_epsilon^2) -depth = an odd integer >= ceil(18 * ln(1 / requested_delta)) -``` +Sketch contracts are derived from the parameters committed to the plan and +must identify any estimator-specific premise. Count-Min Sketch and CountSketch +remain distinct L1- and L2-frequency contracts. TopK membership requires a +selection-margin certificate; a point-frequency guarantee is insufficient. +Hydra must include both its inner error and shared-grid collision error. -Zero or even depth has no modeled guarantee. Width/depth caps are checked -against the target after sizing. - -### TopK membership certificate - -A heap attached to CMS or CountSketch can provide point-frequency estimates, -but those estimates do not prove that the returned keys are the true TopK set. - -`PropagationStats` therefore accepts three pieces of evidence: - -- the lower confidence bound of the kth selected item; -- the greatest upper confidence bound among excluded items; and -- the union-bound failure probability of all intervals used by the - certificate. - -The intervals must already be widened by the underlying sketch error. Exact -membership is certified only when: - -```text -selected_kth_lower_bound > excluded_max_upper_bound -``` - -The result then has metric `TopKMembership`, zero membership error, and the -provided interval failure probability. Missing values, non-finite values, -invalid probabilities, equality, or overlap reject the candidate. - -Static planning currently has no ordinary source for these boundary intervals, -so TopK sketch candidates remain fail-closed until planning-time or runtime -evidence is supplied. - -### Hydra shared-grid composition - -A Hydra result includes both the inner per-subpopulation sketch error and an -outer shared-grid collision term: - -```text -B_hydra = B_inner + B_shared_grid -delta_hydra <= delta_inner + delta_shared_grid -``` - -The second expression is a union bound. The shared-grid collision bound and -failure probability depend on deployment/data statistics, so the default -planner records them as: - -```text -hydra_shared_grid_collision_bound -hydra_shared_grid_failure_probability -``` - -The resulting expression and provenance are visible in the IR. Because these -leaves cannot yet be evaluated, an accuracy-targeted Hydra candidate is not -admitted. Copying the inner guarantee onto Hydra without the shared term would -be unsound. +The concrete interfaces, formulas, evidence fields, and extension procedure +are defined in the +[developer guide](../developer_docs/end-to-end-accuracy-guarantees.md). ### Accuracy targets and allocation @@ -549,34 +323,13 @@ a candidate was rejected. - **Security and robustness:** malformed non-finite or out-of-range numeric leaves fail closed. -## Acceptance and test design +## Verification requirements -Acceptance behavior was derived from the required formulas and fail-closed -boundaries. The current tests were written alongside or after implementation, -so they are not an independent test design; final review is likewise not -independent unless performed by another reviewer. - -Tests cover: - -- exact, additive, relative, Lipschitz, sum, and extremum propagation; -- incompatible metrics and unsupported composition; -- malformed and unknown symbolic values; -- target checking after parameter clamps; -- CountSketch L2 sizing distinct from CMS L1 sizing; -- accepted separated and rejected overlapping TopK intervals; -- Hydra inner-plus-shared-grid symbolic composition; -- KLL/HLL/KMV/Theta confidence and `EpsilonDelta` behavior; -- rejection before cost ranking and global selection; and -- DAG export and frontend-to-post-ASAP integration. - -The repository validation commands are: - -```text -cargo fmt --all -cargo test --workspace -cargo clippy --workspace --all-targets -- -D warnings -git diff --check -``` +Tests must cover every registered local contract and composition rule, their +invalid and unknown boundaries, target checking after parameter clamps, +rejection before cost ranking, and exported proof or rejection data. The +detailed test matrix and repository validation commands are maintained in the +[developer guide](../developer_docs/end-to-end-accuracy-guarantees.md#validation). ## Risks, rollout, and exit criteria diff --git a/docs/developer_docs/end-to-end-accuracy-guarantees.md b/docs/developer_docs/end-to-end-accuracy-guarantees.md new file mode 100644 index 0000000..3cb8784 --- /dev/null +++ b/docs/developer_docs/end-to-end-accuracy-guarantees.md @@ -0,0 +1,283 @@ +# End-to-End Accuracy Guarantees Developer Guide + +This guide explains how to implement, extend, and validate the end-to-end +accuracy model. Read the +[design document](../design_docs/end-to-end-accuracy-guarantees.md) first. The +design document owns architectural decisions and correctness invariants; this +guide owns concrete interfaces, formulas, evidence requirements, and developer +workflow. + +## Implementation model + +Accuracy reasoning and allocation are separate from cost modeling: + +```rust +trait AccuracyModel { + fn local_guarantee(/* family, query, parameters */) + -> Option; + + fn propagate( + &self, + op: &CompositionOperator, + inputs: &[ResultGuarantee], + local: Option<&ResultGuarantee>, + stats: &PropagationStats, + ) -> Result; + + fn satisfies( + &self, + guarantee: &ResultGuarantee, + target: &AccuracyTarget, + ) -> bool; +} +``` + +`AccuracyBudgetAllocator` proposes finite parameter allocations for nested +approximate layers. Every candidate must then be resized, propagated, and +checked. `CostModel` may see only candidates that pass this check. + +## Composition contracts + +### Exact values + +An exact value contributes zero error and zero failure probability: + +```text +B_exact = 0 +delta_exact = 0 +``` + +An exact input does not exempt a local approximate sketch from target checking. + +### Additive and Lipschitz composition + +Compatible absolute bounds compose without an independence assumption: + +```text +B_total <= B_input + B_local +delta_total <= delta_input + delta_local +``` + +For a registered `L`-Lipschitz transformation: + +```text +B_output <= L * B_input + B_local +delta_output <= delta_input + delta_local +``` + +Failure probabilities use a union bound. + +### Relative composition + +For a registered multiplicative rule whose values are known to be +non-negative: + +```text +epsilon_total = + epsilon_input + + epsilon_local + + epsilon_input * epsilon_local +``` + +Reject the rule when sign information is missing or values may cross zero. + +### Exact aggregation over approximate values + +For an exact sum, sum the input bounds and union-bound input failures. If the +number of folded rows is required but unknown, keep the result symbolic. + +For exact minimum or maximum, use the maximum input value-error bound and +union-bound failures. This does not prove the identity of a winning key. + +### Unsupported composition + +Accept approximate-over-approximate composition only when a rule exists for +the operator and metric. Cross-metric and unregistered same-metric composition +must return `UnsupportedComposition`; never treat an approximate child as +exact. + +## Built-in sketch contracts + +These formulas are planner contracts derived from committed parameters. +Always check the resulting guarantee against the target after applying +parameter clamps. + +### KLL + +For quantile and rank queries, the built-in contract follows Apache +DataSketches' empirical 99th-percentile single-sided normalized rank-error fit: + +```text +epsilon_rank = 2.296 / k^0.9723 +k = ceil((2.296 / requested_epsilon)^(1 / 0.9723)) +delta = 0.01 +``` + +A tighter `delta` needs a stronger implementation-specific or amplification +contract and otherwise fails closed. The coefficients are specific to the +cited implementation contract. See +[Apache DataSketches KLL accuracy](https://datasketches.apache.org/docs/KLL/KLLAccuracyAndSize.html) +and its +[C++ contract](https://github.com/apache/datasketches-cpp/blob/master/kll/include/kll_sketch.hpp). + +### DDSketch + +```text +epsilon_relative = alpha +delta = 0 +``` + +### HLL + +The modeled estimator contract is: + +```text +RSE = 1.04 / sqrt(2^p) +epsilon_cardinality = 10 * 1.04 / sqrt(2^p) +delta = 0.01 +``` + +Sizing inverts the bound. Reject HLL when the supported precision cap misses +the target. The confidence claim requires an estimator whose bias and variance +premises justify the modeled interval; do not attach it to a generic HLL family +without that implementation contract. + +### KMV and Theta + +```text +RSE <= 1 / sqrt(k - 2) +epsilon_cardinality = 10 / sqrt(max(k - 2, 1)) +delta = 0.01 +k = ceil(100 / requested_epsilon^2 + 2) +``` + +This is a conservative Chebyshev 99% contract. A tighter confidence target +fails closed unless another model proves it. The variance premise follows the +[Theta/KMV equations](https://datasketches.apache.org/docs/pdf/ThetaSketchEquations.pdf). + +### Count-Min Sketch + +CMS uses an L1-normalized one-sided frequency bound: + +```text +epsilon_l1 = e / width +delta <= exp(-depth) +absolute_error <= epsilon_l1 * ||f||_1 +width = ceil(e / requested_epsilon) +depth = ceil(ln(1 / requested_delta)) +``` + +Posterior CMS relaxation may reduce width only under its documented L1 +assumptions. + +### CountSketch + +CountSketch has a separate L2-normalized point-frequency contract and must not +reuse CMS sizing or posterior relaxation: + +```text +epsilon_l2 = sqrt(3 / width) +absolute_error <= epsilon_l2 * ||f||_2 +delta <= exp(-depth / 18) +width = ceil(3 / requested_epsilon^2) +depth = an odd integer >= ceil(18 * ln(1 / requested_delta)) +``` + +The failure bound is for the median of independent rows when one row is bad +with probability at most `1/3`. Zero or even depth has no modeled guarantee. + +## Runtime or statistics evidence + +ASAPPlanner contains the guarantee algebra and parameter-derived contracts; it +does not import `asap_sketchlib`. Data- or runtime-dependent evidence enters +through explicit statistics fields and is recorded in provenance. + +### TopK membership + +`PropagationStats` supplies: + +- the lower confidence bound of the kth selected item; +- the greatest upper confidence bound among excluded items; and +- the union-bound failure probability of all certificate intervals. + +The intervals must already include underlying sketch error. Certify exact +membership only when: + +```text +selected_kth_lower_bound > excluded_max_upper_bound +``` + +The result uses `TopKMembership`, zero membership error, and the supplied +failure probability. Missing, non-finite, invalid, equal, or overlapping +evidence rejects the candidate. + +### Hydra shared grid + +Compose the inner and shared-grid terms as: + +```text +B_hydra = B_inner + B_shared_grid +delta_hydra <= delta_inner + delta_shared_grid +``` + +When observations are unavailable, preserve +`hydra_shared_grid_collision_bound` and +`hydra_shared_grid_failure_probability` as symbolic leaves and reject an +accuracy-targeted candidate. Never copy the inner guarantee onto Hydra alone. + +## Adding or changing an accuracy target + +`Exact`, `Epsilon`, and `EpsilonDelta` are the current vocabulary, not a closed +set. If a new summary has a caller-visible requirement they cannot express: + +1. Define the target's result semantics and compatible `ErrorMetric`. +2. Define the parameters and evidence needed to prove satisfaction. +3. Add the satisfaction rule, including invalid, unknown, and unsupported + behavior. +4. Add allocation behavior where the target can be divided across layers; do + not invent allocation for non-divisible semantics. +5. Add serialization, DAG explanation, and compatibility behavior. +6. Add local and composition contracts for summaries that claim the target. +7. Keep candidates fail-closed until the end-to-end proof path is implemented + and tested. + +Do not coerce membership, distributional, or another non-epsilon requirement +into an existing variant merely to reuse its API. + +## Adding a summary or composition + +For a new summary, register its error metric and a local contract tied to the +same parameters that sizing commits. Record estimator/version premises in +provenance. For a new composition, specify compatible input and output metrics, +the bound rule, failure composition, required statistics, and unsupported +cases. Missing evidence must stay symbolic or produce a structured rejection. + +Update DAG export whenever new proof or rejection data is introduced. + +## Validation + +Tests must cover: + +- exact, additive, relative, Lipschitz, sum, and extremum propagation; +- incompatible metrics and unsupported composition; +- malformed and unknown symbolic values; +- target checking after parameter clamps; +- CountSketch L2 sizing distinct from CMS L1 sizing; +- accepted separated and rejected overlapping TopK intervals; +- Hydra inner-plus-shared-grid composition; +- implementation-qualified KLL/HLL/KMV/Theta confidence behavior; +- rejection before cost ranking and global selection; and +- DAG export and frontend-to-post-ASAP integration. + +For a new target or summary, add a positive proof case and boundary cases for +missing evidence, invalid values, incompatible metrics, and insufficient +parameters. + +Run: + +```text +cargo fmt --all +cargo test --workspace +cargo clippy --workspace --all-targets -- -D warnings +git diff --check +``` From a816cfbcc42f0f433a7c59b281c46c00815ebde0 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 28 Aug 2026 13:22:54 -0600 Subject: [PATCH 10/12] docs: colocate accuracy design with mapping docs --- docs/design_docs/asap-aware-mapping/README.md | 2 +- .../end-to-end-accuracy-guarantees.md | 6 +++--- docs/developer_docs/end-to-end-accuracy-guarantees.md | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) rename docs/design_docs/{ => asap-aware-mapping}/end-to-end-accuracy-guarantees.md (98%) diff --git a/docs/design_docs/asap-aware-mapping/README.md b/docs/design_docs/asap-aware-mapping/README.md index eb0b045..1d32afe 100644 --- a/docs/design_docs/asap-aware-mapping/README.md +++ b/docs/design_docs/asap-aware-mapping/README.md @@ -88,7 +88,7 @@ The design is split into focused documents: subpopulation and time organization, roll-ups, sharing, semantic rewrites, and hybrid execution. - [Summary properties](summary_properties.md) lists the capabilities used to determine whether summaries and optimizations can be composed safely. -- [End-to-end accuracy guarantees](../end-to-end-accuracy-guarantees.md) specifies the typed +- [End-to-end accuracy guarantees](end-to-end-accuracy-guarantees.md) specifies the typed guarantee IR, sketch contracts, composition rules, target checking, and fail-closed boundaries. - [Explainability](explainability.md) describes how the planner reports available replacements using the same candidate space it optimizes. diff --git a/docs/design_docs/end-to-end-accuracy-guarantees.md b/docs/design_docs/asap-aware-mapping/end-to-end-accuracy-guarantees.md similarity index 98% rename from docs/design_docs/end-to-end-accuracy-guarantees.md rename to docs/design_docs/asap-aware-mapping/end-to-end-accuracy-guarantees.md index 372ebfb..4fcafa7 100644 --- a/docs/design_docs/end-to-end-accuracy-guarantees.md +++ b/docs/design_docs/asap-aware-mapping/end-to-end-accuracy-guarantees.md @@ -8,7 +8,7 @@ guarantees for post-ASAP plans, including nested summaries. Implementation contracts, sketch formulas, extension steps, and validation commands live in the -[developer guide](../developer_docs/end-to-end-accuracy-guarantees.md). This +[developer guide](../../developer_docs/end-to-end-accuracy-guarantees.md). This document is the authority for architectural decisions and correctness invariants; the developer guide is the authority for implementing them. @@ -206,7 +206,7 @@ Hydra must include both its inner error and shared-grid collision error. The concrete interfaces, formulas, evidence fields, and extension procedure are defined in the -[developer guide](../developer_docs/end-to-end-accuracy-guarantees.md). +[developer guide](../../developer_docs/end-to-end-accuracy-guarantees.md). ### Accuracy targets and allocation @@ -329,7 +329,7 @@ Tests must cover every registered local contract and composition rule, their invalid and unknown boundaries, target checking after parameter clamps, rejection before cost ranking, and exported proof or rejection data. The detailed test matrix and repository validation commands are maintained in the -[developer guide](../developer_docs/end-to-end-accuracy-guarantees.md#validation). +[developer guide](../../developer_docs/end-to-end-accuracy-guarantees.md#validation). ## Risks, rollout, and exit criteria diff --git a/docs/developer_docs/end-to-end-accuracy-guarantees.md b/docs/developer_docs/end-to-end-accuracy-guarantees.md index 3cb8784..d421a81 100644 --- a/docs/developer_docs/end-to-end-accuracy-guarantees.md +++ b/docs/developer_docs/end-to-end-accuracy-guarantees.md @@ -2,7 +2,8 @@ This guide explains how to implement, extend, and validate the end-to-end accuracy model. Read the -[design document](../design_docs/end-to-end-accuracy-guarantees.md) first. The +[design document](../design_docs/asap-aware-mapping/end-to-end-accuracy-guarantees.md) +first. The design document owns architectural decisions and correctness invariants; this guide owns concrete interfaces, formulas, evidence requirements, and developer workflow. From eff9982e5b88629d875cc2b32b5158d5b6410fb2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 28 Aug 2026 13:27:51 -0600 Subject: [PATCH 11/12] docs: define summary parameter configuration modes --- .../end-to-end-accuracy-guarantees.md | 88 ++++++++++++++++++- 1 file changed, 84 insertions(+), 4 deletions(-) diff --git a/docs/design_docs/asap-aware-mapping/end-to-end-accuracy-guarantees.md b/docs/design_docs/asap-aware-mapping/end-to-end-accuracy-guarantees.md index 4fcafa7..2a295f4 100644 --- a/docs/design_docs/asap-aware-mapping/end-to-end-accuracy-guarantees.md +++ b/docs/design_docs/asap-aware-mapping/end-to-end-accuracy-guarantees.md @@ -4,7 +4,9 @@ This document is for ASAPPlanner developers, architects, and researchers. It defines how the planner represents, composes, checks, and explains approximation -guarantees for post-ASAP plans, including nested summaries. +guarantees for post-ASAP plans. The parameter-configuration model applies to +both single-summary and nested-summary plans; nesting is one consumer of the +model, not its scope boundary. Implementation contracts, sketch formulas, extension steps, and validation commands live in the @@ -42,9 +44,11 @@ operations, such as TopK selection, need evidence that cannot be expressed by adding point-estimation epsilons. Without an end-to-end model, the planner can select a locally well-sized sketch -whose caller-visible result violates the requested accuracy. Nested summaries, -shared grouping, and cross-query reuse make this a planning concern rather than -an isolated sketch-implementation detail. +whose caller-visible result violates the requested accuracy. Even a single +summary must keep its parameter source and correctness evidence aligned. +Nested summaries, shared grouping, and cross-query reuse add composition +requirements, but do not define the scope of parameter configuration. This is +a planning concern rather than an isolated sketch-implementation detail. ## Inputs, outputs, and end-to-end behavior @@ -123,6 +127,9 @@ The design must: reasons in the post-ASAP IR and DAG export. 7. Fail closed for invalid numeric values, unknown required statistics, incompatible metrics, and unsupported compositions. +8. Record whether committed parameters came from a mathematical model, + empirical input, or a future combination of both, without treating the + parameter-selection method as correctness evidence by itself. ## Proposed design @@ -208,6 +215,74 @@ The concrete interfaces, formulas, evidence fields, and extension procedure are defined in the [developer guide](../../developer_docs/end-to-end-accuracy-guarantees.md). +### Parameter-configuration modes + +Parameter configuration is a planner-wide concern for every summary candidate, +not a mechanism specific to nested queries. ASAPPlanner must distinguish the +source of a candidate's committed parameters from the guarantee used to prove +that candidate legal. + +The design recognizes three modes: + +| Mode | Parameter input | Current status | +| --- | --- | --- | +| Mathematical | `AccuracyTarget` plus an algorithm contract, inverted into parameters | Implemented and wired into candidate generation | +| Empirical-input | Observed or estimated workload/data characteristics supplied as planning input | Designed as an extension point; not wired into end-to-end candidate generation yet | +| Combined | Mathematical constraints and empirical input jointly choose parameters | Future work | + +#### Mathematical configuration + +The mathematical path selects parameters by inverting a registered sketch +contract. For example, an epsilon and delta may determine width and depth. The +planner derives the local guarantee again from the parameters it actually +commits, including clamps, and checks that guarantee against the target. This +is the default path currently available in ASAPPlanner. + +#### Empirical-input configuration + +The empirical path uses explicit information about the expected input or +workload, such as cardinality, frequency distribution, skew, stream norms, or +observed collision behavior. The evidence may come from a catalog, a prior +measurement, or a runtime-facing integration, but it must enter the planner as +typed input with provenance, freshness, and applicability semantics. + +This mode is not wired into the current end-to-end candidate-generation path. +Existing symbolic statistics and posterior-related helpers do not constitute +that integration. Until the planner can carry empirical input through sizing, +guarantee derivation, target checking, and export, the built-in planner must +not claim that an empirical configuration was considered or selected. + +Empirical input can recommend smaller or differently shaped parameters, but a +recommendation is not automatically a correctness proof. If the empirical +method supplies only an expectation or heuristic, legality still requires an +independent guarantee satisfying the target. If it supplies a statistical +certificate, the model must state its population, confidence, validity window, +and failure behavior. + +#### Combined configuration + +The future combined path may use empirical input to refine a mathematically +safe configuration, select among multiple proven contracts, or allocate an +accuracy budget more efficiently. It must define an explicit composition rule +between mathematical and empirical evidence. The planner must not silently +take the smaller of two bounds, assume independence, or use empirical input to +weaken a mathematical requirement. + +All modes converge on the same downstream contract: + +```text +parameter inputs + -> committed summary parameters + -> guarantee derived for those committed parameters + -> AccuracyTarget satisfaction + -> cost ranking +``` + +The post-ASAP IR and DAG export should identify the configuration mode, input +provenance, committed parameters, resulting guarantee, and any unavailable +evidence. This keeps parameter choice auditable and allows future empirical or +combined implementations without creating a second legality pipeline. + ### Accuracy targets and allocation `AccuracyTarget::Exact` accepts only a zero bound and zero failure probability. @@ -343,6 +418,11 @@ the exact/pre-ASAP path until their required evidence is available. This is the intended rollback behavior: removing or disabling a questionable rule reduces optimization opportunities without weakening correctness. +Empirical-input parameterization remains a follow-up until typed empirical +inputs are threaded through candidate sizing, guarantee derivation, target +checking, provenance, and export. Combined parameterization remains future work +until its evidence-composition rule is specified and tested. + The design is ready for use when all workspace tests and warnings-as-errors checks pass, every selected approximate result has an evaluable satisfying guarantee, and exported rejection data identifies unavailable evidence. Runtime From b51476897ca9f869d08e4c8fbee675383a97abf4 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 28 Aug 2026 13:41:30 -0600 Subject: [PATCH 12/12] fix(accuracy): wire evidence and fail closed on HLL confidence --- crates/asap-aware-mapping/src/accuracy.rs | 65 ++++++- crates/asap-aware-mapping/src/grouping.rs | 163 ++++++++++++++---- crates/asap-aware-mapping/src/lib.rs | 5 +- crates/asap-aware-mapping/src/replacement.rs | 155 ++++++++++++----- .../tests/sql_to_post_asap.rs | 4 +- crates/types/src/post_asap/guarantee.rs | 11 +- .../asap-aware-mapping/summary_properties.md | 11 +- .../end-to-end-accuracy-guarantees.md | 21 ++- 8 files changed, 332 insertions(+), 103 deletions(-) diff --git a/crates/asap-aware-mapping/src/accuracy.rs b/crates/asap-aware-mapping/src/accuracy.rs index 99685ba..a9c0224 100644 --- a/crates/asap-aware-mapping/src/accuracy.rs +++ b/crates/asap-aware-mapping/src/accuracy.rs @@ -78,6 +78,8 @@ use asap_types::types::AccuracyTarget; /// [`BoundExpr::Unknown`] leaf (or rejects) rather than guessing. #[derive(Debug, Clone, Default, PartialEq)] pub struct PropagationStats { + /// Provenance for supplied evidence (source, observation identity, etc.). + pub evidence_provenance: Vec, /// Whether every input value is known to be non-negative — required by /// the multiplicative relative-error rule, which is unsound across a /// sign change. @@ -95,8 +97,29 @@ pub struct PropagationStats { /// Union-bound failure probability of all intervals used by the margin /// certificate. pub topk_interval_failure_probability: Option, + /// Hydra shared-grid collision error in the inner guarantee's metric. + pub hydra_shared_grid_collision_bound: Option, + /// Failure probability assigned to the Hydra shared-grid term. + pub hydra_shared_grid_failure_probability: Option, } +/// Supplies typed planning-time evidence required by propagation rules. +pub trait AccuracyEvidenceProvider { + fn propagation_stats( + &self, + _op: &CompositionOperator, + _family: &SummaryFamilyType, + _query: Option<&SketchQuery>, + ) -> PropagationStats { + PropagationStats::default() + } +} + +#[derive(Debug, Default, Clone, Copy)] +pub struct NoAccuracyEvidence; + +impl AccuracyEvidenceProvider for NoAccuracyEvidence {} + /// The deployment-extensible accuracy algebra. `asap-aware-mapping` ships /// [`DefaultAccuracyModel`]; a deployment with a proof for a composition the /// default rejects (a registered cross-metric conversion, say) implements @@ -176,12 +199,15 @@ impl DefaultAccuracyModel { SketchParams::DDSketch { alpha } => { (ErrorMetric::RelativeValue, *alpha, ProbabilityExpr::Zero) } - // HLL: RSE = 1.04/√(2^p). Chebyshev turns ten standard - // deviations into a distribution-free 99% confidence bound. + // HLL parameters encode precision, not a confidence-level budget. + // They therefore provide an RSE magnitude here but no failure + // probability against true cardinality. SketchParams::Hll { precision } => ( ErrorMetric::Cardinality, - 10.0 * 1.04 / 2f64.powi(i32::from(*precision)).sqrt(), - ProbabilityExpr::Constant { value: 0.01 }, + 1.04 / 2f64.powi(i32::from(*precision)).sqrt(), + ProbabilityExpr::Unknown { + statistic: "hll_estimator_failure_probability".into(), + }, ), // KMV / Theta: RSE <= 1/√(k-2); the same Chebyshev conversion // gives a conservative parameter-derived 99% confidence bound. @@ -215,6 +241,20 @@ impl DefaultAccuracyModel { }; let provenance = vec![GuaranteeSource::SketchReadout { algorithm: format!("{algorithm:?}"), + contract: match params { + SketchParams::Kll { .. } => "apache_datasketches_kll_empirical_99_a9b42755072b", + SketchParams::DDSketch { .. } => "ddsketch_relative_error_alpha_v1", + SketchParams::Hll { .. } => "generic_hll_rse_only_no_confidence_v1", + SketchParams::Kmv { .. } => "kmv_unbiased_variance_chebyshev_99_v1", + SketchParams::Theta { .. } => "theta_variance_chebyshev_99_v1", + SketchParams::Cms { .. } | SketchParams::CmsWithHeap { .. } => { + "count_min_l1_markov_v1" + } + SketchParams::CountSketch { .. } | SketchParams::CountSketchWithHeap { .. } => { + "count_sketch_l2_median_hoeffding_v1" + } + } + .into(), params: serde_json::to_value(params).unwrap_or(serde_json::Value::Null), query: format!("{query:?}"), }]; @@ -656,6 +696,7 @@ impl AccuracyModel for DefaultAccuracyModel { guarantee: Box::new(guarantee.clone()), }) .collect::>(); + provenance.extend(stats.evidence_provenance.clone()); if let Some(local) = local { provenance.extend(local.provenance.clone()); } @@ -1075,6 +1116,11 @@ mod tests { } )); assert_eq!(g.approximate_layer_count(), 1); + assert!(g.provenance.iter().any(|source| matches!( + source, + GuaranteeSource::SketchReadout { contract, .. } + if contract == "apache_datasketches_kll_empirical_99_a9b42755072b" + ))); let c = default_cardinality(); let params = default_size_params(SketchAlgorithm::Hll, &c, 0.01, 0.01); @@ -1088,8 +1134,15 @@ mod tests { ) .unwrap(); assert_eq!(g.metric, ErrorMetric::Cardinality); - assert_eq!(g.failure_probability.evaluate(), Some(0.01)); - assert!(!DefaultAccuracyModel.satisfies(&g, &AccuracyTarget::Epsilon(0.01))); + assert_eq!(g.failure_probability.evaluate(), None); + assert!(DefaultAccuracyModel.satisfies(&g, &AccuracyTarget::Epsilon(0.01))); + assert!(!DefaultAccuracyModel.satisfies( + &g, + &AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.01, + } + )); let params = default_size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001); let g = DefaultAccuracyModel diff --git a/crates/asap-aware-mapping/src/grouping.rs b/crates/asap-aware-mapping/src/grouping.rs index 863dea7..4171d07 100644 --- a/crates/asap-aware-mapping/src/grouping.rs +++ b/crates/asap-aware-mapping/src/grouping.rs @@ -80,11 +80,14 @@ use asap_types::post_asap::{ use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::query_expr::{QueryExpr, Reduction}; +use crate::accuracy::{ + AccuracyBudgetAllocator, AccuracyEvidenceProvider, AccuracyModel, PropagationStats, +}; use crate::cost_model::{CostModel, DefaultCostModel}; use crate::replacement::{ - accuracy_target, bindable_intent, construct_summary, describe_intent, implementations_for_with, - summary_candidates, Implementation, Replacement, ReplacementStrategy, ReplacementSubDAG, - TargetSubDAG, + accuracy_target, bindable_intent, construct_summary_with, describe_intent, + implementations_for_with, summary_candidates, Implementation, Models, Replacement, + ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; /// Whether `reduction` has a genuine subpopulation concept for @@ -124,7 +127,7 @@ static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; /// latter, matching every other strategy in this crate's "one strategy, one /// concern" shape. pub struct HydraGroupingStrategy<'a> { - cost_model: &'a dyn CostModel, + models: Models<'a>, } impl HydraGroupingStrategy<'static> { @@ -134,7 +137,7 @@ impl HydraGroupingStrategy<'static> { /// offers. pub fn default_cost_model() -> Self { Self { - cost_model: &DEFAULT_COST_MODEL, + models: Models::with_default_accuracy(&DEFAULT_COST_MODEL), } } } @@ -144,7 +147,25 @@ impl<'a> HydraGroupingStrategy<'a> { /// static preference order — the same customization point /// [`crate::replacement::SketchAlgorithmStrategy::new`] already offers. pub fn new(cost_model: &'a dyn CostModel) -> Self { - Self { cost_model } + Self { + models: Models::with_default_accuracy(cost_model), + } + } + + pub fn with_models_and_evidence( + cost_model: &'a dyn CostModel, + accuracy_model: &'a dyn AccuracyModel, + allocator: &'a dyn AccuracyBudgetAllocator, + evidence: &'a dyn AccuracyEvidenceProvider, + ) -> Self { + Self { + models: Models { + cost: cost_model, + accuracy: accuracy_model, + allocator, + evidence, + }, + } } /// Every legal `SharedMultiSubpopulation` candidate for `target` — empty @@ -161,12 +182,6 @@ impl<'a> HydraGroupingStrategy<'a> { let Some(intent) = bindable_intent(target.root) else { return Vec::new(); }; - // Hydra's shared-grid term is symbolic until deployment/data - // statistics instantiate it. Keep that unevaluable guarantee out of - // cost ranking when this node has an accuracy requirement. - if accuracy_target(intent).is_some() { - return Vec::new(); - } summary_candidates(intent) .iter() .filter_map(|kind| hydra_kind_for(kind).map(|hydra_kind| (kind.clone(), hydra_kind))) @@ -179,7 +194,7 @@ impl<'a> HydraGroupingStrategy<'a> { /// Find the already-ranked candidate [`Implementation::Sketch`] matching /// `sketch_kind` among [`implementations_for_with`]'s exhaustive list for /// `intent`, bind `root` to that exact, already-decided candidate via - /// [`crate::replacement::construct_summary`] (no steering/forcing — see + /// [`crate::replacement::construct_summary_with`] (no steering/forcing — see /// the module docs' "No `ForceSketchKind`-style steering"), then swap the /// resulting `SummaryAgg`'s `grouping` field from the default /// `PerSubpopulationInstance` to @@ -194,12 +209,13 @@ impl<'a> HydraGroupingStrategy<'a> { sketch_kind: SketchAlgorithm, hydra_kind: HydraKind, ) -> Option { - let implementation = implementations_for_with(intent, self.cost_model) + let implementation = implementations_for_with(intent, self.models.cost) .into_iter() .find(|candidate| { matches!(candidate, Implementation::Sketch(kind) if *kind.algorithm() == sketch_kind) })?; - let node = construct_summary(root, implementation, self.cost_model).ok()?; + let node = + construct_summary_with(root, intent, implementation, self.models, None, None).ok()?; let per_subpopulation_params = per_subpopulation_sketch_params(&node)?; let params = default_hydra_params(hydra_kind.clone(), &per_subpopulation_params)?; let grouping = GroupingStrategy::SharedMultiSubpopulation { @@ -207,7 +223,27 @@ impl<'a> HydraGroupingStrategy<'a> { params, }; - let patched = with_grouping(node, grouping); + let (family, query) = match &node.expr { + SummaryExpr::SummaryEstimate { + summary_input, + query, + } => match &summary_input.expr { + SummaryExpr::SummaryAgg { family, .. } => (family, Some(query)), + _ => return None, + }, + _ => return None, + }; + let stats = self.models.evidence.propagation_stats( + &CompositionOperator::ApproximateAggregate, + family, + query, + ); + let patched = with_grouping(node, grouping, &stats); + if let (Some(target), Some(guarantee)) = (accuracy_target(intent), &patched.guarantee) { + if !self.models.accuracy.satisfies(guarantee, target) { + return None; + } + } Some(ReplacementSubDAG { strategy: "HydraGroupingStrategy", replacement: Replacement::Summary(patched), @@ -273,18 +309,22 @@ fn per_subpopulation_sketch_params(node: &SummaryNode) -> Option { /// Recurses through a `SummaryEstimate` readout wrapper (the shape every /// sketch candidate this module builds actually has) to reach the /// `SummaryAgg` underneath. -fn with_grouping(node: Rc, grouping: GroupingStrategy) -> Rc { +fn with_grouping( + node: Rc, + grouping: GroupingStrategy, + stats: &PropagationStats, +) -> Rc { match &node.expr { SummaryExpr::SummaryEstimate { summary_input, query, } => Rc::new(SummaryNode { expr: SummaryExpr::SummaryEstimate { - summary_input: with_grouping(Rc::clone(summary_input), grouping), + summary_input: with_grouping(Rc::clone(summary_input), grouping, stats), query: query.clone(), }, schema: node.schema.clone(), - guarantee: node.guarantee.as_ref().map(hydra_guarantee), + guarantee: node.guarantee.as_ref().map(|g| hydra_guarantee(g, stats)), }), SummaryExpr::SummaryAgg { child, @@ -318,7 +358,7 @@ fn with_grouping(node: Rc, grouping: GroupingStrategy) -> Rc, grouping: GroupingStrategy) -> Rc ResultGuarantee { +fn hydra_guarantee(inner: &ResultGuarantee, stats: &PropagationStats) -> ResultGuarantee { let mut provenance = inner.provenance.clone(); + provenance.extend(stats.evidence_provenance.clone()); provenance.push(GuaranteeSource::ChildGuarantee { input_index: 0, guarantee: Box::new(inner.clone()), }); - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: "hydra_shared_grid_collision_bound".into(), - }); - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: "hydra_shared_grid_failure_probability".into(), - }); + if stats.hydra_shared_grid_collision_bound.is_none() { + provenance.push(GuaranteeSource::UnavailableStatistic { + statistic: "hydra_shared_grid_collision_bound".into(), + }); + } + if stats.hydra_shared_grid_failure_probability.is_none() { + provenance.push(GuaranteeSource::UnavailableStatistic { + statistic: "hydra_shared_grid_failure_probability".into(), + }); + } provenance.push(GuaranteeSource::CompositionStep { operator: CompositionOperator::ApproximateAggregate, rule: "hydra_shared_grid_union_bound".into(), @@ -352,17 +397,23 @@ fn hydra_guarantee(inner: &ResultGuarantee) -> ResultGuarantee { bound: BoundExpr::Sum { terms: vec![ inner.bound.clone(), - BoundExpr::Unknown { - statistic: "hydra_shared_grid_collision_bound".into(), - }, + stats.hydra_shared_grid_collision_bound.map_or_else( + || BoundExpr::Unknown { + statistic: "hydra_shared_grid_collision_bound".into(), + }, + |value| BoundExpr::Constant { value }, + ), ], }, failure_probability: ProbabilityExpr::UnionBound { terms: vec![ inner.failure_probability.clone(), - ProbabilityExpr::Unknown { - statistic: "hydra_shared_grid_failure_probability".into(), - }, + stats.hydra_shared_grid_failure_probability.map_or_else( + || ProbabilityExpr::Unknown { + statistic: "hydra_shared_grid_failure_probability".into(), + }, + |value| ProbabilityExpr::Constant { value }, + ), ], }, provenance, @@ -372,6 +423,7 @@ fn hydra_guarantee(inner: &ResultGuarantee) -> ResultGuarantee { #[cfg(test)] mod tests { use super::*; + use crate::accuracy::{DefaultAccuracyModel, EqualSplitAllocator}; use asap_types::post_asap::ErrorMetric; use asap_types::pre_asap::agg_intent::{default_cardinality, default_quantile}; use asap_types::pre_asap::query_expr::Source; @@ -446,7 +498,7 @@ mod tests { failure_probability: ProbabilityExpr::Constant { value: 0.02 }, provenance: vec![], }; - let composed = hydra_guarantee(&inner); + let composed = hydra_guarantee(&inner, &PropagationStats::default()); assert_eq!(composed.metric, ErrorMetric::Frequency); assert!(matches!( @@ -533,6 +585,49 @@ mod tests { assert!(replacements.is_empty(), "{replacements:?}"); } + struct ZeroSharedGridEvidence; + + impl AccuracyEvidenceProvider for ZeroSharedGridEvidence { + fn propagation_stats( + &self, + _op: &CompositionOperator, + _family: &SummaryFamilyType, + _query: Option<&asap_types::post_asap::SketchQuery>, + ) -> PropagationStats { + PropagationStats { + hydra_shared_grid_collision_bound: Some(0.0), + hydra_shared_grid_failure_probability: Some(0.0), + ..Default::default() + } + } + } + + #[test] + fn hydra_shared_grid_evidence_is_consumed_by_candidate_construction() { + let intent = AggIntent::Count { + accuracy: AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.01, + }, + }; + let q = Rc::new(agg(vec![2], intent, metric_scan(&["job"]))); + let strategy = HydraGroupingStrategy::with_models_and_evidence( + &DefaultCostModel, + &DefaultAccuracyModel, + &EqualSplitAllocator, + &ZeroSharedGridEvidence, + ); + let replacements = strategy.replacements(&TargetSubDAG::new(&q)); + assert_eq!(replacements.len(), 2, "{replacements:?}"); + assert!(replacements.iter().all(|candidate| matches!( + &candidate.replacement, + Replacement::Summary(node) + if node.guarantee.as_ref().is_some_and(|g| + g.bound.evaluate().is_some() + && g.failure_probability.evaluate().is_some()) + ))); + } + #[test] fn cardinality_has_no_hydra_candidate_yet() { // summary_candidates(Cardinality) = [Hll, Theta, Kmv] — none have a diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 69a5b4a..d8adda7 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -193,8 +193,9 @@ pub mod rollup; pub mod topk_reuse; pub use accuracy::{ - AccuracyAllocation, AccuracyBudgetAllocator, AccuracyModel, CompositionShape, - DefaultAccuracyModel, EqualSplitAllocator, PropagationStats, + AccuracyAllocation, AccuracyBudgetAllocator, AccuracyEvidenceProvider, AccuracyModel, + CompositionShape, DefaultAccuracyModel, EqualSplitAllocator, NoAccuracyEvidence, + PropagationStats, }; pub use accuracy_reconciliation::AccuracyReconciliationStrategy; pub use cost_model::{CostModel, DefaultCostModel}; diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 60b3bed..5c40e01 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -365,8 +365,8 @@ use std::rc::Rc; use thiserror::Error; use crate::accuracy::{ - AccuracyBudgetAllocator, AccuracyModel, CompositionShape, DefaultAccuracyModel, - EqualSplitAllocator, PropagationStats, KLL_RANK_ERROR_COEFFICIENT_99, + AccuracyBudgetAllocator, AccuracyEvidenceProvider, AccuracyModel, CompositionShape, + DefaultAccuracyModel, EqualSplitAllocator, NoAccuracyEvidence, KLL_RANK_ERROR_COEFFICIENT_99, KLL_RANK_ERROR_EXPONENT_99, }; use crate::accuracy_reconciliation::AccuracyReconciliationStrategy; @@ -908,7 +908,7 @@ pub fn default_size_params( depth: cms_depth(delta), }, SketchAlgorithm::Hll => SketchParams::Hll { - precision: hll_precision_99(eps), + precision: hll_precision(eps), }, SketchAlgorithm::CmsWithHeap => { let k = match intent { @@ -1079,17 +1079,11 @@ fn kll_k(eps: f64) -> u32 { ) } -/// HLL standard-error inversion. The committed guarantee applies the -/// 99%-confidence conversion below before calling this helper. +/// HLL RSE-magnitude inversion. Generic HLL has no modeled confidence target. fn hll_precision(eps: f64) -> u8 { saturating_ceil((1.04 / eps).powi(2).log2(), 4, 18) as u8 } -/// 99%-confidence HLL relative bound via Chebyshev: ten times the RSE. -fn hll_precision_99(eps: f64) -> u8 { - hll_precision(eps / 10.0) -} - /// CMS: over-count ≤ ε·N with width `w = ⌈e/ε⌉` columns. fn cms_width(eps: f64) -> u32 { saturating_ceil(std::f64::consts::E / eps, 2, 1 << 26) @@ -1144,6 +1138,7 @@ fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; static DEFAULT_ACCURACY_MODEL: DefaultAccuracyModel = DefaultAccuracyModel; static DEFAULT_ALLOCATOR: EqualSplitAllocator = EqualSplitAllocator; +static NO_ACCURACY_EVIDENCE: NoAccuracyEvidence = NoAccuracyEvidence; /// The three deployment-pluggable models one candidate construction /// consults, bundled so the construction path threads one argument rather @@ -1155,6 +1150,7 @@ pub(crate) struct Models<'a> { pub cost: &'a dyn CostModel, pub accuracy: &'a dyn AccuracyModel, pub allocator: &'a dyn AccuracyBudgetAllocator, + pub evidence: &'a dyn AccuracyEvidenceProvider, } impl<'a> Models<'a> { @@ -1166,6 +1162,7 @@ impl<'a> Models<'a> { cost, accuracy: &DEFAULT_ACCURACY_MODEL, allocator: &DEFAULT_ALLOCATOR, + evidence: &NO_ACCURACY_EVIDENCE, } } } @@ -1228,6 +1225,25 @@ impl<'a> SketchAlgorithmStrategy<'a> { cost: cost_model, accuracy: accuracy_model, allocator, + evidence: &NO_ACCURACY_EVIDENCE, + }, + } + } + + /// Like [`Self::with_models`], with typed planning-time evidence for + /// rules such as TopK membership and Hydra shared-grid composition. + pub fn with_models_and_evidence( + cost_model: &'a dyn CostModel, + accuracy_model: &'a dyn AccuracyModel, + allocator: &'a dyn AccuracyBudgetAllocator, + evidence: &'a dyn AccuracyEvidenceProvider, + ) -> Self { + Self { + models: Models { + cost: cost_model, + accuracy: accuracy_model, + allocator, + evidence, }, } } @@ -1595,11 +1611,6 @@ pub fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { None } -/// Construct `expr`'s [`ReplacementSubDAG`] payload for one already-decided -/// [`Implementation`] of its top intent — the mechanical half of -/// [`SketchAlgorithmStrategy::replacements`], called once per candidate that -/// method enumerates. -/// /// `expr` must still be the [`bindable_intent`] shape for `implementation` to /// have any effect; anything else falls back to [`keep_pre_asap`]. /// Only `expr`'s own top-level decision is forced — recursion into `expr`'s @@ -1614,19 +1625,7 @@ pub fn bindable_intent(node: &QueryExpr) -> Option<&AggIntent> { /// candidate gets exactly the same schema derivation/column /// resolution/readout construction as every other candidate, patching only /// the `grouping` field this axis owns. -pub(crate) fn construct_summary( - expr: &QueryExpr, - implementation: Implementation, - cost_model: &dyn CostModel, -) -> Result, ImplementError> { - let models = Models::with_default_accuracy(cost_model); - match bindable_intent(expr) { - Some(intent) => construct_summary_with(expr, intent, implementation, models, None, None), - None => keep_pre_asap_rc(Rc::new(expr.clone())), - } -} - -/// [`construct_summary`] with every model explicit (issue #172). `intent` +/// Construct a summary with every model explicit (issue #172). `intent` /// is `expr`'s own [`bindable_intent`], or a copy of it with an allocated /// `AccuracyTarget` substituted (see [`realize_child_with`]). /// `child_target`, when set, is the end-to-end budget the child subtree is @@ -1741,6 +1740,7 @@ fn construct_summary_agg( &bound_child, intent, models.accuracy, + models.evidence, allocation, )?; @@ -1793,6 +1793,7 @@ fn compose_guarantee( child: &SummaryNode, intent: &AggIntent, accuracy: &dyn AccuracyModel, + evidence: &dyn AccuracyEvidenceProvider, allocation: Option, ) -> Result, AccuracyError> { let (op, local) = match (family, query) { @@ -1845,12 +1846,9 @@ fn compose_guarantee( if local.is_none() && input.is_exact() { return Ok(None); } - let mut guarantee = accuracy.propagate( - &op, - std::slice::from_ref(&input), - local.as_ref(), - &PropagationStats::default(), - )?; + let stats = evidence.propagation_stats(&op, family, query); + let mut guarantee = + accuracy.propagate(&op, std::slice::from_ref(&input), local.as_ref(), &stats)?; if let Some(note) = allocation { guarantee.provenance.push(note); } @@ -3783,6 +3781,7 @@ fn walk_children( #[cfg(test)] mod tests { use super::*; + use crate::accuracy::PropagationStats; use crate::cost_model::Cost; use asap_types::pre_asap::agg_intent::{ agg_is_exact, default_cardinality, default_quantile, MathFunc, TimeFunc, @@ -4005,12 +4004,12 @@ mod tests { } #[test] - fn default_cardinality_uses_the_tightest_supported_hll_precision() { + fn default_cardinality_sizes_hll_to_its_rse_magnitude() { assert_eq!( preferred(&default_cardinality()), Implementation::Sketch(SketchKind::new( SketchAlgorithm::Hll, - SketchParams::Hll { precision: 18 }, + SketchParams::Hll { precision: 14 }, )) ); } @@ -4392,7 +4391,7 @@ mod tests { } #[test] - fn cardinality_rejects_hll_when_its_parameter_cap_misses_the_target() { + fn cardinality_epsilon_keeps_hll_but_epsilon_delta_rejects_unknown_confidence() { let q = Rc::new(agg(vec![2], default_cardinality(), metric_scan(&["job"]))); let target = TargetSubDAG::new(&q); let replacements = SketchAlgorithmStrategy::default_cost_model().replacements(&target); @@ -4405,9 +4404,33 @@ mod tests { .collect(); assert_eq!( kinds, - vec![SketchAlgorithm::Theta, SketchAlgorithm::Kmv], - "the conservative 99% HLL bound cannot meet the target at precision <= 18" + vec![ + SketchAlgorithm::Hll, + SketchAlgorithm::Theta, + SketchAlgorithm::Kmv + ] ); + + let q = Rc::new(agg( + vec![2], + AggIntent::Cardinality { + col: None, + accuracy: AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.01, + }, + }, + metric_scan(&["job"]), + )); + let kinds: Vec<_> = SketchAlgorithmStrategy::default_cost_model() + .replacements(&TargetSubDAG::new(&q)) + .iter() + .map(|r| match &r.replacement { + Replacement::Summary(node) => summary_family_algorithm(node), + Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + }) + .collect(); + assert_eq!(kinds, vec![SketchAlgorithm::Theta, SketchAlgorithm::Kmv]); } #[test] @@ -4808,7 +4831,7 @@ mod tests { .groups() .find(|g| matches!(g.target.as_ref(), QueryExpr::Aggregate { .. })) .unwrap(); - assert_eq!(agg_group.candidates.len(), 2); + assert_eq!(agg_group.candidates.len(), 3); } #[test] @@ -6333,6 +6356,58 @@ mod tests { assert!(matches!(root.expr, SummaryExpr::KeepPreAsap(_))); } + struct SeparatedTopKEvidence; + + impl AccuracyEvidenceProvider for SeparatedTopKEvidence { + fn propagation_stats( + &self, + op: &CompositionOperator, + _family: &SummaryFamilyType, + _query: Option<&PostAsapSketchQuery>, + ) -> PropagationStats { + if matches!(op, CompositionOperator::TopKSelection) { + PropagationStats { + topk_selected_lower_bound: Some(101.0), + topk_excluded_upper_bound: Some(100.0), + topk_interval_failure_probability: Some(0.005), + ..Default::default() + } + } else { + PropagationStats::default() + } + } + } + + #[test] + fn topk_margin_evidence_is_consumed_by_candidate_construction() { + let q = Rc::new(agg( + vec![2], + AggIntent::TopK { + k: 5, + accuracy: AccuracyTarget::EpsilonDelta { + epsilon: 0.0, + delta: 0.01, + }, + }, + metric_scan(&["job"]), + )); + let strategy = SketchAlgorithmStrategy::with_models_and_evidence( + &DefaultCostModel, + &DefaultAccuracyModel, + &EqualSplitAllocator, + &SeparatedTopKEvidence, + ); + let replacements = strategy.replacements(&TargetSubDAG::new(&q)); + assert!(!replacements.is_empty()); + assert!(replacements.iter().all(|candidate| matches!( + &candidate.replacement, + Replacement::Summary(node) + if node.guarantee.as_ref().is_some_and(|g| + g.metric == ErrorMetric::TopKMembership + && g.failure_probability.evaluate() == Some(0.005)) + ))); + } + #[test] fn sql_reducer_resolves_named_input_column() { // SUM(bytes) over a tabular scan: `col` resolves positionally to the diff --git a/crates/integration-tests/tests/sql_to_post_asap.rs b/crates/integration-tests/tests/sql_to_post_asap.rs index f869cbb..43fae36 100644 --- a/crates/integration-tests/tests/sql_to_post_asap.rs +++ b/crates/integration-tests/tests/sql_to_post_asap.rs @@ -240,7 +240,7 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { /// distinct branch of the sketch-vs-exact decision than the quantile test /// above. #[tokio::test] -async fn sql_count_distinct_binds_a_confidence_certified_sketch_over_named_column() { +async fn sql_count_distinct_with_epsilon_binds_hll_rse_over_named_column() { let pre_asap = lower( "SELECT COUNT(DISTINCT service) FROM metrics", AccuracyTarget::Epsilon(0.01), @@ -275,7 +275,7 @@ async fn sql_count_distinct_binds_a_confidence_certified_sketch_over_named_colum assert_eq!( family, &SummaryFamilyType::Sketch( - SketchKind::new(SketchAlgorithm::Theta, SketchParams::Theta { k: 1_000_002 }), + SketchKind::new(SketchAlgorithm::Hll, SketchParams::Hll { precision: 14 }), GroupingStrategy::default() ) ); diff --git a/crates/types/src/post_asap/guarantee.rs b/crates/types/src/post_asap/guarantee.rs index 016e82e..696c6cc 100644 --- a/crates/types/src/post_asap/guarantee.rs +++ b/crates/types/src/post_asap/guarantee.rs @@ -79,11 +79,9 @@ pub enum ErrorMetric { /// CountSketch uses this normalization; it is intentionally distinct /// from CMS's [`ErrorMetric::Frequency`] (`L1`) guarantee. L2Frequency, - /// The returned key set equals the true top-k set. No shipped model - /// produces this yet: it needs a per-key interval margin certificate - /// (issue #172, PR 3). Present so the vocabulary can name the metric a - /// `TopK` readout would need without pretending a frequency bound is - /// one. + /// The returned key set equals the true top-k set. The built-in model + /// produces this only from a supplied per-key interval margin certificate; + /// a frequency bound alone is insufficient. TopKMembership, } @@ -253,6 +251,9 @@ pub enum GuaranteeSource { /// The concrete sketch a readout's local guarantee was derived from. SketchReadout { algorithm: String, + /// Stable estimator/analysis contract used to derive this guarantee. + #[serde(default)] + contract: String, params: serde_json::Value, query: String, }, diff --git a/docs/design_docs/asap-aware-mapping/summary_properties.md b/docs/design_docs/asap-aware-mapping/summary_properties.md index 63bfa37..e662241 100644 --- a/docs/design_docs/asap-aware-mapping/summary_properties.md +++ b/docs/design_docs/asap-aware-mapping/summary_properties.md @@ -123,15 +123,16 @@ semantics: - CMS uses an L1-frequency bound, while CountSketch uses an L2-frequency bound and is sized from both width and odd median depth. The two are not interchangeable. -- KLL, HLL, KMV, and Theta expose conservative 99%-confidence guarantees - derived from their committed parameters. A tighter requested `delta` is - rejected unless another implementation supplies a stronger contract. +- KLL, KMV, and Theta expose identified 99%-confidence contracts derived from + committed parameters. HLL retains its RSE magnitude but has unknown failure + probability because its current parameters encode precision, not a + confidence-level budget; it therefore cannot satisfy `EpsilonDelta`. - TopK membership is certified only when the widened confidence interval of the kth selected item is strictly above every excluded item's widened interval. Missing or overlapping interval evidence fails closed. - Hydra adds its shared-grid collision error to the inner sketch error and - union-bounds their failure probabilities. Until deployment/data statistics - instantiate the shared-grid terms, the expressions remain symbolic and an + union-bounds their failure probabilities. A typed evidence provider may + instantiate the shared-grid terms; without it they remain symbolic and an accuracy-targeted Hydra candidate is not admitted. The default `AccuracyModel` is deliberately conservative. It supports diff --git a/docs/developer_docs/end-to-end-accuracy-guarantees.md b/docs/developer_docs/end-to-end-accuracy-guarantees.md index d421a81..5512f13 100644 --- a/docs/developer_docs/end-to-end-accuracy-guarantees.md +++ b/docs/developer_docs/end-to-end-accuracy-guarantees.md @@ -119,7 +119,7 @@ contract and otherwise fails closed. The coefficients are specific to the cited implementation contract. See [Apache DataSketches KLL accuracy](https://datasketches.apache.org/docs/KLL/KLLAccuracyAndSize.html) and its -[C++ contract](https://github.com/apache/datasketches-cpp/blob/master/kll/include/kll_sketch.hpp). +[C++ contract pinned at `a9b42755072b`](https://github.com/apache/datasketches-cpp/blob/a9b42755072b079fd90b29b9851adc121015c58e/kll/include/kll_sketch.hpp). ### DDSketch @@ -130,18 +130,19 @@ delta = 0 ### HLL -The modeled estimator contract is: +Generic HLL exposes only its RSE magnitude: ```text RSE = 1.04 / sqrt(2^p) -epsilon_cardinality = 10 * 1.04 / sqrt(2^p) -delta = 0.01 +epsilon_cardinality = 1.04 / sqrt(2^p) +delta = unknown ``` -Sizing inverts the bound. Reject HLL when the supported precision cap misses -the target. The confidence claim requires an estimator whose bias and variance -premises justify the modeled interval; do not attach it to a generic HLL family -without that implementation contract. +Sizing inverts the magnitude. HLL's current parameter model contains precision +only; it does not encode a confidence-level budget. HLL may therefore satisfy +`Epsilon` but cannot satisfy `EpsilonDelta`. Supporting a confidence target +would require an explicitly identified estimator contract and corresponding +parameters whose semantics prove the requested failure probability. ### KMV and Theta @@ -191,7 +192,9 @@ with probability at most `1/3`. Zero or even depth has no modeled guarantee. ASAPPlanner contains the guarantee algebra and parameter-derived contracts; it does not import `asap_sketchlib`. Data- or runtime-dependent evidence enters -through explicit statistics fields and is recorded in provenance. +through an `AccuracyEvidenceProvider`, is exposed to propagation as typed +`PropagationStats`, and is recorded in provenance. `NoAccuracyEvidence` is the +default and preserves fail-closed behavior. ### TopK membership