diff --git a/crates/asap-aware-mapping/src/accuracy.rs b/crates/asap-aware-mapping/src/accuracy.rs index a9c02249..4e6b2d40 100644 --- a/crates/asap-aware-mapping/src/accuracy.rs +++ b/crates/asap-aware-mapping/src/accuracy.rs @@ -88,6 +88,10 @@ pub struct PropagationStats { /// of groups a `sum` folds), for `ExactSum`/`ExactExtremum`'s union /// bound over per-input failures. pub input_row_count: Option, + /// Fresh key-frequency distribution evidence from the data workload. + /// Built-in rules preserve it for deployment-specific accuracy models; + /// they do not assume a favorable distribution when it is absent. + pub data_distribution: Option, /// Lower confidence bound of the kth selected TopK item, after widening /// the interval by the sketch's own estimation error. pub topk_selected_lower_bound: Option, @@ -120,6 +124,29 @@ pub struct NoAccuracyEvidence; impl AccuracyEvidenceProvider for NoAccuracyEvidence {} +/// Accuracy evidence backed by the normalized data workload. Freshness is +/// checked at the planning time before values reach any accuracy rule. +#[derive(Debug, Clone, Copy)] +pub struct WorkloadAccuracyEvidence<'a> { + pub data: &'a asap_types::workload::DataWorkload, + pub now_ms: u64, +} + +impl AccuracyEvidenceProvider for WorkloadAccuracyEvidence<'_> { + fn propagation_stats( + &self, + _op: &CompositionOperator, + _family: &SummaryFamilyType, + _query: Option<&SketchQuery>, + ) -> PropagationStats { + PropagationStats { + input_row_count: self.data.input_cardinality.value_at(self.now_ms).copied(), + data_distribution: self.data.distribution.value_at(self.now_ms).cloned(), + ..PropagationStats::default() + } + } +} + /// The deployment-extensible accuracy algebra. `asap-aware-mapping` ships /// [`DefaultAccuracyModel`]; a deployment with a proof for a composition the /// default rejects (a registered cross-metric conversion, say) implements @@ -839,6 +866,7 @@ impl AccuracyBudgetAllocator for EqualSplitAllocator { mod tests { use super::*; use asap_types::post_asap::{GroupingStrategy, SketchKind}; + use asap_types::workload::{DataDistribution, DataWorkload, Evidence, EvidenceSource}; fn abs(bound: f64, delta: f64) -> ResultGuarantee { ResultGuarantee { @@ -865,6 +893,54 @@ mod tests { } } + #[test] + fn workload_accuracy_evidence_uses_only_fresh_data_characteristics() { + let data = DataWorkload { + input_cardinality: Evidence { + value: Some(42), + source: EvidenceSource::Observed, + observed_at_ms: Some(1_000), + valid_for_ms: Some(500), + }, + distribution: Evidence { + value: Some(DataDistribution::Bursty), + source: EvidenceSource::Observed, + observed_at_ms: Some(1_000), + valid_for_ms: Some(500), + }, + ..Default::default() + }; + let provider = WorkloadAccuracyEvidence { + data: &data, + now_ms: 1_500, + }; + let fresh = provider.propagation_stats( + &CompositionOperator::ExactSum, + &SummaryFamilyType::ExactAggregate( + asap_types::post_asap::ExactKind::Sum, + asap_types::post_asap::ExactParams::Sum, + ), + None, + ); + assert_eq!(fresh.input_row_count, Some(42)); + assert_eq!(fresh.data_distribution, Some(DataDistribution::Bursty)); + + let stale = WorkloadAccuracyEvidence { + data: &data, + now_ms: 1_501, + } + .propagation_stats( + &CompositionOperator::ExactSum, + &SummaryFamilyType::ExactAggregate( + asap_types::post_asap::ExactKind::Sum, + asap_types::post_asap::ExactParams::Sum, + ), + None, + ); + assert_eq!(stale.input_row_count, None); + assert_eq!(stale.data_distribution, None); + } + #[test] fn exact_child_contributes_zero_error() { let local = abs(0.05, 0.01); diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index b1780c8d..df060801 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -35,11 +35,14 @@ //! ## CSE sharing (issue #237, #223 stage 4) //! //! [`CseCandidate`]/[`ShareDecision`]/[`CostModel::cse_share_decision`] below -//! decide whether a CSE-detected shared subtree +//! provide the context-free fallback for whether a CSE-detected shared subtree //! ([`asap_types::pre_asap::cse::share_common_subtrees`], issue #223 stages //! 1-2, PR #235) is actually worth sharing, via a real Volcano/Cascades-style -//! cost comparison rather than a fixed rule. See -//! `docs/design_docs/cse-cost-model-decision.md` for the full design discussion (why +//! cost comparison rather than a fixed rule. Workload-aware selection uses +//! [`CostModel::cse_share_decision_with_recurrence`]; the target design also +//! expands each share candidate with its legal summary-maintenance lifecycles +//! before whole-plan ranking. See +//! `docs/design_docs/cost-model.md` for the full design discussion (why //! cost-based, why not a full plan-search engine, the layering constraint //! that forces detection to stay cost-agnostic). //! [`PlanSpace::cost_sorted`](crate::replacement::PlanSpace::cost_sorted) @@ -56,13 +59,221 @@ use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::QueryExpr; +use crate::exact_composition::{CompositionPlacement, ExactComposition}; use crate::recurrence::{ - self, Horizon, RecurrenceCostExplanation, RecurrenceError, RecurrenceProfile, + self, CostRate, EvaluationRate, Horizon, RecurrenceCostExplanation, RecurrenceError, + RecurrenceProfile, }; use crate::replacement::{ realize_child, Implementation, Replacement, ReplacementProvenance, ReplacementSubDAG, TargetSubDAG, }; +use crate::summary_maintenance_lifecycle::{ + SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCostInputs, +}; + +// ── Recurring-cost vocabulary for mixed exact/summary plans (issue #171) ── + +/// The unit a recurring cost is expressed in. One variant today; an enum so +/// a JSON/DAG export names the unit explicitly instead of a consumer +/// assuming it, and so a future per-resource unit can be added without +/// changing every hook's signature. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum CostUnit { + /// Abstract cost units per wall-clock second — the common currency + /// every recurring alternative (maintain-and-read vs. recompute-per-eval) + /// is compared in. + CostUnitsPerSecond, +} + +impl CostUnit { + /// Stable name for export (`"cost_units_per_second"`). + pub fn as_str(self) -> &'static str { + match self { + Self::CostUnitsPerSecond => "cost_units_per_second", + } + } +} + +/// Who produced a set of [`ExactCompositionCostInputs`], and under which +/// model version — carried into every composed decision's explanation and +/// DAG export so a reviewer can tell a deployment's measured numbers from +/// a placeholder. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CostProvenance { + /// The cost model's own name (e.g. `"DefaultCostModel"`). + pub model: String, + /// The model's own version string, whatever scheme it uses. + pub version: String, +} + +/// Which mixed-execution shapes the downstream runtime can actually +/// execute (issue #171). [`crate::exact_composition::ExactCompositionStrategy`] +/// proposes an `ExactPostProcess` candidate only when +/// `exact_post_process` is set, and an `ExactTransform` candidate only +/// when `exact_update_transform` is — a runtime that cannot run an exact +/// operator on the update path must never be handed one. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct MixedExecutionCapabilities { + /// The runtime can apply an exact operator to summary readouts at + /// query evaluation time. + pub exact_post_process: bool, + /// The runtime can apply an exact row transform on the update path, + /// feeding its output into maintained summary state. + pub exact_update_transform: bool, +} + +impl MixedExecutionCapabilities { + /// Neither shape supported. + pub const NONE: Self = Self { + exact_post_process: false, + exact_update_transform: false, + }; + /// Both shapes supported. + pub const ALL: Self = Self { + exact_post_process: true, + exact_update_transform: true, + }; + + pub fn supports(self, placement: CompositionPlacement) -> bool { + match placement { + CompositionPlacement::PostProcess => self.exact_post_process, + CompositionPlacement::Transform => self.exact_update_transform, + } + } +} + +/// What [`CostModel::exact_composition_cost_inputs`] is asked about: one +/// composed alternative at one site, paired with the concrete summary it +/// composes with. +#[derive(Debug, Clone, Copy)] +pub struct ExactCompositionCostRequest<'a> { + /// The pre-ASAP target the composed candidate replaces. + pub target: &'a QueryExpr, + /// The composition itself — phase, operator, child target. + pub composition: &'a ExactComposition, + /// For [`CompositionPlacement::PostProcess`]: the child target's *selected* + /// summary readout candidate the exact operator consumes. For + /// [`CompositionPlacement::Transform`]: the maintained summary *above* the + /// transform that consumes its output (the `SummaryAgg` this transform + /// feeds). Either way, the summary whose maintenance/read cost the + /// formula charges. + pub summary: &'a SummaryNode, + /// How many times this site actually runs once ancestors' own choices + /// are accounted for (see `PlanSpace::global_selection`). + pub effective_consumer_count: usize, +} + +/// Every input the issue #171 cost formulas need, each individually +/// optional: **an unknown stays `None` — never a zero** — so a formula +/// with a missing input yields no rate at all rather than a spuriously +/// cheap one, and global selection then keeps the conservative +/// `KeepPreAsap` behavior. A deployment model that wants defaults supplies +/// them explicitly by overriding [`CostModel::exact_composition_cost_inputs`]. +#[derive(Debug, Clone, PartialEq)] +pub struct ExactCompositionCostInputs { + /// Exact operator cost per row it processes — per readout row for a + /// post-process, per input row for an update-path transform. + pub exact_cost_per_row: Option, + /// Rows the exact operator consumes per evaluation (post-process) or + /// per update (transform). + pub expected_input_rows: Option, + /// Rows the exact operator emits per evaluation/update. + pub expected_output_rows: Option, + /// Cost of one update to the composed-with summary's maintained state. + pub summary_maintenance_cost_per_update: Option, + /// Cost of one readout of that summary at evaluation time. + pub summary_read_cost: Option, + /// Update (ingest) events per second reaching this site. + pub update_rate: Option, + /// Evaluations per second across every consumer of this site. + pub evaluation_rate: Option, + /// Cost of one full raw recompute of the target from pre-ASAP data — + /// the `KeepPreAsap` baseline's per-evaluation cost. + pub raw_recompute_cost: Option, + pub unit: CostUnit, + pub provenance: CostProvenance, +} + +impl ExactCompositionCostInputs { + /// Every input unknown, attributed to `provenance` — what a model that + /// has no statistics for a site returns. + pub fn unknown(provenance: CostProvenance) -> Self { + Self { + exact_cost_per_row: None, + expected_input_rows: None, + expected_output_rows: None, + summary_maintenance_cost_per_update: None, + summary_read_cost: None, + update_rate: None, + evaluation_rate: None, + raw_recompute_cost: None, + unit: CostUnit::CostUnitsPerSecond, + provenance, + } + } + + /// The rate for whichever phase `phase` names — + /// [`postprocess_plan_cost_rate`] or [`pretransform_plan_cost_rate`]. + pub fn composed_plan_cost_rate(&self, placement: CompositionPlacement) -> Option { + match placement { + CompositionPlacement::PostProcess => postprocess_plan_cost_rate(self), + CompositionPlacement::Transform => pretransform_plan_cost_rate(self), + } + } +} + +/// Outer exact post-process over a maintained summary: +/// +/// ```text +/// postprocess_plan_cost_rate = +/// update_rate * summary_maintenance_cost_per_update +/// + evaluation_rate * (summary_read_cost +/// + output_rows_per_eval * exact_postprocess_cost_per_row) +/// ``` +/// +/// `None` if any input is unknown — see [`ExactCompositionCostInputs`]. +pub fn postprocess_plan_cost_rate(inputs: &ExactCompositionCostInputs) -> Option { + let maintenance = inputs.update_rate? * inputs.summary_maintenance_cost_per_update?; + let per_eval = + inputs.summary_read_cost? + inputs.expected_output_rows? * inputs.exact_cost_per_row?; + let evaluation = inputs.evaluation_rate?.0 * per_eval; + finite_rate(maintenance + evaluation) +} + +/// Outer maintained summary over an exact update-time transform: +/// +/// ```text +/// pretransform_plan_cost_rate = +/// update_rate * (exact_transform_cost_per_input_row +/// + summary_maintenance_cost_per_update) +/// + evaluation_rate * summary_read_cost +/// ``` +/// +/// `None` if any input is unknown — see [`ExactCompositionCostInputs`]. +pub fn pretransform_plan_cost_rate(inputs: &ExactCompositionCostInputs) -> Option { + let per_update = inputs.exact_cost_per_row? + inputs.summary_maintenance_cost_per_update?; + let maintenance = inputs.update_rate? * per_update; + let evaluation = inputs.evaluation_rate?.0 * inputs.summary_read_cost?; + finite_rate(maintenance + evaluation) +} + +/// The raw/pre-ASAP fallback baseline: +/// +/// ```text +/// raw_recompute_cost_rate = evaluation_rate * raw_recompute_cost +/// ``` +/// +/// `None` if either input is unknown — see [`ExactCompositionCostInputs`]. +pub fn raw_recompute_cost_rate(inputs: &ExactCompositionCostInputs) -> Option { + finite_rate(inputs.evaluation_rate?.0 * inputs.raw_recompute_cost?) +} + +fn finite_rate(units_per_second: f64) -> Option { + units_per_second + .is_finite() + .then_some(CostRate(units_per_second)) +} /// A CSE-detected, legality-gated shared subtree with two or more consumers /// — the unit [`CostModel::cse_share_decision`] decides over. Built by @@ -71,7 +282,7 @@ use crate::replacement::{ /// needs a representative bound node for a subtree that /// [`asap_types::pre_asap::cse::share_common_subtrees`] already collapsed /// onto one `Rc` for two or more workload roots. See -/// `docs/design_docs/cse-cost-model-decision.md`. +/// `docs/design_docs/cost-model.md`. pub struct CseCandidate<'a> { /// The shared pre-ASAP subtree itself. pub subtree: &'a QueryExpr, @@ -152,12 +363,12 @@ pub fn default_cse_recompute_cost(subtree: &QueryExpr) -> Cost { Cost(asap_types::pre_asap::cse::dag_node_count(subtree) as f64) } -/// Default [`CostModel::cse_shared_maintenance_cost`]: a small +/// Default context-free [`CostModel::cse_shared_maintenance_cost`]: a small /// per-[`SummaryFamilyType`] weight, scaled to the same order of magnitude /// as [`default_cse_recompute_cost`]'s typical output (a small node /// count, not a byte length), reflecting that families differ in how -/// expensive they are to keep *continuously updated* for the life of a -/// workload — an exact accumulator is the cheapest (an O(1) merge), +/// expensive they are to maintain as shared state — an exact accumulator is +/// the cheapest (an O(1) merge), /// sketches/samples cost more (a whole data structure to update per new /// row), wavelets/fitted models cost the most (coefficient/parameter /// maintenance). These weights are illustrative, not measured — a @@ -305,19 +516,22 @@ pub trait CostModel { /// Estimate the one-time cost of recomputing `candidate.subtree` /// independently at a single use site. Default: /// [`default_cse_recompute_cost`] (a structural-size proxy). See - /// `docs/design_docs/cse-cost-model-decision.md`. + /// `docs/design_docs/cost-model.md`. fn cse_recompute_cost(&self, candidate: &CseCandidate) -> Cost { default_cse_recompute_cost(candidate.subtree) } - /// Estimate the cost of maintaining `candidate.bound_summary` as one - /// continuously-updated shared summary for the life of the workload. + /// Estimate a context-free proxy for maintaining `candidate.bound_summary` + /// as shared state. This fallback has no query recurrence, data arrival, + /// or horizon; workload-aware selection uses + /// [`Self::cse_share_decision_with_recurrence`], and full physical + /// selection additionally uses [`Self::summary_maintenance_lifecycle_cost_inputs`]. /// Default: [`default_cse_shared_maintenance_cost`] (a per-family /// weight table), applied to whichever field of /// `candidate.bound_summary`'s output schema actually carries summary /// state (falls back to the cheapest, `Plain`, weight if none does — /// e.g. `bound_summary` is a passthrough `KeepPreAsap` node with nothing - /// summary-shaped to maintain). See `docs/design_docs/cse-cost-model-decision.md`. + /// summary-shaped to maintain). See `docs/design_docs/cost-model.md`. fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> Cost { let family = candidate .bound_summary @@ -336,7 +550,7 @@ pub trait CostModel { /// Decide whether to reuse one shared `SummaryNode` across every /// consumer of `candidate`, or bind each occurrence independently — a /// Volcano/Cascades-style cost comparison (issue #237, #223 stage 4; see - /// `docs/design_docs/cse-cost-model-decision.md`): share iff the estimated cost of + /// `docs/design_docs/cost-model.md`): share iff the estimated cost of /// maintaining one shared summary is no greater than the estimated total /// cost of recomputing it independently everywhere it's used. /// @@ -504,6 +718,77 @@ pub trait CostModel { let _ = (candidate, target); f64::NAN } + + /// Which mixed exact/summary execution shapes the downstream runtime + /// advertises (issue #171). Gates candidate *generation* in + /// [`crate::exact_composition::ExactCompositionStrategy`]: a shape the + /// runtime can't execute is never proposed, so it can't be selected + /// either. + /// + /// Default: [`MixedExecutionCapabilities::ALL`]. The built-in model + /// describes no particular runtime, and leaving both shapes *visible* + /// in `PlanSpace` (for explanations and the DAG viewer) is the more + /// informative default; selection is still gated separately by + /// [`Self::exact_composition_cost_inputs`], whose default supplies no + /// statistics, so nothing is ever *committed* to under the built-in + /// model. A deployment whose runtime lacks a shape narrows this. + fn mixed_execution_capabilities(&self) -> MixedExecutionCapabilities { + MixedExecutionCapabilities::ALL + } + + /// The statistics the issue #171 recurring-cost formulas need for one + /// composed alternative — see [`ExactCompositionCostInputs`] for each + /// input and [`postprocess_plan_cost_rate`]/ + /// [`pretransform_plan_cost_rate`]/[`raw_recompute_cost_rate`] for how + /// they combine. One structured hook rather than eight scalar ones, so + /// a deployment answers them all from one place (and can attach its own + /// [`CostProvenance`]). + /// + /// Default: every input unknown ([`ExactCompositionCostInputs::unknown`]) + /// — unknown is never zero, and with no rate derivable + /// `PlanSpace::global_selection` keeps the conservative `KeepPreAsap` + /// behavior for the site. A deployment that wants defaults must supply + /// them here explicitly. + fn exact_composition_cost_inputs( + &self, + request: &ExactCompositionCostRequest<'_>, + ) -> ExactCompositionCostInputs { + let _ = request; + ExactCompositionCostInputs::unknown(CostProvenance { + model: "CostModel::exact_composition_cost_inputs (default)".into(), + version: "unknown".into(), + }) + } + + /// Primitive build, update, read, retention, and retirement costs used to + /// compare physical summary-state lifecycles. This is part of the same + /// cost model as candidate ranking and recurrence; summary maintenance + /// lifecycle planning does not introduce a second optimizer. + /// + /// The default leaves every value unknown, which prevents a long-lived + /// deployment from winning through optimistic zeroes. + fn summary_maintenance_lifecycle_cost_inputs( + &self, + _summary: &SummaryNode, + ) -> SummaryMaintenanceLifecycleCostInputs { + SummaryMaintenanceLifecycleCostInputs::default() + } + + /// Physical update/merge/delete support for one concrete summary. The + /// conservative default advertises no long-lived maintenance capability. + fn summary_maintenance_capabilities( + &self, + _summary: &SummaryNode, + ) -> SummaryMaintenanceCapabilities { + SummaryMaintenanceCapabilities::default() + } + + /// Cost of evaluating `target` directly from its logical/raw inputs once. + /// When known, summary-maintenance-aware materialization compares this + /// fallback with the aggregate cost of the selected summary deployments. + fn raw_query_recompute_cost(&self, _target: &QueryExpr) -> Option { + None + } } fn sketch_state( @@ -661,6 +946,12 @@ impl CostModel for DefaultCostModel { (self.cse_recompute_cost(&cse) * consumer_count).0 } } + // A composed candidate is costed in cost-units-per-second by + // `PlanSpace::global_selection` against the child decision it + // is committed with — a different unit from this structural + // estimate, and unknowable here without that child. `NaN` + // keeps it from ever out-ranking a real estimate by accident. + Replacement::ExactComposition(_) => f64::NAN, } } } @@ -805,6 +1096,73 @@ mod tests { ); } + // ── Recurring-cost formulas (issue #171) ───────────────────────────── + + fn known_inputs() -> ExactCompositionCostInputs { + ExactCompositionCostInputs { + exact_cost_per_row: Some(0.1), + expected_input_rows: Some(50.0), + expected_output_rows: Some(10.0), + summary_maintenance_cost_per_update: Some(0.01), + summary_read_cost: Some(1.0), + update_rate: Some(100.0), + evaluation_rate: Some(EvaluationRate(2.0)), + raw_recompute_cost: Some(100.0), + unit: CostUnit::CostUnitsPerSecond, + provenance: CostProvenance { + model: "test".into(), + version: "1".into(), + }, + } + } + + #[test] + fn composition_formulas_match_the_issue_definitions() { + let inputs = known_inputs(); + // 100 * 0.01 + 2 * (1 + 10 * 0.1) = 1 + 4 = 5 + assert_eq!(postprocess_plan_cost_rate(&inputs).unwrap().0, 5.0); + // 100 * (0.1 + 0.01) + 2 * 1 = 11 + 2 = 13 + assert!((pretransform_plan_cost_rate(&inputs).unwrap().0 - 13.0).abs() < 1e-9); + // 2 * 100 + assert_eq!(raw_recompute_cost_rate(&inputs).unwrap().0, 200.0); + assert_eq!( + crate::recurrence::total_cost(CostRate(5.0), Horizon(10.0), Cost(3.0)), + Cost(53.0) + ); + } + + #[test] + fn a_missing_input_yields_no_rate_not_zero() { + let mut inputs = known_inputs(); + inputs.summary_maintenance_cost_per_update = None; + assert_eq!(postprocess_plan_cost_rate(&inputs), None); + assert_eq!(pretransform_plan_cost_rate(&inputs), None); + // The baseline doesn't need maintenance and is still known. + assert!(raw_recompute_cost_rate(&inputs).is_some()); + let unknown = ExactCompositionCostInputs::unknown(known_inputs().provenance); + assert_eq!(raw_recompute_cost_rate(&unknown), None); + } + + #[test] + fn default_model_advertises_capabilities_but_no_statistics() { + assert_eq!( + DefaultCostModel.mixed_execution_capabilities(), + MixedExecutionCapabilities::ALL + ); + assert!(MixedExecutionCapabilities::NONE + .supports(CompositionPlacement::PostProcess) + .not()); + } + + trait Not { + fn not(self) -> bool; + } + impl Not for bool { + fn not(self) -> bool { + !self + } + } + // ── CSE sharing (issue #237, #223 stage 4) ────────────────────────── use asap_types::post_asap::{ diff --git a/crates/asap-aware-mapping/src/exact_composition.rs b/crates/asap-aware-mapping/src/exact_composition.rs new file mode 100644 index 00000000..5b23d854 --- /dev/null +++ b/crates/asap-aware-mapping/src/exact_composition.rs @@ -0,0 +1,673 @@ +//! [`ExactCompositionStrategy`] — composing an exact operator with a +//! summary plan across an explicit update/readout boundary (issue #171). +//! +//! ## The gap this closes +//! +//! `construct_summary_agg` already nests: a `SummaryAgg` recursively +//! realizes its child, so a KLL over an exact `Sum` accumulator, or a +//! `quantile(rate(...))` over a `Rate` accumulator, come out as one bound +//! DAG. What it cannot represent is an exact operator that is **not** +//! maintained summary state sitting next to a summary: +//! +//! - `max by (zone) (quantile_over_time(0.99, latency[5m]))` — the outer +//! `max` is an exact fold over the inner summary's *readout*. A `MinMax` +//! accumulator over that readout is data_state-illegal (a maintained summary +//! can't consume query-time values — see +//! `asap_types::post_asap::execution_data_state`), +//! and `avg` has no accumulator at all, so today either shape collapses +//! into one opaque `KeepPreAsap` that swallows the realizable inner +//! quantile. +//! - `quantile(0.99, deriv(m[5m]))` — the inner `deriv` is an exact, +//! per-sample transform with no accumulator form; the outer summary can +//! only consume it as an opaque raw `KeepPreAsap` blob today, with no +//! explicit "this row transform runs on the update path" node. +//! +//! [`SummaryExpr::ReadoutPostProcess`] and [`SummaryExpr::UpdateTransform`] +//! are the two data_state-explicit representations; this strategy is what +//! proposes them. +//! +//! ## Reference, don't select +//! +//! A composed candidate needs a child plan to compose *with* — the inner +//! quantile's own summary readout, say. This strategy deliberately does +//! **not** pick that child itself (the way `construct_summary_agg`'s +//! `realize_child` takes the head of the child's own ranking): a +//! [`Replacement::ExactComposition`] carries only the child *target* +//! (`ExactComposition::child_target`, the same `Rc` whose +//! `MemoGroup` in `PlanSpace` already holds every candidate for it). It is +//! [`PlanSpace::global_selection`](crate::replacement::PlanSpace::global_selection) +//! that commits the compatible parent/child pair — so the child's own +//! cost-model ranking, workload-wide effective consumer count, and shared +//! `Rc` identity (one inner summary serving two outer folds) all stay +//! correct, and a child that is also shared by an unrelated consumer is +//! maintained exactly once. `GlobalSelection::materialize` then links the +//! committed pair into one validated post-ASAP DAG. +//! +//! ## Proposal conditions +//! +//! A candidate is proposed only when all of these hold: +//! +//! - the target is a single-measure, `HAVING`-free exact aggregate; +//! - post-process: the child is a bindable aggregate that has at least one +//! readout-producing summary implementation (a sketch/sample/wavelet/ +//! model — the shapes a maintained accumulator can't sit above), and the +//! target's grouping keys resolve in the child's output schema; +//! transform: the target is a per-entity exact transform with no +//! accumulator form (its only implementation is `PassThrough`); +//! - the exact operator consumes only `Plain` values in its data_state — checked +//! again, structurally, when the pair is composed; +//! - the plugged-in [`CostModel`] advertises the matching +//! [`MixedExecutionCapabilities`](crate::cost_model::MixedExecutionCapabilities). +//! +//! `avg` gets a post-process candidate *and* keeps +//! [`crate::rewrite::AvgToSumOverCountStrategy`]'s rewrite in the same +//! group; the cost model picks between them, nothing here hard-codes one. +//! +//! ## What this strategy never does +//! +//! - Propose an `ExactPostProcess` for a position beneath a maintained +//! summary — data_state validation at composition rejects it as a typed +//! `ImplementError` regardless. +//! - Decide whether a composition is *worth it*: that is +//! `global_selection`'s job, using the issue's cost-units-per-second +//! formulas (see `crate::cost_model::postprocess_plan_cost_rate` and +//! siblings). Missing statistics keep the conservative `KeepPreAsap`. + +use std::rc::Rc; + +use asap_types::post_asap::execution_data_state::validate_execution_data_states_at; +use asap_types::post_asap::{ + exact_operator_output_schema, produced_data_state, CompositionOperator, ExactOperator, + ExecutionDataState, ExecutionDataStateError, SummaryExpr, SummaryNode, SummarySchema, + ValueOperator, +}; +use asap_types::pre_asap::agg_intent::AggIntent; +use asap_types::pre_asap::query_expr::{QueryExpr, Reduction}; +use asap_types::types::AccuracyTarget; + +use crate::cost_model::CostModel; +use crate::replacement::{ + bindable_intent, describe_intent, implementations_for_with, ImplementError, Implementation, + Replacement, ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, +}; +use crate::{AccuracyModel, DefaultAccuracyModel, PropagationStats}; + +/// Which side of the update/readout boundary an [`ExactComposition`]'s +/// exact operator executes on — selects the `SummaryExpr` variant +/// [`ExactComposition::compose`] builds. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum CompositionPlacement { + /// [`SummaryExpr::ReadoutPostProcess`]: after the child's readout. + PostProcess, + /// [`SummaryExpr::UpdateTransform`]: on the update path, feeding + /// maintained state above. + Transform, +} + +impl CompositionPlacement { + /// The availability the composed operator consumes and produces. + pub fn data_state(self) -> ExecutionDataState { + match self { + Self::PostProcess => ExecutionDataState::READ_ROWS, + Self::Transform => ExecutionDataState::MAINTENANCE_ROWS, + } + } + + pub fn provenance(self) -> ReplacementProvenance { + match self { + Self::PostProcess => ReplacementProvenance::ExactPostProcess, + Self::Transform => ReplacementProvenance::ExactTransform, + } + } +} + +/// The payload of a [`Replacement::ExactComposition`] candidate: an exact +/// operator, the placement it runs at, and a *reference* to the child target +/// it composes over — never an already-selected child plan (see the module +/// docs' "Reference, don't select"). +#[derive(Debug, Clone)] +pub struct ExactComposition { + pub placement: CompositionPlacement, + pub op: ExactOperator, + /// The pre-ASAP child the operator consumes; its `MemoGroup` holds the + /// candidates `global_selection` may commit this composition with. + pub child_target: Rc, + /// The composed node's output schema — the target's own pre-ASAP + /// output schema, lifted with every column `Plain` (an exact operator + /// only ever produces plain values). + pub schema: SummarySchema, +} + +impl ExactComposition { + /// Can `child` legally be this composition's input? Phase legality + /// (the child's produced data_state — a `KeepPreAsap` leaf takes the + /// phase this edge assigns) plus the plain-operand rule, checked + /// through the same schema derivation [`Self::compose`] uses. + pub fn accepts_child(&self, child: &SummaryNode) -> bool { + let phase_ok = match produced_data_state(&child.expr) { + None => true, + Some(avail) => avail == self.placement.data_state(), + }; + phase_ok && exact_operator_output_schema(&self.op, &child.schema).is_ok() + } + + /// Build the composed, data_state-validated node over `child`. Every edge of + /// the result (including everything beneath `child`) is checked by + /// `asap_types::post_asap::validate_execution_data_states`; an illegal + /// placement is a typed [`ImplementError::ExecutionDataState`], never deferred to a + /// runtime. + pub fn compose(&self, child: Rc) -> Result, ImplementError> { + self.compose_with_accuracy(child, &DefaultAccuracyModel) + } + + /// Compose using the caller's accuracy algebra. Exact operators do not + /// erase an approximate child's error: supported folds propagate it; + /// unsupported folds fail closed with a typed accuracy error. + pub fn compose_with_accuracy( + &self, + child: Rc, + accuracy_model: &dyn AccuracyModel, + ) -> Result, ImplementError> { + if let Some(produced) = produced_data_state(&child.expr) { + if produced != self.placement.data_state() { + let edge = match self.placement { + CompositionPlacement::Transform => "UpdateTransform.child", + CompositionPlacement::PostProcess => "ReadoutPostProcess.child", + }; + return Err(ImplementError::ExecutionDataState( + ExecutionDataStateError::IllegalChildPhase { + edge, + child: produced, + }, + )); + } + } + let schema = exact_operator_output_schema(&self.op, &child.schema)?; + let guarantee = match &child.guarantee { + None => None, + Some(input) => { + let operator = match &self.op { + ExactOperator::Aggregate { measures, .. } => match measures.as_slice() { + [AggIntent::Sum { .. }] => CompositionOperator::ExactSum, + [AggIntent::Min { .. } | AggIntent::Max { .. } | AggIntent::Avg { .. }] => { + CompositionOperator::ExactExtremum + } + // This placeholder is only used by the model's exact-input + // fast path. Approximate inputs correctly fail closed. + [_] => CompositionOperator::Lipschitz { constant: 1.0 }, + _ => CompositionOperator::Lipschitz { constant: 1.0 }, + }, + _ => CompositionOperator::Lipschitz { constant: 1.0 }, + }; + Some(accuracy_model.propagate( + &operator, + std::slice::from_ref(input), + None, + &PropagationStats::default(), + )?) + } + }; + let expr = match self.placement { + CompositionPlacement::PostProcess => SummaryExpr::ReadoutPostProcess { + child, + op: ValueOperator::Exact(self.op.clone()), + }, + CompositionPlacement::Transform => SummaryExpr::UpdateTransform { + child, + op: ValueOperator::Exact(self.op.clone()), + }, + }; + let node = Rc::new(SummaryNode { + expr, + schema, + guarantee, + }); + validate_execution_data_states_at(&node, self.placement.data_state())?; + Ok(node) + } + + /// Structural identity for `MemoGroup` dedup: same placement, same + /// operator, same child `Rc`. + pub(crate) fn same_as(&self, other: &Self) -> bool { + self.placement == other.placement + && self.op == other.op + && Rc::ptr_eq(&self.child_target, &other.child_target) + } +} + +/// Which exact reducers may run as a query-time fold over readout rows. +/// `Count` only at `Exact` accuracy (an approximate count is a sketch +/// target, not an exact fold). +fn is_post_process_reducer(intent: &AggIntent) -> bool { + matches!( + intent, + AggIntent::Sum { .. } + | AggIntent::Min { .. } + | AggIntent::Max { .. } + | AggIntent::Avg { .. } + | AggIntent::StdDev { .. } + | AggIntent::Variance { .. } + | AggIntent::Count { + accuracy: AccuracyTarget::Exact + } + ) +} + +/// Does `implementation` need a `SummaryEstimate` readout to yield a value +/// — i.e. is it a shape a maintained accumulator can't legally sit above? +fn needs_readout(implementation: &Implementation) -> bool { + matches!( + implementation, + Implementation::Sketch(_) + | Implementation::Sample { .. } + | Implementation::Wavelet { .. } + | Implementation::StatModel { .. } + ) +} + +/// The `(op, child)` of a post-process-shaped target, or `None`. +fn post_process_shape( + root: &QueryExpr, + cost_model: &dyn CostModel, +) -> Option<(ExactOperator, Rc, AggIntent)> { + let QueryExpr::Aggregate { + reduction, + measures, + output_names, + having: None, + child, + } = root + else { + return None; + }; + let Reduction::Reduce(by) = reduction else { + return None; + }; + if by.is_without() { + return None; + } + let [intent] = measures.as_slice() else { + return None; + }; + if !is_post_process_reducer(intent) { + return None; + } + let child_intent = bindable_intent(child)?; + if !implementations_for_with(child_intent, cost_model) + .iter() + .any(needs_readout) + { + return None; + } + // Grouping keys must resolve in the child's output schema — the same + // derivation the composed node's own schema will use. + root.output_schema().ok()?; + Some(( + ExactOperator::Aggregate { + reduction: reduction.clone(), + measures: measures.clone(), + output_names: output_names.clone(), + having: None, + }, + Rc::clone(child), + intent.clone(), + )) +} + +/// The `(op, child)` of a transform-shaped target — a per-entity exact +/// transform with no accumulator form — or `None`. +fn transform_shape( + root: &QueryExpr, + cost_model: &dyn CostModel, +) -> Option<(ExactOperator, Rc, AggIntent)> { + let QueryExpr::Aggregate { + reduction: Reduction::PerEntity, + measures, + output_names, + having: None, + child, + } = root + else { + return None; + }; + let [intent] = measures.as_slice() else { + return None; + }; + if !intent.is_per_series() { + return None; + } + // Exact accumulators (`Rate`/`Increase`) are already directly nestable + // as `SummaryAgg(ExactAggregate)`; only a pass-through transform needs + // an explicit update-path node. + if implementations_for_with(intent, cost_model) + .iter() + .any(|i| *i != Implementation::PassThrough) + { + return None; + } + root.output_schema().ok()?; + Some(( + ExactOperator::Aggregate { + reduction: Reduction::PerEntity, + measures: measures.clone(), + output_names: output_names.clone(), + having: None, + }, + Rc::clone(child), + intent.clone(), + )) +} + +/// Proposes [`Replacement::ExactComposition`] candidates — see the module +/// docs. Holds a [`CostModel`] only to ask it which mixed-execution shapes +/// the runtime advertises and which implementations the child has; it +/// never uses it to *rank* anything. +pub struct ExactCompositionStrategy<'a> { + cost_model: &'a dyn CostModel, +} + +static DEFAULT_COST_MODEL: crate::cost_model::DefaultCostModel = + crate::cost_model::DefaultCostModel; + +impl ExactCompositionStrategy<'static> { + /// A strategy consulting the built-in [`DefaultCostModel`](crate::cost_model::DefaultCostModel). + pub fn default_cost_model() -> Self { + Self { + cost_model: &DEFAULT_COST_MODEL, + } + } +} + +impl<'a> ExactCompositionStrategy<'a> { + pub fn new(cost_model: &'a dyn CostModel) -> Self { + Self { cost_model } + } + + fn candidates(&self, target: &TargetSubDAG<'_>) -> Vec { + let capabilities = self.cost_model.mixed_execution_capabilities(); + let Ok(schema) = target.root.output_schema() else { + return Vec::new(); + }; + let schema = asap_types::post_asap::execution_data_state::lift_plain(&schema); + let mut out = Vec::new(); + + if capabilities.supports(CompositionPlacement::PostProcess) { + if let Some((op, child, intent)) = post_process_shape(target.root, self.cost_model) { + let child_desc = describe_intent( + bindable_intent(&child).expect("checked by post_process_shape"), + ); + out.push(ReplacementSubDAG { + strategy: "ExactCompositionStrategy", + replacement: Replacement::ExactComposition(ExactComposition { + placement: CompositionPlacement::PostProcess, + op, + child_target: child, + schema: schema.clone(), + }), + provenance: ReplacementProvenance::ExactPostProcess, + rationale: format!( + "{} is an exact fold whose input is the readout of {} — a maintained \ + accumulator cannot consume query-time values, so instead of collapsing \ + the whole tree into KeepPreAsap this applies the fold as an \ + ExactPostProcess over whichever summary readout global_selection \ + commits for the child target (asap_aware_mapping::exact_composition)", + describe_intent(&intent), + child_desc + ), + }); + } + } + + if capabilities.supports(CompositionPlacement::Transform) { + if let Some((op, child, intent)) = transform_shape(target.root, self.cost_model) { + out.push(ReplacementSubDAG { + strategy: "ExactCompositionStrategy", + replacement: Replacement::ExactComposition(ExactComposition { + placement: CompositionPlacement::Transform, + op, + child_target: child, + schema, + }), + provenance: ReplacementProvenance::ExactTransform, + rationale: format!( + "{} is an exact per-entity transform with no accumulator form; as an \ + explicit ExactTransform on the update path its output can feed a \ + maintained summary above it instead of being handed over as an opaque \ + raw KeepPreAsap blob (asap_aware_mapping::exact_composition)", + describe_intent(&intent) + ), + }); + } + } + out + } +} + +impl ReplacementStrategy for ExactCompositionStrategy<'_> { + fn matches(&self, target: &TargetSubDAG<'_>) -> bool { + !self.candidates(target).is_empty() + } + + fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { + self.candidates(target) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cost_model::{DefaultCostModel, MixedExecutionCapabilities}; + use crate::replacement::keep_pre_asap; + use asap_types::post_asap::{ExecutionDataStateError, SketchAlgorithm, SummaryFamilyType}; + use asap_types::pre_asap::agg_intent::default_quantile; + use asap_types::pre_asap::query_expr::Source; + use asap_types::pre_asap::schema::{Column, DataType, Schema}; + + fn metric_scan(labels: &[&str]) -> QueryExpr { + let mut columns = vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + ]; + columns.extend(labels.iter().map(|n| Column::new(*n, DataType::Utf8, true))); + QueryExpr::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index(columns, 0, vec![]), + } + } + + fn agg(by: Vec, intent: AggIntent, child: QueryExpr) -> QueryExpr { + QueryExpr::Aggregate { + reduction: Reduction::by(by), + measures: vec![intent], + output_names: vec![], + having: None, + child: Rc::new(child), + } + } + + fn per_entity(intent: AggIntent, child: QueryExpr) -> QueryExpr { + QueryExpr::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![intent], + output_names: vec![], + having: None, + child: Rc::new(child), + } + } + + /// `max by (zone) (quantile by (zone, host) (m))`. + fn max_over_quantile() -> Rc { + let inner = agg( + vec![2, 3], + default_quantile(0.99), + metric_scan(&["zone", "host"]), + ); + Rc::new(agg(vec![0], AggIntent::Max { col: None }, inner)) + } + + #[test] + fn proposes_post_process_for_max_over_quantile() { + let root = max_over_quantile(); + let target = TargetSubDAG::new(&root); + let strategy = ExactCompositionStrategy::default_cost_model(); + assert!(strategy.matches(&target)); + let candidates = strategy.replacements(&target); + assert_eq!(candidates.len(), 1); + let Replacement::ExactComposition(comp) = &candidates[0].replacement else { + panic!( + "expected a composition, got {:?}", + candidates[0].replacement + ); + }; + assert_eq!(comp.placement, CompositionPlacement::PostProcess); + assert_eq!( + candidates[0].provenance, + ReplacementProvenance::ExactPostProcess + ); + let QueryExpr::Aggregate { child, .. } = root.as_ref() else { + unreachable!() + }; + assert!( + Rc::ptr_eq(&comp.child_target, child), + "the candidate references the child target's own Rc — nothing selected" + ); + let names: Vec<_> = comp.schema.fields.iter().map(|f| f.name.as_str()).collect(); + assert_eq!(names, vec!["zone", "max"]); + } + + #[test] + fn proposes_post_process_for_avg_over_quantile_alongside_the_rewrite() { + let inner = agg(vec![2], default_quantile(0.99), metric_scan(&["zone"])); + let root = Rc::new(agg(vec![0], AggIntent::Avg { col: None }, inner)); + let target = TargetSubDAG::new(&root); + assert_eq!( + ExactCompositionStrategy::default_cost_model() + .replacements(&target) + .len(), + 1 + ); + // `avg` competes with AvgToSumOverCountStrategy in the same group. + assert!(crate::rewrite::AvgToSumOverCountStrategy.matches(&target)); + } + + #[test] + fn proposes_transform_for_a_per_entity_pass_through_over_raw_input() { + let root = Rc::new(per_entity(AggIntent::Deriv, metric_scan(&["zone"]))); + let target = TargetSubDAG::new(&root); + let candidates = ExactCompositionStrategy::default_cost_model().replacements(&target); + assert_eq!(candidates.len(), 1); + assert_eq!( + candidates[0].provenance, + ReplacementProvenance::ExactTransform + ); + } + + #[test] + fn does_not_propose_for_shapes_already_covered_by_accumulators() { + // sum by (zone) over an exact Sum child: the child has no readout, + // so SummaryAgg(Sum) over SummaryAgg(Sum) is already legal. + let inner = agg( + vec![2, 3], + AggIntent::Sum { col: None }, + metric_scan(&["zone", "host"]), + ); + let root = Rc::new(agg(vec![0], AggIntent::Sum { col: None }, inner)); + assert!(!ExactCompositionStrategy::default_cost_model().matches(&TargetSubDAG::new(&root))); + // rate is an exact accumulator — directly nestable, no transform. + let rate = Rc::new(per_entity(AggIntent::Rate, metric_scan(&[]))); + assert!(!ExactCompositionStrategy::default_cost_model().matches(&TargetSubDAG::new(&rate))); + // A sketch-capable outer intent is not an exact fold. + let inner = agg(vec![2], default_quantile(0.5), metric_scan(&["zone"])); + let root = Rc::new(agg(vec![0], default_quantile(0.99), inner)); + assert!(!ExactCompositionStrategy::default_cost_model().matches(&TargetSubDAG::new(&root))); + } + + struct NoMixedExecution; + impl CostModel for NoMixedExecution { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + fn mixed_execution_capabilities(&self) -> MixedExecutionCapabilities { + MixedExecutionCapabilities::NONE + } + } + + #[test] + fn a_runtime_without_the_capability_gets_no_candidate() { + let root = max_over_quantile(); + let target = TargetSubDAG::new(&root); + let strategy = ExactCompositionStrategy::new(&NoMixedExecution); + assert!(!strategy.matches(&target)); + assert!(strategy.replacements(&target).is_empty()); + let deriv = Rc::new(per_entity(AggIntent::Deriv, metric_scan(&[]))); + assert!(!strategy.matches(&TargetSubDAG::new(&deriv))); + } + + #[test] + fn compose_rejects_a_maintained_state_child_for_a_post_process() { + let root = max_over_quantile(); + let target = TargetSubDAG::new(&root); + let candidates = ExactCompositionStrategy::default_cost_model().replacements(&target); + let Replacement::ExactComposition(comp) = &candidates[0].replacement else { + unreachable!() + }; + // A bare SummaryAgg (state, no readout) is not a legal post-process + // input — the operator would be consuming sketch state. + let state_child = + crate::replacement::realize_child(&comp.child_target, &DefaultCostModel).unwrap(); + let SummaryExpr::SummaryEstimate { summary_input, .. } = &state_child.expr else { + panic!("expected the child to realize to a readout"); + }; + assert!(!comp.accepts_child(summary_input)); + assert!(matches!( + comp.compose(Rc::clone(summary_input)), + Err(ImplementError::ExecutionDataState( + ExecutionDataStateError::IllegalChildPhase { .. } + )) + )); + // The readout itself is accepted and composes to a plain schema. + assert!(comp.accepts_child(&state_child)); + let composed = comp.compose(state_child).unwrap(); + assert!(matches!( + composed.expr, + SummaryExpr::ReadoutPostProcess { .. } + )); + assert!(composed + .schema + .fields + .iter() + .all(|f| matches!(f.dtype, SummaryFamilyType::Plain(_)))); + } + + #[test] + fn compose_rejects_a_readout_child_for_a_transform() { + let inner = agg(vec![2], default_quantile(0.99), metric_scan(&["zone"])); + let root = Rc::new(per_entity(AggIntent::Deriv, inner)); + let candidates = + ExactCompositionStrategy::default_cost_model().replacements(&TargetSubDAG::new(&root)); + let Replacement::ExactComposition(comp) = &candidates[0].replacement else { + unreachable!() + }; + let readout = + crate::replacement::realize_child(&comp.child_target, &DefaultCostModel).unwrap(); + assert!(!comp.accepts_child(&readout)); + assert!(matches!( + comp.compose(readout), + Err(ImplementError::ExecutionDataState( + ExecutionDataStateError::IllegalChildPhase { .. } + )) + )); + // Raw update input is fine. + let raw = keep_pre_asap(&comp.child_target).unwrap(); + assert!(comp.accepts_child(&raw)); + assert!(matches!( + comp.compose(raw).unwrap().expr, + SummaryExpr::UpdateTransform { .. } + )); + } +} diff --git a/crates/asap-aware-mapping/src/explanation.rs b/crates/asap-aware-mapping/src/explanation.rs index bcfee70a..3900c729 100644 --- a/crates/asap-aware-mapping/src/explanation.rs +++ b/crates/asap-aware-mapping/src/explanation.rs @@ -216,6 +216,13 @@ pub enum ExplanationKind { /// cross-subpopulation reuse entries, all the same underlying structural /// fact. CommonSubexpressionReuse, + /// A `TargetSubDAG`'s candidate list contains at least one + /// [`Replacement::ExactComposition`] — + /// [`crate::exact_composition::ExactCompositionStrategy`] found an exact + /// operator that can be composed with a summary plan across an explicit + /// update/readout boundary instead of collapsing the whole tree into + /// `KeepPreAsap` (issue #171). + ExactComposition, } /// Why a [`Replacement`] of `kind` exists at `location` (a human-readable @@ -319,6 +326,15 @@ fn findings_from_plan_space(space: &PlanSpace) -> Vec) -> Vec Option { + let reasons: Vec<&str> = group + .candidates + .iter() + .filter(|c| matches!(c.replacement, Replacement::ExactComposition(_))) + .map(|c| c.rationale.as_str()) + .collect(); + if reasons.is_empty() { + None + } else { + Some(reasons.join("; ")) + } +} + /// Does `group`'s candidate list contain a genuine sketch-family realization? /// If so, the finding's `reason` is every such candidate's own `rationale`, /// joined — this module does not invent new prose to restate why a candidate diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 613d5061..04dafdde 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -184,21 +184,28 @@ pub mod accuracy; pub mod accuracy_reconciliation; pub mod cost_model; +pub mod exact_composition; pub mod explanation; pub mod grouping; pub mod recurrence; pub mod replacement; pub mod rewrite; pub mod rollup; +pub mod summary_maintenance_lifecycle; pub mod topk_reuse; pub use accuracy::{ AccuracyAllocation, AccuracyBudgetAllocator, AccuracyEvidenceProvider, AccuracyModel, CompositionShape, DefaultAccuracyModel, EqualSplitAllocator, NoAccuracyEvidence, - PropagationStats, + PropagationStats, WorkloadAccuracyEvidence, }; pub use accuracy_reconciliation::AccuracyReconciliationStrategy; -pub use cost_model::{CostModel, DefaultCostModel}; +pub use cost_model::{ + postprocess_plan_cost_rate, pretransform_plan_cost_rate, raw_recompute_cost_rate, CostModel, + CostProvenance, CostUnit, DefaultCostModel, ExactCompositionCostInputs, + ExactCompositionCostRequest, MixedExecutionCapabilities, +}; +pub use exact_composition::{CompositionPlacement, ExactComposition, ExactCompositionStrategy}; pub use explanation::{ explain_replacements, explain_replacements_with, ExplanationKind, ReplacementExplanation, }; @@ -210,11 +217,21 @@ pub use recurrence::{ }; pub use replacement::{ default_strategies, default_strategies_with, search_workload, search_workload_with, - search_workload_with_targets, summary_candidates, GlobalSelection, ImplementError, - Implementation, Matcher, MemoGroup, PlanSpace, Proposals, RankedGroup, RecurrenceProfileMap, - RejectedCandidate, Replacement, ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, - SelectedGroup, SharedSubtreeStrategy, SketchAlgorithmStrategy, TargetSubDAG, - MAX_SEARCH_ITERATIONS, + search_workload_with_targets, summary_candidates, CompositionDecision, GlobalSelection, + ImplementError, Implementation, Matcher, MemoGroup, PlanSpace, Proposals, RankedGroup, + RecurrenceProfileMap, RejectedCandidate, Replacement, ReplacementProvenance, + ReplacementStrategy, ReplacementSubDAG, SelectedGroup, SharedSubtreeStrategy, + SketchAlgorithmStrategy, TargetSubDAG, MAX_SEARCH_ITERATIONS, }; pub use rewrite::AvgToSumOverCountStrategy; +pub use summary_maintenance_lifecycle::{ + global_selection_with_summary_maintenance_lifecycles, + materialize_with_summary_maintenance_lifecycles, plan_summary_maintenance_lifecycles, + MaterializeSummaryMaintenanceLifecycleError, SummaryMaintenanceCapabilities, + SummaryMaintenanceDeployment, SummaryMaintenanceLifecycleAlternative, + SummaryMaintenanceLifecycleCapabilities, SummaryMaintenanceLifecycleCostInputs, + SummaryMaintenanceLifecyclePlan, SummaryMaintenanceLifecyclePlanError, + SummaryMaintenanceLifecycleRejection, SummaryMaintenanceLifecycleSelectionError, + WorkloadDemand, +}; pub use topk_reuse::TopKLimitReuseStrategy; diff --git a/crates/asap-aware-mapping/src/recurrence.rs b/crates/asap-aware-mapping/src/recurrence.rs index 8b380584..01f43e59 100644 --- a/crates/asap-aware-mapping/src/recurrence.rs +++ b/crates/asap-aware-mapping/src/recurrence.rs @@ -73,7 +73,7 @@ //! //! ## Provenance of each new cost input //! -//! - [`EvaluationRate`]: derived from [`asap_types::workload::RepeatingEntry::interval`] +//! - [`EvaluationRate`]: derived from [`asap_types::workload::RepeatingEntry::demand`] //! values of every repeating consumer reaching a target (via //! [`evaluation_rate_of`], or [`crate::replacement::PlanSpace::recurrence_profiles`] //! for a whole workload). A one-shot ([`asap_types::workload::BatchEntry`]) @@ -200,6 +200,12 @@ pub enum RecurrenceError { well-defined maintained_cost_rate" )] InvalidUpdateRate(UpdateRate), + #[error("invalid EvaluationRate({0:?}Hz): an evaluation rate must be finite and >= 0")] + InvalidEvaluationRate(EvaluationRate), + #[error(transparent)] + InvalidWorkload(#[from] asap_types::workload::WorkloadError), + #[error("workload entry index {index} is out of bounds for {entry_count} entries")] + InvalidWorkloadEntry { index: usize, entry_count: usize }, /// A [`Horizon`] that isn't finite and strictly positive (NaN, /// infinite, zero, or negative) was supplied — a non-positive or /// infinite horizon would silently drop or invert the recurring @@ -374,16 +380,24 @@ impl RecurrenceProfile { /// already-opaque `Id` granularity `search_workload`'s callers already use /// — this crate needs no more of a caller's own query identity than "which /// of these two recurrence kinds is this root". -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum RootRecurrence { /// A one-shot (batch) root — contributes to a reached target's /// [`RecurrenceProfile::one_shot_consumers`], never to its /// `evaluation_rate`. OneShot, + /// A declared number of one-time invocations for this root. + OneShotCount(usize), /// A repeating root firing every `RepetitionInterval` — contributes to /// a reached target's `evaluation_rate` (`1 / interval`, aggregated via /// [`evaluation_rate_of`]). Repeating(RepetitionInterval), + /// A repeated root whose schedule or estimate has already been + /// normalized to evaluations per second. + RepeatingRate(EvaluationRate), + /// No reliable recurrence evidence was supplied. It contributes no read + /// count or evaluation rate, but remains distinct from zero demand. + Unknown, } // ── Explanation ────────────────────────────────────────────────────────── diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 5835adac..98984145 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -241,7 +241,7 @@ //! //! [`PlanSpace::cost_sorted`] is the `sorted_by(cost_model)` step, and it //! reuses this crate's existing [`CostModel`] trait rather than inventing a -//! second cost interface (`docs/design_docs/cse-cost-model-decision.md`, +//! second cost interface (`docs/design_docs/cost-model.md`, //! issue #237, explicitly reasoned about *why* a narrow, direct cost //! comparison was enough for the CSE share/recompute decision alone, and //! flagged that a real search engine — this module — is where that stops @@ -345,22 +345,26 @@ //! multi-group joint optimization beyond this per-site recurrence is left //! for whenever that changes. +use std::cell::RefCell; use std::collections::{HashMap, HashSet, VecDeque}; -use asap_types::post_asap::{AccuracyError, CompositionOperator, GuaranteeSource, ResultGuarantee}; use asap_types::post_asap::{ - ExactKind, ExactParams, GroupingStrategy, SamplingKind, SamplingParams, SketchAlgorithm, - SketchKind, SketchParams, SketchQuery as PostAsapSketchQuery, StatModelKind, StatModelParams, - SummaryExpr, SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, WaveletKind, - WaveletParams, + validate_execution_data_states_at, ExactKind, ExactOperatorSchemaError, ExactParams, + ExecutionDataState, ExecutionDataStateError, GroupingStrategy, SamplingKind, SamplingParams, + SketchAlgorithm, SketchKind, SketchParams, SketchQuery as PostAsapSketchQuery, StatModelKind, + StatModelParams, SummaryExpr, SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, + WaveletKind, WaveletParams, }; +use asap_types::post_asap::{AccuracyError, CompositionOperator, GuaranteeSource, ResultGuarantee}; use asap_types::pre_asap::agg_intent::{agg_is_mergeable, AggIntent}; use asap_types::pre_asap::cse::{share_common_subtrees, structural_hash, HashCache}; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::{QueryExpr, QueryExprError, Reduction}; use asap_types::pre_asap::schema::Schema; use asap_types::types::AccuracyTarget; -use asap_types::workload::RepetitionInterval; +use asap_types::workload::{ + ExpectedDemand, QueryRecurrence, QueryWorkload, RepeatedDemand, RepetitionInterval, +}; use std::rc::Rc; use thiserror::Error; @@ -370,10 +374,15 @@ use crate::accuracy::{ KLL_RANK_ERROR_EXPONENT_99, }; use crate::accuracy_reconciliation::AccuracyReconciliationStrategy; -use crate::cost_model::{CostModel, CseCandidate, DefaultCostModel, ShareDecision}; +use crate::cost_model::{ + raw_recompute_cost_rate, Cost, CostModel, CseCandidate, DefaultCostModel, + ExactCompositionCostInputs, ExactCompositionCostRequest, ShareDecision, +}; +use crate::exact_composition::{CompositionPlacement, ExactComposition, ExactCompositionStrategy}; use crate::grouping::HydraGroupingStrategy; use crate::recurrence::{ - evaluation_rate_of, Horizon, RecurrenceError, RecurrenceProfile, RootRecurrence, UpdateRate, + evaluation_rate_of, CostRate, Horizon, RecurrenceError, RecurrenceProfile, RootRecurrence, + UpdateRate, }; use crate::rollup::RollupStrategy; use crate::topk_reuse::TopKLimitReuseStrategy; @@ -396,6 +405,15 @@ pub enum ImplementError { /// records it as a [`RejectedCandidate`] instead of a candidate. #[error("accuracy-illegal candidate: {0}")] Accuracy(#[from] AccuracyError), + /// A constructed plan violates the update/readout phase contract + /// (issue #171) — e.g. a summary readout placed beneath a maintained + /// `SummaryAgg`. Detected at construction, never at runtime. + #[error("execution-data_state violation in post-ASAP plan: {0}")] + ExecutionDataState(#[from] ExecutionDataStateError), + /// An `ExactOperator`'s output schema could not be derived over its + /// child — the child carries summary state the operator can't read. + #[error("exact operator schema derivation failed: {0}")] + ExactOperatorSchema(#[from] ExactOperatorSchemaError), } /// A pre-ASAP sub-DAG a [`ReplacementStrategy`] knows how to replace. @@ -455,6 +473,15 @@ pub enum Replacement { /// different from the target's own `root` (e.g. sharing vs. not sharing /// a subtree) but semantically equivalent to it. Rewrite(Rc), + /// An exact operator composed over another target's *own* selected + /// decision across an explicit update/readout boundary (issue #171): + /// `ExactPostProcess` over a child's summary readout, or + /// `ExactTransform` feeding a maintained summary above. Carries only a + /// reference to the child target — [`PlanSpace::global_selection`] + /// commits the compatible parent/child pair and + /// [`GlobalSelection::materialize`] links it into one validated + /// `SummaryNode`. See [`crate::exact_composition`]. + ExactComposition(ExactComposition), } /// One candidate replacement for a [`TargetSubDAG`], plus a human-readable @@ -496,6 +523,12 @@ pub enum ReplacementProvenance { /// regardless, so pricing it like a full independent rebuild would be /// the wrong shape of cost, not just the wrong number. AccuracyReconciliation, + /// [`Replacement::ExactComposition`] with + /// [`CompositionPlacement::PostProcess`] (issue #171). + ExactPostProcess, + /// [`Replacement::ExactComposition`] with + /// [`CompositionPlacement::Transform`] (issue #171). + ExactTransform, } /// A candidate a strategy considered for a target but refused to propose on @@ -521,6 +554,7 @@ pub struct RejectedCandidate { pub struct Proposals { pub candidates: Vec, pub rejected: Vec, + domain_error: Option, } /// A replacement strategy: given a [`TargetSubDAG`], does this strategy have @@ -566,6 +600,7 @@ pub trait ReplacementStrategy { Proposals { candidates: self.replacements(target), rejected: Vec::new(), + domain_error: None, } } } @@ -1369,6 +1404,22 @@ impl<'a> SketchAlgorithmStrategy<'a> { ); } } + if proposals.candidates.is_empty() { + if let Some(error) = &proposals.domain_error { + if let Ok(node) = keep_pre_asap(root) { + proposals.candidates.push(ReplacementSubDAG { + strategy: "SketchAlgorithmStrategy", + replacement: Replacement::Summary(node), + provenance: ReplacementProvenance::SummaryImplementation, + rationale: format!( + "{} stays pre-ASAP because summary construction crosses an illegal \ + execution-data_state boundary ({error})", + describe_intent(intent) + ), + }); + } + } + } proposals } } @@ -1390,7 +1441,10 @@ impl Proposals { description: rationale, error, }), - Err(ImplementError::Schema(_)) => {} + Err(ImplementError::ExecutionDataState(error)) => { + self.domain_error.get_or_insert(error); + } + Err(ImplementError::Schema(_) | ImplementError::ExactOperatorSchema(_)) => {} } } } @@ -1538,10 +1592,10 @@ pub(crate) fn realize_child_with( .. }) => Ok(node), Some(ReplacementSubDAG { - replacement: Replacement::Rewrite(_), + replacement: Replacement::Rewrite(_) | Replacement::ExactComposition(_), .. }) => { - unreachable!("SketchAlgorithmStrategy never returns a Rewrite candidate") + unreachable!("SketchAlgorithmStrategy never returns a Rewrite/composition candidate") } // No candidate at all: `root` isn't `bindable_intent` shape (or its // intent has no realization `implementations_for_with` can't @@ -1762,6 +1816,10 @@ fn construct_summary_agg( // finalized value does. An exact accumulator's state is its value. guarantee: if estimate { None } else { guarantee.clone() }, }); + // Phase contract (issue #171): a maintained summary consumes update-path + // values or exact accumulator state — never a query-time readout. A + // typed error here, at construction; the caller decides the fallback. + validate_execution_data_states_at(&agg, ExecutionDataState::MAINTENANCE_SUMMARY)?; match query { // The readout: downstream of the estimate the schema is the plain // pre-ASAP row shape again (the summary-state type does not @@ -2079,10 +2137,13 @@ impl MemoGroup { (Replacement::Summary(existing_node), Replacement::Summary(node)) => { is_duplicate_summary(existing_node, node) } - // A `Rewrite` and a `Summary` are never the same candidate — - // they're different `Replacement` variants entirely. - (Replacement::Rewrite(_), Replacement::Summary(_)) - | (Replacement::Summary(_), Replacement::Rewrite(_)) => false, + ( + Replacement::ExactComposition(existing), + Replacement::ExactComposition(candidate), + ) => existing.same_as(candidate), + // Different `Replacement` variants are never the same + // candidate. + _ => false, } }); if is_duplicate { @@ -2179,6 +2240,34 @@ pub struct PlanSpace { order: Vec<*const QueryExpr>, } +/// Lifecycle-aware whole-subplan costs keyed by target and candidate pointer. +/// Built by `lifecycle` before final selection; kept internal so pointer keys +/// never become part of the public planner API. +#[derive(Default)] +pub(crate) struct CandidateCostOverrides { + costs: HashMap<(*const QueryExpr, *const ReplacementSubDAG), Cost>, +} + +impl CandidateCostOverrides { + pub(crate) fn insert( + &mut self, + target: &Rc, + candidate: &ReplacementSubDAG, + cost: Cost, + ) { + self.costs.insert( + (Rc::as_ptr(target), candidate as *const ReplacementSubDAG), + cost, + ); + } + + fn get(&self, target: &Rc, candidate: &ReplacementSubDAG) -> Option { + self.costs + .get(&(Rc::as_ptr(target), candidate as *const ReplacementSubDAG)) + .copied() + } +} + impl PlanSpace { /// Every discovered group, in discovery order. pub fn groups(&self) -> impl Iterator { @@ -2310,7 +2399,7 @@ impl PlanSpace { // ── Recurrence-aware cost context (issue #287) ────────────────────────── /// One [`RecurrenceProfile`] per discovered [`MemoGroup`] target, built by -/// [`PlanSpace::recurrence_profiles`] — the "carry `RepeatingEntry.interval` +/// [`PlanSpace::recurrence_profiles`] — the "carry `RepeatingEntry.demand` /// and relevant `DataWorkload` into ASAP-aware search/cost context" /// half of issue #287. Looked up by `Rc` pointer identity, the same /// currency [`PlanSpace::group_for`]/[`GlobalSelection::for_target`] already @@ -2421,8 +2510,18 @@ impl PlanSpace { if let Some(rate) = update_rate { crate::recurrence::validate_update_rate(rate)?; } + for recurrence in root_recurrence { + if let RootRecurrence::RepeatingRate(rate) = recurrence { + if !rate.0.is_finite() || rate.0 < 0.0 { + return Err(crate::recurrence::RecurrenceError::InvalidEvaluationRate( + *rate, + )); + } + } + } let mut intervals: HashMap<*const QueryExpr, Vec> = HashMap::new(); + let mut rates: HashMap<*const QueryExpr, f64> = HashMap::new(); let mut one_shot_counts: HashMap<*const QueryExpr, usize> = HashMap::new(); // Sites actually reached by at least one root's own recurrence tag // during the walk below — see this method's own "Unreachable @@ -2446,6 +2545,7 @@ impl PlanSpace { path_count, recurrence, &mut intervals, + &mut rates, &mut one_shot_counts, &mut reached, ); @@ -2470,7 +2570,12 @@ impl PlanSpace { let mut profiles = HashMap::with_capacity(self.order.len()); for ptr in &self.order { let site_intervals = intervals.get(ptr).unwrap_or(&empty_intervals); - let evaluation_rate = evaluation_rate_of(site_intervals.iter().copied())?; + let interval_rate = + evaluation_rate_of(site_intervals.iter().copied())?.map_or(0.0, |rate| rate.0); + let direct_rate = rates.get(ptr).copied().unwrap_or(0.0); + let evaluation_rate = ((interval_rate + direct_rate) > 0.0).then_some( + crate::recurrence::EvaluationRate(interval_rate + direct_rate), + ); let one_shot_consumers = one_shot_counts.get(ptr).copied().unwrap_or(0); // Bug 2 fix (see "Unreachable sites" above): only a reached // site carries the caller-supplied `update_rate`. @@ -2495,6 +2600,142 @@ impl PlanSpace { Ok(RecurrenceProfileMap { profiles }) } + + /// Derive per-target recurrence profiles directly from the normalized + /// query and data workloads. This is the authoritative bridge from the + /// public workload model into recurrence-aware candidate costing. + /// `root_workload_entries[i]` explicitly identifies the normalized + /// workload entry for `self.roots[i]`; callers need not arrange roots in + /// the batch-then-repeating storage order. + pub fn recurrence_profiles_from_workload( + &self, + workload: &QueryWorkload, + // For each `PlanSpace::roots[i]`, the explicit index of its + // corresponding normalized workload entry. + root_workload_entries: &[usize], + now_ms: u64, + horizon: Option, + ) -> Result { + workload.validate()?; + if let Some(horizon) = horizon { + if !horizon.0.is_finite() || horizon.0 <= 0.0 { + return Err(crate::recurrence::RecurrenceError::InvalidHorizon(horizon)); + } + } + if root_workload_entries.len() != self.roots.len() { + return Err(crate::recurrence::RecurrenceError::RootCountMismatch { + expected: self.roots.len(), + got: root_workload_entries.len(), + }); + } + let entries: Vec<_> = workload.entries().collect(); + let mut recurrences = Vec::with_capacity(root_workload_entries.len()); + for &index in root_workload_entries { + let entry = entries.get(index).ok_or( + crate::recurrence::RecurrenceError::InvalidWorkloadEntry { + index, + entry_count: entries.len(), + }, + )?; + let recurrence = match &entry.recurrence { + QueryRecurrence::OneTime { invocations, .. } => RootRecurrence::OneShotCount( + usize::try_from(*invocations).unwrap_or(usize::MAX), + ), + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(interval)) => { + RootRecurrence::Repeating(*interval) + } + QueryRecurrence::Repeated(RepeatedDemand::Scheduled(schedule)) => { + let Some(horizon) = horizon else { + return Err(crate::recurrence::RecurrenceError::MissingHorizon); + }; + let end_ms = now_ms.saturating_add((horizon.0 * 1000.0) as u64); + let count = schedule + .iter() + .filter(|at| at.0 >= now_ms && at.0 <= end_ms) + .count(); + RootRecurrence::RepeatingRate(crate::recurrence::EvaluationRate( + count as f64 / horizon.0, + )) + } + QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(estimate)) => { + if !estimate.is_fresh_at(now_ms) { + RootRecurrence::Unknown + } else { + let rate = match estimate.expected { + ExpectedDemand::AverageRate(rate) => rate.0, + ExpectedDemand::InvocationCount(count) => { + let millis = estimate + .observation_window + .end + .0 + .saturating_sub(estimate.observation_window.start.0); + count as f64 / (millis as f64 / 1000.0) + } + }; + RootRecurrence::RepeatingRate(crate::recurrence::EvaluationRate(rate)) + } + } + QueryRecurrence::Unknown => RootRecurrence::Unknown, + }; + recurrences.push(recurrence); + } + let update_rate = workload + .data_workload + .as_ref() + .and_then(|data| data.ingestion_rate.value_at(now_ms)) + .map(|rate| UpdateRate(rate.0)); + self.recurrence_profiles(&recurrences, update_rate) + } + + /// Map every discovered target to the normalized workload entries whose + /// roots can reach it. Each entry appears at most once per target even + /// when a root has several paths to that target; path multiplicity is a + /// separate recurrence/effective-use concern. + pub(crate) fn workload_entries_by_target( + &self, + workload: &QueryWorkload, + root_workload_entries: &[usize], + ) -> Result>, RecurrenceError> { + let entry_count = workload.entries().count(); + if root_workload_entries.len() != self.roots.len() { + return Err(RecurrenceError::RootCountMismatch { + expected: self.roots.len(), + got: root_workload_entries.len(), + }); + } + let mut bindings: HashMap<*const QueryExpr, HashSet> = HashMap::new(); + for ((_, root), &entry_index) in self.roots.iter().zip(root_workload_entries) { + if entry_index >= entry_count { + return Err(RecurrenceError::InvalidWorkloadEntry { + index: entry_index, + entry_count, + }); + } + let mut seen = HashSet::new(); + let mut queue = VecDeque::from([Rc::as_ptr(root)]); + while let Some(ptr) = queue.pop_front() { + if !seen.insert(ptr) { + continue; + } + bindings.entry(ptr).or_default().insert(entry_index); + if let Some(group) = self.groups.get(&ptr) { + queue.extend( + direct_child_counts(&group.target) + .into_iter() + .map(|(child, _)| child), + ); + } + } + } + Ok(bindings + .into_iter() + .map(|(ptr, entries)| { + let mut entries: Vec<_> = entries.into_iter().collect(); + entries.sort_unstable(); + (ptr, entries) + }) + .collect()) + } } /// Record `times` occurrences of `recurrence` against `ptr` — `times > 1` @@ -2508,6 +2749,7 @@ fn contribute( times: usize, recurrence: RootRecurrence, intervals: &mut HashMap<*const QueryExpr, Vec>, + rates: &mut HashMap<*const QueryExpr, f64>, one_shot_counts: &mut HashMap<*const QueryExpr, usize>, reached: &mut HashSet<*const QueryExpr>, ) { @@ -2522,9 +2764,16 @@ fn contribute( .or_default() .extend(std::iter::repeat_n(interval, times)); } + RootRecurrence::RepeatingRate(rate) => { + *rates.entry(ptr).or_insert(0.0) += rate.0 * times as f64; + } RootRecurrence::OneShot => { *one_shot_counts.entry(ptr).or_insert(0) += times; } + RootRecurrence::OneShotCount(count) => { + *one_shot_counts.entry(ptr).or_insert(0) += count.saturating_mul(times); + } + RootRecurrence::Unknown => {} } } @@ -2618,7 +2867,7 @@ fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a R .iter() .map(|c| match &c.replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) => None, + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, }) .collect(); if let Some(kinds) = kinds { @@ -2626,7 +2875,7 @@ fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a R ranked.sort_by_key(|c| { let kind = match &c.replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) => None, + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, }; kind.and_then(|k| order.iter().position(|o| *o == k)) .unwrap_or(usize::MAX) @@ -2649,6 +2898,59 @@ fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a R ranked } +/// Apply lifecycle-aware costs to summary siblings after the ordinary +/// strategy-specific ordering. Known lifecycle totals sort before unknown +/// totals; non-summary alternatives keep their existing relative order and +/// continue through their dedicated CSE/composition selection paths. +fn rank_group_with_candidate_costs<'a>( + group: &'a MemoGroup, + cost_model: &dyn CostModel, + overrides: Option<&CandidateCostOverrides>, +) -> Vec<&'a ReplacementSubDAG> { + let mut ranked = rank_group(group, cost_model); + let Some(overrides) = overrides else { + return ranked; + }; + let positions: Vec = ranked + .iter() + .enumerate() + .filter_map(|(index, candidate)| { + matches!(candidate.replacement, Replacement::Summary(_)).then_some(index) + }) + .collect(); + let mut summaries: Vec<_> = positions.iter().map(|&index| ranked[index]).collect(); + summaries.sort_by(|a, b| { + match ( + overrides.get(&group.target, a), + overrides.get(&group.target, b), + ) { + (Some(a), Some(b)) => a.0.total_cmp(&b.0), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + } + }); + for (index, candidate) in positions.into_iter().zip(summaries) { + ranked[index] = candidate; + } + ranked +} + +fn estimated_candidate_cost( + group: &MemoGroup, + candidate: &ReplacementSubDAG, + target: &TargetSubDAG<'_>, + cost_model: &dyn CostModel, + overrides: Option<&CandidateCostOverrides>, +) -> f64 { + overrides + .and_then(|costs| costs.get(&group.target, candidate)) + .map_or_else( + || cost_model.estimate_cost(candidate, target), + |cost| cost.0, + ) +} + /// For a group whose candidates are all [`Replacement::Rewrite`] (the /// [`SharedSubtreeStrategy`] shape): does [`CostModel::cse_share_decision`] /// prefer the candidate that shares `group.target`'s own `Rc` (`true`), or @@ -2754,6 +3056,32 @@ pub struct SelectedGroup<'a> { /// registered strategy proposed anything for (mirrors /// [`MemoGroup::candidates`] being possibly empty). pub chosen: Option<&'a ReplacementSubDAG>, + /// When `chosen` is a [`Replacement::ExactComposition`]: the child + /// decision it was committed together with, and the cost comparison + /// that justified it — the explicit target-to-decision provenance + /// chain (issue #171). + pub composition: Option>, +} + +/// Why [`PlanSpace::global_selection`] committed an exact composition at a +/// site: which child candidate it composes with, and the +/// cost-units-per-second comparison against the raw fallback that it won. +#[derive(Debug)] +pub struct CompositionDecision<'a> { + /// The child target the composed operator consumes. + pub child_target: &'a Rc, + /// For a post-process: the child's own candidate committed alongside + /// (the summary readout the operator folds). `None` for an update-path + /// transform, whose input is raw update data — its cost is charged to + /// the maintained summary *above* it instead. + pub child_candidate: Option<&'a ReplacementSubDAG>, + /// The composed plan's recurring rate — `postprocess_plan_cost_rate` + /// or `pretransform_plan_cost_rate`. + pub cost_rate: CostRate, + /// `raw_recompute_cost_rate` — the `KeepPreAsap` baseline it beat. + pub baseline_rate: CostRate, + /// The statistics (and their provenance) both rates were computed from. + pub inputs: ExactCompositionCostInputs, } /// [`PlanSpace::global_selection`]'s result: one [`SelectedGroup`] per @@ -2763,6 +3091,10 @@ pub struct SelectedGroup<'a> { pub struct GlobalSelection<'a> { order: Vec<*const QueryExpr>, groups: HashMap<*const QueryExpr, SelectedGroup<'a>>, + /// [`Self::materialize`]'s memo — one bound node per target for the + /// life of this selection, so two parents composing over one shared + /// child get the *same* `Rc`. + materialized: RefCell>>, } impl<'a> GlobalSelection<'a> { @@ -2777,6 +3109,272 @@ impl<'a> GlobalSelection<'a> { pub fn for_target(&self, target: &Rc) -> Option<&SelectedGroup<'a>> { self.groups.get(&Rc::as_ptr(target)) } + + /// Link this selection's per-site decisions into one data_state-validated + /// post-ASAP DAG rooted at `target` — the one place a committed + /// composition's child *reference* becomes an actual `Rc` + /// edge (issue #171). `None` if `target` is not a discovered site. + /// + /// Per site: a [`Replacement::ExactComposition`] composes over its + /// child target's own materialization; a [`Replacement::Summary`] is + /// re-linked so its `SummaryAgg` child is the child target's own + /// materialization whenever that is phase-legal beneath maintenance + /// (so a child that chose an `ExactTransform` actually ends up under + /// the summary); a [`Replacement::Rewrite`] or an unmatched site stays + /// the conservative `KeepPreAsap`. Memoized by target identity, so a + /// shared inner summary is one `Rc` no matter how many roots reach it. + pub fn materialize( + &self, + target: &Rc, + ) -> Result>, ImplementError> { + if !self.groups.contains_key(&Rc::as_ptr(target)) { + return Ok(None); + } + self.materialize_inner(target).map(Some) + } + + fn materialize_inner(&self, target: &Rc) -> Result, ImplementError> { + let ptr = Rc::as_ptr(target); + if let Some(node) = self.materialized.borrow().get(&ptr) { + return Ok(Rc::clone(node)); + } + let node = match self + .groups + .get(&ptr) + .and_then(|sel| sel.chosen) + .map(|c| &c.replacement) + { + None => keep_pre_asap(target)?, + Some(Replacement::Rewrite(rewritten)) => keep_pre_asap(rewritten)?, + Some(Replacement::Summary(node)) => self.relink_summary(node, target)?, + Some(Replacement::ExactComposition(composition)) => { + let child = self.materialize_inner(&composition.child_target)?; + let child = if composition.accepts_child(&child) { + child + } else { + keep_pre_asap(&composition.child_target)? + }; + composition.compose(child)? + } + }; + self.materialized.borrow_mut().insert(ptr, Rc::clone(&node)); + Ok(node) + } + + /// Re-link a bound `Summary` candidate's `SummaryAgg` child to the + /// child target's own materialization when that is legal beneath + /// maintenance; otherwise keep the candidate exactly as constructed. + fn relink_summary( + &self, + node: &Rc, + target: &Rc, + ) -> Result, ImplementError> { + let QueryExpr::Aggregate { + child: pre_child, .. + } = target.as_ref() + else { + return Ok(Rc::clone(node)); + }; + if !self.groups.contains_key(&Rc::as_ptr(pre_child)) { + return Ok(Rc::clone(node)); + } + let new_child = self.materialize_inner(pre_child)?; + Ok(relink_agg_child(node, &new_child)) + } +} + +/// Rebuild `node` (a `SummaryAgg`, possibly under a `SummaryEstimate`) with +/// `new_child` as the `SummaryAgg`'s child, if the result still validates +/// as maintained state; otherwise return `node` unchanged. +fn relink_agg_child(node: &Rc, new_child: &Rc) -> Rc { + match &node.expr { + SummaryExpr::SummaryEstimate { + summary_input, + query, + } => { + let inner = relink_agg_child(summary_input, new_child); + if Rc::ptr_eq(&inner, summary_input) { + return Rc::clone(node); + } + Rc::new(SummaryNode { + expr: SummaryExpr::SummaryEstimate { + summary_input: inner, + query: query.clone(), + }, + schema: node.schema.clone(), + guarantee: node.guarantee.clone(), + }) + } + SummaryExpr::SummaryAgg { + child, + family, + col, + reduction, + grouping, + } => { + if Rc::ptr_eq(child, new_child) { + return Rc::clone(node); + } + let rebuilt = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child: Rc::clone(new_child), + family: family.clone(), + col: col.clone(), + reduction: reduction.clone(), + grouping: grouping.clone(), + }, + schema: node.schema.clone(), + guarantee: node.guarantee.clone(), + }); + match validate_execution_data_states_at( + &rebuilt, + ExecutionDataState::MAINTENANCE_SUMMARY, + ) { + Ok(_) => rebuilt, + Err(_) => Rc::clone(node), + } + } + _ => Rc::clone(node), + } +} + +/// The maintained `SummaryAgg` a bound `Summary` candidate builds (under +/// its `SummaryEstimate` readout, if any) — the summary an `ExactTransform` +/// beneath it feeds, for `pretransform_plan_cost_rate`. +fn maintained_summary(node: &Rc) -> Option<&Rc> { + match &node.expr { + SummaryExpr::SummaryEstimate { summary_input, .. } => maintained_summary(summary_input), + SummaryExpr::SummaryAgg { .. } => Some(node), + _ => None, + } +} + +fn is_composition_candidate(candidate: &ReplacementSubDAG) -> bool { + matches!(candidate.replacement, Replacement::ExactComposition(_)) +} + +/// Everything [`PlanSpace::global_selection`] threads between sites for +/// exact compositions (issue #171): child candidates already committed by +/// an earlier parent, and the maintained summary above each site. +#[derive(Default)] +struct CompositionContext { + /// child target ptr → the child's candidate an ancestor's composition + /// already committed to (a later parent must compose with the *same* + /// one, and the child's own selection is forced to it). + committed_child: HashMap<*const QueryExpr, *const ReplacementSubDAG>, + /// site ptr → the maintained `SummaryAgg` directly above it, when its + /// parent chose a bound `Summary` — what an `ExactTransform` here feeds. + maintaining_parent: HashMap<*const QueryExpr, Rc>, +} + +/// One eligible composed alternative at a site, before the cheapest wins. +struct CompositionOption<'a> { + candidate: &'a ReplacementSubDAG, + decision: CompositionDecision<'a>, +} + +/// Every [`Replacement::ExactComposition`] candidate of `group` whose +/// composed-plan rate is *known* and beats the raw-recompute baseline — +/// costed against each compatible child candidate already in `PlanSpace` +/// (or the one an earlier parent committed). Unknown statistics yield no +/// option at all: the conservative `KeepPreAsap` path stays. +fn composition_options<'a>( + group: &'a MemoGroup, + groups: &'a HashMap<*const QueryExpr, MemoGroup>, + effective: usize, + cost_model: &dyn CostModel, + context: &CompositionContext, +) -> Vec> { + let mut options = Vec::new(); + for candidate in &group.candidates { + let Replacement::ExactComposition(composition) = &candidate.replacement else { + continue; + }; + let child_ptr = Rc::as_ptr(&composition.child_target); + let Some(child_group) = groups.get(&child_ptr) else { + continue; + }; + let already_committed = context.committed_child.get(&child_ptr).copied(); + let cost = |summary: &SummaryNode, shared: bool| { + let request = ExactCompositionCostRequest { + target: &group.target, + composition, + summary, + effective_consumer_count: effective, + }; + let mut inputs = cost_model.exact_composition_cost_inputs(&request); + if shared { + // Shared state is counted once: an earlier parent already + // pays this child's maintenance, so the marginal cost here + // is zero — a *known* zero, unlike an unknown input. + if let Some(maintenance) = inputs.summary_maintenance_cost_per_update.as_mut() { + *maintenance = 0.0; + } + } + let rate = inputs.composed_plan_cost_rate(composition.placement)?; + let baseline = raw_recompute_cost_rate(&inputs)?; + (rate < baseline).then_some((rate, baseline, inputs)) + }; + match composition.placement { + CompositionPlacement::PostProcess => { + let child_candidates: Vec<&'a ReplacementSubDAG> = match already_committed { + // SAFETY-free: the pointer was taken from `groups`'s own + // candidate storage, which outlives this borrow. + Some(ptr) => child_group + .candidates + .iter() + .filter(|c| std::ptr::eq(*c, ptr)) + .collect(), + None => child_group.candidates.iter().collect(), + }; + for child_candidate in child_candidates { + let Replacement::Summary(summary) = &child_candidate.replacement else { + continue; + }; + if !composition.accepts_child(summary) { + continue; + } + let Some((rate, baseline, inputs)) = cost(summary, already_committed.is_some()) + else { + continue; + }; + options.push(CompositionOption { + candidate, + decision: CompositionDecision { + child_target: &composition.child_target, + child_candidate: Some(child_candidate), + cost_rate: rate, + baseline_rate: baseline, + inputs, + }, + }); + } + } + CompositionPlacement::Transform => { + // An update-path transform only pays off beneath a + // maintained summary; with nothing above it, its output is + // never read and the raw fallback is the same computation. + let Some(parent) = context.maintaining_parent.get(&Rc::as_ptr(&group.target)) + else { + continue; + }; + let Some((rate, baseline, inputs)) = cost(parent, false) else { + continue; + }; + options.push(CompositionOption { + candidate, + decision: CompositionDecision { + child_target: &composition.child_target, + child_candidate: None, + cost_rate: rate, + baseline_rate: baseline, + inputs, + }, + }); + } + } + } + options } impl PlanSpace { @@ -2788,7 +3386,7 @@ impl PlanSpace { /// [`Self::cost_sorted`], whose per-group ranking only ever sees a /// group's own raw [`MemoGroup::consumer_count`]. pub fn global_selection(&self, cost_model: &dyn CostModel) -> GlobalSelection<'_> { - self.global_selection_impl(cost_model, None, None) + self.global_selection_impl(cost_model, None, None, None) .expect("structural global selection cannot produce a recurrence error") } @@ -2802,7 +3400,20 @@ impl PlanSpace { profiles: &RecurrenceProfileMap, horizon: Option, ) -> Result, RecurrenceError> { - self.global_selection_impl(cost_model, Some(profiles), horizon) + self.global_selection_impl(cost_model, Some(profiles), horizon, None) + } + + /// Final selection with lifecycle-aware whole-subplan cost overrides. + /// `lifecycle` builds the overrides from normalized workload evidence and + /// calls this only after candidate legality and accuracy validation. + pub(crate) fn global_selection_with_candidate_costs( + &self, + cost_model: &dyn CostModel, + profiles: &RecurrenceProfileMap, + horizon: Option, + candidate_costs: &CandidateCostOverrides, + ) -> Result, RecurrenceError> { + self.global_selection_impl(cost_model, Some(profiles), horizon, Some(candidate_costs)) } fn global_selection_impl( @@ -2810,6 +3421,7 @@ impl PlanSpace { cost_model: &dyn CostModel, profiles: Option<&RecurrenceProfileMap>, horizon: Option, + candidate_costs: Option<&CandidateCostOverrides>, ) -> Result, RecurrenceError> { let graph = reference_graph(self); let topo = topological_order(&self.order, &graph); @@ -2817,6 +3429,7 @@ impl PlanSpace { let mut effective_uses = graph.external_root_uses.clone(); let mut chosen_share: HashMap<*const QueryExpr, ShareDecision> = HashMap::new(); let mut groups: HashMap<*const QueryExpr, SelectedGroup<'_>> = HashMap::new(); + let mut context = CompositionContext::default(); for ptr in &topo { let group = &self.groups[ptr]; @@ -2824,7 +3437,50 @@ impl PlanSpace { let effective = effective_uses.get(ptr).copied().unwrap_or(0); effective_uses.insert(*ptr, effective); - let chosen = if effective >= 2 && cse_candidate_pair(group).is_some() { + // ── Exact compositions (issue #171) ───────────────────────── + // A child an earlier parent's composition committed to is + // forced to exactly that candidate — the parent/child pair is + // one decision. Otherwise, a composition here wins only when + // its cost-units-per-second rate is *known* and beats the raw + // recompute baseline; missing statistics keep the conservative + // path below. + let mut composition_decision = None; + let forced = context + .committed_child + .get(ptr) + .and_then(|&cptr| group.candidates.iter().find(|c| std::ptr::eq(*c, cptr))); + let composed = if forced.is_some() { + None + } else { + composition_options(group, &self.groups, effective, cost_model, &context) + .into_iter() + .min_by(|a, b| a.decision.cost_rate.0.total_cmp(&b.decision.cost_rate.0)) + }; + if let Some(option) = &composed { + if let Some(child_candidate) = option.decision.child_candidate { + context.committed_child.insert( + Rc::as_ptr(option.decision.child_target), + child_candidate as *const ReplacementSubDAG, + ); + } + if let Replacement::ExactComposition(composition) = &option.candidate.replacement { + if composition.placement == CompositionPlacement::Transform { + // A chain of transforms feeds the same summary. + if let Some(parent) = context.maintaining_parent.get(ptr).cloned() { + context + .maintaining_parent + .insert(Rc::as_ptr(&composition.child_target), parent); + } + } + } + } + + let chosen = if let Some(forced) = forced { + Some(forced) + } else if let Some(option) = composed { + composition_decision = Some(option.decision); + Some(option.candidate) + } else if effective >= 2 && cse_candidate_pair(group).is_some() { let decision = if let Some(profiles) = profiles { decide_group_with_recurrence( group, @@ -2844,18 +3500,44 @@ impl PlanSpace { let logical = group .candidates .iter() - .filter(|candidate| !is_cse_candidate(candidate)) + .filter(|candidate| { + !is_cse_candidate(candidate) && !is_composition_candidate(candidate) + }) .min_by(|a, b| { - cost_model - .estimate_cost(a, &effective_target) - .total_cmp(&cost_model.estimate_cost(b, &effective_target)) + estimated_candidate_cost( + group, + a, + &effective_target, + cost_model, + candidate_costs, + ) + .total_cmp( + &estimated_candidate_cost( + group, + b, + &effective_target, + cost_model, + candidate_costs, + ), + ) }); match (cse, logical) { (Some(cse), Some(logical)) - if cost_model - .estimate_cost(logical, &effective_target) - .total_cmp(&cost_model.estimate_cost(cse, &effective_target)) - .is_lt() => + if estimated_candidate_cost( + group, + logical, + &effective_target, + cost_model, + candidate_costs, + ) + .total_cmp(&estimated_candidate_cost( + group, + cse, + &effective_target, + cost_model, + candidate_costs, + )) + .is_lt() => { Some(logical) } @@ -2875,15 +3557,32 @@ impl PlanSpace { // valid answer, just not a cross-group-aware one; this // group also contributes no Share collapse to its own // children (see `multiplier`'s `_ => effective` arm). - None => rank_group(group, cost_model).into_iter().next(), + None => rank_group_with_candidate_costs(group, cost_model, candidate_costs) + .into_iter() + .find(|candidate| !is_composition_candidate(candidate)), } } else { - rank_group(group, cost_model) + rank_group_with_candidate_costs(group, cost_model, candidate_costs) .into_iter() - .find(|candidate| !is_cse_candidate(candidate)) + .find(|candidate| { + !is_cse_candidate(candidate) && !is_composition_candidate(candidate) + }) .or_else(|| cse_candidate_pair(group).map(|(share, _)| share)) }; + // Record the maintained summary this site's bound candidate + // builds, for a child that may compose an `ExactTransform` + // beneath it. + if let (Some(Replacement::Summary(node)), QueryExpr::Aggregate { child, .. }) = + (chosen.map(|c| &c.replacement), group.target.as_ref()) + { + if let Some(summary) = maintained_summary(node) { + context + .maintaining_parent + .insert(Rc::as_ptr(child), Rc::clone(summary)); + } + } + let outgoing_multiplier = multiplier(*ptr, &effective_uses, &chosen_share); match chosen { Some(ReplacementSubDAG { @@ -2901,7 +3600,9 @@ impl PlanSpace { _ => { let selected_rewrite = match chosen.map(|candidate| &candidate.replacement) { Some(Replacement::Rewrite(rewrite)) => rewrite, - Some(Replacement::Summary(_)) | None => &group.target, + Some(Replacement::Summary(_) | Replacement::ExactComposition(_)) | None => { + &group.target + } }; for (child, edge_count) in direct_child_counts(selected_rewrite) { *effective_uses.entry(child).or_insert(0) += @@ -2917,6 +3618,7 @@ impl PlanSpace { consumer_count: group.consumer_count, effective_consumer_count: effective, chosen, + composition: composition_decision, }, ); } @@ -2924,6 +3626,7 @@ impl PlanSpace { Ok(GlobalSelection { order: self.order.clone(), groups, + materialized: RefCell::new(HashMap::new()), }) } } @@ -3299,6 +4002,7 @@ pub fn default_strategies() -> Vec> { Box::new(HydraGroupingStrategy::default_cost_model()), Box::new(SharedSubtreeStrategy), Box::new(crate::rewrite::AvgToSumOverCountStrategy), + Box::new(ExactCompositionStrategy::default_cost_model()), ] } @@ -3313,6 +4017,7 @@ pub fn default_strategies_with<'a>( Box::new(HydraGroupingStrategy::new(cost_model)), Box::new(SharedSubtreeStrategy), Box::new(crate::rewrite::AvgToSumOverCountStrategy), + Box::new(ExactCompositionStrategy::new(cost_model)), ] } @@ -3406,6 +4111,7 @@ pub fn search_workload_with_targets<'s, Id>( .as_ref() .is_some_and(|g| accuracy_model.satisfies(g, &target)), Replacement::Rewrite(_) => true, + Replacement::ExactComposition(_) => false, }); group.candidates = legal; group.rejected.extend(illegal.into_iter().map(|candidate| { @@ -3426,6 +4132,11 @@ pub fn search_workload_with_targets<'s, Id>( None, )), Replacement::Rewrite(_) => unreachable!("rewrites are never rejected here"), + Replacement::ExactComposition(_) => ( + asap_types::post_asap::ErrorMetric::AbsoluteValue, + None, + None, + ), }; RejectedCandidate { strategy: candidate.strategy, @@ -4379,7 +5090,9 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { + panic!("expected a Summary replacement") + } }) .collect(); assert!(kinds.contains(&SketchAlgorithm::Kll), "{kinds:?}"); @@ -4399,7 +5112,9 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { + panic!("expected a Summary replacement") + } }) .collect(); assert_eq!( @@ -4427,7 +5142,9 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { + panic!("expected a Summary replacement") + } }) .collect(); assert_eq!(kinds, vec![SketchAlgorithm::Theta, SketchAlgorithm::Kmv]); @@ -4505,7 +5222,9 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { + panic!("expected a Summary replacement") + } }) .collect(); assert!(kinds.contains(&SketchAlgorithm::Kll)); @@ -4513,13 +5232,13 @@ mod tests { assert_eq!(kinds.len(), 2); } - /// Enumerating candidates for the *target* node must only steer that - /// node's own decision — a nested aggregate underneath it still gets its - /// own independent (`cost_model`-ranked) enumeration, not whatever the - /// caller happened to pick for the outer target. This is the behavior - /// [`construct_summary`]'s recursion (via [`realize_child`]) - /// gets for free: only the top node's `Implementation` is ever forced - /// from outside; the child is always re-enumerated fresh. + /// Constructing the outer target's candidates never leaks the outer + /// choice into the nested aggregate — and, since issue #171's phase + /// contract, a maintained outer sketch can no longer sit above the + /// inner sketch's *readout* at all: the outer target degrades to the + /// conservative `KeepPreAsap` fallback (reported once, not once per + /// dropped family), while the inner quantile keeps its own, + /// independently cost-ranked candidates in its own `MemoGroup`. #[test] fn enumerating_the_targets_candidates_does_not_leak_into_a_nested_aggregate() { // outer: quantile(0.99, ...) over inner: quantile(0.5, m) — both @@ -4539,35 +5258,41 @@ mod tests { ) .replacements(&target); - let ddsketch = replacements - .iter() - .find(|r| { - matches!(&r.replacement, Replacement::Summary(node) - if summary_family_algorithm(node) == SketchAlgorithm::DDSketch) - }) - .expect("the outer target's DDSketch candidate must be present"); - let Replacement::Summary(node) = &ddsketch.replacement else { - unreachable!("filtered on Replacement::Summary above"); + assert_eq!(replacements.len(), 1, "{replacements:?}"); + let Replacement::Summary(node) = &replacements[0].replacement else { + unreachable!("SketchAlgorithmStrategy only returns Summary candidates"); }; - assert_eq!( - summary_family_algorithm(node), - SketchAlgorithm::DDSketch, - "the outer (target) node must be the DDSketch candidate" + assert!( + matches!(node.expr, SummaryExpr::KeepPreAsap(ref e) if Rc::ptr_eq(e, &outer)), + "a sketch over a sketch readout is data_state-illegal; expected the conservative \ + fallback, got {:?}", + node.expr ); - - let asap_types::post_asap::SummaryExpr::SummaryEstimate { summary_input, .. } = &node.expr - else { - panic!("expected SummaryEstimate root, got {:?}", node.expr); - }; - let asap_types::post_asap::SummaryExpr::SummaryAgg { child, .. } = &summary_input.expr - else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); + assert!(replacements[0].rationale.contains("readout")); + + // The inner target is still independently enumerated and ranked — + // a custom cost model that prefers DDSketch for it is honored, and + // nothing about the outer target's choice reaches it. + let space = search_workload_with( + vec![("q", Rc::clone(&outer))], + &default_strategies_with(&PreferDDSketchViaCostModel), + ); + let QueryExpr::Aggregate { child, .. } = space.roots[0].1.as_ref() else { + unreachable!() }; + let inner_group = space.group_for(child).expect("inner quantile is a target"); + let inner_kinds: Vec = inner_group + .candidates + .iter() + .filter_map(|c| match &c.replacement { + Replacement::Summary(node) => sketch_kind_of(node), + _ => None, + }) + .collect(); assert_eq!( - summary_family_algorithm(child), - SketchAlgorithm::Kll, - "the nested inner aggregate must still get the cost-model-ranked \ - default (Kll), not inherit the outer target's DDSketch candidate" + inner_kinds, + vec![SketchAlgorithm::DDSketch, SketchAlgorithm::Kll], + "the nested inner aggregate keeps its own cost-model-ranked candidates" ); } @@ -5094,7 +5819,7 @@ mod tests { assert_eq!(rewrites.len(), 2); let first_shares_target = match &rewrites[0].replacement { Replacement::Rewrite(rc) => Rc::ptr_eq(rc, &group.target), - Replacement::Summary(_) => false, + Replacement::Summary(_) | Replacement::ExactComposition(_) => false, }; assert!( first_shares_target, @@ -5130,7 +5855,7 @@ mod tests { assert_eq!(agg_group.candidates.len(), 2); let first_kind = match &agg_group.candidates[0].replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) => None, + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, }; assert_eq!(first_kind, Some(SketchAlgorithm::DDSketch)); } @@ -5326,7 +6051,7 @@ mod tests { .unwrap(); let kind = match &agg_group.chosen.unwrap().replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) => None, + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, }; assert_eq!(kind, Some(SketchAlgorithm::DDSketch)); } @@ -6437,7 +7162,7 @@ mod tests { // ── Accuracy guarantees and fail-closed composition (issue #172) ───── - use asap_types::post_asap::{BoundExpr, ErrorMetric}; + use asap_types::post_asap::ErrorMetric; /// A test-only `AccuracyModel` that *registers* a rule the default /// deliberately lacks — a sketch over rank-bounded inputs composes @@ -6492,21 +7217,6 @@ mod tests { } } - /// The `SketchParams::Kll { k }` of the top `SummaryAgg` under `node`. - fn kll_k_of(node: &SummaryNode) -> u32 { - match &node.expr { - SummaryExpr::SummaryEstimate { summary_input, .. } => kll_k_of(summary_input), - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::Sketch(kind, _), - .. - } => match kind.params() { - SketchParams::Kll { k } => *k, - other => panic!("expected KLL params, got {other:?}"), - }, - other => panic!("expected a sketch SummaryAgg, got {other:?}"), - } - } - fn summary_child(node: &SummaryNode) -> &Rc { match &node.expr { SummaryExpr::SummaryEstimate { summary_input, .. } => summary_child(summary_input), @@ -6603,25 +7313,19 @@ mod tests { } #[test] - fn exact_sum_over_approximate_child_keeps_the_row_count_unknown() { - // sum(count_distinct by (job) (m)): an exact sum over HLL estimates - // is representable (Σ B_i) but its bound depends on the group count - // and the true cardinalities — unknown at planning time, so the - // guarantee exists, says what it needs, and satisfies nothing. + fn exact_sum_over_approximate_readout_falls_back_to_the_logical_plan() { + // sum(count_distinct by (job) (m)) cannot be maintained over the + // inner HLL's query-time readout. The phase contract therefore keeps + // the whole expression logical instead of manufacturing an accuracy + // guarantee for an execution shape the runtime cannot schedule. let inner = agg(vec![2], default_cardinality(), metric_scan(&["job"])); let outer = agg(vec![], AggIntent::Sum { col: None }, inner); let root = realize(&outer).unwrap(); - let guarantee = root + assert!(matches!(root.expr, SummaryExpr::KeepPreAsap(_))); + assert!(root .guarantee .as_ref() - .expect("an exact sum carries a guarantee"); - assert_eq!(guarantee.metric, ErrorMetric::AbsoluteValue); - assert_eq!(guarantee.bound.evaluate(), None); - assert!(guarantee.provenance.iter().any(|s| matches!( - s, - GuaranteeSource::UnavailableStatistic { statistic } if statistic == "input_row_count" - ))); - assert!(!DefaultAccuracyModel.satisfies(guarantee, &AccuracyTarget::Epsilon(1e9))); + .is_some_and(ResultGuarantee::is_exact)); // count(...) over the same child is exact: a row count does not // depend on the rows' values. @@ -6641,11 +7345,11 @@ mod tests { } #[test] - fn equal_split_allocation_makes_a_legal_tighter_candidate_and_rejects_the_declared_one() { - // With a registered rank-additive rule: outer ε=0.1 over inner ε=0.1 - // composes above 0.1 as declared (cheap: k=26 each) — illegal. - // The equal split re-sizes both layers to k=52 — - // pricier, and the only legal way to meet the outer target. + fn equal_split_allocation_does_not_override_phase_legality() { + // Even a registered rank-additive rule and a valid budget split do + // not make a maintained sketch over another sketch's query-time + // readout schedulable. Accuracy legality cannot override phase + // legality, so the conservative logical fallback is the only plan. let inner = agg(vec![2], quantile_eps(0.5, 0.1), metric_scan(&["job"])); let outer = Rc::new(agg(vec![], quantile_eps(0.99, 0.1), inner)); let strategy = SketchAlgorithmStrategy::with_models( @@ -6655,75 +7359,18 @@ mod tests { ); let proposals = strategy.propose(&TargetSubDAG::new(&outer)); - let declared = proposals - .rejected - .iter() - .filter(|r| { - matches!( - &r.error, - AccuracyError::TargetNotSatisfied { - metric: ErrorMetric::Rank, - bound: Some(b), - target: AccuracyTarget::Epsilon(e), - .. - } if (b - 2.0 * crate::accuracy::kll_rank_error_99(26)).abs() < 1e-12 - && *e == 0.1 - ) - }) - .count(); - assert_eq!( - declared, 1, - "the as-declared KLL composition is rejected: {:?}", - proposals.rejected - ); - - let kll: Vec<_> = proposals - .candidates - .iter() - .filter_map(|c| match &c.replacement { - Replacement::Summary(node) - if summary_family_algorithm(node) == SketchAlgorithm::Kll => - { - Some(node) - } - _ => None, - }) - .collect(); - assert_eq!( - kll.len(), - 1, - "exactly one legal KLL candidate (the allocated one)" - ); - let node = kll[0]; - assert_eq!(kll_k_of(node), 52, "outer re-sized to ε/2"); - assert_eq!(kll_k_of(summary_child(node)), 52, "inner re-sized to ε/2"); - let guarantee = node.guarantee.as_ref().unwrap(); - assert_eq!(guarantee.metric, ErrorMetric::Rank); - let composed_bound = guarantee.bound.evaluate().unwrap(); - assert!(composed_bound <= 0.1); - assert!((composed_bound - 2.0 * crate::accuracy::kll_rank_error_99(52)).abs() < 1e-12); - assert_eq!(guarantee.approximate_layer_count(), 2); - assert!(guarantee.provenance.iter().any(|s| matches!( - s, - GuaranteeSource::BudgetAllocation { allocator, layer_count: 2, .. } - if allocator == "EqualSplitAllocator" - ))); - assert!(matches!(guarantee.bound, BoundExpr::Sum { .. })); - // No candidate with the cheaper illegal sizing exists anywhere. - assert!(proposals.candidates.iter().all(|c| match &c.replacement { - Replacement::Summary(node) - if summary_family_algorithm(node) == SketchAlgorithm::Kll => - kll_k_of(node) != 20, - _ => true, - })); + assert_eq!(proposals.candidates.len(), 1); + let Replacement::Summary(node) = &proposals.candidates[0].replacement else { + panic!() + }; + assert!(matches!(node.expr, SummaryExpr::KeepPreAsap(_))); } #[test] - fn legality_precedes_cost_in_search_and_global_selection() { - // Same fixture through the workload search: the illegal cheaper - // candidate is absent from the group *before* any cost ranking, the - // rejection is recorded on the group, and global selection commits - // to the legal, more expensive one. + fn phase_legality_precedes_accuracy_and_cost_in_global_selection() { + // The same phase-illegal nesting through workload search remains a + // logical fallback before cost ranking. Neither a favorable cost nor + // a valid accuracy allocation can resurrect it. let inner = agg(vec![2], quantile_eps(0.5, 0.1), metric_scan(&["job"])); let outer = Rc::new(agg(vec![], quantile_eps(0.99, 0.1), inner)); let strategies: Vec> = @@ -6735,13 +7382,7 @@ mod tests { let space = search_workload_with(vec![("q", Rc::clone(&outer))], &strategies); let root = &space.roots[0].1; let group = space.group_for(root).unwrap(); - assert!(!group.rejected.is_empty()); - assert!(group.candidates.iter().all(|c| match &c.replacement { - Replacement::Summary(node) => node.guarantee.as_ref().is_some_and(|g| { - DefaultAccuracyModel.satisfies(g, &AccuracyTarget::Epsilon(0.1)) - }), - Replacement::Rewrite(_) => false, - })); + assert_eq!(group.candidates.len(), 1); let ranked = space.cost_sorted(&DefaultCostModel); let root_ranked = ranked.iter().find(|g| Rc::ptr_eq(g.target, root)).unwrap(); assert_eq!(root_ranked.candidates.len(), group.candidates.len()); @@ -6751,11 +7392,11 @@ mod tests { .for_target(root) .unwrap() .chosen - .expect("a legal candidate wins"); + .expect("the conservative fallback wins"); let Replacement::Summary(node) = &chosen.replacement else { panic!() }; - assert_eq!(kll_k_of(node), 52); + assert!(matches!(node.expr, SummaryExpr::KeepPreAsap(_))); } #[test] @@ -6808,6 +7449,7 @@ mod tests { .as_ref() .is_some_and(ResultGuarantee::is_exact), Replacement::Rewrite(_) => true, + Replacement::ExactComposition(_) => false, })); } diff --git a/crates/asap-aware-mapping/src/rollup.rs b/crates/asap-aware-mapping/src/rollup.rs index 5dc3c81e..a5abae8a 100644 --- a/crates/asap-aware-mapping/src/rollup.rs +++ b/crates/asap-aware-mapping/src/rollup.rs @@ -690,7 +690,7 @@ mod tests { .iter() .find_map(|candidate| match &candidate.replacement { Replacement::Rewrite(rewrite) => Some(rewrite), - Replacement::Summary(_) => None, + Replacement::Summary(_) | Replacement::ExactComposition(_) => None, }) .expect("default search must include the roll-up rewrite"); let QueryExpr::Aggregate { child, .. } = rewrite.as_ref() else { diff --git a/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs b/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs new file mode 100644 index 00000000..5dc3a774 --- /dev/null +++ b/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs @@ -0,0 +1,1734 @@ +//! Workload-aware physical summary-maintenance lifecycle planning. +//! +//! Phase validation from PR #300 answers whether a post-ASAP DAG can execute. +//! This module answers how each unique `SummaryAgg` state is deployed for the +//! supplied query and data workloads. Unknown evidence stays unknown and +//! therefore cannot make a long-lived summary maintenance lifecycle win. + +use std::collections::{HashMap, HashSet}; +use std::rc::Rc; + +use asap_types::post_asap::{ + produced_data_state, validate_execution_data_states, ExecutionDataState, SummaryExpr, + SummaryMaintenanceLifecycle, SummaryNode, +}; +use asap_types::post_asap::{ + EvaluationSchedule, OutputRepresentation, SummaryMaintenanceLifecycleGuarantee, +}; +use asap_types::pre_asap::QueryExpr; +use asap_types::workload::{ + DataArrival, Predictability, QueryRecurrence, QueryWorkload, RepeatedDemand, TimestampMs, + WorkloadError, +}; + +use crate::cost_model::{Cost, CostModel}; +use crate::recurrence::{ + CostRate, EvaluationRate, Horizon, RecurrenceError, RecurrenceProfile, UpdateRate, +}; +use crate::replacement::{ + CandidateCostOverrides, GlobalSelection, ImplementError, PlanSpace, Replacement, +}; + +/// Summary maintenance lifecycle shapes available to the runtime planner. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SummaryMaintenanceLifecycleCapabilities { + pub ephemeral: bool, + pub prepared: bool, + pub shared: bool, + pub continuously_maintained: bool, +} + +/// Capabilities of one concrete summary family/state representation. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SummaryMaintenanceCapabilities { + pub incremental_update: bool, + pub merge: bool, + pub delete: bool, +} + +impl SummaryMaintenanceLifecycleCapabilities { + pub const ALL: Self = Self { + ephemeral: true, + prepared: true, + shared: true, + continuously_maintained: true, + }; +} + +impl Default for SummaryMaintenanceLifecycleCapabilities { + fn default() -> Self { + Self::ALL + } +} + +/// Primitive costs for one concrete summary state. Every field is optional: +/// missing statistics produce an uncosted alternative, never a zero. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct SummaryMaintenanceLifecycleCostInputs { + pub build_cost: Option, + pub maintenance_cost_per_update: Option, + pub summary_read_cost: Option, + pub retention_cost_rate: Option, + pub retirement_cost: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SummaryMaintenanceLifecycleRejection { + UnsupportedByRuntime, + RequiresPredictableOneTimeQuery, + RequiresMultipleReads, + RequiresHorizon, + RequiresContinuousData, + MissingOrStaleIngestionRate, + SummaryDoesNotSupportIncrementalUpdates, + SummaryDoesNotSupportDeletion, + MissingCostEvidence, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct SummaryMaintenanceLifecycleAlternative { + pub summary_maintenance_lifecycle: SummaryMaintenanceLifecycle, + pub total_cost: Option, + pub rejection: Option, + pub assumptions: Vec, +} + +impl SummaryMaintenanceLifecycleAlternative { + fn selectable(&self) -> bool { + self.rejection.is_none() && self.total_cost.is_some() + } +} + +/// One unique summary-state deployment. Shared `Rc` nodes are emitted once. +#[derive(Debug, Clone)] +pub struct SummaryMaintenanceDeployment { + pub summary_index: usize, + pub summary: Rc, + pub summary_maintenance_lifecycle_guarantee: Option, + pub alternatives: Vec, +} + +#[derive(Debug, Clone)] +pub struct SummaryMaintenanceLifecyclePlan { + pub root: Rc, + pub deployments: Vec, + pub horizon: Option, + pub evaluation_rate: Option, + pub update_rate: Option, + pub expected_reads: Option, + pub selected_raw_recompute: bool, + pub summary_total_cost: Option, + pub raw_recompute_total_cost: Option, +} + +/// Explicit association between a materialized target and the normalized +/// workload entries whose demand consumes it. +#[derive(Debug, Clone, Copy)] +pub struct WorkloadDemand<'a> { + pub workload: &'a QueryWorkload, + pub entry_indices: &'a [usize], +} + +impl<'a> WorkloadDemand<'a> { + pub const fn new(workload: &'a QueryWorkload, entry_indices: &'a [usize]) -> Self { + Self { + workload, + entry_indices, + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum SummaryMaintenanceLifecyclePlanError { + #[error(transparent)] + InvalidWorkload(#[from] WorkloadError), + #[error(transparent)] + InvalidExecutionDataStates(#[from] asap_types::post_asap::ExecutionDataStateError), + #[error("optimization horizon must be finite and strictly positive")] + InvalidHorizon, + #[error("workload entry index {index} is out of bounds for {entry_count} entries")] + InvalidWorkloadEntry { index: usize, entry_count: usize }, + #[error("a workload-demand binding must contain at least one entry")] + EmptyWorkloadDemand, + #[error("workload entry index {index} appears more than once in one demand binding")] + DuplicateWorkloadEntry { index: usize }, +} + +#[derive(Debug, thiserror::Error)] +pub enum MaterializeSummaryMaintenanceLifecycleError { + #[error(transparent)] + Materialize(#[from] ImplementError), + #[error(transparent)] + SummaryMaintenance(#[from] SummaryMaintenanceLifecyclePlanError), +} + +/// Failure while deriving workload-aware candidate costs before global +/// selection. +#[derive(Debug, thiserror::Error)] +pub enum SummaryMaintenanceLifecycleSelectionError { + #[error(transparent)] + Recurrence(#[from] RecurrenceError), + #[error(transparent)] + SummaryMaintenance(#[from] SummaryMaintenanceLifecyclePlanError), +} + +#[derive(Debug)] +struct WorkloadFacts { + reads: Option, + one_time_invocations: u64, + evaluation_rate: Option, + update_rate: Option, + arrival: DataArrival, + prepared_window: Option<(TimestampMs, TimestampMs)>, + prepared_eligible: bool, + requires_deletion: bool, +} + +/// Validate a materialized plan, enumerate lifecycle alternatives for each +/// unique summary state, and select the cheapest legal alternative whose cost +/// is fully known. +pub fn plan_summary_maintenance_lifecycles( + root: Rc, + demand: WorkloadDemand<'_>, + now_ms: u64, + horizon: Option, + capabilities: SummaryMaintenanceLifecycleCapabilities, + cost_model: &dyn CostModel, +) -> Result { + plan_summary_maintenance_lifecycles_with_profile( + root, + demand, + now_ms, + horizon, + capabilities, + cost_model, + None, + ) +} + +/// Internal candidate-costing form. The workload binding supplies temporal +/// eligibility and data-arrival facts; `profile` supplies effective uses after +/// DAG path multiplicity has been propagated by `PlanSpace`. +fn plan_summary_maintenance_lifecycles_with_profile( + root: Rc, + demand: WorkloadDemand<'_>, + now_ms: u64, + horizon: Option, + capabilities: SummaryMaintenanceLifecycleCapabilities, + cost_model: &dyn CostModel, + profile: Option, +) -> Result { + demand.workload.validate()?; + validate_execution_data_states(&root)?; + if horizon.is_some_and(|h| !h.0.is_finite() || h.0 <= 0.0) { + return Err(SummaryMaintenanceLifecyclePlanError::InvalidHorizon); + } + let mut facts = workload_facts(demand.workload, demand.entry_indices, now_ms, horizon)?; + if let Some(profile) = profile { + facts.one_time_invocations = u64::try_from(profile.one_shot_consumers).unwrap_or(u64::MAX); + facts.evaluation_rate = profile.evaluation_rate; + facts.update_rate = profile.update_rate; + facts.reads = match (profile.evaluation_rate, horizon) { + (Some(rate), Some(horizon)) => { + Some(profile.one_shot_consumers as f64 + rate.0 * horizon.0) + } + (Some(_), None) => None, + (None, _) if profile.one_shot_consumers > 0 => Some(profile.one_shot_consumers as f64), + // Preserve unknown recurrence from the normalized workload. An + // empty profile does not prove that the target is never read. + (None, _) => facts.reads, + }; + } + let mut summaries = Vec::new(); + collect_summary_aggs(&root, &mut HashSet::new(), &mut summaries); + let components = summary_state_components(&summaries); + let mut deployments: Vec = summaries + .into_iter() + .enumerate() + .map(|(summary_index, summary)| { + let alternatives = alternatives_for( + &facts, + horizon, + capabilities, + cost_model.summary_maintenance_capabilities(&summary), + cost_model.summary_maintenance_lifecycle_cost_inputs(&summary), + ); + SummaryMaintenanceDeployment { + summary_index, + summary, + summary_maintenance_lifecycle_guarantee: None, + alternatives, + } + }) + .collect(); + select_compatible_lifecycles(&mut deployments, &components, facts.arrival); + let summary_total_cost = deployments.iter().try_fold(Cost::ZERO, |sum, deployment| { + let selected = &deployment + .summary_maintenance_lifecycle_guarantee + .as_ref()? + .summary_maintenance_lifecycle; + let cost = deployment + .alternatives + .iter() + .find(|alternative| &alternative.summary_maintenance_lifecycle == selected)? + .total_cost?; + Some(Cost(sum.0 + cost.0)) + }); + Ok(SummaryMaintenanceLifecyclePlan { + root, + deployments, + horizon, + evaluation_rate: facts.evaluation_rate, + update_rate: facts.update_rate, + expected_reads: facts.reads, + selected_raw_recompute: false, + summary_total_cost, + raw_recompute_total_cost: None, + }) +} + +/// Rank semantic summary siblings using the cheapest legal +/// summary-maintenance lifecycle for each candidate before final global +/// selection. The candidate space stays compact; only cost overrides are +/// attached, so shared `Rc` identity and exact-composition commitments remain +/// the responsibility of `GlobalSelection`. +pub fn global_selection_with_summary_maintenance_lifecycles<'a, Id>( + space: &'a PlanSpace, + workload: &QueryWorkload, + root_workload_entries: &[usize], + now_ms: u64, + horizon: Option, + capabilities: SummaryMaintenanceLifecycleCapabilities, + cost_model: &dyn CostModel, +) -> Result, SummaryMaintenanceLifecycleSelectionError> { + let profiles = space.recurrence_profiles_from_workload( + workload, + root_workload_entries, + now_ms, + horizon, + )?; + let bindings = space.workload_entries_by_target(workload, root_workload_entries)?; + let mut costs = CandidateCostOverrides::default(); + for group in space.groups() { + let Some(entry_indices) = bindings.get(&Rc::as_ptr(&group.target)) else { + continue; + }; + for candidate in &group.candidates { + let Replacement::Summary(summary) = &candidate.replacement else { + continue; + }; + let plan = plan_summary_maintenance_lifecycles_with_profile( + Rc::clone(summary), + WorkloadDemand::new(workload, entry_indices), + now_ms, + horizon, + capabilities, + cost_model, + Some(profiles.for_target(&group.target)), + )?; + if !plan.deployments.is_empty() { + if let Some(total) = plan.summary_total_cost { + costs.insert(&group.target, candidate, total); + } + } + } + } + Ok(space.global_selection_with_candidate_costs(cost_model, &profiles, horizon, &costs)?) +} + +/// Materialize a globally selected phase-valid DAG and immediately attach +/// workload-aware summary maintenance deployments. +pub fn materialize_with_summary_maintenance_lifecycles( + selection: &GlobalSelection<'_>, + target: &Rc, + demand: WorkloadDemand<'_>, + now_ms: u64, + horizon: Option, + capabilities: SummaryMaintenanceLifecycleCapabilities, + cost_model: &dyn CostModel, +) -> Result, MaterializeSummaryMaintenanceLifecycleError> { + selection + .materialize(target)? + .map(|root| { + let mut plan = plan_summary_maintenance_lifecycles( + root, + demand, + now_ms, + horizon, + capabilities, + cost_model, + )?; + plan.raw_recompute_total_cost = cost_model + .raw_query_recompute_cost(target) + .zip(plan.expected_reads) + .map(|(per_read, reads)| Cost(per_read.0 * reads)); + if plan.raw_recompute_total_cost.is_some_and(|raw| { + plan.summary_total_cost + .is_none_or(|summary| raw.0 <= summary.0) + }) { + plan.root = crate::replacement::keep_pre_asap(target)?; + plan.deployments.clear(); + plan.selected_raw_recompute = true; + } + Ok(plan) + }) + .transpose() +} + +fn workload_facts( + workload: &QueryWorkload, + workload_entry_indices: &[usize], + now_ms: u64, + horizon: Option, +) -> Result { + let mut one_time_invocations = 0u64; + let mut recurring_reads = 0.0; + let mut recurring_known = true; + let mut evaluation_rate = 0.0; + let mut has_evaluation_rate = false; + let mut prepared_start: Option = None; + let mut prepared_end: Option = None; + let mut prepared_eligible = true; + let mut requires_deletion = false; + + let entries: Vec<_> = workload.entries().collect(); + if workload_entry_indices.is_empty() { + return Err(SummaryMaintenanceLifecyclePlanError::EmptyWorkloadDemand); + } + let mut seen_indices = HashSet::new(); + for &index in workload_entry_indices { + if !seen_indices.insert(index) { + return Err(SummaryMaintenanceLifecyclePlanError::DuplicateWorkloadEntry { index }); + } + let entry = entries.get(index).ok_or( + SummaryMaintenanceLifecyclePlanError::InvalidWorkloadEntry { + index, + entry_count: entries.len(), + }, + )?; + requires_deletion |= entry.time_selection.lookback.is_some() + && entry.time_selection.as_of.is_none() + && matches!( + entry.time_selection.scope, + asap_types::workload::QueryTimeScope::RealTime + | asap_types::workload::QueryTimeScope::Mixed + ); + match &entry.recurrence { + QueryRecurrence::OneTime { + invocations, + execute_at, + } => { + one_time_invocations = one_time_invocations.saturating_add(*invocations); + let covered = if let ( + Predictability::Predictable { + known_at: Some(known), + }, + Some(execute), + ) = (&entry.predictability, execute_at) + { + if known < execute { + prepared_start = Some(prepared_start.map_or(*known, |old| old.min(*known))); + prepared_end = Some(prepared_end.map_or(*execute, |old| old.max(*execute))); + true + } else { + false + } + } else { + false + }; + prepared_eligible &= covered; + } + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(interval)) => { + prepared_eligible = false; + let rate = 1000.0 / f64::from(interval.0); + evaluation_rate += rate; + has_evaluation_rate = true; + if let Some(h) = horizon { + recurring_reads += h.0 * rate; + } else { + recurring_known = false; + } + } + QueryRecurrence::Repeated(RepeatedDemand::Scheduled(schedule)) => { + prepared_eligible = false; + if let Some(h) = horizon { + let end_ms = now_ms.saturating_add((h.0 * 1000.0) as u64); + let reads_in_horizon = schedule + .iter() + .filter(|at| at.0 >= now_ms && at.0 <= end_ms) + .count() as f64; + recurring_reads += reads_in_horizon; + evaluation_rate += reads_in_horizon / h.0; + has_evaluation_rate = true; + } else { + recurring_known = false; + } + } + QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(estimate)) => { + prepared_eligible = false; + if !estimate.is_fresh_at(now_ms) { + recurring_known = false; + continue; + } + let rate = match estimate.expected { + asap_types::workload::ExpectedDemand::AverageRate(rate) => Some(rate.0), + asap_types::workload::ExpectedDemand::InvocationCount(count) => { + let millis = estimate + .observation_window + .end + .0 + .saturating_sub(estimate.observation_window.start.0); + (millis > 0).then_some(count as f64 / (millis as f64 / 1000.0)) + } + }; + if let Some(rate) = rate { + evaluation_rate += rate; + has_evaluation_rate = true; + if let Some(h) = horizon { + recurring_reads += h.0 * rate; + } else { + recurring_known = false; + } + } else { + recurring_known = false; + } + } + QueryRecurrence::Unknown => { + prepared_eligible = false; + recurring_known = false; + } + } + } + + let data = workload.data_workload.as_ref(); + let arrival = data.map_or(DataArrival::Unknown, |data| data.arrival); + let update_rate = data + .and_then(|data| data.ingestion_rate.value_at(now_ms)) + .map(|rate| UpdateRate(rate.0)); + let reads = recurring_known.then_some(one_time_invocations as f64 + recurring_reads); + Ok(WorkloadFacts { + reads, + one_time_invocations, + evaluation_rate: has_evaluation_rate.then_some(EvaluationRate(evaluation_rate)), + update_rate, + arrival, + prepared_window: prepared_start.zip(prepared_end), + prepared_eligible, + requires_deletion, + }) +} + +fn alternatives_for( + facts: &WorkloadFacts, + horizon: Option, + capabilities: SummaryMaintenanceLifecycleCapabilities, + summary_capabilities: SummaryMaintenanceCapabilities, + costs: SummaryMaintenanceLifecycleCostInputs, +) -> Vec { + let alternatives = vec![ + ephemeral(facts, capabilities, &costs), + prepared(facts, capabilities, summary_capabilities, &costs), + shared(facts, horizon, capabilities, summary_capabilities, &costs), + continuous(facts, horizon, capabilities, summary_capabilities, &costs), + ]; + alternatives +} + +fn ephemeral( + facts: &WorkloadFacts, + capabilities: SummaryMaintenanceLifecycleCapabilities, + costs: &SummaryMaintenanceLifecycleCostInputs, +) -> SummaryMaintenanceLifecycleAlternative { + let lifecycle = SummaryMaintenanceLifecycle::Ephemeral; + if !capabilities.ephemeral { + return rejected( + lifecycle, + SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime, + ); + } + let total_cost = zip_costs(&[ + costs.build_cost, + costs.summary_read_cost, + costs.retirement_cost, + ]) + .zip(facts.reads) + .map(|(per_read, reads)| Cost(per_read * reads)); + costed_or_unknown( + lifecycle, + total_cost, + vec!["state is rebuilt per invocation".into()], + ) +} + +fn prepared( + facts: &WorkloadFacts, + capabilities: SummaryMaintenanceLifecycleCapabilities, + summary_capabilities: SummaryMaintenanceCapabilities, + costs: &SummaryMaintenanceLifecycleCostInputs, +) -> SummaryMaintenanceLifecycleAlternative { + if !facts.prepared_eligible { + return rejected( + SummaryMaintenanceLifecycle::Prepared { + activate_at: TimestampMs(0), + retire_at: TimestampMs(0), + }, + SummaryMaintenanceLifecycleRejection::RequiresPredictableOneTimeQuery, + ); + } + let Some((activate_at, retire_at)) = facts.prepared_window else { + return rejected( + SummaryMaintenanceLifecycle::Prepared { + activate_at: TimestampMs(0), + retire_at: TimestampMs(0), + }, + SummaryMaintenanceLifecycleRejection::RequiresPredictableOneTimeQuery, + ); + }; + let lifecycle = SummaryMaintenanceLifecycle::Prepared { + activate_at, + retire_at, + }; + if !capabilities.prepared { + return rejected( + lifecycle, + SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime, + ); + } + if let Some(rejection) = maintenance_capability_rejection(facts, summary_capabilities) { + return rejected(lifecycle, rejection); + } + let seconds = retire_at.0.saturating_sub(activate_at.0) as f64 / 1000.0; + let maintenance = maintenance_cost(facts, costs, seconds); + let total_cost = match ( + costs.build_cost, + costs.summary_read_cost, + costs.retention_cost_rate, + costs.retirement_cost, + maintenance, + ) { + (Some(build), Some(read), Some(retention), Some(retire), Some(maintenance)) => Some(Cost( + build.0 + + read.0 * facts.one_time_invocations as f64 + + retention.0 * seconds + + retire.0 + + maintenance, + )), + _ => None, + }; + costed_or_unknown( + lifecycle, + total_cost, + vec!["activation and retirement come from the declared schedule".into()], + ) +} + +fn shared( + facts: &WorkloadFacts, + horizon: Option, + capabilities: SummaryMaintenanceLifecycleCapabilities, + summary_capabilities: SummaryMaintenanceCapabilities, + costs: &SummaryMaintenanceLifecycleCostInputs, +) -> SummaryMaintenanceLifecycleAlternative { + let lifecycle = SummaryMaintenanceLifecycle::Shared { + retention: asap_types::workload::DurationMs(horizon.map_or(0, |h| (h.0 * 1000.0) as u64)), + }; + if !capabilities.shared { + return rejected( + lifecycle, + SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime, + ); + } + if let Some(rejection) = maintenance_capability_rejection(facts, summary_capabilities) { + return rejected(lifecycle, rejection); + } + if facts.reads.is_none_or(|reads| reads <= 1.0) { + return rejected( + lifecycle, + SummaryMaintenanceLifecycleRejection::RequiresMultipleReads, + ); + } + let Some(horizon) = horizon else { + return rejected( + lifecycle, + SummaryMaintenanceLifecycleRejection::RequiresHorizon, + ); + }; + let total_cost = retained_cost(facts, costs, horizon.0); + costed_or_unknown( + lifecycle, + total_cost, + vec!["one state is shared across reads".into()], + ) +} + +fn continuous( + facts: &WorkloadFacts, + horizon: Option, + capabilities: SummaryMaintenanceLifecycleCapabilities, + summary_capabilities: SummaryMaintenanceCapabilities, + costs: &SummaryMaintenanceLifecycleCostInputs, +) -> SummaryMaintenanceLifecycleAlternative { + let lifecycle = SummaryMaintenanceLifecycle::ContinuouslyMaintained; + if !capabilities.continuously_maintained { + return rejected( + lifecycle, + SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime, + ); + } + if !matches!( + facts.arrival, + DataArrival::ContinuouslyIngesting | DataArrival::Mixed + ) { + return rejected( + lifecycle, + SummaryMaintenanceLifecycleRejection::RequiresContinuousData, + ); + } + if facts.update_rate.is_none() { + return rejected( + lifecycle, + SummaryMaintenanceLifecycleRejection::MissingOrStaleIngestionRate, + ); + } + if let Some(rejection) = maintenance_capability_rejection(facts, summary_capabilities) { + return rejected(lifecycle, rejection); + } + let Some(horizon) = horizon else { + return rejected( + lifecycle, + SummaryMaintenanceLifecycleRejection::RequiresHorizon, + ); + }; + let total_cost = retained_cost(facts, costs, horizon.0); + costed_or_unknown( + lifecycle, + total_cost, + vec!["updates are applied for the optimization horizon".into()], + ) +} + +fn maintenance_capability_rejection( + facts: &WorkloadFacts, + capabilities: SummaryMaintenanceCapabilities, +) -> Option { + if matches!( + facts.arrival, + DataArrival::ContinuouslyIngesting | DataArrival::Mixed + ) && !capabilities.incremental_update + { + Some(SummaryMaintenanceLifecycleRejection::SummaryDoesNotSupportIncrementalUpdates) + } else if matches!( + facts.arrival, + DataArrival::ContinuouslyIngesting | DataArrival::Mixed + ) && facts.requires_deletion + && !capabilities.delete + { + Some(SummaryMaintenanceLifecycleRejection::SummaryDoesNotSupportDeletion) + } else { + None + } +} + +fn retained_cost( + facts: &WorkloadFacts, + costs: &SummaryMaintenanceLifecycleCostInputs, + seconds: f64, +) -> Option { + let reads = facts.reads?; + let maintenance = maintenance_cost(facts, costs, seconds)?; + Some(Cost( + costs.build_cost?.0 + + maintenance + + reads * costs.summary_read_cost?.0 + + seconds * costs.retention_cost_rate?.0 + + costs.retirement_cost?.0, + )) +} + +fn maintenance_cost( + facts: &WorkloadFacts, + costs: &SummaryMaintenanceLifecycleCostInputs, + seconds: f64, +) -> Option { + match facts.arrival { + DataArrival::AtRest => Some(0.0), + DataArrival::ContinuouslyIngesting | DataArrival::Mixed => { + Some(seconds * facts.update_rate?.0 * costs.maintenance_cost_per_update?.0) + } + DataArrival::Unknown => None, + } +} + +fn zip_costs(costs: &[Option]) -> Option { + costs + .iter() + .try_fold(0.0, |sum, cost| Some(sum + cost.as_ref()?.0)) +} + +fn costed_or_unknown( + summary_maintenance_lifecycle: SummaryMaintenanceLifecycle, + total_cost: Option, + assumptions: Vec, +) -> SummaryMaintenanceLifecycleAlternative { + SummaryMaintenanceLifecycleAlternative { + summary_maintenance_lifecycle, + total_cost, + rejection: total_cost + .is_none() + .then_some(SummaryMaintenanceLifecycleRejection::MissingCostEvidence), + assumptions, + } +} + +fn rejected( + summary_maintenance_lifecycle: SummaryMaintenanceLifecycle, + rejection: SummaryMaintenanceLifecycleRejection, +) -> SummaryMaintenanceLifecycleAlternative { + SummaryMaintenanceLifecycleAlternative { + summary_maintenance_lifecycle, + total_cost: None, + rejection: Some(rejection), + assumptions: Vec::new(), + } +} + +fn collect_summary_aggs( + node: &Rc, + seen: &mut HashSet<*const SummaryNode>, + output: &mut Vec>, +) { + if !seen.insert(Rc::as_ptr(node)) { + return; + } + match &node.expr { + SummaryExpr::SummaryAgg { child, .. } => { + output.push(Rc::clone(node)); + collect_summary_aggs(child, seen, output); + } + SummaryExpr::SummaryJoin { outer, inner, .. } + | SummaryExpr::SummarySubtract { + left: outer, + right: inner, + } => { + collect_summary_aggs(outer, seen, output); + collect_summary_aggs(inner, seen, output); + } + SummaryExpr::SummaryDelete { summary_input, .. } + | SummaryExpr::SummaryEstimate { summary_input, .. } => { + collect_summary_aggs(summary_input, seen, output) + } + SummaryExpr::SummaryMerge { children } => { + for child in children { + collect_summary_aggs(child, seen, output); + } + } + SummaryExpr::UpdateTransform { child, .. } + | SummaryExpr::ReadoutPostProcess { child, .. } => { + collect_summary_aggs(child, seen, output) + } + SummaryExpr::KeepPreAsap(_) => {} + } +} + +fn evaluation_schedule( + lifecycle: &SummaryMaintenanceLifecycle, + arrival: DataArrival, +) -> EvaluationSchedule { + match lifecycle { + SummaryMaintenanceLifecycle::Ephemeral => EvaluationSchedule::OneShot, + SummaryMaintenanceLifecycle::Prepared { .. } + | SummaryMaintenanceLifecycle::Shared { .. } + if matches!( + arrival, + DataArrival::ContinuouslyIngesting | DataArrival::Mixed + ) => + { + EvaluationSchedule::PerUpdate + } + SummaryMaintenanceLifecycle::Prepared { .. } => EvaluationSchedule::OneShot, + SummaryMaintenanceLifecycle::Shared { .. } => EvaluationSchedule::OnRead, + SummaryMaintenanceLifecycle::ContinuouslyMaintained => EvaluationSchedule::PerUpdate, + } +} + +/// Summary states composed on one maintenance path must be produced on the +/// same schedule. Return a component id for each collected `SummaryAgg`. +fn summary_state_components(summaries: &[Rc]) -> Vec { + let indices: HashMap<_, _> = summaries + .iter() + .enumerate() + .map(|(index, summary)| (Rc::as_ptr(summary), index)) + .collect(); + let mut parents: Vec<_> = (0..summaries.len()).collect(); + + fn find(parents: &mut [usize], index: usize) -> usize { + if parents[index] != index { + parents[index] = find(parents, parents[index]); + } + parents[index] + } + + for (parent_index, summary) in summaries.iter().enumerate() { + let SummaryExpr::SummaryAgg { child, .. } = &summary.expr else { + continue; + }; + if produced_data_state(&child.expr) != Some(ExecutionDataState::MAINTENANCE_SUMMARY) { + continue; + } + let mut descendants = Vec::new(); + collect_summary_aggs(child, &mut HashSet::new(), &mut descendants); + for descendant in descendants { + let child_index = indices[&Rc::as_ptr(&descendant)]; + let parent_root = find(&mut parents, parent_index); + let child_root = find(&mut parents, child_index); + parents[child_root] = parent_root; + } + } + (0..parents.len()) + .map(|index| find(&mut parents, index)) + .collect() +} + +fn select_compatible_lifecycles( + deployments: &mut [SummaryMaintenanceDeployment], + components: &[usize], + arrival: DataArrival, +) { + let component_ids: HashSet<_> = components.iter().copied().collect(); + for component in component_ids { + let members: Vec<_> = components + .iter() + .enumerate() + .filter_map(|(index, &id)| (id == component).then_some(index)) + .collect(); + let selected_schedule = [ + EvaluationSchedule::OneShot, + EvaluationSchedule::PerUpdate, + EvaluationSchedule::OnRead, + ] + .into_iter() + .filter_map(|schedule| { + members + .iter() + .try_fold(0.0, |sum, &index| { + deployments[index] + .alternatives + .iter() + .filter(|candidate| { + candidate.selectable() + && evaluation_schedule( + &candidate.summary_maintenance_lifecycle, + arrival, + ) == schedule + }) + .map(|candidate| candidate.total_cost.unwrap().0) + .min_by(f64::total_cmp) + .map(|cost| sum + cost) + }) + .map(|cost| (schedule, cost)) + }) + .min_by(|(_, a), (_, b)| a.total_cmp(b)) + .map(|(schedule, _)| schedule); + + let Some(schedule) = selected_schedule else { + continue; + }; + for index in members { + let selected = deployments[index] + .alternatives + .iter() + .filter(|candidate| { + candidate.selectable() + && evaluation_schedule(&candidate.summary_maintenance_lifecycle, arrival) + == schedule + }) + .min_by(|a, b| a.total_cost.unwrap().0.total_cmp(&b.total_cost.unwrap().0)); + deployments[index].summary_maintenance_lifecycle_guarantee = + selected.map(|candidate| SummaryMaintenanceLifecycleGuarantee { + summary_maintenance_lifecycle: candidate.summary_maintenance_lifecycle.clone(), + evaluation_schedule: schedule, + output_representation: OutputRepresentation::SummaryState, + }); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::post_asap::{ + ExactKind, ExactParams, GroupingStrategy, ResultGuarantee, SketchAlgorithm, + SummaryFamilyType, SummaryField, SummarySchema, + }; + use asap_types::pre_asap::AggIntent; + use asap_types::pre_asap::{Column, ColumnRef, DataType, QueryExpr, Reduction, Schema, Source}; + use asap_types::types::AccuracyTarget; + use asap_types::workload::{ + BatchEntry, DataWorkload, DurationMs, Evidence, EvidenceSource, Predictability, Query, + QueryLanguage, QueryRequirements, Rate, RepeatingEntry, RepetitionInterval, TimeSelection, + }; + + struct UnitCosts; + + impl CostModel for UnitCosts { + fn rank_candidates( + &self, + _intent: &asap_types::pre_asap::AggIntent, + candidates: &[asap_types::post_asap::SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + + fn summary_maintenance_lifecycle_cost_inputs( + &self, + _summary: &SummaryNode, + ) -> SummaryMaintenanceLifecycleCostInputs { + SummaryMaintenanceLifecycleCostInputs { + build_cost: Some(Cost(10.0)), + maintenance_cost_per_update: Some(Cost(1.0)), + summary_read_cost: Some(Cost(1.0)), + retention_cost_rate: Some(CostRate(0.1)), + retirement_cost: Some(Cost(1.0)), + } + } + + fn summary_maintenance_capabilities( + &self, + _summary: &SummaryNode, + ) -> SummaryMaintenanceCapabilities { + SummaryMaintenanceCapabilities { + incremental_update: true, + merge: true, + delete: true, + } + } + } + + struct RawCheaper; + + impl CostModel for RawCheaper { + fn rank_candidates( + &self, + _intent: &asap_types::pre_asap::AggIntent, + candidates: &[asap_types::post_asap::SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + + fn summary_maintenance_lifecycle_cost_inputs( + &self, + summary: &SummaryNode, + ) -> SummaryMaintenanceLifecycleCostInputs { + UnitCosts.summary_maintenance_lifecycle_cost_inputs(summary) + } + + fn summary_maintenance_capabilities( + &self, + summary: &SummaryNode, + ) -> SummaryMaintenanceCapabilities { + UnitCosts.summary_maintenance_capabilities(summary) + } + + fn raw_query_recompute_cost(&self, _target: &QueryExpr) -> Option { + Some(Cost(1.0)) + } + } + + struct NoDelete; + + impl CostModel for NoDelete { + fn rank_candidates( + &self, + _intent: &asap_types::pre_asap::AggIntent, + candidates: &[asap_types::post_asap::SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + + fn summary_maintenance_lifecycle_cost_inputs( + &self, + summary: &SummaryNode, + ) -> SummaryMaintenanceLifecycleCostInputs { + UnitCosts.summary_maintenance_lifecycle_cost_inputs(summary) + } + + fn summary_maintenance_capabilities( + &self, + _summary: &SummaryNode, + ) -> SummaryMaintenanceCapabilities { + SummaryMaintenanceCapabilities { + incremental_update: true, + merge: true, + delete: false, + } + } + } + + struct SummaryMaintenancePrefersDdSketch; + + impl CostModel for SummaryMaintenancePrefersDdSketch { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + // Preserve semantic mapping's KLL-first order. The lifecycle + // total below must be what changes the final choice. + candidates.to_vec() + } + + fn summary_maintenance_lifecycle_cost_inputs( + &self, + summary: &SummaryNode, + ) -> SummaryMaintenanceLifecycleCostInputs { + let build = match sketch_algorithm(summary) { + Some(SketchAlgorithm::Kll) => 100.0, + Some(SketchAlgorithm::DDSketch) => 1.0, + _ => 10.0, + }; + SummaryMaintenanceLifecycleCostInputs { + build_cost: Some(Cost(build)), + maintenance_cost_per_update: Some(Cost(1.0)), + summary_read_cost: Some(Cost(1.0)), + retention_cost_rate: Some(CostRate(0.1)), + retirement_cost: Some(Cost(1.0)), + } + } + } + + struct IncompatibleNestedCosts; + + impl CostModel for IncompatibleNestedCosts { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + + fn summary_maintenance_lifecycle_cost_inputs( + &self, + summary: &SummaryNode, + ) -> SummaryMaintenanceLifecycleCostInputs { + let is_leaf = matches!( + summary.expr, + SummaryExpr::SummaryAgg { ref child, .. } + if matches!(child.expr, SummaryExpr::KeepPreAsap(_)) + ); + SummaryMaintenanceLifecycleCostInputs { + build_cost: Some(Cost(if is_leaf { 1.0 } else { 100.0 })), + maintenance_cost_per_update: Some(Cost(if is_leaf { 100.0 } else { 0.0 })), + summary_read_cost: Some(Cost::ZERO), + retention_cost_rate: Some(CostRate(0.0)), + retirement_cost: Some(Cost::ZERO), + } + } + + fn summary_maintenance_capabilities( + &self, + _summary: &SummaryNode, + ) -> SummaryMaintenanceCapabilities { + SummaryMaintenanceCapabilities { + incremental_update: true, + merge: true, + delete: true, + } + } + } + + fn sketch_algorithm(node: &SummaryNode) -> Option { + match &node.expr { + SummaryExpr::SummaryEstimate { summary_input, .. } => sketch_algorithm(summary_input), + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::Sketch(kind, _), + .. + } => Some(kind.algorithm().clone()), + _ => None, + } + } + + fn query_root() -> Rc { + query_root_for("m") + } + + fn query_root_for(metric: &str) -> Rc { + Rc::new(QueryExpr::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + ], + 0, + vec![], + ), + }) + } + + fn sum_query() -> Rc { + Rc::new(QueryExpr::Aggregate { + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Sum { col: None }], + output_names: vec![], + having: None, + child: query_root(), + }) + } + + fn quantile_query() -> Rc { + Rc::new(QueryExpr::Aggregate { + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Quantile { + col: None, + q: 0.99, + accuracy: AccuracyTarget::Epsilon(0.1), + }], + output_names: vec![], + having: None, + child: query_root(), + }) + } + + fn summary() -> Rc { + let child = Rc::new(SummaryNode { + expr: SummaryExpr::KeepPreAsap(query_root()), + schema: SummarySchema { + fields: vec![], + time_index: None, + }, + guarantee: Some(ResultGuarantee::exact("raw")), + }); + let family = SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); + Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child, + family: family.clone(), + col: ColumnRef::Named("value".into()), + reduction: Reduction::by(vec![]), + grouping: GroupingStrategy::default(), + }, + schema: SummarySchema { + fields: vec![SummaryField { + name: "state".into(), + dtype: family, + nullable: false, + }], + time_index: None, + }, + guarantee: Some(ResultGuarantee::exact("sum")), + }) + } + + fn nested_summary() -> Rc { + let child = summary(); + let family = SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); + Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child, + family: family.clone(), + col: ColumnRef::Named("state".into()), + reduction: Reduction::by(vec![]), + grouping: GroupingStrategy::default(), + }, + schema: SummarySchema { + fields: vec![SummaryField { + name: "state".into(), + dtype: family, + nullable: false, + }], + time_index: None, + }, + guarantee: Some(ResultGuarantee::exact("nested sum")), + }) + } + + fn batch(predictability: Predictability) -> BatchEntry { + BatchEntry { + query: Query("sum(m)".into()), + requirements: QueryRequirements::default(), + predictability, + invocations: 1, + execute_at: None, + time_selection: TimeSelection::default(), + } + } + + fn workload( + batches: Vec, + repeating: Vec, + data: DataWorkload, + ) -> QueryWorkload { + QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: (!batches.is_empty()).then_some(batches), + repeating_queries: (!repeating.is_empty()).then_some(repeating), + data_workload: Some(data), + } + } + + fn at_rest() -> DataWorkload { + DataWorkload { + arrival: DataArrival::AtRest, + ..Default::default() + } + } + + fn continuous(observed_at_ms: u64, valid_for_ms: u64) -> DataWorkload { + DataWorkload { + arrival: DataArrival::ContinuouslyIngesting, + ingestion_rate: Evidence { + value: Some(Rate(1.0)), + source: EvidenceSource::Observed, + observed_at_ms: Some(observed_at_ms), + valid_for_ms: Some(valid_for_ms), + }, + ..Default::default() + } + } + + fn repeating() -> RepeatingEntry { + RepeatingEntry { + query: Query("sum(m)".into()), + demand: RepeatedDemand::FixedInterval(RepetitionInterval(1_000)), + requirements: QueryRequirements::default(), + predictability: Predictability::Predictable { known_at: None }, + time_selection: TimeSelection::default(), + } + } + + fn selected_summary_maintenance_lifecycle( + deployment: &SummaryMaintenanceDeployment, + ) -> Option<&SummaryMaintenanceLifecycle> { + deployment + .summary_maintenance_lifecycle_guarantee + .as_ref() + .map(|guarantee| &guarantee.summary_maintenance_lifecycle) + } + + #[test] + fn unpredictable_one_time_at_rest_selects_ephemeral() { + let plan = plan_summary_maintenance_lifecycles( + summary(), + WorkloadDemand::new( + &workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()), + &[0], + ), + 1_000, + None, + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + assert_eq!(plan.deployments.len(), 1); + assert_eq!( + selected_summary_maintenance_lifecycle(&plan.deployments[0]), + Some(&SummaryMaintenanceLifecycle::Ephemeral) + ); + let guarantee = plan.deployments[0] + .summary_maintenance_lifecycle_guarantee + .as_ref() + .unwrap(); + assert_eq!(guarantee.evaluation_schedule, EvaluationSchedule::OneShot); + assert_eq!( + guarantee.output_representation, + OutputRepresentation::SummaryState + ); + assert_eq!( + plan.deployments[0].alternatives[0].total_cost, + Some(Cost(12.0)) + ); + } + + #[test] + fn predictable_scheduled_one_time_offers_prepared_state() { + let mut entry = batch(Predictability::Predictable { + known_at: Some(TimestampMs(1_000)), + }); + entry.execute_at = Some(TimestampMs(11_000)); + let plan = plan_summary_maintenance_lifecycles( + summary(), + WorkloadDemand::new(&workload(vec![entry], vec![], at_rest()), &[0]), + 1_000, + None, + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + let prepared = &plan.deployments[0].alternatives[1]; + assert!(prepared.rejection.is_none()); + assert_eq!(prepared.total_cost, Some(Cost(13.0))); + } + + #[test] + fn nested_summary_lifecycles_have_compatible_evaluation_schedules() { + let workload = workload(vec![], vec![repeating()], continuous(1_000, 20_000)); + let plan = plan_summary_maintenance_lifecycles( + nested_summary(), + WorkloadDemand::new(&workload, &[0]), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &IncompatibleNestedCosts, + ) + .unwrap(); + + assert_eq!(plan.deployments.len(), 2); + let schedules: HashSet<_> = plan + .deployments + .iter() + .map(|deployment| { + deployment + .summary_maintenance_lifecycle_guarantee + .as_ref() + .unwrap() + .evaluation_schedule + }) + .collect(); + assert_eq!(schedules.len(), 1); + } + + #[test] + fn repeated_at_rest_selects_shared_without_inventing_updates() { + let plan = plan_summary_maintenance_lifecycles( + summary(), + WorkloadDemand::new(&workload(vec![], vec![repeating()], at_rest()), &[0]), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + assert_eq!( + selected_summary_maintenance_lifecycle(&plan.deployments[0]), + Some(&SummaryMaintenanceLifecycle::Shared { + retention: DurationMs(10_000) + }) + ); + assert_eq!( + plan.deployments[0].alternatives[3].rejection, + Some(SummaryMaintenanceLifecycleRejection::RequiresContinuousData) + ); + assert_eq!(plan.update_rate, None); + } + + #[test] + fn repeated_continuous_workload_can_select_continuous_maintenance() { + let capabilities = SummaryMaintenanceLifecycleCapabilities { + shared: false, + ..SummaryMaintenanceLifecycleCapabilities::ALL + }; + let plan = plan_summary_maintenance_lifecycles( + summary(), + WorkloadDemand::new( + &workload(vec![], vec![repeating()], continuous(1_000, 60_000)), + &[0], + ), + 1_000, + Some(Horizon(10.0)), + capabilities, + &UnitCosts, + ) + .unwrap(); + assert_eq!( + selected_summary_maintenance_lifecycle(&plan.deployments[0]), + Some(&SummaryMaintenanceLifecycle::ContinuouslyMaintained) + ); + assert_eq!(plan.evaluation_rate, Some(EvaluationRate(1.0))); + assert_eq!(plan.update_rate, Some(UpdateRate(1.0))); + } + + #[test] + fn stale_ingestion_evidence_cannot_enable_continuous_maintenance() { + let plan = plan_summary_maintenance_lifecycles( + summary(), + WorkloadDemand::new( + &workload(vec![], vec![repeating()], continuous(1_000, 1_000)), + &[0], + ), + 3_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + assert_eq!( + plan.deployments[0].alternatives[3].rejection, + Some(SummaryMaintenanceLifecycleRejection::MissingOrStaleIngestionRate) + ); + assert_eq!(plan.update_rate, None); + } + + #[test] + fn unknown_costs_do_not_make_a_long_lived_lifecycle_win() { + let plan = plan_summary_maintenance_lifecycles( + summary(), + WorkloadDemand::new( + &workload(vec![], vec![repeating()], continuous(1_000, 60_000)), + &[0], + ), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &crate::cost_model::DefaultCostModel, + ) + .unwrap(); + assert_eq!( + selected_summary_maintenance_lifecycle(&plan.deployments[0]), + None + ); + assert!(plan.deployments[0] + .alternatives + .iter() + .all(|alternative| alternative.rejection.is_some())); + } + + #[test] + fn unrelated_workload_entries_do_not_create_reuse_for_a_target() { + let plan = plan_summary_maintenance_lifecycles( + summary(), + WorkloadDemand::new( + &workload( + vec![batch(Predictability::AdHoc), batch(Predictability::AdHoc)], + vec![], + at_rest(), + ), + &[0], + ), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + assert_eq!( + selected_summary_maintenance_lifecycle(&plan.deployments[0]), + Some(&SummaryMaintenanceLifecycle::Ephemeral) + ); + assert_eq!( + plan.deployments[0].alternatives[2].rejection, + Some(SummaryMaintenanceLifecycleRejection::RequiresMultipleReads) + ); + } + + #[test] + fn scheduled_rate_counts_only_executions_inside_the_horizon() { + let mut entry = repeating(); + entry.demand = RepeatedDemand::Scheduled(vec![ + TimestampMs(999), + TimestampMs(5_000), + TimestampMs(20_000), + ]); + let plan = plan_summary_maintenance_lifecycles( + summary(), + WorkloadDemand::new(&workload(vec![], vec![entry], at_rest()), &[0]), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + assert_eq!(plan.evaluation_rate, Some(EvaluationRate(0.1))); + } + + #[test] + fn demand_binding_rejects_empty_and_duplicate_entries() { + let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); + assert!(matches!( + plan_summary_maintenance_lifecycles( + summary(), + WorkloadDemand::new(&workload, &[]), + 1_000, + None, + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ), + Err(SummaryMaintenanceLifecyclePlanError::EmptyWorkloadDemand) + )); + assert!(matches!( + plan_summary_maintenance_lifecycles( + summary(), + WorkloadDemand::new(&workload, &[0, 0]), + 1_000, + None, + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ), + Err(SummaryMaintenanceLifecyclePlanError::DuplicateWorkloadEntry { index: 0 }) + )); + } + + #[test] + fn prepared_requires_every_bound_consumer_to_be_scheduled_and_predictable() { + let mut predictable = batch(Predictability::Predictable { + known_at: Some(TimestampMs(1_000)), + }); + predictable.execute_at = Some(TimestampMs(2_000)); + let workload = workload( + vec![predictable, batch(Predictability::AdHoc)], + vec![], + at_rest(), + ); + let plan = plan_summary_maintenance_lifecycles( + summary(), + WorkloadDemand::new(&workload, &[0, 1]), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + assert_eq!( + plan.deployments[0].alternatives[1].rejection, + Some(SummaryMaintenanceLifecycleRejection::RequiresPredictableOneTimeQuery) + ); + } + + #[test] + fn moving_realtime_maintenance_requires_summary_deletion_support() { + let mut entry = repeating(); + entry.time_selection = TimeSelection { + scope: asap_types::workload::QueryTimeScope::RealTime, + lookback: Some(DurationMs(60_000)), + as_of: None, + }; + let plan = plan_summary_maintenance_lifecycles( + summary(), + WorkloadDemand::new( + &workload(vec![], vec![entry], continuous(1_000, 60_000)), + &[0], + ), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &NoDelete, + ) + .unwrap(); + assert_eq!( + plan.deployments[0].alternatives[3].rejection, + Some(SummaryMaintenanceLifecycleRejection::SummaryDoesNotSupportDeletion) + ); + } + + #[test] + fn lifecycle_cost_can_fall_back_to_raw_recomputation() { + let target = sum_query(); + let space = crate::replacement::search_workload(vec![("q", Rc::clone(&target))]); + let selection = space.global_selection(&RawCheaper); + let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); + let plan = materialize_with_summary_maintenance_lifecycles( + &selection, + &space.roots[0].1, + WorkloadDemand::new(&workload, &[0]), + 1_000, + None, + SummaryMaintenanceLifecycleCapabilities::ALL, + &RawCheaper, + ) + .unwrap() + .unwrap(); + assert!(plan.selected_raw_recompute); + assert_eq!(plan.raw_recompute_total_cost, Some(Cost(1.0))); + assert!(plan.deployments.is_empty()); + assert!(matches!(plan.root.expr, SummaryExpr::KeepPreAsap(_))); + } + + #[test] + fn lifecycle_cost_reorders_semantic_summary_candidates_before_materialization() { + let target = quantile_query(); + let space = crate::replacement::search_workload(vec![("q", target)]); + let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); + + let selection = global_selection_with_summary_maintenance_lifecycles( + &space, + &workload, + &[0], + 1_000, + None, + SummaryMaintenanceLifecycleCapabilities::ALL, + &SummaryMaintenancePrefersDdSketch, + ) + .unwrap(); + let materialized = selection.materialize(&space.roots[0].1).unwrap().unwrap(); + + assert_eq!( + sketch_algorithm(&materialized), + Some(SketchAlgorithm::DDSketch) + ); + } + + #[test] + fn lifecycle_cost_counts_one_shared_summary_node_once() { + let shared = summary(); + let root = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryMerge { + children: vec![Rc::clone(&shared), Rc::clone(&shared)], + }, + schema: shared.schema.clone(), + guarantee: None, + }); + let workload = workload( + vec![batch(Predictability::AdHoc), batch(Predictability::AdHoc)], + vec![], + at_rest(), + ); + let horizon = Some(Horizon(10.0)); + let plan = plan_summary_maintenance_lifecycles( + root, + WorkloadDemand::new(&workload, &[0, 1]), + 1_000, + horizon, + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + assert_eq!(plan.deployments.len(), 1); + assert!(matches!( + selected_summary_maintenance_lifecycle(&plan.deployments[0]), + Some(SummaryMaintenanceLifecycle::Shared { .. }) + )); + } + + #[test] + fn normalized_workload_drives_plan_space_recurrence_profiles() { + let root = query_root(); + let space = crate::replacement::search_workload(vec![("dashboard", Rc::clone(&root))]); + let workload = workload(vec![], vec![repeating()], continuous(1_000, 60_000)); + let profiles = space + .recurrence_profiles_from_workload(&workload, &[0], 1_000, Some(Horizon(10.0))) + .unwrap(); + // `search_workload` canonicalizes roots through CSE; recurrence + // profiles are keyed by that canonical post-CSE node. + let profile = profiles.for_target(&space.roots[0].1); + assert_eq!(profile.evaluation_rate, Some(EvaluationRate(1.0))); + assert_eq!(profile.update_rate, Some(UpdateRate(1.0))); + assert_eq!(profile.one_shot_consumers, 0); + } + + #[test] + fn recurrence_binding_is_explicit_when_root_order_differs_from_workload_order() { + let repeating_root = query_root_for("dashboard"); + let batch_root = query_root_for("batch"); + let space = crate::replacement::search_workload(vec![ + ("dashboard", repeating_root), + ("batch", batch_root), + ]); + let workload = workload( + vec![batch(Predictability::AdHoc)], + vec![repeating()], + at_rest(), + ); + let profiles = space + .recurrence_profiles_from_workload(&workload, &[1, 0], 1_000, Some(Horizon(10.0))) + .unwrap(); + let dashboard = profiles.for_target(&space.roots[0].1); + let batch = profiles.for_target(&space.roots[1].1); + assert_eq!(dashboard.evaluation_rate, Some(EvaluationRate(1.0))); + assert_eq!(dashboard.one_shot_consumers, 0); + assert_eq!(batch.evaluation_rate, None); + assert_eq!(batch.one_shot_consumers, 1); + } +} diff --git a/crates/devtools/src/bin/dag_export.rs b/crates/devtools/src/bin/dag_export.rs index 071c4a40..589e5d93 100644 --- a/crates/devtools/src/bin/dag_export.rs +++ b/crates/devtools/src/bin/dag_export.rs @@ -251,6 +251,56 @@ struct Winner<'a> { target: &'a Rc, candidate: &'a ReplacementSubDAG, cost: f64, + /// The unit `cost` is in — `None` for the legacy unitless structural + /// estimate, `Some("cost_units_per_second")` for a composed decision + /// `global_selection` costed as a recurring rate (issue #171). + cost_unit: Option<&'static str>, + /// For a `Replacement::ExactComposition` winner: the composed node + /// `GlobalSelection::materialize` linked over the child's own committed + /// decision — the shape actually exported. `None` otherwise. + materialized: Option>, + /// For a composed winner: the child target it was committed with. + child_target: Option<&'a Rc>, +} + +impl Winner<'_> { + /// The post-ASAP node to export for this winner, if it is a bound + /// (`Summary` or materialized composition) shape. + fn summary_node(&self) -> Option> { + match &self.candidate.replacement { + Replacement::Summary(node) => Some(Rc::clone(node)), + Replacement::ExactComposition(_) => self.materialized.clone(), + Replacement::Rewrite(_) => None, + } + } +} + +/// The `DagDecision` for winner `i`, with explicit provenance/unit and — +/// for a composed winner — the child decision it was committed with, so a +/// viewer never infers composition from graph shape (issue #171). +fn decision_for(i: usize, winners: &[Winner<'_>], role: &'static str) -> DagDecision { + let winner = &winners[i]; + let child_decisions = winner + .child_target + .into_iter() + .filter_map(|child| { + winners + .iter() + .position(|w| Rc::ptr_eq(w.target, child)) + .map(|j| j as u32) + }) + .collect(); + DagDecision { + id: i as u32, + strategy: winner.candidate.strategy.to_string(), + rationale: decision_rationale(winner), + rank: 0, + cost: winner.cost, + role, + provenance: Some(format!("{:?}", winner.candidate.provenance)), + cost_unit: winner.cost_unit.map(str::to_string), + child_decisions, + } } /// Short explanation intended for a selected winner in node-level UI. The @@ -280,6 +330,18 @@ fn decision_rationale(winner: &Winner<'_>) -> String { "Derives this smaller top-k from a compatible larger top-k result shared by the workload." .to_string() } + "ExactCompositionStrategy" => match winner.candidate.provenance { + asap_aware_mapping::replacement::ReplacementProvenance::ExactPostProcess => { + "Applies the exact fold at query time over the child's committed summary readout." + .to_string() + } + asap_aware_mapping::replacement::ReplacementProvenance::ExactTransform => { + "Runs the exact row transform on the update path, feeding the maintained summary above it." + .to_string() + } + _ => "Composes an exact operator with a summary plan across an explicit domain boundary." + .to_string(), + }, _ => { let summary = winner .candidate @@ -325,13 +387,14 @@ fn target_replacement( ) -> TargetReplacement { let strategy = winner.candidate.strategy.to_string(); let before = dag_export::export(winner.target); - let after = match &winner.candidate.replacement { - Replacement::Summary(node) => { - TargetReplacementAfter::Summary(dag_export::export_summary(node)) - } - Replacement::Rewrite(rewritten) => { + let after = match (&winner.candidate.replacement, winner.summary_node()) { + (Replacement::Rewrite(rewritten), _) => { TargetReplacementAfter::Rewrite(dag_export::export(rewritten)) } + (_, Some(node)) => TargetReplacementAfter::Summary(dag_export::export_summary(&node)), + (_, None) => { + unreachable!("a composed winner always carries its materialized node") + } }; TargetReplacement { decision_id, @@ -428,6 +491,10 @@ fn run_post_asap_with_progress( .collect(); let space = search_workload(roots); let ranked_groups = space.cost_sorted(&DefaultCostModel); + // Composed decisions (issue #171) are only ever committed by + // `global_selection`, together with the child decision they compose + // over — never by per-group `cost_sorted` ranking. + let selection = space.global_selection(&DefaultCostModel); // A group's top candidate can be `keep_pre_asap`'s own conservative // fallback — `Replacement::Summary(SummaryNode { expr: @@ -454,7 +521,26 @@ fn run_post_asap_with_progress( let winners: Vec> = ranked_groups .iter() .filter_map(|group| { - let candidate = group.candidates.first()?; + if let Some(selected) = selection.for_target(group.target) { + if let (Some(chosen), Some(decision)) = (selected.chosen, &selected.composition) { + let materialized = selection.materialize(group.target).ok().flatten()?; + return Some(Winner { + target: group.target, + candidate: chosen, + cost: decision.cost_rate.0, + cost_unit: Some(decision.inputs.unit.as_str()), + materialized: Some(materialized), + child_target: Some(decision.child_target), + }); + } + } + // A composition not committed by `global_selection` is never a + // winner on its own — it has no child to compose over. + let (i, candidate) = group + .candidates + .iter() + .enumerate() + .find(|(_, c)| !matches!(c.replacement, Replacement::ExactComposition(_)))?; if matches!( &candidate.replacement, Replacement::Summary(node) if matches!(node.expr, SummaryExpr::KeepPreAsap(_)) @@ -464,7 +550,10 @@ fn run_post_asap_with_progress( Some(Winner { target: group.target, candidate, - cost: group.costs[0], + cost: group.costs[i], + cost_unit: None, + materialized: None, + child_target: None, }) }) .collect(); @@ -507,24 +596,20 @@ fn run_post_asap_with_progress( let mut find_winner = |expr: &QueryExpr| -> Option { let i = lookup_winner(&by_hash, &winners, &mut post_graph_cache, expr)?; let winner = &winners[i]; - let decision = DagDecision { - id: i as u32, - strategy: winner.candidate.strategy.to_string(), - rationale: decision_rationale(winner), - rank: 0, - cost: winner.cost, - role: "replacement_region", - }; - Some(match &winners[i].candidate.replacement { - Replacement::Rewrite(rc) => PostAsapSubstitution::Rewrite { - replacement: Rc::clone(rc), - decision, + let decision = decision_for(i, &winners, "replacement_region"); + Some( + match (&winner.candidate.replacement, winner.summary_node()) { + (Replacement::Rewrite(rc), _) => PostAsapSubstitution::Rewrite { + replacement: Rc::clone(rc), + decision, + }, + (_, Some(node)) => PostAsapSubstitution::Summary { + replacement: node, + decision, + }, + (_, None) => unreachable!("a composed winner always carries its materialized node"), }, - Replacement::Summary(rc) => PostAsapSubstitution::Summary { - replacement: Rc::clone(rc), - decision, - }, - }) + ) }; let post_graphs: Vec<(String, DagGraph)> = lowered_queries .iter() diff --git a/crates/frontend-promql/src/lib.rs b/crates/frontend-promql/src/lib.rs index 08fec09a..ff29379a 100644 --- a/crates/frontend-promql/src/lib.rs +++ b/crates/frontend-promql/src/lib.rs @@ -71,11 +71,7 @@ pub fn lower_promql_batch(workload: &QueryWorkload) -> Vec QueryExpr { + let mut columns = vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + ]; + columns.extend(labels.iter().map(|n| Column::new(*n, DataType::Utf8, true))); + QueryExpr::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: vec![], + schema: Schema::with_time_index(columns, 0, vec![]), + } +} + +fn agg(by: Vec, intent: AggIntent, child: Rc) -> Rc { + Rc::new(QueryExpr::Aggregate { + reduction: Reduction::by(by), + measures: vec![intent], + output_names: vec![], + having: None, + child, + }) +} + +fn per_entity(intent: AggIntent, child: Rc) -> Rc { + Rc::new(QueryExpr::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![intent], + output_names: vec![], + having: None, + child, + }) +} + +/// `quantile by (zone, host) (latency)` — the fine-grained inner summary. +fn fine_quantile() -> Rc { + agg( + vec![2, 3], + default_quantile(0.99), + Rc::new(metric_scan(&["zone", "host"])), + ) +} + +/// A deployment cost model that supplies every statistic the issue's +/// formulas need, so a composition can actually win — and advertises both +/// mixed-execution shapes. +struct StatsModel; + +impl CostModel for StatsModel { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + fn exact_composition_cost_inputs( + &self, + _request: &ExactCompositionCostRequest<'_>, + ) -> ExactCompositionCostInputs { + ExactCompositionCostInputs { + exact_cost_per_row: Some(0.1), + expected_input_rows: Some(50.0), + expected_output_rows: Some(10.0), + summary_maintenance_cost_per_update: Some(0.01), + summary_read_cost: Some(1.0), + update_rate: Some(100.0), + evaluation_rate: Some(EvaluationRate(1.0)), + raw_recompute_cost: Some(100.0), + unit: CostUnit::CostUnitsPerSecond, + provenance: CostProvenance { + model: "StatsModel".into(), + version: "test-1".into(), + }, + } + } +} + +/// Same statistics, but the runtime advertises no mixed-execution shape. +struct NoCapabilityModel; + +impl CostModel for NoCapabilityModel { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + fn mixed_execution_capabilities(&self) -> MixedExecutionCapabilities { + MixedExecutionCapabilities::NONE + } + fn exact_composition_cost_inputs( + &self, + request: &ExactCompositionCostRequest<'_>, + ) -> ExactCompositionCostInputs { + StatsModel.exact_composition_cost_inputs(request) + } +} + +fn plan( + roots: Vec<(&'static str, Rc)>, + cost_model: &dyn CostModel, +) -> asap_aware_mapping::PlanSpace<&'static str> { + search_workload_with(roots, &default_strategies_with(cost_model)) +} + +fn is_plain(node: &SummaryNode) -> bool { + node.schema + .fields + .iter() + .all(|f| matches!(f.dtype, SummaryFamilyType::Plain(_))) +} + +fn names(node: &SummaryNode) -> Vec<&str> { + node.schema.fields.iter().map(|f| f.name.as_str()).collect() +} + +// ── step 1: pin every already-supported exact-accumulator nesting ─────── + +#[test] +fn every_exact_accumulator_nests_directly_under_an_outer_sketch() { + use std::time::Duration; + let cases: Vec<(Rc, ExactKind)> = vec![ + ( + agg( + vec![2], + AggIntent::Sum { col: None }, + Rc::new(metric_scan(&["zone"])), + ), + ExactKind::Sum, + ), + ( + agg( + vec![2], + AggIntent::Count { + accuracy: AccuracyTarget::Exact, + }, + Rc::new(metric_scan(&["zone"])), + ), + ExactKind::Count, + ), + ( + agg( + vec![2], + AggIntent::Min { col: None }, + Rc::new(metric_scan(&["zone"])), + ), + ExactKind::MinMax, + ), + ( + agg( + vec![2], + AggIntent::Max { col: None }, + Rc::new(metric_scan(&["zone"])), + ), + ExactKind::MinMax, + ), + ( + per_entity( + AggIntent::Rate, + Rc::new(QueryExpr::TimeRange { + range: Duration::from_secs(300), + child: Rc::new(metric_scan(&["zone"])), + }), + ), + ExactKind::Rate, + ), + ( + per_entity( + AggIntent::Increase, + Rc::new(QueryExpr::TimeRange { + range: Duration::from_secs(300), + child: Rc::new(metric_scan(&["zone"])), + }), + ), + ExactKind::Increase, + ), + ]; + for (inner, kind) in cases { + let outer = agg(vec![], default_quantile(0.9), inner); + let target = TargetSubDAG::new(&outer); + let candidates = SketchAlgorithmStrategy::default_cost_model().replacements(&target); + let Replacement::Summary(root) = &candidates[0].replacement else { + unreachable!() + }; + let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { + panic!("expected KLL readout, got {:?}", root.expr); + }; + let SummaryExpr::SummaryAgg { child, .. } = &summary_input.expr else { + panic!("expected outer SummaryAgg"); + }; + assert!( + matches!( + &child.expr, + SummaryExpr::SummaryAgg { family: SummaryFamilyType::ExactAggregate(k, _), .. } if *k == kind + ), + "{kind:?}: expected the exact accumulator directly under the outer sketch, got {:?}", + child.expr + ); + validate_execution_data_states(root).expect("accumulator state composes under maintenance"); + } +} + +// ── direction 1: outer exact fold over an inner summary readout ──────── + +/// Before this PR both `max`/`avg` over a quantile collapsed into one +/// opaque `KeepPreAsap`. Now: the outer group holds an `ExactPostProcess` +/// candidate referencing the inner target, the inner group keeps its own +/// sketch candidates, and with statistics the pair is committed and +/// materializes as `ExactPostProcess → SummaryEstimate → SummaryAgg`. +#[test] +fn max_and_avg_over_quantile_compose_as_post_process_with_statistics() { + for intent in [AggIntent::Max { col: None }, AggIntent::Avg { col: None }] { + let root = agg(vec![0], intent.clone(), fine_quantile()); + let space = plan(vec![("q", Rc::clone(&root))], &StatsModel); + let root = Rc::clone(&space.roots[0].1); + let QueryExpr::Aggregate { child: inner, .. } = root.as_ref() else { + unreachable!() + }; + + let outer_group = space.group_for(&root).unwrap(); + assert!( + outer_group + .candidates + .iter() + .any(|c| c.provenance == ReplacementProvenance::ExactPostProcess), + "{intent:?}: outer group must hold an ExactPostProcess candidate" + ); + let inner_group = space.group_for(inner).unwrap(); + assert!( + inner_group + .candidates + .iter() + .any(|c| matches!(&c.replacement, Replacement::Summary(n) + if matches!(n.expr, SummaryExpr::SummaryEstimate { .. }))), + "{intent:?}: the inner quantile keeps its own readout candidates" + ); + + let selection = space.global_selection(&StatsModel); + let selected = selection.for_target(&root).unwrap(); + let chosen = selected.chosen.expect("a decision"); + assert_eq!(chosen.provenance, ReplacementProvenance::ExactPostProcess); + let decision = selected + .composition + .as_ref() + .expect("composition provenance"); + assert!(Rc::ptr_eq(decision.child_target, inner)); + assert!(decision.cost_rate < decision.baseline_rate); + assert_eq!(decision.inputs.unit, CostUnit::CostUnitsPerSecond); + assert_eq!(decision.inputs.provenance.model, "StatsModel"); + // The child was committed to a compatible candidate *from its own + // group* — the same candidate its own selection reports. + let child_candidate = decision.child_candidate.expect("post-process child"); + let inner_selected = selection.for_target(inner).unwrap(); + assert!(std::ptr::eq( + inner_selected.chosen.unwrap(), + child_candidate + )); + + let composed = selection.materialize(&root).unwrap().unwrap(); + let SummaryExpr::ReadoutPostProcess { child, .. } = &composed.expr else { + panic!( + "{intent:?}: expected ExactPostProcess root, got {:?}", + composed.expr + ); + }; + assert!(matches!(child.expr, SummaryExpr::SummaryEstimate { .. })); + let child_guarantee = child.guarantee.as_ref().expect("child guarantee"); + let composed_guarantee = composed + .guarantee + .as_ref() + .expect("exact post-process must propagate the child's guarantee"); + assert_eq!(composed_guarantee.metric, child_guarantee.metric); + assert_eq!( + composed_guarantee.bound.evaluate(), + child_guarantee.bound.evaluate(), + "an exact max/average fold retains the modeled error magnitude" + ); + assert!(is_plain(&composed)); + assert_eq!( + names(&composed), + root.output_schema() + .unwrap() + .columns + .iter() + .map(|c| c.name.as_str()) + .collect::>(), + "the composed plan's schema is the pre-ASAP target's own" + ); + validate_execution_data_states(&composed).unwrap(); + } +} + +/// `avg` keeps competing with `AvgToSumOverCountStrategy`: both candidates +/// live in the same group; nothing hard-codes the winner. +#[test] +fn avg_over_quantile_keeps_the_sum_over_count_rewrite_as_a_competitor() { + // `by (zone)` over `by (zone)`: the averaged column resolves to the + // non-null quantile output, which is what the rewrite requires. + let inner = agg( + vec![2], + default_quantile(0.99), + Rc::new(metric_scan(&["zone"])), + ); + let root = agg(vec![0], AggIntent::Avg { col: None }, inner); + let space = plan(vec![("q", root)], &StatsModel); + let group = space.group_for(&space.roots[0].1).unwrap(); + let provenances: Vec<_> = group.candidates.iter().map(|c| c.provenance).collect(); + assert!(provenances.contains(&ReplacementProvenance::LogicalRewrite)); + assert!(provenances.contains(&ReplacementProvenance::ExactPostProcess)); +} + +/// Grouped fine-to-coarse fold (`by (zone)` over `by (zone, host)`) and the +/// identity fold (`by (zone)` over `by (zone)`) both compose; the operator +/// is the same, only the fold's row multiplicity differs. +#[test] +fn identity_and_genuine_multi_row_folds_both_compose() { + let identity_inner = agg( + vec![2], + default_quantile(0.99), + Rc::new(metric_scan(&["zone"])), + ); + for (label, inner) in [ + ("identity", identity_inner), + ("fine-to-coarse", fine_quantile()), + ] { + let root = agg(vec![0], AggIntent::Max { col: None }, inner); + let space = plan(vec![("q", root)], &StatsModel); + let root = &space.roots[0].1; + let composed = space + .global_selection(&StatsModel) + .materialize(root) + .unwrap() + .unwrap(); + assert!( + matches!(composed.expr, SummaryExpr::ReadoutPostProcess { .. }), + "{label}: {:?}", + composed.expr + ); + assert_eq!(names(&composed), vec!["zone", "max"], "{label}"); + } +} + +/// One inner quantile consumed by two outer folds in two queries: CSE +/// collapses the inner target onto one `Rc`, both compositions commit to +/// the *same* child candidate, and both materializations share one +/// `Rc` for it — the summary is maintained once. +#[test] +fn a_shared_inner_summary_is_materialized_once_for_several_outer_folds() { + let max = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); + let min = agg(vec![0], AggIntent::Min { col: None }, fine_quantile()); + let space = plan(vec![("max", max), ("min", min)], &StatsModel); + let selection = space.global_selection(&StatsModel); + + let roots: Vec> = space.roots.iter().map(|(_, r)| Rc::clone(r)).collect(); + let inner_of = |r: &Rc| match r.as_ref() { + QueryExpr::Aggregate { child, .. } => Rc::clone(child), + _ => unreachable!(), + }; + assert!( + Rc::ptr_eq(&inner_of(&roots[0]), &inner_of(&roots[1])), + "CSE must intern the shared inner quantile" + ); + let inner = inner_of(&roots[0]); + assert_eq!(space.group_for(&inner).unwrap().consumer_count, 2); + + let decisions: Vec<_> = roots + .iter() + .map(|r| { + selection + .for_target(r) + .unwrap() + .composition + .as_ref() + .expect("both roots compose") + }) + .collect(); + assert!(std::ptr::eq( + decisions[0].child_candidate.unwrap(), + decisions[1].child_candidate.unwrap() + )); + // Shared state counted once: the second parent sees zero marginal + // maintenance, so its rate is strictly lower than the first's. + assert!(decisions[1].cost_rate < decisions[0].cost_rate); + + let composed: Vec<_> = roots + .iter() + .map(|r| selection.materialize(r).unwrap().unwrap()) + .collect(); + let child_of = |n: &Rc| match &n.expr { + SummaryExpr::ReadoutPostProcess { child, .. } => Rc::clone(child), + other => panic!("expected ExactPostProcess, got {other:?}"), + }; + assert!( + Rc::ptr_eq(&child_of(&composed[0]), &child_of(&composed[1])), + "both folds compose over the same Rc" + ); +} + +// ── direction 2: outer summary over an inner exact update-path transform ─ + +/// `quantile(0.99, deriv(latency[5m]))`: `deriv` has no accumulator form. +/// The transform target gets an `ExactTransform` candidate; with a +/// maintained summary above it and statistics, it is committed, and the +/// outer summary's materialization is re-linked over it. +#[test] +fn outer_summary_over_an_exact_transform_composes_on_the_update_path() { + use std::time::Duration; + let deriv = per_entity( + AggIntent::Deriv, + Rc::new(QueryExpr::TimeRange { + range: Duration::from_secs(300), + child: Rc::new(metric_scan(&["zone"])), + }), + ); + let root = agg(vec![], default_quantile(0.99), deriv); + let space = plan(vec![("q", root)], &StatsModel); + let root = Rc::clone(&space.roots[0].1); + let QueryExpr::Aggregate { child: deriv, .. } = root.as_ref() else { + unreachable!() + }; + assert!(space + .group_for(deriv) + .unwrap() + .candidates + .iter() + .any(|c| c.provenance == ReplacementProvenance::ExactTransform)); + + let selection = space.global_selection(&StatsModel); + let deriv_sel = selection.for_target(deriv).unwrap(); + assert_eq!( + deriv_sel.chosen.unwrap().provenance, + ReplacementProvenance::ExactTransform + ); + let decision = deriv_sel.composition.as_ref().unwrap(); + assert!(decision.child_candidate.is_none(), "transform input is raw"); + assert!(decision.cost_rate < decision.baseline_rate); + + let composed = selection.materialize(&root).unwrap().unwrap(); + let SummaryExpr::SummaryEstimate { summary_input, .. } = &composed.expr else { + panic!("expected readout root, got {:?}", composed.expr); + }; + let SummaryExpr::SummaryAgg { child, .. } = &summary_input.expr else { + panic!("expected SummaryAgg"); + }; + let SummaryExpr::UpdateTransform { child: raw, .. } = &child.expr else { + panic!( + "expected ExactTransform under the maintained summary, got {:?}", + child.expr + ); + }; + assert!(matches!(raw.expr, SummaryExpr::KeepPreAsap(_))); + let assignment = validate_execution_data_states(&composed).unwrap(); + assert_eq!( + assignment.data_state_of(child), + Some(ExecutionDataState::MAINTENANCE_ROWS) + ); + assert_eq!( + assignment.data_state_of(raw), + Some(ExecutionDataState::MAINTENANCE_ROWS) + ); +} + +// ── rejection, capability, statistics ─────────────────────────────────── + +/// A maintained summary above a query-time readout is a typed plan-time +/// error, both for the construction path and for a hand-built plan. +#[test] +fn readout_under_maintenance_is_rejected_at_construction() { + let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); + let candidates = + SketchAlgorithmStrategy::default_cost_model().replacements(&TargetSubDAG::new(&root)); + // The MinMax accumulator over the quantile readout is not constructible; + // the strategy reports the conservative fallback once instead. + assert_eq!(candidates.len(), 1); + let Replacement::Summary(node) = &candidates[0].replacement else { + unreachable!() + }; + assert!(matches!(node.expr, SummaryExpr::KeepPreAsap(_))); + + // ExactPostProcess can never be placed under a SummaryAgg: compose a + // post-process, then try to maintain a summary over it. + let space = plan(vec![("q", Rc::clone(&root))], &StatsModel); + let post = space + .global_selection(&StatsModel) + .materialize(&space.roots[0].1) + .unwrap() + .unwrap(); + let illegal = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child: post, + family: SummaryFamilyType::ExactAggregate( + ExactKind::MinMax, + asap_types::post_asap::ExactParams::MinMax, + ), + col: asap_types::pre_asap::ColumnRef::SampleValue, + reduction: Reduction::by(vec![]), + grouping: Default::default(), + }, + schema: asap_types::post_asap::SummarySchema { + fields: vec![], + time_index: None, + }, + guarantee: None, + }); + assert!(matches!( + validate_execution_data_states(&illegal), + Err(ExecutionDataStateError::ReadoutUnderMaintenance { .. }) + )); + let err: ImplementError = validate_execution_data_states(&illegal).unwrap_err().into(); + assert!(matches!(err, ImplementError::ExecutionDataState(_))); +} + +#[test] +fn a_runtime_without_mixed_execution_gets_no_composition_candidates() { + let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); + let space = plan(vec![("q", root)], &NoCapabilityModel); + let root = Rc::clone(&space.roots[0].1); + let group = space.group_for(&root).unwrap(); + assert!(group + .candidates + .iter() + .all(|c| !matches!(c.replacement, Replacement::ExactComposition(_)))); + let selection = space.global_selection(&NoCapabilityModel); + assert!(selection.for_target(&root).unwrap().composition.is_none()); + let node = selection.materialize(&root).unwrap().unwrap(); + assert!(matches!(node.expr, SummaryExpr::KeepPreAsap(_))); + // The inner quantile is still independently selectable. + let QueryExpr::Aggregate { child, .. } = root.as_ref() else { + unreachable!() + }; + assert!(selection.for_target(child).unwrap().chosen.is_some()); +} + +/// Without statistics (the built-in model) the composition is *proposed* +/// — visible in `PlanSpace` and explanations — but never *selected*: the +/// site keeps the conservative `KeepPreAsap`, and the inner summary stays +/// independently selectable. +#[test] +fn missing_cost_statistics_preserve_the_conservative_keep_pre_asap() { + let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); + let space = plan(vec![("q", root)], &DefaultCostModel); + let root = Rc::clone(&space.roots[0].1); + assert!(space + .group_for(&root) + .unwrap() + .candidates + .iter() + .any(|c| c.provenance == ReplacementProvenance::ExactPostProcess)); + let selection = space.global_selection(&DefaultCostModel); + let selected = selection.for_target(&root).unwrap(); + assert!(selected.composition.is_none()); + assert!(!matches!( + selected.chosen.map(|c| &c.replacement), + Some(Replacement::ExactComposition(_)) + )); + let node = selection.materialize(&root).unwrap().unwrap(); + assert!(matches!(node.expr, SummaryExpr::KeepPreAsap(_))); + + let explanations = asap_aware_mapping::explain_replacements(vec![("q", (*root).clone())]); + assert!(explanations + .iter() + .any(|e| e.kind == ExplanationKind::ExactComposition)); +} + +// ── DAG export: explicit data_state, schema, provenance ────────────────────── + +#[test] +fn dag_export_carries_explicit_domain_and_plain_schema_for_a_composed_plan() { + let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); + let space = plan(vec![("q", root)], &StatsModel); + let root = &space.roots[0].1; + let composed = space + .global_selection(&StatsModel) + .materialize(root) + .unwrap() + .unwrap(); + let graph = dag_export::export_summary(&composed); + let node = &graph.nodes[graph.root as usize]; + assert_eq!(node.kind, "ReadoutPostProcess"); + assert_eq!(node.detail["execution_data_state"]["timing"], "read_time"); + assert_eq!(node.detail["execution_data_state"]["primitive"], "rows"); + assert_eq!(node.detail["op"], "Aggregate"); + let domains: Vec<(&str, &str, &str)> = graph + .nodes + .iter() + .map(|n| { + ( + n.kind, + n.detail["execution_data_state"]["timing"].as_str().unwrap(), + n.detail["execution_data_state"]["primitive"] + .as_str() + .unwrap(), + ) + }) + .collect(); + assert!(domains.contains(&("SummaryEstimate", "read_time", "rows"))); + assert!(domains.contains(&("SummaryAgg", "maintenance_time", "summary_state"))); + assert!(domains.contains(&("KeepPreAsap", "maintenance_time", "rows"))); + + // Pre-ASAP export of the same target still describes the same columns. + let pre = dag_export::export(root); + let pre_root = &pre.nodes[pre.root as usize]; + let pre_cols: Vec = pre_root.schema.as_ref().unwrap()["columns"] + .as_array() + .unwrap() + .iter() + .map(|c| c["name"].as_str().unwrap().to_string()) + .collect(); + assert_eq!(pre_cols, names(&composed)); +} + +/// The PromQL front end produces the exact issue shape and it composes. +#[test] +fn promql_max_by_zone_over_quantile_over_time_composes() { + let expr = lower_promql( + "max by (zone) (quantile_over_time(0.99, latency[5m]))", + AccuracyTarget::Epsilon(0.01), + ) + .unwrap(); + let space = plan(vec![("q", Rc::new(expr))], &StatsModel); + let root = &space.roots[0].1; + let selection = space.global_selection(&StatsModel); + let selected = selection.for_target(root).unwrap(); + assert_eq!( + selected.chosen.map(|c| c.provenance), + Some(ReplacementProvenance::ExactPostProcess), + "{:?}", + space + .group_for(root) + .unwrap() + .candidates + .iter() + .map(|c| (c.strategy, c.provenance)) + .collect::>() + ); + let composed = selection.materialize(root).unwrap().unwrap(); + assert!(matches!( + composed.expr, + SummaryExpr::ReadoutPostProcess { .. } + )); + assert_eq!( + selected.composition.as_ref().map(|d| d.inputs.unit), + Some(CostUnit::CostUnitsPerSecond) + ); + let _ = CompositionPlacement::PostProcess; +} diff --git a/crates/integration-tests/tests/workload_lifecycle_e2e.rs b/crates/integration-tests/tests/workload_lifecycle_e2e.rs new file mode 100644 index 00000000..dacfc037 --- /dev/null +++ b/crates/integration-tests/tests/workload_lifecycle_e2e.rs @@ -0,0 +1,173 @@ +//! End-to-end coverage for workload-aware summary-maintenance planning: +//! source workload -> PromQL lowering -> candidate search -> lifecycle-aware +//! global selection -> materialized deployment guarantees. + +use std::rc::Rc; + +use asap_aware_mapping::cost_model::Cost; +use asap_aware_mapping::CostRate; +use asap_aware_mapping::{ + global_selection_with_summary_maintenance_lifecycles, + materialize_with_summary_maintenance_lifecycles, search_workload_with, CostModel, Horizon, + SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCapabilities, + SummaryMaintenanceLifecycleCostInputs, SummaryMaintenanceLifecycleRejection, WorkloadDemand, +}; +use asap_frontend_promql::lower_promql_batch; +use asap_types::post_asap::{EvaluationSchedule, SummaryMaintenanceLifecycle, SummaryNode}; +use asap_types::pre_asap::agg_intent::AggIntent; +use asap_types::types::AccuracyTarget; +use asap_types::workload::{ + AccuracyRequirement, BatchEntry, DataArrival, DataWorkload, Evidence, EvidenceSource, + Predictability, Query, QueryLanguage, QueryRequirements, QueryTimeScope, QueryWorkload, Rate, + RepeatedDemand, RepeatingEntry, RepetitionInterval, TimeSelection, +}; + +const NOW_MS: u64 = 1_000_000; + +struct FullyCostedRuntime; + +impl CostModel for FullyCostedRuntime { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[asap_types::post_asap::SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + + fn summary_maintenance_lifecycle_cost_inputs( + &self, + _summary: &SummaryNode, + ) -> SummaryMaintenanceLifecycleCostInputs { + SummaryMaintenanceLifecycleCostInputs { + build_cost: Some(Cost(10.0)), + maintenance_cost_per_update: Some(Cost(1.0)), + summary_read_cost: Some(Cost(1.0)), + retention_cost_rate: Some(CostRate(0.1)), + retirement_cost: Some(Cost(1.0)), + } + } + + fn summary_maintenance_capabilities( + &self, + _summary: &SummaryNode, + ) -> SummaryMaintenanceCapabilities { + SummaryMaintenanceCapabilities { + incremental_update: true, + merge: true, + delete: true, + } + } +} + +fn dashboard_workload() -> QueryWorkload { + let query = Query("quantile_over_time(0.99, latency[5m])".into()); + let requirements = QueryRequirements { + accuracy: AccuracyRequirement::Explicit(AccuracyTarget::Epsilon(0.01)), + ..QueryRequirements::default() + }; + QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: Some(vec![BatchEntry { + query: query.clone(), + requirements: requirements.clone(), + predictability: Predictability::AdHoc, + invocations: 1, + execute_at: None, + time_selection: TimeSelection::default(), + }]), + repeating_queries: Some(vec![RepeatingEntry { + query, + demand: RepeatedDemand::FixedInterval(RepetitionInterval(1_000)), + requirements, + predictability: Predictability::Predictable { known_at: None }, + time_selection: TimeSelection { + scope: QueryTimeScope::RealTime, + ..TimeSelection::default() + }, + }]), + data_workload: Some(DataWorkload { + arrival: DataArrival::ContinuouslyIngesting, + ingestion_rate: Evidence { + value: Some(Rate(1.0)), + source: EvidenceSource::Observed, + observed_at_ms: Some(NOW_MS), + valid_for_ms: Some(60_000), + }, + ..DataWorkload::default() + }), + } +} + +#[test] +fn promql_dashboard_materializes_continuous_summary_with_explained_rejections() { + let workload = dashboard_workload(); + workload.validate().unwrap(); + + let lowered = lower_promql_batch(&workload) + .into_iter() + .next() + .expect("one normalized workload entry") + .expect("valid PromQL"); + let root = Rc::new(lowered); + let strategies = asap_aware_mapping::default_strategies_with(&FullyCostedRuntime); + let space = search_workload_with(vec![("dashboard", Rc::clone(&root))], &strategies); + let target = Rc::clone(&space.roots[0].1); + let capabilities = SummaryMaintenanceLifecycleCapabilities { + ephemeral: true, + prepared: false, + shared: false, + continuously_maintained: true, + }; + + let selection = global_selection_with_summary_maintenance_lifecycles( + &space, + &workload, + &[1], + NOW_MS, + Some(Horizon(100.0)), + capabilities, + &FullyCostedRuntime, + ) + .unwrap(); + let plan = materialize_with_summary_maintenance_lifecycles( + &selection, + &target, + WorkloadDemand::new(&workload, &[1]), + NOW_MS, + Some(Horizon(100.0)), + capabilities, + &FullyCostedRuntime, + ) + .unwrap() + .expect("selected summary plan"); + + assert!(!plan.selected_raw_recompute); + assert_eq!(plan.expected_reads, Some(100.0)); + assert_eq!(plan.deployments.len(), 1); + + let deployment = &plan.deployments[0]; + let guarantee = deployment + .summary_maintenance_lifecycle_guarantee + .as_ref() + .expect("selected lifecycle guarantee"); + assert_eq!( + guarantee.summary_maintenance_lifecycle, + SummaryMaintenanceLifecycle::ContinuouslyMaintained + ); + assert_eq!(guarantee.evaluation_schedule, EvaluationSchedule::PerUpdate); + assert!(deployment.alternatives.iter().any(|alternative| { + matches!( + alternative.summary_maintenance_lifecycle, + SummaryMaintenanceLifecycle::Prepared { .. } + ) && alternative.rejection + == Some(SummaryMaintenanceLifecycleRejection::RequiresPredictableOneTimeQuery) + })); + assert!(deployment.alternatives.iter().any(|alternative| { + matches!( + alternative.summary_maintenance_lifecycle, + SummaryMaintenanceLifecycle::Shared { .. } + ) && alternative.rejection + == Some(SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime) + })); +} diff --git a/crates/types/src/dag_export.rs b/crates/types/src/dag_export.rs index 5df2ed8c..3672c658 100644 --- a/crates/types/src/dag_export.rs +++ b/crates/types/src/dag_export.rs @@ -14,7 +14,7 @@ //! //! This is literally the same hashing //! [`share_common_subtrees`](crate::pre_asap::cse::share_common_subtrees) -//! uses to bucket candidates in its `InternTable` (issue #223 stage 3) — not +//! uses to bucket candidates in its `InternTable` (issue #223 data_state 3) — not //! a parallel reimplementation. `tools/dag-viewer`'s "shared subtree" //! highlighting is still a *proxy* for real CSE, though: a hash match here //! only means two nodes are legal `InternTable` bucket-mates (same coarse @@ -46,7 +46,10 @@ use std::rc::Rc; use serde::Serialize; -use crate::post_asap::{AccuracyError, ResultGuarantee, SummaryExpr, SummaryNode}; +use crate::post_asap::{ + assigned_child_data_state, produced_data_state, AccuracyError, ExactOperator, + ExecutionDataState, ResultGuarantee, SummaryExpr, SummaryNode, ValueOperator, +}; use crate::pre_asap::cse::{structural_hash, HashCache}; use crate::pre_asap::query_expr::{QueryExpr, Source}; @@ -152,6 +155,24 @@ pub struct DagDecision { /// `replacement_root` for the node replacing the pre-ASAP target; /// `replacement_region` for its generated or carried descendants. pub role: &'static str, + /// Machine-readable origin of the winning candidate (a `Debug`-formatted + /// `asap_aware_mapping::ReplacementProvenance`, e.g. + /// `"ExactPostProcess"`), so a viewer never infers *how* a node was + /// composed from its label or shape (issue #171). Omitted when the + /// producing layer predates this field. + #[serde(skip_serializing_if = "Option::is_none")] + pub provenance: Option, + /// The unit `cost` is expressed in — e.g. `"cost_units_per_second"` for + /// a recurring-rate comparison, or absent for the legacy unitless + /// structural estimate. Additive; consumers must not assume one unit. + #[serde(skip_serializing_if = "Option::is_none")] + pub cost_unit: Option, + /// For a composed decision (an exact operator over another target's + /// own selected decision), the `id`s of the child decisions this one + /// was committed together with — the explicit target-to-decision + /// provenance chain, never reconstructed from graph adjacency. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub child_decisions: Vec, } /// One query's exported graph. `nodes[root as usize]` is the tree's root. @@ -338,10 +359,63 @@ pub struct SummaryDagGraph { /// top-level `DagNode::kind`. pub fn export_summary(node: &SummaryNode) -> SummaryDagGraph { let mut nodes = Vec::new(); - let root = build_summary(node, &mut nodes); + let root = build_summary(node, &mut nodes, root_domain(node)); SummaryDagGraph { nodes, root } } +/// The explicit execution data_state of an exported plan's root — its own +/// produced data_state, or query-time readout for a bare `KeepPreAsap` +/// (the same convention `post_asap::execution_data_state::validate_execution_data_states` +/// uses for a root). +fn root_domain(node: &SummaryNode) -> ExecutionDataState { + produced_data_state(&node.expr).unwrap_or(ExecutionDataState::READ_ROWS) +} + +/// `detail` for an [`ExactOperator`] payload — its own fields, rendered the +/// same way the pre-ASAP `Aggregate` node renders them. +fn exact_operator_detail(op: &ExactOperator) -> serde_json::Value { + match op { + ExactOperator::Aggregate { + reduction, + measures, + output_names, + having, + } => serde_json::json!({ + "op": "Aggregate", + "reduction": reduction, + "measures": measures, + "output_names": output_names, + "having": having.as_ref().map(|p| export(&p.0)), + }), + } +} + +fn exact_operator_label(op: &ExactOperator) -> String { + match op { + ExactOperator::Aggregate { measures, .. } => { + let funcs: Vec = measures.iter().map(|m| format!("{m:?}")).collect(); + format!("Aggregate[{}]", funcs.join(", ")) + } + } +} + +fn value_operator_detail(op: &ValueOperator) -> serde_json::Value { + match op { + ValueOperator::Exact(op) => exact_operator_detail(op), + ValueOperator::Extension { name } => serde_json::json!({ + "op": "Extension", + "name": name, + }), + } +} + +fn value_operator_label(op: &ValueOperator) -> String { + match op { + ValueOperator::Exact(op) => exact_operator_label(op), + ValueOperator::Extension { name } => name.clone(), + } +} + fn push_summary_node( nodes: &mut Vec, kind: &'static str, @@ -391,8 +465,16 @@ fn family_label(family: &crate::post_asap::SummaryFamilyType) -> String { /// shared [`DagGraph`] node list — see [`export_post_asap`]) can't drift /// apart on how every *other* variant's own shape is described, since /// nothing about that description differs between the two. -fn summary_shape(expr: &SummaryExpr) -> (&'static str, String, serde_json::Value) { - match expr { +/// +/// `data_state` is the node's explicit execution data_state (issue #171) — its own +/// [`produced_data_state`], or the edge-assigned data_state for a `KeepPreAsap` +/// — and is written into `detail.execution_data_state` on every post-ASAP node so a viewer +/// reads it rather than inferring it from the node's kind. +fn summary_shape( + expr: &SummaryExpr, + data_state: ExecutionDataState, +) -> (&'static str, String, serde_json::Value) { + let (kind, label, mut detail) = match expr { SummaryExpr::KeepPreAsap(_) => { unreachable!("summary_shape's callers special-case KeepPreAsap before calling it") } @@ -438,7 +520,25 @@ fn summary_shape(expr: &SummaryExpr) -> (&'static str, String, serde_json::Value let label = format!("SummaryMerge({} children)", children.len()); ("SummaryMerge", label, serde_json::json!({})) } + SummaryExpr::UpdateTransform { op, .. } => { + let label = format!("UpdateTransform({})", value_operator_label(op)); + ("UpdateTransform", label, value_operator_detail(op)) + } + SummaryExpr::ReadoutPostProcess { op, .. } => { + let label = format!("ReadoutPostProcess({})", value_operator_label(op)); + ("ReadoutPostProcess", label, value_operator_detail(op)) + } + }; + if let serde_json::Value::Object(map) = &mut detail { + map.insert( + "execution_data_state".into(), + serde_json::json!({ + "timing": data_state.timing.as_str(), + "primitive": data_state.primitive.as_str(), + }), + ); } + (kind, label, detail) } /// `expr`'s own `Rc` children, in the variant's field order @@ -455,6 +555,10 @@ fn summary_children(expr: &SummaryExpr) -> Vec<&Rc> { SummaryExpr::SummaryDelete { summary_input, .. } => vec![summary_input], SummaryExpr::SummaryEstimate { summary_input, .. } => vec![summary_input], SummaryExpr::SummaryMerge { children } => children.iter().collect(), + SummaryExpr::UpdateTransform { child, .. } + | SummaryExpr::ReadoutPostProcess { child, .. } => { + vec![child] + } } } @@ -462,12 +566,22 @@ fn summary_children(expr: &SummaryExpr) -> Vec<&Rc> { /// post-order (children pushed before their parent), and return the pushed /// root's id. Exhaustive over every [`SummaryExpr`] variant, matching this /// file's own exhaustive style for `QueryExpr` in [`build`]. -fn build_summary(node: &SummaryNode, nodes: &mut Vec) -> u32 { +fn build_summary( + node: &SummaryNode, + nodes: &mut Vec, + data_state: ExecutionDataState, +) -> u32 { if let SummaryExpr::KeepPreAsap(inner) = &node.expr { let pre_asap_subgraph = export(inner); let inner_kind = pre_asap_subgraph.nodes[pre_asap_subgraph.root as usize].kind; let label = format!("KeepPreAsap({inner_kind})"); - let detail = serde_json::json!({ "pre_asap_subgraph": pre_asap_subgraph }); + let detail = serde_json::json!({ + "pre_asap_subgraph": pre_asap_subgraph, + "execution_data_state": { + "timing": data_state.timing.as_str(), + "primitive": data_state.primitive.as_str(), + }, + }); return push_summary_node( nodes, "KeepPreAsap", @@ -479,9 +593,9 @@ fn build_summary(node: &SummaryNode, nodes: &mut Vec) -> u32 { } let children: Vec = summary_children(&node.expr) .into_iter() - .map(|child| build_summary(child, nodes)) + .map(|child| build_summary(child, nodes, assigned_child_data_state(&node.expr, child))) .collect(); - let (kind, label, detail) = summary_shape(&node.expr); + let (kind, label, detail) = summary_shape(&node.expr, data_state); push_summary_node(nodes, kind, label, detail, children, node.guarantee.clone()) } @@ -759,15 +873,24 @@ fn build_summary_hybrid( nodes: &mut Vec, cache: &mut HashCache, find_winner: &mut dyn FnMut(&QueryExpr) -> Option, + data_state: ExecutionDataState, ) -> u32 { if let SummaryExpr::KeepPreAsap(inner) = &node.expr { return build(inner, nodes, cache, find_winner); } let children: Vec = summary_children(&node.expr) .into_iter() - .map(|child| build_summary_hybrid(child, nodes, cache, find_winner)) + .map(|child| { + build_summary_hybrid( + child, + nodes, + cache, + find_winner, + assigned_child_data_state(&node.expr, child), + ) + }) .collect(); - let (kind, label, mut detail) = summary_shape(&node.expr); + let (kind, label, mut detail) = summary_shape(&node.expr, data_state); // The merged graph's `DagNode` has no dedicated guarantee field (it is // the pre-ASAP node shape); the guarantee rides in `detail` under the // same key/shape `SummaryDagNode::guarantee` uses, additively. @@ -853,7 +976,13 @@ fn build( decision, }) => { let first = nodes.len(); - let root = build_summary_hybrid(&replacement, nodes, cache, find_winner); + let root = build_summary_hybrid( + &replacement, + nodes, + cache, + find_winner, + root_domain(&replacement), + ); for node in &mut nodes[first..] { if node.decision.is_none() { let mut node_decision = decision.clone(); @@ -1413,7 +1542,7 @@ mod tests { ); } - // ── Issue #223 stage 3: dag_export's hash literally *is* cse's hash ──── + // ── Issue #223 data_state 3: dag_export's hash literally *is* cse's hash ──── #[test] fn root_hash_matches_cse_structural_hash_for_the_same_node() { diff --git a/crates/types/src/post_asap/execution_data_state.rs b/crates/types/src/post_asap/execution_data_state.rs new file mode 100644 index 00000000..417a329f --- /dev/null +++ b/crates/types/src/post_asap/execution_data_state.rs @@ -0,0 +1,907 @@ +//! Execution-data-state contract for mixed exact/summary plans (issue #171). +//! +//! A post-ASAP DAG mixes two very different moments of execution: the +//! **update/ingest path** (rows arrive, maintained summary state is updated) +//! and **query evaluation** (maintained state is read out and a final result +//! is produced). A plan that places a query-time residual *underneath* a +//! maintained summary is not merely expensive — it is unexecutable, because +//! the maintenance loop has no readout values to feed into that summary. +//! [`SummaryExpr::ReadoutPostProcess`] is exactly such a residual, which is +//! why it and [`SummaryExpr::UpdateTransform`] are two separate variants +//! rather than one data_state-ambiguous value operation. +//! +//! [`ExecutionDataState`] is what a node's output *is*, at which data_state; +//! [`validate_execution_data_states`] checks every edge of a DAG against the +//! rules below at plan construction, returning a typed [`ExecutionDataStateError`] rather +//! than deferring to a runtime failure. +//! +//! ## Edge rules +//! +//! | Parent | Accepts from `child` | +//! |---|---| +//! | `SummaryAgg.child` | `MAINTENANCE_ROWS`, or `MAINTENANCE_SUMMARY` of an **exact accumulator** family. Never a read-time data_state. | +//! | `SummaryEstimate.summary_input` | `MAINTENANCE_SUMMARY` (any family). Produces `READ_ROWS`. | +//! | `SummaryJoin.outer/inner` | `MAINTENANCE_ROWS` or `MAINTENANCE_SUMMARY`; never a read-time data_state. | +//! | `SummarySubtract`/`SummaryDelete`/`SummaryMerge` | `MAINTENANCE_SUMMARY`. | +//! | `UpdateTransform.child` | `MAINTENANCE_ROWS`. Produces `MAINTENANCE_ROWS`. | +//! | `ReadoutPostProcess.child` | `READ_ROWS`. Produces `READ_ROWS`. | +//! +//! ## `KeepPreAsap` declares its data_state through the derivation +//! +//! A [`SummaryExpr::KeepPreAsap`] leaf is a raw pre-ASAP computation that a +//! runtime can execute at either data_state: as update-path raw input beneath a +//! `SummaryAgg`/`UpdateTransform`, or as a query-time fallback beneath a +//! `ReadoutPostProcess` (or at the root). It carries no data_state field of its own +//! — every existing consumer pattern-matches the one-field shape — so its +//! data_state is *assigned* by [`validate_execution_data_states`] from the edge that +//! reaches it and reported in the returned [`ExecutionDataStateAssignment`]. What it may +//! not do is stay ambiguous inside one mixed plan: the same `Rc` +//! reached once as update input and once as query-time fallback is +//! [`ExecutionDataStateError::AmbiguousKeepPreAsap`], because no single execution of that +//! subtree can serve both roles. + +use std::collections::HashMap; +use std::rc::Rc; + +use thiserror::Error; + +use super::expr::{ExactOperator, SummaryExpr, SummaryNode, ValueOperator}; +use super::schema::{SummaryFamilyType, SummaryField, SummarySchema}; +use crate::pre_asap::query_expr::{aggregate_output_schema, QueryExprError}; +use crate::pre_asap::schema::{Column, Schema}; + +/// When a post-ASAP value is produced. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ExecutionTiming { + MaintenanceTime, + ReadTime, +} + +impl ExecutionTiming { + pub fn as_str(self) -> &'static str { + match self { + Self::MaintenanceTime => "maintenance_time", + Self::ReadTime => "read_time", + } + } +} + +/// The primitive representation carried by a post-ASAP edge. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum DataPrimitive { + Rows, + SummaryState, +} + +impl DataPrimitive { + pub fn as_str(self) -> &'static str { + match self { + Self::Rows => "rows", + Self::SummaryState => "summary_state", + } + } +} + +/// The two-dimensional edge contract: when a value exists and which data +/// primitive it carries. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ExecutionDataState { + pub timing: ExecutionTiming, + pub primitive: DataPrimitive, +} + +impl ExecutionDataState { + pub const MAINTENANCE_ROWS: Self = Self { + timing: ExecutionTiming::MaintenanceTime, + primitive: DataPrimitive::Rows, + }; + pub const MAINTENANCE_SUMMARY: Self = Self { + timing: ExecutionTiming::MaintenanceTime, + primitive: DataPrimitive::SummaryState, + }; + pub const READ_ROWS: Self = Self { + timing: ExecutionTiming::ReadTime, + primitive: DataPrimitive::Rows, + }; +} + +impl std::fmt::Display for ExecutionDataState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}/{}", self.timing.as_str(), self.primitive.as_str()) + } +} + +/// Which parent/edge a [`ExecutionDataStateError`] is about — the variant name of the +/// parent `SummaryExpr` plus its field, for a message a plan author can act +/// on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExecutionDataStateEdge { + SummaryAggChild, + SummaryEstimateInput, + SummaryJoinInput, + SummarySubtractInput, + SummaryDeleteInput, + SummaryMergeInput, + UpdateTransformChild, + ReadoutPostProcessChild, +} + +impl ExecutionDataStateEdge { + fn describe(self) -> &'static str { + match self { + Self::SummaryAggChild => "SummaryAgg.child", + Self::SummaryEstimateInput => "SummaryEstimate.summary_input", + Self::SummaryJoinInput => "SummaryJoin.{outer,inner}", + Self::SummarySubtractInput => "SummarySubtract.{left,right}", + Self::SummaryDeleteInput => "SummaryDelete.summary_input", + Self::SummaryMergeInput => "SummaryMerge.children[]", + Self::UpdateTransformChild => "UpdateTransform.child", + Self::ReadoutPostProcessChild => "ReadoutPostProcess.child", + } + } +} + +/// A plan-construction-time data_state violation. Typed (not a string) so a +/// strategy can degrade to a conservative fallback on the specific variant +/// it expects, and so tests can assert the *reason* a plan was rejected. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum ExecutionDataStateError { + /// A query-time value (`SummaryEstimate` / `ReadoutPostProcess` output) + /// placed beneath a maintained summary — the one shape issue #171's + /// data_state split exists to make unrepresentable. + #[error( + "readout value under maintenance: {edge} received a {child} input, but a maintained \ + summary can only consume update-path values (or exact accumulator state)" + )] + ReadoutUnderMaintenance { + edge: &'static str, + child: ExecutionDataState, + }, + /// Any other edge whose child data_state the parent does not accept + /// (e.g. plain update rows fed straight into a `SummaryEstimate`, or a + /// sketch's opaque state fed into a `ReadoutPostProcess`). + #[error("{edge} does not accept a {child} input")] + IllegalChildPhase { + edge: &'static str, + child: ExecutionDataState, + }, + /// A `SummaryAgg` whose child is summary state of a family other than an + /// exact accumulator — re-accumulating opaque sketch/sample/… state on + /// the update path has no defined semantics here. + #[error( + "SummaryAgg.child carries {family} summary state; only exact accumulator state can be \ + composed into another maintained summary" + )] + UnsupportedStateComposition { family: String }, + /// One shared `KeepPreAsap` node reached both as update-path raw input + /// and as a query-time fallback — see the module docs. + #[error( + "KeepPreAsap subtree is data_state-ambiguous: reached as {first} and as {second} in the same \ + plan" + )] + AmbiguousKeepPreAsap { + first: ExecutionDataState, + second: ExecutionDataState, + }, + /// An update-path-only node (`UpdateTransform`) at the root of a plan: + /// nothing maintains state above it, so its output is never read. + #[error("UpdateTransform cannot be a plan root: its update-path output feeds nothing")] + MaintenanceRowsAtRoot, + /// An `ExactOperator` whose input columns are not all `Plain` at its + /// declared data_state. + #[error("exact operator consumes non-plain column {column:?} ({dtype})")] + NonPlainOperand { column: String, dtype: String }, +} + +/// The data_state assigned to every node of a validated plan, keyed by +/// `Rc` pointer identity — the explicit per-node "execution_data_state" a +/// runtime or a DAG export reads instead of re-deriving it. For every +/// non-`KeepPreAsap` node this equals [`produced_data_state`]; for a +/// `KeepPreAsap` leaf it is the data_state the reaching edge assigned. +#[derive(Debug, Clone, Default)] +pub struct ExecutionDataStateAssignment { + domains: HashMap<*const SummaryNode, ExecutionDataState>, +} + +impl ExecutionDataStateAssignment { + /// The data_state assigned to `node`, if it was part of the validated plan. + pub fn data_state_of(&self, node: &Rc) -> Option { + self.domains.get(&Rc::as_ptr(node)).copied() + } + + /// The data_state assigned to the node at `ptr` — for callers walking a plan + /// by reference rather than by `Rc`. + pub fn data_state_of_ptr(&self, ptr: *const SummaryNode) -> Option { + self.domains.get(&ptr).copied() + } +} + +/// The data_state `expr` *produces*, independent of context — `None` for +/// [`SummaryExpr::KeepPreAsap`], whose data_state is assigned by the edge reaching +/// it (see the module docs). +pub fn produced_data_state(expr: &SummaryExpr) -> Option { + Some(match expr { + SummaryExpr::KeepPreAsap(_) => return None, + SummaryExpr::SummaryAgg { .. } + | SummaryExpr::SummaryJoin { .. } + | SummaryExpr::SummarySubtract { .. } + | SummaryExpr::SummaryDelete { .. } + | SummaryExpr::SummaryMerge { .. } => ExecutionDataState::MAINTENANCE_SUMMARY, + SummaryExpr::SummaryEstimate { .. } | SummaryExpr::ReadoutPostProcess { .. } => { + ExecutionDataState::READ_ROWS + } + SummaryExpr::UpdateTransform { .. } => ExecutionDataState::MAINTENANCE_ROWS, + }) +} + +/// Is `family` the exact-accumulator family whose partial state *is* the +/// value — the one summary state a `SummaryAgg` may re-accumulate? +fn is_exact_accumulator_state(schema: &SummarySchema) -> Result<(), ExecutionDataStateError> { + for field in &schema.fields { + match &field.dtype { + SummaryFamilyType::Plain(_) | SummaryFamilyType::ExactAggregate(..) => {} + other => { + return Err(ExecutionDataStateError::UnsupportedStateComposition { + family: format!("{other:?}"), + }) + } + } + } + Ok(()) +} + +/// Validate every edge of the DAG rooted at `root` against the module-level +/// rules, returning each node's assigned data_state on success. Shared +/// `Rc`s are visited once per reaching edge (the assignment is +/// per node, so a conflict between two edges is what +/// [`ExecutionDataStateError::AmbiguousKeepPreAsap`] detects). +pub fn validate_execution_data_states( + root: &Rc, +) -> Result { + // The root may be a readable value or bare maintained state (a + // deployment may hand an `ExactAggregate` accumulator straight to a + // consumer) — only an update-path-only root is meaningless. + let root_domain = match produced_data_state(&root.expr) { + None => ExecutionDataState::READ_ROWS, + Some(ExecutionDataState::MAINTENANCE_ROWS) => { + return Err(ExecutionDataStateError::MaintenanceRowsAtRoot) + } + Some(data_state) => data_state, + }; + validate_execution_data_states_at(root, root_domain) +} + +/// [`validate_execution_data_states`] for a *sub*-plan whose root is known to +/// sit at `data_state` — e.g. an `UpdateTransform` about to be placed beneath a +/// `SummaryAgg`, which would be rejected as a whole-plan root but is a +/// legal update-path input. Validates every edge beneath `root` exactly +/// as the whole-plan entry point does. +pub fn validate_execution_data_states_at( + root: &Rc, + data_state: ExecutionDataState, +) -> Result { + let mut assignment = ExecutionDataStateAssignment::default(); + visit(root, data_state, &mut assignment)?; + Ok(assignment) +} + +/// Record `data_state` for `node` (detecting a conflicting earlier assignment +/// for a `KeepPreAsap`), then check and recurse into every child edge. +fn visit( + node: &Rc, + data_state: ExecutionDataState, + assignment: &mut ExecutionDataStateAssignment, +) -> Result<(), ExecutionDataStateError> { + let ptr = Rc::as_ptr(node); + if let Some(previous) = assignment.domains.get(&ptr) { + if *previous != data_state { + return Err(ExecutionDataStateError::AmbiguousKeepPreAsap { + first: *previous, + second: data_state, + }); + } + // Already validated through another edge with the same data_state. + return Ok(()); + } + assignment.domains.insert(ptr, data_state); + + match &node.expr { + SummaryExpr::KeepPreAsap(_) => Ok(()), + SummaryExpr::SummaryAgg { child, .. } => { + let child_domain = child_domain( + child, + ExecutionDataStateEdge::SummaryAggChild, + |avail| match avail { + ExecutionDataState::MAINTENANCE_ROWS => Ok(()), + ExecutionDataState::MAINTENANCE_SUMMARY => { + is_exact_accumulator_state(&child.schema) + } + other => Err(ExecutionDataStateError::ReadoutUnderMaintenance { + edge: ExecutionDataStateEdge::SummaryAggChild.describe(), + child: other, + }), + }, + )?; + visit(child, child_domain, assignment) + } + SummaryExpr::SummaryJoin { outer, inner, .. } => { + for input in [outer, inner] { + let s = child_domain(input, ExecutionDataStateEdge::SummaryJoinInput, |avail| { + match avail { + ExecutionDataState::MAINTENANCE_ROWS + | ExecutionDataState::MAINTENANCE_SUMMARY => Ok(()), + other => Err(ExecutionDataStateError::ReadoutUnderMaintenance { + edge: ExecutionDataStateEdge::SummaryJoinInput.describe(), + child: other, + }), + } + })?; + visit(input, s, assignment)?; + } + Ok(()) + } + SummaryExpr::SummarySubtract { left, right } => { + for input in [left, right] { + let s = state_only(input, ExecutionDataStateEdge::SummarySubtractInput)?; + visit(input, s, assignment)?; + } + Ok(()) + } + SummaryExpr::SummaryDelete { summary_input, .. } => { + let s = state_only(summary_input, ExecutionDataStateEdge::SummaryDeleteInput)?; + visit(summary_input, s, assignment) + } + SummaryExpr::SummaryMerge { children } => { + for input in children { + let s = state_only(input, ExecutionDataStateEdge::SummaryMergeInput)?; + visit(input, s, assignment)?; + } + Ok(()) + } + SummaryExpr::SummaryEstimate { summary_input, .. } => { + let s = state_only(summary_input, ExecutionDataStateEdge::SummaryEstimateInput)?; + visit(summary_input, s, assignment) + } + SummaryExpr::UpdateTransform { child, op } => { + let s = child_domain( + child, + ExecutionDataStateEdge::UpdateTransformChild, + |avail| match avail { + ExecutionDataState::MAINTENANCE_ROWS => Ok(()), + other => Err(ExecutionDataStateError::IllegalChildPhase { + edge: ExecutionDataStateEdge::UpdateTransformChild.describe(), + child: other, + }), + }, + )?; + check_plain_operands(op, &child.schema)?; + visit(child, s, assignment) + } + SummaryExpr::ReadoutPostProcess { child, op } => { + let s = child_domain( + child, + ExecutionDataStateEdge::ReadoutPostProcessChild, + |avail| match avail { + ExecutionDataState::READ_ROWS => Ok(()), + other => Err(ExecutionDataStateError::IllegalChildPhase { + edge: ExecutionDataStateEdge::ReadoutPostProcessChild.describe(), + child: other, + }), + }, + )?; + check_plain_operands(op, &child.schema)?; + visit(child, s, assignment) + } + } +} + +/// The data_state `child` takes as a direct input of `parent`, without +/// validating legality — `child`'s own produced data_state, or for a +/// `KeepPreAsap` leaf the data_state `parent`'s edge assigns it (update-path raw +/// input under maintenance/transform edges, query-time fallback under a +/// post-process, and — meaninglessly, but for a stable answer — maintenance rows +/// under a state-only edge). For DAG export and other reporting that needs +/// an explicit per-node data_state even on a plan that +/// [`validate_execution_data_states`] would reject. +pub fn assigned_child_data_state(parent: &SummaryExpr, child: &SummaryNode) -> ExecutionDataState { + if let Some(avail) = produced_data_state(&child.expr) { + return avail; + } + match parent { + SummaryExpr::ReadoutPostProcess { .. } => ExecutionDataState::READ_ROWS, + SummaryExpr::KeepPreAsap(_) + | SummaryExpr::SummaryAgg { .. } + | SummaryExpr::SummaryJoin { .. } + | SummaryExpr::SummarySubtract { .. } + | SummaryExpr::SummaryDelete { .. } + | SummaryExpr::SummaryEstimate { .. } + | SummaryExpr::SummaryMerge { .. } + | SummaryExpr::UpdateTransform { .. } => ExecutionDataState::MAINTENANCE_ROWS, + } +} + +/// The data_state `child` takes on `edge`: its own produced data_state +/// (checked via `accept`), or — for a `KeepPreAsap` leaf — the data_state the +/// edge assigns it, derived from what that edge accepts. +fn child_domain( + child: &Rc, + edge: ExecutionDataStateEdge, + accept: impl Fn(ExecutionDataState) -> Result<(), ExecutionDataStateError>, +) -> Result { + match produced_data_state(&child.expr) { + Some(avail) => { + accept(avail)?; + Ok(avail) + } + None => { + // A raw pre-ASAP subtree executes at whichever data_state its consumer + // needs: update-path input for maintenance/transform edges, + // query-time fallback for a post-process edge. State-only edges + // can't consume plain rows at all. + let assigned = match edge { + ExecutionDataStateEdge::SummaryAggChild + | ExecutionDataStateEdge::SummaryJoinInput + | ExecutionDataStateEdge::UpdateTransformChild => { + ExecutionDataState::MAINTENANCE_ROWS + } + ExecutionDataStateEdge::ReadoutPostProcessChild => ExecutionDataState::READ_ROWS, + ExecutionDataStateEdge::SummaryEstimateInput + | ExecutionDataStateEdge::SummarySubtractInput + | ExecutionDataStateEdge::SummaryDeleteInput + | ExecutionDataStateEdge::SummaryMergeInput => { + return Err(ExecutionDataStateError::IllegalChildPhase { + edge: edge.describe(), + child: ExecutionDataState::MAINTENANCE_ROWS, + }) + } + }; + accept(assigned)?; + Ok(assigned) + } + } +} + +fn state_only( + child: &Rc, + edge: ExecutionDataStateEdge, +) -> Result { + child_domain(child, edge, |avail| match avail { + ExecutionDataState::MAINTENANCE_SUMMARY => Ok(()), + other => Err(ExecutionDataStateError::IllegalChildPhase { + edge: edge.describe(), + child: other, + }), + }) +} + +/// The exact operator must consume only `Plain` columns of its input: for +/// an `Aggregate` payload, every grouping key and every measure's input +/// column. +fn check_plain_operands( + op: &ValueOperator, + input: &SummarySchema, +) -> Result<(), ExecutionDataStateError> { + let ValueOperator::Exact(op) = op else { + return check_all_plain(input); + }; + let ExactOperator::Aggregate { + reduction, + measures, + .. + } = op; + let mut referenced: Vec = reduction + .group_keys() + .map(|keys| keys.keys().to_vec()) + .unwrap_or_default(); + for m in measures { + if let Some(col) = m.input_col() { + referenced.push(col); + } + } + // With no explicit input column (the PromQL sample-value convention) + // the operator reads every non-key column, so all must be plain. + let implicit = measures.iter().any(|m| m.input_col().is_none()); + for (i, field) in input.fields.iter().enumerate() { + if !(implicit || referenced.contains(&i)) { + continue; + } + if !matches!(field.dtype, SummaryFamilyType::Plain(_)) { + return Err(ExecutionDataStateError::NonPlainOperand { + column: field.name.clone(), + dtype: format!("{:?}", field.dtype), + }); + } + } + Ok(()) +} + +fn check_all_plain(input: &SummarySchema) -> Result<(), ExecutionDataStateError> { + for field in &input.fields { + if !matches!(field.dtype, SummaryFamilyType::Plain(_)) { + return Err(ExecutionDataStateError::NonPlainOperand { + column: field.name.clone(), + dtype: format!("{:?}", field.dtype), + }); + } + } + Ok(()) +} + +/// The plain pre-ASAP `Schema` underlying an all-`Plain` `SummarySchema`, or +/// `None` if any column carries summary state. +pub fn plain_schema(schema: &SummarySchema) -> Option { + let mut columns = Vec::with_capacity(schema.fields.len()); + for field in &schema.fields { + let SummaryFamilyType::Plain(dtype) = &field.dtype else { + return None; + }; + columns.push(Column::new(&field.name, dtype.clone(), field.nullable)); + } + Some(Schema { + columns, + time_index: schema.time_index, + unique_keys: Vec::new(), + closed: true, + }) +} + +/// Lift a plain pre-ASAP schema to a `SummarySchema` with every column +/// `Plain` — the output of every exact operator. +pub fn lift_plain(schema: &Schema) -> SummarySchema { + SummarySchema { + fields: schema + .columns + .iter() + .map(|c| SummaryField { + name: c.name.clone(), + dtype: SummaryFamilyType::Plain(c.dtype.clone()), + nullable: c.nullable, + }) + .collect(), + time_index: schema.time_index, + } +} + +/// Output schema of `op` applied to a child whose edge carries `input` — +/// the same canonical derivation the pre-ASAP `Aggregate` node uses, so an +/// exact `ReadoutPostProcess`/`UpdateTransform` never disagrees with the pre-ASAP +/// target it was lowered from. `Err` when the child carries non-plain +/// state the operator cannot read. +pub fn exact_operator_output_schema( + op: &ExactOperator, + input: &SummarySchema, +) -> Result { + let plain = plain_schema(input).ok_or(ExactOperatorSchemaError::NonPlainInput)?; + let ExactOperator::Aggregate { + reduction, + measures, + output_names, + .. + } = op; + let out = aggregate_output_schema(&plain, reduction, measures, output_names)?; + Ok(lift_plain(&out)) +} + +/// Why [`exact_operator_output_schema`] could not derive a schema. +#[derive(Debug, Error)] +pub enum ExactOperatorSchemaError { + #[error("exact operator input carries summary state, not plain columns")] + NonPlainInput, + #[error("schema derivation failed: {0}")] + Schema(#[from] QueryExprError), +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::post_asap::{ExactKind, ExactParams, GroupingStrategy, SketchQuery}; + use crate::pre_asap::agg_intent::AggIntent; + use crate::pre_asap::expr_ir::ColumnRef; + use crate::pre_asap::query_expr::{QueryExpr, Reduction, Source}; + use crate::pre_asap::schema::DataType; + + fn scan() -> Rc { + Rc::new(QueryExpr::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + Column::new("zone", DataType::Utf8, true), + ], + 0, + vec![], + ), + }) + } + + fn keep() -> Rc { + let s = scan(); + let schema = lift_plain(&s.output_schema().unwrap()); + Rc::new(SummaryNode { + expr: SummaryExpr::KeepPreAsap(s), + schema, + guarantee: None, + }) + } + + fn plain(names: &[&str]) -> SummarySchema { + SummarySchema { + fields: names + .iter() + .map(|n| SummaryField { + name: (*n).into(), + dtype: SummaryFamilyType::Plain(DataType::Float64), + nullable: false, + }) + .collect(), + time_index: None, + } + } + + fn agg(child: Rc, family: SummaryFamilyType) -> Rc { + Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child, + family: family.clone(), + col: ColumnRef::SampleValue, + reduction: Reduction::by(vec![]), + grouping: GroupingStrategy::default(), + }, + schema: SummarySchema { + fields: vec![SummaryField { + name: "state".into(), + dtype: family, + nullable: false, + }], + time_index: None, + }, + guarantee: None, + }) + } + + fn kll() -> SummaryFamilyType { + use crate::post_asap::{SketchAlgorithm, SketchKind, SketchParams}; + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + GroupingStrategy::default(), + ) + } + + fn estimate(child: Rc) -> Rc { + Rc::new(SummaryNode { + expr: SummaryExpr::SummaryEstimate { + summary_input: child, + query: SketchQuery::Quantile { q: 0.99 }, + }, + schema: plain(&["quantile_0_99"]), + guarantee: None, + }) + } + + fn max_op() -> ExactOperator { + ExactOperator::Aggregate { + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Max { col: None }], + output_names: vec![], + having: None, + } + } + + #[test] + fn keep_pre_asap_under_summary_agg_is_update_input() { + let leaf = keep(); + let root = agg(Rc::clone(&leaf), kll()); + let assignment = validate_execution_data_states(&root).unwrap(); + assert_eq!( + assignment.data_state_of(&leaf), + Some(ExecutionDataState::MAINTENANCE_ROWS) + ); + assert_eq!( + assignment.data_state_of(&root), + Some(ExecutionDataState::MAINTENANCE_SUMMARY) + ); + } + + #[test] + fn exact_accumulator_state_may_feed_another_summary_agg() { + let inner = agg( + keep(), + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + ); + let root = estimate(agg(inner, kll())); + assert!(validate_execution_data_states(&root).is_ok()); + } + + #[test] + fn readout_under_summary_agg_is_rejected() { + let inner = estimate(agg(keep(), kll())); + let root = agg(inner, kll()); + assert!(matches!( + validate_execution_data_states(&root), + Err(ExecutionDataStateError::ReadoutUnderMaintenance { .. }) + )); + } + + #[test] + fn post_process_over_readout_is_legal_and_root_is_readout() { + let inner = estimate(agg(keep(), kll())); + let root = Rc::new(SummaryNode { + expr: SummaryExpr::ReadoutPostProcess { + child: inner, + op: ValueOperator::Exact(max_op()), + }, + schema: plain(&["max"]), + guarantee: None, + }); + let assignment = validate_execution_data_states(&root).unwrap(); + assert_eq!( + assignment.data_state_of(&root), + Some(ExecutionDataState::READ_ROWS) + ); + } + + #[test] + fn non_exact_operator_uses_the_same_read_domain_contract() { + let inner = estimate(agg(keep(), kll())); + let root = Rc::new(SummaryNode { + expr: SummaryExpr::ReadoutPostProcess { + child: inner, + op: ValueOperator::Extension { + name: "approximate_calibration".into(), + }, + }, + schema: plain(&["calibrated"]), + guarantee: None, + }); + + let assignment = validate_execution_data_states(&root).unwrap(); + assert_eq!( + assignment.data_state_of(&root), + Some(ExecutionDataState::READ_ROWS) + ); + } + + #[test] + fn post_process_under_summary_agg_is_rejected() { + let inner = estimate(agg(keep(), kll())); + let post = Rc::new(SummaryNode { + expr: SummaryExpr::ReadoutPostProcess { + child: inner, + op: ValueOperator::Exact(max_op()), + }, + schema: plain(&["max"]), + guarantee: None, + }); + let root = agg(post, kll()); + assert_eq!( + validate_execution_data_states(&root).err(), + Some(ExecutionDataStateError::ReadoutUnderMaintenance { + edge: "SummaryAgg.child", + child: ExecutionDataState::READ_ROWS, + }) + ); + } + + #[test] + fn transform_under_summary_agg_is_legal_but_not_at_root() { + let transform = Rc::new(SummaryNode { + expr: SummaryExpr::UpdateTransform { + child: keep(), + op: ValueOperator::Exact(max_op()), + }, + schema: plain(&["max"]), + guarantee: None, + }); + assert_eq!( + validate_execution_data_states(&transform).err(), + Some(ExecutionDataStateError::MaintenanceRowsAtRoot) + ); + let root = estimate(agg(Rc::clone(&transform), kll())); + let assignment = validate_execution_data_states(&root).unwrap(); + assert_eq!( + assignment.data_state_of(&transform), + Some(ExecutionDataState::MAINTENANCE_ROWS) + ); + } + + #[test] + fn transform_over_readout_is_rejected() { + let inner = estimate(agg(keep(), kll())); + let transform = Rc::new(SummaryNode { + expr: SummaryExpr::UpdateTransform { + child: inner, + op: ValueOperator::Exact(max_op()), + }, + schema: plain(&["max"]), + guarantee: None, + }); + let root = agg(transform, kll()); + assert!(matches!( + validate_execution_data_states(&root), + Err(ExecutionDataStateError::IllegalChildPhase { + edge: "UpdateTransform.child", + child: ExecutionDataState::READ_ROWS + }) + )); + } + + #[test] + fn a_shared_keep_pre_asap_reached_in_two_domains_is_ambiguous() { + // One raw subtree used both as update input (under a SummaryAgg) and + // as a query-time fallback (under an ExactPostProcess) — no single + // execution can serve both, so the plan is rejected. + let shared = keep(); + let maintained = estimate(agg(Rc::clone(&shared), kll())); + let post_over_raw = Rc::new(SummaryNode { + expr: SummaryExpr::ReadoutPostProcess { + child: Rc::clone(&shared), + op: ValueOperator::Exact(max_op()), + }, + schema: plain(&["max"]), + guarantee: None, + }); + let root = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryMerge { + children: vec![ + Rc::new(SummaryNode { + expr: SummaryExpr::ReadoutPostProcess { + child: maintained, + op: ValueOperator::Exact(max_op()), + }, + schema: plain(&["max"]), + guarantee: None, + }), + post_over_raw, + ], + }, + schema: plain(&["max"]), + guarantee: None, + }); + // SummaryMerge only accepts state, so this fails earlier for a + // different reason; probe the ambiguity through a direct visit. + let mut assignment = ExecutionDataStateAssignment::default(); + visit( + &shared, + ExecutionDataState::MAINTENANCE_ROWS, + &mut assignment, + ) + .unwrap(); + assert_eq!( + visit(&shared, ExecutionDataState::READ_ROWS, &mut assignment), + Err(ExecutionDataStateError::AmbiguousKeepPreAsap { + first: ExecutionDataState::MAINTENANCE_ROWS, + second: ExecutionDataState::READ_ROWS, + }) + ); + assert!(validate_execution_data_states(&root).is_err()); + } + + #[test] + fn exact_operator_schema_matches_pre_asap_aggregate_derivation() { + let child_schema = lift_plain(&scan().output_schema().unwrap()); + let op = ExactOperator::Aggregate { + reduction: Reduction::by(vec![2]), + measures: vec![AggIntent::Max { col: None }], + output_names: vec![], + having: None, + }; + let out = exact_operator_output_schema(&op, &child_schema).unwrap(); + let names: Vec<_> = out.fields.iter().map(|f| f.name.as_str()).collect(); + assert_eq!(names, vec!["zone", "max"]); + assert!(out + .fields + .iter() + .all(|f| matches!(f.dtype, SummaryFamilyType::Plain(_)))); + } + + #[test] + fn exact_operator_rejects_non_plain_input() { + let state = agg(keep(), kll()); + assert!(matches!( + exact_operator_output_schema(&max_op(), &state.schema), + Err(ExactOperatorSchemaError::NonPlainInput) + )); + } +} diff --git a/crates/types/src/post_asap/expr.rs b/crates/types/src/post_asap/expr.rs index b93aa904..5441e832 100644 --- a/crates/types/src/post_asap/expr.rs +++ b/crates/types/src/post_asap/expr.rs @@ -3,8 +3,60 @@ use std::rc::Rc; use super::guarantee::ResultGuarantee; use super::schema::{SummaryFamilyType, SummarySchema}; use super::sketch::{GroupingStrategy, SketchQuery}; +use crate::pre_asap::agg_intent::AggIntent; +use crate::pre_asap::query_expr::Predicate; use crate::pre_asap::{ColumnRef, QueryExpr, Reduction}; +// ── Exact operators composed with summary plans (issue #171) ──────────────── + +/// An exact, plain-row operator that a mixed exact/summary plan executes at +/// an explicit data_state. Exact composition is one producer of the generic +/// [`ValueOperator`] data_state payload. +/// +/// Deliberately **not** an intact pre-ASAP [`QueryExpr`] subtree: a +/// `QueryExpr`'s children are always `Rc`, so embedding one here +/// would point back at pre-ASAP nodes and recreate exactly the opaque +/// boundary [`SummaryExpr::KeepPreAsap`] already has (a logical parent +/// swallowing an otherwise-realizable descendant). Instead this carries only +/// the operator's *own* fields; its input is the post-ASAP `child` of the +/// enclosing `SummaryExpr` variant. +/// +/// `#[non_exhaustive]`: starts with the one payload issue #171 needs. Future +/// PRs add `Filter`/`Project`/`BinaryOp`/`Sort`/`Limit` payloads when a +/// concrete mixed plan needs them — never every relational `QueryExpr` +/// variant at once. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub enum ExactOperator { + /// The same fields a pre-ASAP `QueryExpr::Aggregate` carries, applied + /// exactly (no summary family) over the enclosing node's post-ASAP + /// `child`. Output schema follows + /// `pre_asap::query_expr::aggregate_output_schema` over the child's + /// plain schema. + Aggregate { + reduction: Reduction, + measures: Vec, + output_names: Vec, + having: Option, + }, +} + +/// An operation over values at a declared execution data_state. +/// +/// Phase placement is independent of whether the operation is exact or +/// approximate: [`SummaryExpr::UpdateTransform`] and +/// [`SummaryExpr::ReadoutPostProcess`] describe when their input is +/// available, while this payload describes what is computed. The extension +/// form lets summary families and approximate strategies name operations +/// whose output schema and guarantee are carried by the enclosing +/// [`SummaryNode`]. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub enum ValueOperator { + Exact(ExactOperator), + Extension { name: String }, +} + // ── Post-ASAP DAG node ─────────────────────────────────────────────────────── /// A node in the post-ASAP DAG: wraps the expression and its derived output @@ -134,4 +186,26 @@ pub enum SummaryExpr { /// allocator (not modeled in this crate) on cut edges. /// Output schema: one field (same family + params as inputs). SummaryMerge { children: Vec> }, + + /// Value transformation executed on the **update/ingest + /// path** (issue #171). Consumes `child`'s plain update values and + /// produces plain update values, so its output may feed a downstream + /// [`SummaryAgg`](SummaryExpr::SummaryAgg)'s maintenance — the "outer + /// summary over an inner non-accumulator exact transform" direction. + /// See [`super::execution_data_state::ExecutionDataState`] for the edge contract. + UpdateTransform { + child: Rc, + op: ValueOperator, + }, + + /// Operation executed **after** `child`'s summary has been read + /// out (issue #171). Consumes plain readout values and produces the + /// final plain query result — the "outer exact fold over an inner + /// summary readout" direction. Can never feed maintained state: a + /// `SummaryAgg` above one of these is a plan-time + /// [`super::execution_data_state::ExecutionDataStateError`], never a runtime failure. + ReadoutPostProcess { + child: Rc, + op: ValueOperator, + }, } diff --git a/crates/types/src/post_asap/mod.rs b/crates/types/src/post_asap/mod.rs index 5ce45387..6dcc8b6a 100644 --- a/crates/types/src/post_asap/mod.rs +++ b/crates/types/src/post_asap/mod.rs @@ -27,13 +27,21 @@ //! alongside `reduction` and on sketch-valued edge types //! — see `asap_aware_mapping::grouping`'s module docs for why. +pub mod execution_data_state; pub mod expr; pub mod guarantee; pub mod query_time; pub mod schema; pub mod sketch; +pub mod summary_maintenance_lifecycle; -pub use expr::{SummaryExpr, SummaryNode}; +pub use execution_data_state::{ + assigned_child_data_state, exact_operator_output_schema, produced_data_state, + validate_execution_data_states, validate_execution_data_states_at, DataPrimitive, + ExactOperatorSchemaError, ExecutionDataState, ExecutionDataStateAssignment, + ExecutionDataStateError, ExecutionTiming, +}; +pub use expr::{ExactOperator, SummaryExpr, SummaryNode, ValueOperator}; pub use guarantee::{ AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, GuaranteeSource, ProbabilityExpr, ResultGuarantee, @@ -48,3 +56,7 @@ pub use sketch::{ HydraParams, SamplingKind, SamplingParams, SketchAlgorithm, SketchCategory, SketchKind, SketchParams, SketchQuery, StatModelKind, StatModelParams, WaveletKind, WaveletParams, }; +pub use summary_maintenance_lifecycle::{ + EvaluationSchedule, OutputRepresentation, SummaryMaintenanceLifecycle, + SummaryMaintenanceLifecycleGuarantee, +}; diff --git a/crates/types/src/post_asap/summary_maintenance_lifecycle.rs b/crates/types/src/post_asap/summary_maintenance_lifecycle.rs new file mode 100644 index 00000000..09521bf5 --- /dev/null +++ b/crates/types/src/post_asap/summary_maintenance_lifecycle.rs @@ -0,0 +1,51 @@ +//! Physical summary-maintenance lifecycle vocabulary. +//! +//! These choices are attached by physical planning; a `SummaryAgg` does not +//! imply continuous maintenance by itself. "Summary maintenance lifecycle" +//! is deliberately narrower than the end-to-end data lifecycle (collection, +//! transmission, storage, and analytics). + +use crate::workload::{DurationMs, TimestampMs}; + +/// When an operator is evaluated. This is independent of whether it owns +/// state and how long that state is retained. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum EvaluationSchedule { + OneShot, + PerUpdate, + OnRead, +} + +/// The physical value crossing an execution boundary. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum OutputRepresentation { + PlainRows, + SummaryState, + FinalizedValue, +} + +/// How long one planned summary state deployment exists. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum SummaryMaintenanceLifecycle { + Ephemeral, + Prepared { + activate_at: TimestampMs, + retire_at: TimestampMs, + }, + Shared { + retention: DurationMs, + }, + ContinuouslyMaintained, +} + +/// The lifecycle commitment emitted for one materialized summary deployment. +/// +/// This names the summary-maintenance promise explicitly so consumers do not +/// confuse it with guarantees about the broader data lifecycle. Accuracy is a +/// separate [`super::ResultGuarantee`]. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct SummaryMaintenanceLifecycleGuarantee { + pub summary_maintenance_lifecycle: SummaryMaintenanceLifecycle, + pub evaluation_schedule: EvaluationSchedule, + pub output_representation: OutputRepresentation, +} diff --git a/crates/types/src/pre_asap/cse.rs b/crates/types/src/pre_asap/cse.rs index 593cfea6..529ee105 100644 --- a/crates/types/src/pre_asap/cse.rs +++ b/crates/types/src/pre_asap/cse.rs @@ -97,7 +97,7 @@ //! from `asap_aware_mapping::replacement::PlanSpace::cost_sorted` (via that //! module's own `cse_preference`) — a real, Volcano/Cascades-style cost //! comparison over what this module detects, not a fixed rule. See -//! `docs/design_docs/cse-cost-model-decision.md`. This module's own +//! `docs/design_docs/cost-model.md`. This module's own //! unconditional "share whenever legal" behavior is unchanged: detection //! stays cost-agnostic by construction (this crate cannot depend on //! `asap-aware-mapping`'s `CostModel`), and the cost-aware decision is diff --git a/crates/types/src/workload.rs b/crates/types/src/workload.rs index 7a444fd3..52a0d5c8 100644 --- a/crates/types/src/workload.rs +++ b/crates/types/src/workload.rs @@ -10,6 +10,15 @@ pub struct Query(pub String); #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct RepetitionInterval(pub u32); +/// Milliseconds since the Unix epoch. Workload timestamps use one explicit +/// representation so schedules, observations, and time selections agree. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TimestampMs(pub u64); + +/// A non-negative duration in milliseconds. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct DurationMs(pub u64); + /// SQL dialect variant — different dialects have different syntax and /// function sets that affect how the query string is parsed. #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -31,23 +40,132 @@ pub enum QueryLanguage { // ── Per-query requirements ──────────────────────────────────────────────────── -/// SLA constraints attached to a single query. -/// Both fields are optional: an absent bound means "no constraint on this axis." -#[derive(Debug, Clone)] +/// Whether the caller explicitly requested an accuracy target or inherited +/// the normalized exact default. +#[derive(Debug, Clone, PartialEq)] +pub enum AccuracyRequirement { + Explicit(AccuracyTarget), + ImplicitExact, +} + +impl AccuracyRequirement { + pub fn target(&self) -> AccuracyTarget { + match self { + Self::Explicit(target) => target.clone(), + Self::ImplicitExact => AccuracyTarget::Exact, + } + } +} + +/// Optional maximum wall-clock response time for one query execution. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub enum LatencyRequirement { + ExplicitMaxMs(f64), + #[default] + Unspecified, +} + +/// Independent accuracy and response-latency constraints attached to one +/// query in the workload. +#[derive(Debug, Clone, PartialEq)] pub struct QueryRequirements { - /// Maximum acceptable approximation error. - pub accuracy: Option, - /// Maximum acceptable end-to-end query latency in milliseconds. - pub latency_ms: Option, + pub accuracy: AccuracyRequirement, + pub response_latency: LatencyRequirement, +} + +impl Default for QueryRequirements { + fn default() -> Self { + Self { + accuracy: AccuracyRequirement::ImplicitExact, + response_latency: LatencyRequirement::Unspecified, + } + } } // ── Workload entries ────────────────────────────────────────────────────────── +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum Predictability { + AdHoc, + Predictable { + known_at: Option, + }, + #[default] + Unknown, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum QueryTimeScope { + RealTime, + Longitudinal, + Mixed, + #[default] + Unknown, +} + +/// Concrete event-time interval selected by a query, kept separate from its +/// semantic real-time/longitudinal classification. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TimeSelection { + pub scope: QueryTimeScope, + pub lookback: Option, + /// Fixed upper bound. `None` means the planning/evaluation time. + pub as_of: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Confidence(pub f64); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ObservationWindow { + pub start: TimestampMs, + pub end: TimestampMs, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum ExpectedDemand { + InvocationCount(u64), + AverageRate(Rate), +} + +#[derive(Debug, Clone, PartialEq)] +pub struct DemandEstimate { + pub observation_window: ObservationWindow, + pub expected: ExpectedDemand, + pub peak_rate: Option, + pub max_concurrency: Option, + pub confidence: Confidence, + pub source: EvidenceSource, + pub observed_at: Option, + pub valid_for: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum RepeatedDemand { + FixedInterval(RepetitionInterval), + Scheduled(Vec), + EstimatedRate(DemandEstimate), +} + +#[derive(Debug, Clone, PartialEq)] +pub enum QueryRecurrence { + OneTime { + invocations: u64, + execute_at: Option, + }, + Repeated(RepeatedDemand), + Unknown, +} + /// One entry in a one-shot batch: a query plus its optional SLA constraints. #[derive(Debug, Clone)] pub struct BatchEntry { pub query: Query, - pub requirements: Option, + pub requirements: QueryRequirements, + pub predictability: Predictability, + pub invocations: u64, + pub execute_at: Option, + pub time_selection: TimeSelection, } /// One query that fires every `interval` milliseconds. Its recurrence does @@ -55,9 +173,46 @@ pub struct BatchEntry { #[derive(Debug, Clone)] pub struct RepeatingEntry { pub query: Query, - /// How often the query fires, in milliseconds. - pub interval: RepetitionInterval, - pub requirements: Option, + pub demand: RepeatedDemand, + pub requirements: QueryRequirements, + pub predictability: Predictability, + pub time_selection: TimeSelection, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct QueryWorkloadEntry { + pub query: Query, + pub requirements: QueryRequirements, + pub predictability: Predictability, + pub recurrence: QueryRecurrence, + pub time_selection: TimeSelection, +} + +impl From<&BatchEntry> for QueryWorkloadEntry { + fn from(entry: &BatchEntry) -> Self { + Self { + query: entry.query.clone(), + requirements: entry.requirements.clone(), + predictability: entry.predictability.clone(), + recurrence: QueryRecurrence::OneTime { + invocations: entry.invocations, + execute_at: entry.execute_at, + }, + time_selection: entry.time_selection.clone(), + } + } +} + +impl From<&RepeatingEntry> for QueryWorkloadEntry { + fn from(entry: &RepeatingEntry) -> Self { + Self { + query: entry.query.clone(), + requirements: entry.requirements.clone(), + predictability: entry.predictability.clone(), + recurrence: QueryRecurrence::Repeated(entry.demand.clone()), + time_selection: entry.time_selection.clone(), + } + } } // ── Data workload ───────────────────────────────────────────────────────────── @@ -120,6 +275,36 @@ impl Default for Evidence { } } +impl Evidence { + /// Return the value only while its freshness contract holds. Declared or + /// timeless evidence with no `valid_for_ms` does not expire. + pub fn value_at(&self, now_ms: u64) -> Option<&T> { + let value = self.value.as_ref()?; + match (self.observed_at_ms, self.valid_for_ms) { + (Some(observed), _) if observed > now_ms => None, + (Some(observed), Some(valid_for)) if now_ms > observed.saturating_add(valid_for) => { + None + } + (None, Some(_)) => None, + _ => Some(value), + } + } +} + +impl DemandEstimate { + /// Whether this estimate was already observed and has not expired at + /// `now_ms`. A validity duration without an observation time is not a + /// usable freshness contract. + pub fn is_fresh_at(&self, now_ms: u64) -> bool { + match (self.observed_at, self.valid_for) { + (Some(observed), _) if observed.0 > now_ms => false, + (Some(observed), Some(valid_for)) => now_ms <= observed.0.saturating_add(valid_for.0), + (None, Some(_)) => false, + _ => true, + } + } +} + /// Queries per second, samples per second, or another rate whose unit is /// established by the field that contains it. #[derive(Debug, Clone, Copy, PartialEq)] @@ -141,9 +326,9 @@ pub struct DataWorkload { /// The single normalised input type accepted by every entry point into the /// planner (HTTP POST /plan, YAML file, query-log replay, OpAMP callback). /// -/// `query_batch` and `repeating_queries` are mutually exclusive today; both -/// may be present in the future when mixed batch+streaming workloads are -/// supported. +/// `query_batch` and `repeating_queries` may both be present. [`Self::entries`] +/// normalizes them into one ordered stream without conflating recurrence with +/// data arrival. #[derive(Debug, Clone)] pub struct QueryWorkload { /// Source language shared by all queries in this workload. @@ -156,3 +341,245 @@ pub struct QueryWorkload { /// Applies to all queries in this workload. pub data_workload: Option, } + +impl QueryWorkload { + /// One normalized entry stream, independent of the source's legacy + /// batch/repeating containers. Mixed workloads preserve both kinds. + pub fn entries(&self) -> impl Iterator + '_ { + self.query_batch + .iter() + .flatten() + .map(QueryWorkloadEntry::from) + .chain( + self.repeating_queries + .iter() + .flatten() + .map(QueryWorkloadEntry::from), + ) + } + + pub fn validate(&self) -> Result<(), WorkloadError> { + for entry in self.entries() { + validate_entry(&entry)?; + } + if let Some(data) = &self.data_workload { + if matches!(data.arrival, DataArrival::AtRest) + && data.ingestion_rate.value.is_some_and(|rate| rate.0 > 0.0) + { + return Err(WorkloadError::AtRestWithPositiveIngestionRate); + } + validate_optional_rate(data.ingestion_rate.value)?; + } + Ok(()) + } +} + +fn validate_entry(entry: &QueryWorkloadEntry) -> Result<(), WorkloadError> { + if let LatencyRequirement::ExplicitMaxMs(ms) = entry.requirements.response_latency { + if !ms.is_finite() || ms < 0.0 { + return Err(WorkloadError::InvalidLatency(ms)); + } + } + match &entry.recurrence { + QueryRecurrence::OneTime { invocations: 0, .. } => { + return Err(WorkloadError::ZeroInvocations) + } + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(0))) => { + return Err(WorkloadError::ZeroRepetitionInterval) + } + QueryRecurrence::Repeated(RepeatedDemand::Scheduled(schedule)) if schedule.is_empty() => { + return Err(WorkloadError::EmptySchedule) + } + QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(estimate)) => { + if estimate.observation_window.start >= estimate.observation_window.end { + return Err(WorkloadError::EmptyObservationWindow); + } + if !(estimate.confidence.0.is_finite() && (0.0..=1.0).contains(&estimate.confidence.0)) + { + return Err(WorkloadError::InvalidConfidence(estimate.confidence.0)); + } + if let ExpectedDemand::AverageRate(rate) = estimate.expected { + validate_rate(rate)?; + } + validate_optional_rate(estimate.peak_rate)?; + } + _ => {} + } + Ok(()) +} + +fn validate_optional_rate(rate: Option) -> Result<(), WorkloadError> { + rate.map(validate_rate).transpose().map(|_| ()) +} + +fn validate_rate(rate: Rate) -> Result { + if rate.0.is_finite() && rate.0 >= 0.0 { + Ok(rate) + } else { + Err(WorkloadError::InvalidRate(rate.0)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, thiserror::Error)] +pub enum WorkloadError { + #[error("a one-time query must have at least one invocation")] + ZeroInvocations, + #[error("a fixed repetition interval must be greater than zero")] + ZeroRepetitionInterval, + #[error("a repeated-query schedule must not be empty")] + EmptySchedule, + #[error("a demand-estimate observation window must have start < end")] + EmptyObservationWindow, + #[error("confidence must be finite and in [0, 1], got {0}")] + InvalidConfidence(f64), + #[error("rate must be finite and non-negative, got {0}")] + InvalidRate(f64), + #[error("response latency must be finite and non-negative, got {0} ms")] + InvalidLatency(f64), + #[error("data at rest cannot have a positive ingestion rate")] + AtRestWithPositiveIngestionRate, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn base_workload() -> QueryWorkload { + QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: None, + repeating_queries: None, + data_workload: None, + } + } + + #[test] + fn mixed_batch_and_repeated_entries_normalize_without_conflating_axes() { + let mut workload = base_workload(); + workload.query_batch = Some(vec![BatchEntry { + query: Query("historical".into()), + requirements: QueryRequirements::default(), + predictability: Predictability::AdHoc, + invocations: 1, + execute_at: None, + time_selection: TimeSelection { + scope: QueryTimeScope::Longitudinal, + lookback: Some(DurationMs(300_000)), + as_of: Some(TimestampMs(1_000_000)), + }, + }]); + workload.repeating_queries = Some(vec![RepeatingEntry { + query: Query("dashboard".into()), + demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), + requirements: QueryRequirements::default(), + predictability: Predictability::Predictable { known_at: None }, + time_selection: TimeSelection { + scope: QueryTimeScope::RealTime, + lookback: Some(DurationMs(300_000)), + as_of: None, + }, + }]); + + let entries: Vec<_> = workload.entries().collect(); + assert_eq!(entries.len(), 2); + assert!(matches!( + entries[0].recurrence, + QueryRecurrence::OneTime { .. } + )); + assert!(matches!( + entries[1].recurrence, + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(_)) + )); + assert_eq!( + entries[0].time_selection.scope, + QueryTimeScope::Longitudinal + ); + assert_eq!(entries[1].time_selection.scope, QueryTimeScope::RealTime); + workload.validate().unwrap(); + } + + #[test] + fn stale_evidence_is_unknown_at_planning_time() { + let evidence = Evidence { + value: Some(Rate(10.0)), + source: EvidenceSource::Observed, + observed_at_ms: Some(1_000), + valid_for_ms: Some(500), + }; + assert_eq!(evidence.value_at(1_500), Some(&Rate(10.0))); + assert_eq!(evidence.value_at(1_501), None); + } + + #[test] + fn future_evidence_and_demand_estimates_are_not_fresh() { + let evidence = Evidence { + value: Some(Rate(2.0)), + source: EvidenceSource::Observed, + observed_at_ms: Some(2_000), + valid_for_ms: Some(1_000), + }; + assert_eq!(evidence.value_at(1_999), None); + assert_eq!(evidence.value_at(2_000), Some(&Rate(2.0))); + + let estimate = DemandEstimate { + observation_window: ObservationWindow { + start: TimestampMs(0), + end: TimestampMs(1_000), + }, + expected: ExpectedDemand::AverageRate(Rate(1.0)), + peak_rate: None, + max_concurrency: None, + confidence: Confidence(1.0), + source: EvidenceSource::Observed, + observed_at: Some(TimestampMs(2_000)), + valid_for: Some(DurationMs(1_000)), + }; + assert!(!estimate.is_fresh_at(1_999)); + assert!(estimate.is_fresh_at(2_000)); + } + + #[test] + fn at_rest_rejects_a_positive_ingestion_rate() { + let mut workload = base_workload(); + workload.data_workload = Some(DataWorkload { + arrival: DataArrival::AtRest, + ingestion_rate: Evidence { + value: Some(Rate(1.0)), + ..Default::default() + }, + ..Default::default() + }); + assert_eq!( + workload.validate(), + Err(WorkloadError::AtRestWithPositiveIngestionRate) + ); + } + + #[test] + fn estimated_demand_validates_window_rate_and_confidence() { + let mut workload = base_workload(); + workload.repeating_queries = Some(vec![RepeatingEntry { + query: Query("estimated".into()), + demand: RepeatedDemand::EstimatedRate(DemandEstimate { + observation_window: ObservationWindow { + start: TimestampMs(10), + end: TimestampMs(10), + }, + expected: ExpectedDemand::AverageRate(Rate(1.0)), + peak_rate: None, + max_concurrency: None, + confidence: Confidence(0.9), + source: EvidenceSource::Observed, + observed_at: None, + valid_for: None, + }), + requirements: QueryRequirements::default(), + predictability: Predictability::Unknown, + time_selection: TimeSelection::default(), + }]); + assert_eq!( + workload.validate(), + Err(WorkloadError::EmptyObservationWindow) + ); + } +} diff --git a/docs/design_docs/README.md b/docs/design_docs/README.md new file mode 100644 index 00000000..36713c56 --- /dev/null +++ b/docs/design_docs/README.md @@ -0,0 +1,207 @@ +# ASAPPlanner Design Overview + +ASAPPlanner converts queries in supported source languages into a compact +space of plans that use exact summaries, sketches, samples, wavelets, +statistical models, sharing, and other ASAP-aware alternatives. It removes +illegal alternatives, expands each remaining plan with legal summary +maintenance lifecycles, costs the resulting combinations, and only then +materializes a final plan. + +## Planner component flow + +```mermaid +flowchart LR + subgraph FRONTEND[Query frontend] + Q[Original query-language input] + PARSE[Parse and normalize] + PRE[Pre-ASAP DAG] + Q --> PARSE --> PRE + end + + subgraph WORKLOAD[Workload inputs and model] + QW[Query workload] + DW[Data workload] + H[Explicit planning horizon H] + W[Normalize workload and derive demand,
time scope, recurrence, and data evidence] + QW --> W + DW --> W + H --> W + end + + subgraph MAPPING[Semantic mapping DAG] + MAP[Build a compact space representing all
Post-ASAP DAG candidates] + end + + subgraph ACCURACY[Correctness and accuracy models] + LEGAL[Check semantic, schema,
capability, and phase legality] + PROP[Propagate guarantees through nested summaries] + ACHECK[Keep candidates that satisfy each query's
accuracy requirement; reject unknown guarantees] + LEGAL --> PROP --> ACHECK + end + + subgraph COST[Lifecycle expansion, cost model, and global selection] + LIFE[Expand every candidate with legal summary-maintenance lifecycles:
build once / prepared / shared / incremental / existing state] + EST[Estimate lifecycle-aware candidate cost over H:
build + maintenance + reads + retention + retirement] + RANK[Select the lowest-cost compatible
whole-plan and lifecycle combination] + LIFE --> EST --> RANK + end + + subgraph OUTPUT[Materialization and explanation] + MAT[Materialize the selected Post-ASAP DAG
with its selected summary-maintenance lifecycle] + EMIT[Emit final plan, deployment actions,
guarantees, assumptions, and rejections] + MAT --> EMIT + end + + PRE --> MAP + W --> MAP + MAP --> LEGAL + W --> PROP + ACHECK --> LIFE + W --> LIFE + RANK --> MAT +``` + +The optimizer's decision unit is a compatible whole-plan combination: + +```text +Post-ASAP candidate plan × summary-maintenance lifecycle assignment +``` + +It is not sound to select a summary implementation first and attach a +lifecycle afterward. Workload and lifecycle can reverse the ranking: a +summary that is cheapest to build once may be more expensive than another +summary when maintained for a high-frequency dashboard. + +## Terminology: summary maintenance lifecycle + +This design uses **summary maintenance lifecycle** for the lifetime of planner- +selected summary state: build, prepare, share, incrementally maintain, read, +and retire. A final plan's promises about those actions are its **summary +maintenance lifecycle guarantees**. + +This is narrower than the end-to-end **data lifecycle**, which covers data +collection, transmission, storage, and analytics. Unqualified names such as +"lifecycle guarantee" are avoided because they do not say which lifecycle is +being guaranteed. + +## Major components + +### Query frontend + +The frontend parses an original query-language input, such as PromQL or SQL, +and normalizes it into a Pre-ASAP DAG. The Pre-ASAP DAG represents the query's +semantics without committing to an ASAP summary implementation. + +Detailed designs: + +- [Parsing and canonicalization](parse_and_canonicalize.md) +- [Pre-ASAP IR](pre-asap-ir.md) + +### Workload inputs and model + +The workload model keeps three inputs explicit and separate: + +- the query workload describes one-time and repeated queries, predictability, + accuracy and latency requirements, and concrete time selections; +- the data workload describes data arrival, ingestion volume and rate, input + cardinality, and distribution evidence; +- the planning horizon `H` is the interval over which one-time costs and cost + rates can be compared. + +Normalization derives demand, recurrence, time scope, and freshness-checked +data evidence. Repeated query demand does not imply continuously arriving +data, and a numeric lookback does not by itself determine whether a query is +real-time or longitudinal. + +Detailed design: + +- [Query workloads, data workloads, and summary lifecycle maintenance](asap-aware-mapping/workload-demand-and-summary-lifecycle.md) + +### Semantic mapping DAG + +Semantic mapping takes the Pre-ASAP DAG and constructs a compact candidate +space. Candidates may use different summary families, summary parameters, +semantic rewrites, sharing arrangements, roll-ups, and generic update- or +readout-phase value operations. Shared structure and local alternative groups +represent possible complete Post-ASAP DAGs without eagerly copying every full +DAG. + +Semantic mapping enumerates possibilities; it does not select or deploy one. +Semantic equivalence, schema compatibility, summary capabilities, and phase +contracts remove illegal combinations before costing. + +Detailed designs: + +- [Post-ASAP IR](post-asap-ir.md) +- [ASAP-aware mapping overview](asap-aware-mapping/README.md) +- [Mapping key concepts](asap-aware-mapping/key_concepts.md) +- [Searching over candidate plans](asap-aware-mapping/searching_over_plans.md) +- [Mapping optimizations](asap-aware-mapping/optimizations.md) +- [Summary properties](asap-aware-mapping/summary_properties.md) + +### Accuracy model + +The accuracy model derives a machine-readable guarantee for each complete +candidate plan. It propagates guarantees through nested summaries and post- +processing rather than checking each summary independently. A candidate +remains eligible only when its end-to-end guarantee satisfies the +corresponding query requirement; missing evidence or unsupported propagation +rules fail closed. + +Detailed design: + +- [End-to-end accuracy guarantees](asap-aware-mapping/end-to-end-accuracy-guarantees.md) + +### Cost model + +Every eligible semantic candidate is expanded with its legal summary- +maintenance lifecycle alternatives. A summary may be built once for an +ephemeral query, prepared for predictable demand, shared for a bounded period, +maintained incrementally as data arrives, or read from compatible existing +state. Runtime, summary, and existing-state capabilities determine which +alternatives are legal. + +The cost model estimates; it does not decide legality or silently remove an +alternative. For each legal whole-plan and lifecycle assignment, it combines +build, maintenance, read, retention, and retirement costs over the same +explicit horizon `H`. Unknown costs remain unknown. + +The global optimizer then selects the lowest-cost compatible combination. It +accounts for shared state once, validates lifecycle compatibility across +nested summaries and consumers, and retains raw recomputation as an explicit +fallback. Lifecycle is therefore part of candidate cost and global selection, +not a separate decision made after semantic ranking. + +Detailed designs: + +- [Cost model](cost-model.md) + +### Materialization and explanation + +The final output contains the selected Post-ASAP DAG, deployment actions, +accuracy guarantees, and summary maintenance lifecycle guarantees. It also +records cost evidence, assumptions, and rejected alternatives. + +Conceptually: + +```text +FinalPlan { + post_asap_dag, + deployments, + accuracy_guarantees, + summary_maintenance_lifecycle_guarantees, + cost_estimates, + assumptions, + rejected_alternatives, +} +``` + +Materialization commits the summary implementation and its maintenance +lifecycle. For example, after a KLL summary is deployed for incremental +maintenance, the runtime cannot silently maintain DDSketch instead. A later +replan may select DDSketch, but the resulting deployment must explicitly +build or migrate state, cut over readers, and retire the KLL state. + +Detailed design: + +- [Explainability](asap-aware-mapping/explainability.md) diff --git a/docs/design_docs/asap-aware-mapping/workload-demand-and-summary-lifecycle.md b/docs/design_docs/asap-aware-mapping/workload-demand-and-summary-lifecycle.md index fef2aff1..df85ffc9 100644 --- a/docs/design_docs/asap-aware-mapping/workload-demand-and-summary-lifecycle.md +++ b/docs/design_docs/asap-aware-mapping/workload-demand-and-summary-lifecycle.md @@ -5,8 +5,9 @@ This document is for ASAPPlanner designers, architects, researchers, and developers working on workload-aware plan selection. It defines how the planner should describe query workload, data workload, and the lifecycle of -summary state. It is a design contract, not a description of the current -public Rust API. +summary state. It is the design contract for the public Rust model and the +workload-to-lifecycle planning API; deployments still supply their own cost +statistics and runtime capabilities. The terminology follows the ProjectASAP [glossary](https://github.com/ProjectASAP/internal-docs/blob/03e1c70f5af3ae9221471898541067eee7f86338/glossary.md). @@ -22,6 +23,16 @@ decides whether a candidate is correct enough. Workload demand and state lifecycle decide whether building, maintaining, sharing, or recomputing that candidate is worthwhile. Neither decision may override the other. +### Lifecycle terminology + +This document uses **summary maintenance lifecycle** for the lifetime of +planner-selected summary state: build, prepare, share, incrementally maintain, +read, and retire. The final plan's promises about those actions are its +**summary maintenance lifecycle guarantees**. This term is intentionally +distinct from the broader **data lifecycle**, which covers data collection, +transmission, storage, and analytics. Unqualified "lifecycle guarantees" are +avoided. + ## Problem and why now A summary operator does not imply one execution lifecycle. The same exact or @@ -36,18 +47,49 @@ Likewise, an exact stateless operator may run once over a batch, once per update in an incremental pipeline, or once per readout. Operator statefulness, execution schedule, and output representation are separate properties. +The phase contract is also independent of accuracy semantics. A value +operation may be exact, summary-derived, or approximate. The post-ASAP IR +therefore uses the generic phase nodes `UpdateTransform` (`UpdateValue -> +UpdateValue`) and `ReadoutPostProcess` (`ReadoutValue -> ReadoutValue`). Their +`ValueOperator` payload identifies the computation; the enclosing node carries +its output schema and accuracy guarantee. The exact-composition strategy emits +`ValueOperator::Exact` today, but it is only the first producer of these phase +nodes, not their definition. + The query expression alone cannot determine those properties. The same query may arrive unexpectedly during exploration, run once at a scheduled time, or repeat every ten seconds on a dashboard. Planning summary state from syntax alone either misses reuse or invents reuse that the workload does not justify. -The current normalized workload distinguishes a one-shot `query_batch` from -fixed-interval `repeating_queries`, and the recurrence cost model distinguishes -one-shot consumers from evaluation and update rates. This is a useful base, but -it does not represent predictability, uncertain demand, real-time versus -longitudinal scope, at-rest versus continuously ingesting data, or summary-state -lifecycle. It also risks treating "repeating query" and "streaming data" as the -same fact even though the glossary defines them on different axes. +The normalized workload preserves `query_batch` and `repeating_queries` as +compatibility-shaped inputs, then exposes both through `QueryWorkload::entries` +as recurrence, predictability, requirements, and time-selection axes. Data +arrival and fresh ingestion evidence remain a separate `DataWorkload`; a +repeating query therefore never implies streaming data. + +### Implementation map + +- `asap_types::workload` defines the normalized query/data workload and + evidence freshness contract. +- `PlanSpace::recurrence_profiles_from_workload` derives per-target read and + update recurrence from an explicit root-to-workload-entry binding, without + treating missing evidence as zero or relying on container order. +- `WorkloadAccuracyEvidence` supplies fresh cardinality and distribution to + accuracy models. +- `plan_summary_maintenance_lifecycles` enumerates legal ephemeral, prepared, shared, and + continuously maintained alternatives for the entries explicitly associated + with the target, and compares their costs over the caller's explicit horizon. +- `global_selection_with_summary_maintenance_lifecycles` prices each semantic + summary candidate using its cheapest legal summary maintenance lifecycle + before global selection. Its recurrence profile includes repeated DAG paths, + while the workload binding separately preserves time-selection and + predictability facts. +- `materialize_with_summary_maintenance_lifecycles` materializes that phase-valid selection and + attaches the selected state deployments. Each deployment retains assumptions + and rejected alternatives for explanation. +- `UpdateTransform` and `ReadoutPostProcess` express availability boundaries + for any value operator. Exact, summary-derived, and approximate producers use + the same phase validation rather than defining accuracy-specific phase nodes. ## Inputs, outputs, and end-to-end behavior @@ -60,10 +102,12 @@ The planner receives four logically distinct inputs: distribution; 4. existing summaries and the lifecycle actions available to the deployment. -The output is a legal physical-plan choice plus explicit state deployments. A +The implemented output is a phase-valid selected summary plan (or a +cost-preferred raw-recomputation fallback) plus explicit state deployments. A state deployment states whether a summary is ephemeral, prepared, shared for a -bounded period, or continuously maintained. Its cost explanation identifies -the demand and data evidence used in the decision. +bounded period, or continuously maintained. It retains costs, assumptions, and +structured rejection reasons. Exporting full input provenance remains a later +integration. ```text logical queries ---+ @@ -82,18 +126,20 @@ preparing state in advance with building or recomputing at execution time. For repeated queries, it may amortize build and maintenance cost across reads over an explicit horizon. -### End-to-end decision order +### Target end-to-end decision order ```text normalize query and data workloads -> derive recurrence, time-scope, and data evidence - -> enumerate semantic plan alternatives - -> enumerate legal execution contracts and state lifecycles - -> validate summary capabilities and phase constraints + -> build a compact space of semantic plan alternatives + -> validate semantic, schema, summary-capability, and phase constraints -> derive and check accuracy guarantees + -> expand every legal candidate with summary-maintenance lifecycles -> normalize one-time and rate costs over an explicit horizon - -> rank legal alternatives - -> emit plan, deployments, assumptions, and rejected alternatives + -> globally rank compatible plan-and-lifecycle combinations + -> emit plan, deployments, accuracy guarantees, + summary maintenance lifecycle guarantees, assumptions, + and rejected alternatives ``` ## Goals and non-goals @@ -191,8 +237,8 @@ arbitrary approximation: the current normalization policy makes it whether the caller chose exactness or inherited the default. An unspecified response-latency requirement imposes no response-time constraint; it is not a zero-duration bound or evidence that every latency is acceptable. Accuracy is -checked as a legality constraint, while response latency is used to reject -plans that cannot meet the bound. +checked as a legality constraint. The normalized model preserves response +latency, but the current planner does not yet reject plans against that bound. #### Classification axes @@ -393,7 +439,8 @@ struct Evidence { ``` This reuses the provenance and freshness principles from empirical summary -parameter configuration. A missing or stale value remains unknown. +parameter configuration. Missing, stale, or future-dated evidence remains +unknown. ### Output cardinality is a derived or evidenced cost input @@ -435,10 +482,10 @@ not imply long-lived incremental maintenance. A stateless transform can run `PerUpdate` before a downstream maintained summary. These types describe an execution contract; they do not replace semantic operators in the post-ASAP IR. -### State lifecycle is a plan alternative +### Summary maintenance lifecycle is a plan alternative ```rust -enum StateLifecycle { +enum SummaryMaintenanceLifecycle { Ephemeral, Prepared { activate_at: Timestamp, @@ -461,7 +508,9 @@ enum StateLifecycle { The summary family and its properties constrain which lifecycles are legal. For example, an append-only sketch may support continuous inserts but not a sliding-window lifecycle requiring deletion. Lifecycle legality is checked -before cost ranking, like accuracy legality. +before cost ranking, like accuracy legality. Deployments provide these +per-summary properties through `summary_maintenance_capabilities`; moving +real-time windows require deletion support as well as incremental updates. ### Existing summaries are planning input @@ -509,6 +558,14 @@ For repeated raw recomputation: total(H) = reads(H) * raw_recompute_cost ``` +Before materialization, lifecycle-aware global selection computes the cheapest +legal summary maintenance lifecycle total for every semantic summary sibling +whose cost evidence is complete. Those totals can reorder summary families; +unknown totals remain conservative and cannot win as invented zeroes. After +selection, materialization sums each unique selected summary deployment once +and can replace the selected summary plan with raw recomputation when the raw +cost is lower or the summary maintenance lifecycle is uncostable. + For an ephemeral summary: ```text @@ -552,15 +609,15 @@ The glossary review found the following required coverage and current gaps. | Glossary concept | Current ASAPPlanner representation | Missing design support | | --- | --- | --- | -| Data at rest vs continuously ingesting | Continuous ingest characteristics are available; no explicit arrival mode | Add `DataArrival`; support at-rest statistics without inventing update rate | -| Ingestion volume | Not a first-class workload input | Add evidenced volume with a time basis | -| Ingestion rate | Derived from series count and sample rate | Preserve as evidenced rate; do not conflate with query evaluation rate | -| Input cardinality | Partial `series_count` and distinct-key inputs | Associate each estimate with its dataset, metric, columns, and observation window | -| Data distribution | Small built-in enum | Preserve source/freshness; permit deployment-specific distributions later | -| Ad-hoc vs predictable | Not represented | Add predictability independently from recurrence | -| One-time vs repeated | Batch entries and fixed-interval repeating entries | Add scheduled one-time, unknown recurrence, and estimated/scheduled repetition | -| Query volume and characteristics | Fixed interval or structural consumer count | Add observation window, peak/burst and concurrency evidence where latency or capacity models require it | -| Real-time vs longitudinal | Temporal IR can carry ranges; no workload classification | Add time scope plus concrete selection; avoid inferring scope from lookback alone | +| Data at rest vs continuously ingesting | `DataArrival` is explicit | Runtime/catalog-specific arrival discovery remains external | +| Ingestion volume | `DataWorkload::ingestion_volume` carries evidence | A concrete time basis for volume remains deployment-specific | +| Ingestion rate | Evidenced independently from query evaluation rate | Preserve richer unit/provenance metadata when integrations require it | +| Input cardinality | Evidenced workload-level cardinality feeds accuracy | Per-dataset/metric/column scoping remains future work | +| Data distribution | Evidenced built-in enum | Permit deployment-specific distributions later | +| Ad-hoc vs predictable | `Predictability` is independent from recurrence | Parameterized-template equivalence remains open | +| One-time vs repeated | One-time, fixed, scheduled, estimated, and unknown recurrence | Forecast-policy integration remains future work | +| Query volume and characteristics | Estimates preserve average/count, peak, concurrency, confidence, and freshness | Peak and concurrency are not yet consumed by cost or latency models | +| Real-time vs longitudinal | `TimeSelection` carries scope, lookback, and `as_of` | Conflict policy with temporal IR remains open | | Output cardinality | May be inferred locally; no common evidenced input | Add derived/evidenced value and provenance for costing | | Lookback window | Represented in temporal query shapes/frontends | Establish query IR as authority and expose it to workload costing | | CTSA pipeline | Not explicitly modeled | Keep as architectural context; planner consumes collect/store/analyze facts but does not model transmission topology in the MVP | @@ -655,9 +712,9 @@ and after aggregation. - **Understandability:** explanations use glossary terms and show each axis separately. Proxy: reviewers can distinguish repeated queries from continuous ingestion in exported plan evidence. -- **Debuggability:** selected and rejected lifecycle alternatives record demand, - horizon, data statistics, and provenance. Proxy: no lifecycle decision is - explained only as a scalar cost. +- **Debuggability:** selected and rejected lifecycle alternatives record costs, + horizon-derived decisions, assumptions, and typed rejection reasons. Full + demand/data provenance in exported explanations remains future work. - **Maintainability:** current recurrence types remain the cost authority; normalized workload types remain the source authority. No duplicate formula system is introduced. diff --git a/docs/design_docs/cost-model.md b/docs/design_docs/cost-model.md new file mode 100644 index 00000000..d1bf5564 --- /dev/null +++ b/docs/design_docs/cost-model.md @@ -0,0 +1,483 @@ +# Cost Model + +## Purpose + +The cost model estimates the resource cost of every legal ASAPPlanner +alternative in a common currency. The global optimizer uses those estimates to +select a compatible Post-ASAP plan and a summary-maintenance lifecycle for +each stateful node. + +The unit of optimization is: + +```text +complete Post-ASAP candidate plan + × compatible summary-maintenance lifecycle assignment +``` + +Cost must be evaluated after semantic, schema, phase, capability, and accuracy +validation, but before final selection and materialization. + +See the [overall planner design](README.md) and the +[workload and summary-maintenance lifecycle design](asap-aware-mapping/workload-demand-and-summary-lifecycle.md). + +## Responsibilities + +The cost model is responsible for: + +- defining typed one-time and recurring cost units; +- consuming workload, data, candidate, summary, and runtime evidence; +- estimating primitive build, update, read, retention, retirement, transfer, + and raw-computation costs; +- calculating lifecycle-aware costs over one explicit planning horizon; +- estimating costs for nested and phase-composed plans; +- accounting for shared state once rather than once per reference; +- returning comparable estimates with provenance and assumptions; +- preserving unknown or stale inputs as unknown; +- providing estimates for every legal alternative without filtering the + candidate set. + +The cost model is not responsible for: + +- parsing query languages or constructing the Pre-ASAP DAG; +- deciding semantic equivalence; +- deciding schema, phase, or summary-capability legality; +- deriving or approving accuracy guarantees; +- generating candidate plans; +- choosing which compatible alternatives form the final plan; +- materializing summaries, scheduling jobs, or assigning machines; +- changing a materialized summary family without explicit replanning. + +Those responsibilities belong respectively to the frontend, semantic mapping, +correctness and accuracy models, global optimizer, and deployment/runtime +layers. + +## Cost vocabulary and units + +One-time costs and cost rates are different types and must not be added +directly. + +| Quantity | Unit | Meaning | +|---|---|---| +| `Cost` | cost units | A one-time action such as build or retirement | +| `CostRate` | cost units/second | A recurring cost such as retention or steady maintenance | +| `EvaluationRate` | evaluations/second | How often consumers read a result | +| `UpdateRate` | updates/second | How often incoming data changes maintained state | +| `Horizon` (`H`) | seconds | The interval over which recurring and one-time alternatives are compared | + +The only valid conversion from a rate to a comparable total is: + +```text +total_cost(H) = one_time_cost + H × recurring_cost_rate +``` + +`H` must be finite and strictly positive. Every alternative in one comparison +uses the same `H`. A latency requirement is not a horizon: latency constrains +one query result, while `H` determines how much future activity is included in +the economic comparison. + +## Factors that determine cost + +### Query workload + +Query workload determines: + +- one-time invocation count; +- fixed, scheduled, or estimated repeated demand; +- evaluation rate and expected reads within `H`; +- effective consumer count after sharing decisions; +- predictability and preparation windows; +- concurrency and peak demand when supplied; +- per-query latency and accuracy requirements; +- real-time, longitudinal, mixed, or unknown time scope; +- lookback and concrete `as_of` selection. + +For fixed repeating intervals `t_i`: + +```text +evaluation_rate = Σ_i (1 / t_i) +reads(H) = one_time_invocations + H × evaluation_rate +``` + +Scheduled demand counts only executions inside `H`. Estimated demand may be +used only while its evidence is fresh. Structural references are not a +substitute for execution frequency. + +### Data workload + +Data workload determines: + +- whether data is at rest, continuously ingesting, mixed, or unknown; +- update rate and update count within `H`; +- ingestion volume and input cardinality; +- data distribution and skew; +- whether a moving real-time window requires deletion or expiry; +- whether evidence is fresh enough to use. + +For maintained state: + +```text +updates(H) = H × update_rate +``` + +Repeated queries do not imply continuous data. Data at rest contributes no +invented update cost. Unknown arrival or stale ingestion evidence cannot make +continuous maintenance appear free. + +### Candidate plan structure + +Cost depends on the complete Post-ASAP DAG, including: + +- summary family and parameters; +- exact versus approximate implementation; +- grouping and subpopulation organization; +- nested summaries and post-processing; +- update-path transforms and readout-time operations; +- roll-ups and semantic rewrites; +- CSE sharing and number of effective consumers; +- shared node identity and whether state already exists. + +The same logical query can therefore have different costs for KLL, DDSketch, +an exact accumulator, raw recomputation, or a nested composition. + +### Summary physical properties + +Summary physical properties affect numeric cost because they determine how +much state and work a legal candidate requires: + +- parameter-dependent state size and retention footprint; +- update, merge, deletion, and readout complexity; +- input and output cardinality; +- number of physical instances created by grouping; +- bytes transferred or stored; +- rows processed by update-path transforms and readout post-processing. + +These properties are converted into primitive build, update, read, retention, +retirement, and transfer estimates using runtime performance evidence. + +### Summary and runtime capabilities + +Capabilities determine legality, not numeric cost. They include: + +- incremental-update, merge, subtract, and deletion support; +- supported update/readout execution phases; +- available ephemeral, prepared, shared, and continuous lifecycles; +- supported state placement, transfer, and storage operations. + +An unsupported alternative is rejected before costing. The planner must not +represent an unsupported operation by assigning it an arbitrarily high cost: +that would incorrectly allow it to win if every other estimate were even +higher or unknown. + +### Runtime performance evidence + +Measured or modeled runtime performance may determine numeric cost, for +example: + +- CPU time per summary update or readout; +- storage cost per byte-second; +- network cost per transferred byte; +- fixed deployment and retirement overhead; +- machine-, region-, or execution-stage-specific operator throughput. + +This evidence is distinct from capability flags. “The runtime supports KLL +deletion” is a legality fact; “one KLL deletion costs X CPU units on this +runtime” is cost evidence. + +### Accuracy requirements and data characteristics + +Accuracy affects cost indirectly by changing legal summary families and their +parameters. A tighter error requirement may require a larger sketch, more +samples, or exact computation. Cardinality, distribution, skew, and other +fresh data evidence may affect both sizing and read/post-processing cost. + +The accuracy model derives guarantees; the cost model prices candidates that +already carry valid guarantees. + +### Existing materialized state + +Existing state may avoid a new build cost only when a catalog establishes: + +- semantic and parameter compatibility; +- ownership and shareability; +- freshness and coverage; +- accuracy guarantee; +- representation and execution phase; +- summary maintenance lifecycle guarantees. + +Existing state is a distinct alternative, not a newly built summary with an +assumed zero build cost. + +## Primitive cost inputs + +For one concrete summary state, the model may provide: + +```text +build_cost +maintenance_cost_per_update +summary_read_cost +retention_cost_rate +retirement_cost +``` + +For raw and stateless execution it may additionally provide: + +```text +raw_recompute_cost_per_read +operator_cost_per_input_row +expected_input_rows +expected_output_rows +transfer_cost +``` + +Every estimate must name its model/version provenance. Deployment-specific +measurements may override documented heuristic defaults. + +## Cost calculations + +### Raw recomputation + +For a query evaluated directly from raw or Pre-ASAP input: + +```text +raw_total(H) + = reads(H) × raw_recompute_cost_per_read +``` + +The result has no summary build, retention, maintenance, or retirement term. + +### Ephemeral summary + +Ephemeral state is rebuilt and retired for every invocation: + +```text +ephemeral_total(H) + = reads(H) + × (build_cost + summary_read_cost + retirement_cost) +``` + +This is appropriate for one-time or unpredictable demand and does not imply +future reuse. + +### Prepared summary + +For a predictable activation window of `T` seconds: + +```text +prepared_total(T) + = build_cost + + updates(T) × maintenance_cost_per_update + + reads(T) × summary_read_cost + + T × retention_cost_rate + + retirement_cost +``` + +For data at rest, `updates(T) = 0`. Preparation is legal only when the declared +window covers every consumer that relies on the state. + +### Bounded shared summary + +For one state shared across multiple reads over horizon `H`: + +```text +shared_total(H) + = build_cost + + updates(H) × maintenance_cost_per_update + + reads(H) × summary_read_cost + + H × retention_cost_rate + + retirement_cost +``` + +The build, maintenance, retention, and retirement terms are charged once for +the shared state. Read cost is charged for every evaluation. + +### Continuously maintained summary + +For continuously ingesting or mixed data: + +```text +continuous_total(H) + = build_cost + + H × update_rate × maintenance_cost_per_update + + reads(H) × summary_read_cost + + H × retention_cost_rate + + retirement_cost +``` + +This alternative requires fresh update-rate evidence and incremental-update +support. Moving real-time windows additionally require deletion or equivalent +expiry support. + +### Existing summary + +For compatible existing state: + +```text +existing_total(H) + = remaining_update_cost(H) + + reads(H) × summary_read_cost + + remaining_retention_cost(H) + + transition_or_retirement_cost +``` + +A new build term is omitted only when catalog provenance proves that the state +already exists and is reusable. Migration or cutover costs are included when +the selected plan changes representation or summary family. + +### Update-path transform feeding a summary + +For a value transform executed per update before summary maintenance: + +```text +pretransform_cost_rate + = update_rate + × (transform_cost_per_input_row + + summary_maintenance_cost_per_update) + + evaluation_rate × summary_read_cost +``` + +The transform consumes update values; it cannot consume a query-time readout. + +### Readout-time post-processing + +For an operation applied after reading a summary: + +```text +postprocess_cost_rate + = update_rate × summary_maintenance_cost_per_update + + evaluation_rate + × (summary_read_cost + + expected_output_rows × postprocess_cost_per_row) +``` + +These phase formulas apply to exact, summary-derived, or approximate value +operators. Accuracy semantics are carried separately by the candidate's +guarantee. + +### CSE sharing versus independent recomputation + +CSE contributes semantic alternatives; it is not a separate lifecycle. + +```text +independent_total(H) + = Σ_consumer raw_or_candidate_cost(consumer, H) + +shared_total(H) + = cost(one shared candidate and lifecycle, H) + + Σ_consumer read_or_postprocess_cost(consumer, H) +``` + +The optimizer compares these whole-plan totals. A context-free fallback may +use: + +```text +consumer_count × structural_recompute_weight + versus +shared_family_maintenance_weight +``` + +but this is only a heuristic when recurrence, lifecycle, and horizon evidence +are unavailable. It must not be presented as full lifecycle-aware cost. + +### Grouping and shared-subpopulation organization + +Grouping changes the number and size of physical summary instances. A simple +state-size estimate is: + +```text +per-subpopulation_state + = subpopulation_count × inner_summary_state_size + +shared_grid_state + = shared_grid_cells × inner_summary_state_size +``` + +For CMS-like structures, inner state size may be proportional to +`width × depth`; other families use their own parameter-dependent sizing +formula. State size then affects build, update, retention, transfer, and read +cost rather than acting as a disconnected preference score. + +### Whole-plan aggregation + +The total cost of a candidate plan is the sum of its unique physical actions: + +```text +plan_total(H) + = Σ unique summary deployments + lifecycle_total(summary, H) + + Σ stateless update/readout operations + operator_total(operation, H) + + transfer_and_transition_costs +``` + +Shared DAG nodes are counted once by physical identity. References to the same +state contribute their read or post-processing work but do not duplicate +build or maintenance cost. Nested plans must preserve phase compatibility and +must not double-count an internally shared descendant. + +## Lifecycle expansion and global selection + +For every legal semantic candidate, the optimizer enumerates legal summary- +maintenance lifecycles before ranking: + +```text +semantic candidate + × ephemeral + × prepared + × bounded shared + × continuously maintained + × compatible existing state +``` + +This is conceptual multiplication: incompatible combinations are removed by +capability, workload, phase, and schedule checks. The cost model estimates each +remaining combination. The global optimizer, not the cost model, selects the +lowest-cost compatible whole plan and retains raw recomputation as an explicit +fallback. + +Selecting a semantic summary first and attaching a lifecycle afterward is +insufficient because lifecycle cost can reverse the summary-family or CSE +ranking. + +## Unknown evidence and fail-closed behavior + +Unknown is not zero. A total remains unknown when a required term is missing, +stale, non-finite, or has incompatible units. + +The following cannot make a candidate win by assumption: + +- missing horizon when one-time and rate costs must be combined; +- missing or stale evaluation or update rate; +- missing build, update, read, retention, retirement, or raw cost; +- unknown runtime or summary capability; +- unsupported accuracy propagation; +- a `NaN`, infinite, negative rate, or non-positive horizon. + +An unknown-cost alternative remains visible for explanation but is not ranked +as cheaper than a fully costed legal alternative. If no summary alternative is +legally and completely costed, the planner preserves a conservative fallback +or reports that selection requires more evidence. + +## Cost provenance and explanation + +Every selected estimate should expose: + +- cost-model name and version; +- primitive input values and their units; +- evidence source, observation window, and freshness; +- horizon and derived reads/updates; +- lifecycle and capability assumptions; +- one-time and recurring terms before normalization; +- total cost and alternatives compared; +- missing inputs and typed rejection reasons. + +This allows users to distinguish measured deployment costs from heuristic +defaults and to understand why the same query receives a different plan under +a different workload or data distribution. + +## Summary-maintenance commitment + +The selected lifecycle becomes part of the emitted plan's summary maintenance +lifecycle guarantees. Once a KLL summary is materialized for incremental +maintenance, the runtime cannot silently maintain DDSketch instead. A replan +may choose a different family, but its cost must include explicit build or +migration, reader cutover, and retirement actions. diff --git a/docs/design_docs/cse-cost-model-decision.md b/docs/design_docs/cse-cost-model-decision.md deleted file mode 100644 index 3203a441..00000000 --- a/docs/design_docs/cse-cost-model-decision.md +++ /dev/null @@ -1,115 +0,0 @@ -# CSE sharing: rule-based vs. cost-based framework (issue #237) - -## Context - -[`asap_types::pre_asap::cse::share_common_subtrees`](../../crates/types/src/pre_asap/cse.rs) -(issue #223 stages 1-2, PR #235) already *detects* every structurally-identical, -legally-shareable (`Schema::unique_keys`-gated) subtree and shares it -**unconditionally** — there is no cost gate on top of legality. This document -decides the framework for stage 4, "wire workload-level CSE credit into -`CostModel`" — turning "these two subtrees are the same computation" into -"and it's actually worth maintaining one shared summary for them." - -## The two textbook framings (as posed in #237) - -| Framework | Mechanism | CSE policy | -|---|---|---| -| Volcano/Cascades (SQL Server, Snowflake, Calcite) | cost-based: explores a plan space via DP + memo | share iff a real cost comparison (materialize/maintain vs. recompute-per-site) favors it | -| System R (classic) | heuristic: fixed rules over basic statistics | share whenever a fixed rule says to (e.g. "referenced more than once"), no per-case comparison | - -## Decision: cost-based (Volcano/Cascades), implemented for real - -This lands as an actual cost comparison, not a documented-but-unimplemented -shape. [`CostModel::cse_share_decision`](../../crates/asap-aware-mapping/src/cost_model.rs) -compares two real, overridable cost estimates for every CSE candidate with -two or more consumers: - -- `cse_recompute_cost(candidate) * candidate.consumer_count` — the total cost - of recomputing the subtree independently at every use site. -- `cse_shared_maintenance_cost(candidate)` — the cost of keeping one shared - summary alive and continuously updated for the workload's lifetime. - -Share iff the shared-maintenance cost is no greater than the total recompute -cost. This is a genuine Volcano/Cascades-style decision: a real, per-candidate -cost comparison, not a fixed "always share when legal" rule. - -Why cost-based and not pure System R: a shared summary here is not a free win -the way sharing a relational scan is in a textbook OLTP optimizer — it is a -sketch/accumulator that (per this crate's stated purpose: *workload*-level -planning, not single-query) is typically kept **continuously updated** as new -data arrives, for as long as the workload runs, regardless of how often it's -actually read. A structurally-shareable subtree that is cheap to recompute on -demand, or rarely queried, can cost more to keep alive as a standing shared -summary than to just recompute independently at each of its (few, or cheap) -use sites. A blanket "always share" rule cannot express that trade-off; a -cost comparison does, without needing a separately hardcoded cheap-threshold -carve-out — a cheap-to-recompute candidate naturally loses the comparison on -its own. - -This decision does not need search infrastructure of its own. Issue #252's -MEMO-based search engine (`PlanSpace`/`MemoGroup` in `replacement.rs`) already -enumerates and ranks the larger, workload-wide candidate space. The choice -between sharing and recomputing one already-detected CSE candidate is binary, -so `PlanSpace::cost_sorted` reuses one direct -`CostModel::cse_share_decision` comparison per group. This preserves the -policy described here—compare costs rather than applying a fixed rule—inside -the larger search engine. `search_workload_with`'s -target-discovery pass still computes the true `consumer_count` for each -candidate via a whole-workload traversal before any ranking happens — the -decision is made from full knowledge of the workload's sharing structure, the -same way a real cost-based optimizer would. - -## Layering constraint - -`share_common_subtrees` lives in `asap-types::pre_asap` — a lower layer that -`asap-aware-mapping` (which owns `CostModel`) depends on, never the reverse. -Detection therefore cannot consult cost even if it wanted to. This is why -stage 1/2's detection stays unconditional (correctly, as a legality-only -gate) and the cost-aware decision is applied downstream, in -`asap-aware-mapping`, after detection rather than fused into it. - -## Where it hooks in - -[`PlanSpace::cost_sorted`](../../crates/asap-aware-mapping/src/replacement.rs) -is where this hooks in today. `search_workload_with` computes each shared -subtree's true `consumer_count` across the whole workload up front (the same -role `implement_workload_with`'s pre-pass used to play, before that function -was retired along with `bind.rs` — this crate no longer commits to one -physically-materialized answer at all; picking and building one final -`SummaryNode` per shared subtree is a downstream deployment's job, not this -crate's). For a `MemoGroup` whose candidates are a -[`SharedSubtreeStrategy`](../../crates/asap-aware-mapping/src/replacement.rs) -share-vs-recompute pair, `cost_sorted`'s ranking step (`rank_group`/ -`cse_preference`) asks `CostModel::cse_share_decision` once per group — using -one representative bound `SummaryNode` built just for that comparison, not -cached anywhere — and sorts the pair so the preferred candidate (`Share` or -`RecomputeIndependently`) comes first. Both candidates are still returned; -ranking never drops one: a `CostModel` orders and parameterizes candidates; it -does not prune them. - -## Defaults - -`cse_recompute_cost`'s default is a structural-size proxy: `cse::dag_node_count`, -the number of *unique* nodes in the subtree's DAG (deduplicated by `Rc` -pointer identity), not a raw serialization length. This distinction matters -here specifically — a `CseCandidate`'s subtree is, by definition, something -CSE already found sharing in, so it's generally a DAG, not a tree; a naive -tree-shaped size measure (a full `serde_json` serialization, or a recursive -walk with no identity tracking) would re-count any descendant the subtree -already shares internally once per parent that reaches it, over-stating the -real cost of holding or recomputing it once. `cse_shared_maintenance_cost`'s default -is a small per-`SummaryFamilyType` weight table (exact accumulators cheapest, -sketches/samples/wavelets/stat-models progressively more expensive to keep -continuously updated) scaled to the same order of magnitude as typical -subtree sizes. Both are documented as coarse heuristic proxies — a real -deployment with actual memory/update-cost/query-frequency knowledge overrides -either or both, same as `size_params` already lets a deployment override -`asap-plan`'s built-in sizing formulas without forking anything else. - -## Scope - -This decision, and `cse_share_decision`'s wiring into `PlanSpace::cost_sorted` -(originally into `implement_workload_with`, before `bind.rs` was retired — -see above), close out #223's stage 4 and #212's original "add CSE" tracking -issue. Stage 3 (`dag_export::structural_hash` unification) landed separately -in PR #244.