diff --git a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_end_to_end.rs b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_end_to_end.rs index 1f6881a87..9939abf38 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_end_to_end.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_end_to_end.rs @@ -24,6 +24,14 @@ use nodedb_types::DatabaseId; use crate::common::cluster_harness::{TestCluster, wait_for}; +/// The instant [`a_shuffle_join_renders_a_time_key_as_the_stored_instant`] +/// stores in its timeseries collection. +const EARLY: &str = "2020-03-05 10:00:00"; +/// `EARLY` as a declared `TIMESTAMP` time key renders it. The engine stores +/// 1583402400000 epoch milliseconds and emits the cell as a typed instant, +/// which the pgwire encoder writes as ISO-8601 UTC. +const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; + /// Run `sql` and collect the `id` column of every returned data row, sorted, so /// the result is order-independent for equality assertions. async fn collect_ids(client: &tokio_postgres::Client, sql: &str, col: &str) -> Vec { @@ -211,3 +219,132 @@ async fn distributed_shuffle_join_matches_inner_join() { cluster.shutdown().await; } + +/// A timeseries `TIMESTAMP` time key projected through a shuffle join denotes +/// the stored instant, the same as a direct read does. The shuffle path +/// repartitions rows through its own coordinator-side encode/decode hop, so +/// it is a route to the cell distinct from the local join and the direct +/// `SELECT` both. +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_shuffle_join_renders_a_time_key_as_the_stored_instant() { + const LEFT: &str = "ts_shuffle_events"; + const RIGHT: &str = "ts_shuffle_hosts"; + assert_ne!( + vshard_for_collection(DatabaseId::DEFAULT, LEFT), + vshard_for_collection(DatabaseId::DEFAULT, RIGHT), + "test collections must hash to different vShards to exercise cross-node shuffle" + ); + + let cluster = TestCluster::spawn_three().await.expect("3-node cluster"); + + cluster + .exec_ddl_on_any_leader(&format!( + "CREATE COLLECTION {LEFT} \ + (captured_at TIMESTAMP TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .expect("CREATE COLLECTION for the timeseries side"); + cluster + .exec_ddl_on_any_leader(&format!( + "CREATE COLLECTION {RIGHT} (id TEXT PRIMARY KEY, region TEXT) \ + WITH (engine='document_strict')" + )) + .await + .expect("CREATE COLLECTION for the document side"); + + wait_for( + "all 3 nodes see both collections", + Duration::from_secs(10), + Duration::from_millis(50), + || { + cluster + .nodes + .iter() + .all(|n| n.cached_collection_count() >= 2) + }, + ) + .await; + + cluster.nodes[0] + .client + .simple_query(&format!( + "INSERT INTO {LEFT} (captured_at, host, v) VALUES ('{EARLY}', 'h1', 1.5)" + )) + .await + .expect("insert timeseries row"); + cluster.nodes[0] + .client + .simple_query(&format!( + "INSERT INTO {RIGHT} (id, region) VALUES ('h1', 'eu')" + )) + .await + .expect("insert document row"); + + for (idx, node) in cluster.nodes.iter().enumerate() { + wait_for( + &format!("node {idx} sees the timeseries row"), + Duration::from_secs(15), + Duration::from_millis(50), + || { + let n = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(count_rows( + &node.client, + &format!("SELECT host FROM {LEFT}"), + )) + }); + n >= 1 + }, + ) + .await; + wait_for( + &format!("node {idx} sees the document row"), + Duration::from_secs(15), + Duration::from_millis(50), + || { + let n = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current() + .block_on(count_rows(&node.client, &format!("SELECT id FROM {RIGHT}"))) + }); + n >= 1 + }, + ) + .await; + } + + cluster.nodes[1] + .client + .simple_query("SET nodedb.force_shuffle_join = on") + .await + .expect("SET nodedb.force_shuffle_join"); + cluster.nodes[1] + .client + .simple_query("SET nodedb.shuffle_num_parts = 4") + .await + .expect("SET nodedb.shuffle_num_parts"); + + let join_sql = format!( + "SELECT {LEFT}.captured_at FROM {LEFT} \ + INNER JOIN {RIGHT} ON {LEFT}.host = {RIGHT}.id" + ); + let msgs = cluster.nodes[1] + .client + .simple_query(&join_sql) + .await + .expect("a shuffle join over a time key must succeed"); + let values: Vec = msgs + .into_iter() + .filter_map(|m| match m { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_string), + _ => None, + }) + .collect(); + + assert_eq!(values.len(), 1, "one event matches one host: {values:?}"); + assert_eq!( + values[0], EARLY_ISO, + "a shuffle-joined time key must denote {EARLY}: got {values:?}" + ); + + cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/join_cross_node.rs b/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/join_cross_node.rs index be92431a2..452708b45 100644 --- a/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/join_cross_node.rs +++ b/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/join_cross_node.rs @@ -26,6 +26,18 @@ async fn count_rows(client: &tokio_postgres::Client, sql: &str) -> usize { .count() } +/// Helper: run a simple query and return the first column of every data row, +/// as the pgwire text encoder rendered it. +async fn first_column_text(client: &tokio_postgres::Client, sql: &str) -> Vec { + let msgs = client.simple_query(sql).await.expect("simple_query"); + msgs.into_iter() + .filter_map(|m| match m { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_string), + _ => None, + }) + .collect() +} + /// A cross-node inner join between two single-vShard-homed collections must /// return every matching row, regardless of which node the SELECT is issued /// from. Before the build-side gather fix this returned 0 rows when the build @@ -151,3 +163,144 @@ async fn cross_node_join_returns_all_matches() { cluster.shutdown().await; } + +/// A distributed join gathers the remote (build) side through the coordinator +/// while scanning the local (probe) side directly. When the `ON` predicate +/// compares two TIME_KEY columns, both sides must reach that comparison +/// expressed in the same unit — the gathered side must not arrive pre-decoded +/// into a different representation than the locally scanned side. This holds +/// for every node the query runs from, whichever side that node hosts. +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn cross_node_join_compares_time_keys_in_one_unit() { + use nodedb_cluster::routing::vshard_for_collection; + use nodedb_types::DatabaseId; + const EVENTS: &str = "ts_events"; + const FEATURES: &str = "ts_features"; + assert_ne!( + vshard_for_collection(DatabaseId::DEFAULT, EVENTS), + vshard_for_collection(DatabaseId::DEFAULT, FEATURES), + "test collections must hash to different vShards to exercise cross-node join" + ); + + let cluster = TestCluster::spawn_three().await.expect("3-node cluster"); + + cluster + .exec_ddl_on_any_leader(&format!( + "CREATE COLLECTION {EVENTS} \ + (captured_at TIMESTAMP TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .expect("CREATE COLLECTION ts_events"); + cluster + .exec_ddl_on_any_leader(&format!( + "CREATE COLLECTION {FEATURES} \ + (captured_at TIMESTAMP TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .expect("CREATE COLLECTION ts_features"); + + wait_for( + "all 3 nodes see both collections", + Duration::from_secs(10), + Duration::from_millis(50), + || { + cluster + .nodes + .iter() + .all(|n| n.cached_collection_count() >= 2) + }, + ) + .await; + + cluster.nodes[0] + .client + .simple_query(&format!( + "INSERT INTO {EVENTS} (captured_at, host, v) VALUES ('2020-03-05 10:00:00', 'h1', 1.5)" + )) + .await + .expect("insert event row"); + cluster.nodes[0] + .client + .simple_query(&format!( + "INSERT INTO {FEATURES} (captured_at, host, v) VALUES ('2020-03-05 09:00:00', 'h1', 2.5)" + )) + .await + .expect("insert feature row"); + + for (idx, node) in cluster.nodes.iter().enumerate() { + wait_for( + &format!("node {idx} sees the event row"), + Duration::from_secs(15), + Duration::from_millis(50), + || { + let n = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(count_rows( + &node.client, + &format!("SELECT host FROM {EVENTS}"), + )) + }); + n >= 1 + }, + ) + .await; + wait_for( + &format!("node {idx} sees the feature row"), + Duration::from_secs(15), + Duration::from_millis(50), + || { + let n = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(count_rows( + &node.client, + &format!("SELECT host FROM {FEATURES}"), + )) + }); + n >= 1 + }, + ) + .await; + } + + // The join's ON predicate compares the two TIME_KEY columns directly + // (`FEATURES.captured_at <= EVENTS.captured_at`). This only matches the + // single feature row when both sides land in the comparison expressed + // in the same unit, regardless of which node the coordinator gathers the + // build side from. + let value_sql = format!( + "SELECT {FEATURES}.v FROM {EVENTS} INNER JOIN {FEATURES} \ + ON {EVENTS}.host = {FEATURES}.host \ + AND {FEATURES}.captured_at <= {EVENTS}.captured_at" + ); + for (idx, node) in cluster.nodes.iter().enumerate() { + let rows = first_column_text(&node.client, &value_sql).await; + assert_eq!( + rows, + vec!["2.5".to_string()], + "node {idx}: join comparing time keys must return the one matching feature row" + ); + } + + // The joined TIME_KEY is a typed instant on every node, so the + // coordinator's gather step renders it as ISO-8601 UTC. The stored + // 1583402400000 milliseconds read as a number would denote 1970-01-19. + const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; + let time_key_sql = format!( + "SELECT {EVENTS}.captured_at FROM {EVENTS} INNER JOIN {FEATURES} \ + ON {EVENTS}.host = {FEATURES}.host" + ); + for (idx, node) in cluster.nodes.iter().enumerate() { + let rows = first_column_text(&node.client, &time_key_sql).await; + assert_eq!( + rows.len(), + 1, + "node {idx}: the join must yield one row: {rows:?}" + ); + assert_eq!( + rows[0], EARLY_ISO, + "node {idx}: joined time key must denote 2020-03-05T10:00:00Z: got {rows:?}" + ); + } + + cluster.shutdown().await; +} diff --git a/nodedb-columnar/src/materialize_rows/extract.rs b/nodedb-columnar/src/materialize_rows/extract.rs index cd8d7a3b7..faab06c22 100644 --- a/nodedb-columnar/src/materialize_rows/extract.rs +++ b/nodedb-columnar/src/materialize_rows/extract.rs @@ -22,11 +22,15 @@ pub(crate) fn extract_row_value( use nodedb_types::value::Value; let v = match col { - DecodedColumn::Int64 { values, valid } => { + // The reader infers a column's physical kind from its codec, so a + // time column decodes as `Int64`. The declared type decides the cell: + // an instant column yields the variant its declared type names; + // every other type yields the integer stored. + DecodedColumn::Int64 { values, valid } | DecodedColumn::Timestamp { values, valid } => { if !valid[row_idx] { Value::Null } else { - Value::Integer(values[row_idx]) + col_type.time_cell(values[row_idx]) } } DecodedColumn::Float64 { values, valid } => { @@ -36,20 +40,6 @@ pub(crate) fn extract_row_value( Value::Float(values[row_idx]) } } - DecodedColumn::Timestamp { values, valid } => { - if !valid[row_idx] { - Value::Null - } else { - let micros = values[row_idx]; - let dt = nodedb_types::datetime::NdbDateTime::from_micros(micros); - match col_type { - nodedb_types::columnar::ColumnType::Timestamptz - | nodedb_types::columnar::ColumnType::SystemTimestamp => Value::DateTime(dt), - // Timestamp (naive) and anything else that maps to i64 storage. - _ => Value::NaiveDateTime(dt), - } - } - } DecodedColumn::Bool { values, valid } => { if !valid[row_idx] { Value::Null diff --git a/nodedb-columnar/src/memtable/column_data/access.rs b/nodedb-columnar/src/memtable/column_data/access.rs index 1be4bb1c7..2d6a769ad 100644 --- a/nodedb-columnar/src/memtable/column_data/access.rs +++ b/nodedb-columnar/src/memtable/column_data/access.rs @@ -2,6 +2,7 @@ //! Read-only access methods on `ColumnData`: validity checks, value extraction. +use nodedb_types::columnar::ColumnType; use nodedb_types::value::Value; use nodedb_types::value_from_msgpack; @@ -57,7 +58,13 @@ impl ColumnData { } /// Extract a single row's value as `nodedb_types::Value`. - pub(crate) fn get_value(&self, row: usize) -> Value { + /// + /// A time cell is typed by the column's declared type: an instant column + /// yields `Value::NaiveDateTime` (`Timestamp`) or `Value::DateTime` + /// (`Timestamptz`) from the stored epoch microseconds, and every other + /// declared type backed by time storage (`SystemTimestamp`, `Duration`) + /// yields the integer stored. + pub(crate) fn get_value(&self, row: usize, declared: &ColumnType) -> Value { if self.is_null(row) { return Value::Null; } @@ -65,9 +72,7 @@ impl ColumnData { Self::Int64 { values, .. } => Value::Integer(values[row]), Self::Float64 { values, .. } => Value::Float(values[row]), Self::Bool { values, .. } => Value::Bool(values[row]), - Self::Timestamp { values, .. } => Value::DateTime( - nodedb_types::datetime::NdbDateTime::from_micros(values[row]), - ), + Self::Timestamp { values, .. } => declared.time_cell(values[row]), Self::Decimal { values, .. } => { Value::Decimal(rust_decimal::Decimal::deserialize(values[row])) } diff --git a/nodedb-columnar/src/memtable/iter.rs b/nodedb-columnar/src/memtable/iter.rs index a80236c40..d411a6b6b 100644 --- a/nodedb-columnar/src/memtable/iter.rs +++ b/nodedb-columnar/src/memtable/iter.rs @@ -2,6 +2,7 @@ //! Row-oriented iteration and single-row lookup over the memtable. +use nodedb_types::columnar::ColumnDef; use nodedb_types::value::Value; use super::column_data::ColumnData; @@ -9,9 +10,13 @@ use super::core::ColumnarMemtable; impl ColumnarMemtable { /// Iterate rows as `Vec`. For scan/read operations. + /// + /// Every cell is typed by its column's declared type, so a time cell + /// comes back as the instant variant the schema declares. pub fn iter_rows(&self) -> MemtableRowIter<'_> { MemtableRowIter { columns: &self.columns, + column_defs: &self.schema.columns, row_count: self.row_count, current: 0, } @@ -23,8 +28,8 @@ impl ColumnarMemtable { return None; } let mut row = Vec::with_capacity(self.columns.len()); - for col in &self.columns { - row.push(col.get_value(row_idx)); + for (col, def) in self.columns.iter().zip(&self.schema.columns) { + row.push(col.get_value(row_idx, &def.column_type)); } Some(row) } @@ -33,6 +38,7 @@ impl ColumnarMemtable { /// Row iterator over a columnar memtable. pub struct MemtableRowIter<'a> { columns: &'a [ColumnData], + column_defs: &'a [ColumnDef], row_count: usize, current: usize, } @@ -45,8 +51,8 @@ impl Iterator for MemtableRowIter<'_> { return None; } let mut row = Vec::with_capacity(self.columns.len()); - for col in self.columns { - row.push(col.get_value(self.current)); + for (col, def) in self.columns.iter().zip(self.column_defs) { + row.push(col.get_value(self.current, &def.column_type)); } self.current += 1; Some(row) @@ -59,3 +65,47 @@ impl Iterator for MemtableRowIter<'_> { } impl ExactSizeIterator for MemtableRowIter<'_> {} + +#[cfg(test)] +mod tests { + use nodedb_types::NdbDateTime; + use nodedb_types::columnar::{ColumnDef, ColumnType, ColumnarSchema}; + use nodedb_types::value::Value; + + use super::super::core::ColumnarMemtable; + + const MICROS: i64 = 1_583_402_400_000_000; + + /// A row read back from the memtable types each time cell by its declared + /// column: `Timestamp` is a naive instant, `Timestamptz` a UTC instant, + /// `SystemTimestamp` the integer stored. + #[test] + fn a_time_cell_reads_back_as_its_declared_type() { + let schema = ColumnarSchema::new(vec![ + ColumnDef::required("id", ColumnType::Int64).with_primary_key(), + ColumnDef::required("at", ColumnType::Timestamp), + ColumnDef::required("at_tz", ColumnType::Timestamptz), + ColumnDef::required("sys", ColumnType::SystemTimestamp), + ]) + .expect("valid schema"); + let mut mt = ColumnarMemtable::new(&schema); + let dt = NdbDateTime::from_micros(MICROS); + mt.append_row(&[ + Value::Integer(1), + Value::NaiveDateTime(dt), + Value::DateTime(dt), + Value::Integer(7), + ]) + .expect("append"); + + let expected = vec![ + Value::Integer(1), + Value::NaiveDateTime(dt), + Value::DateTime(dt), + Value::Integer(7), + ]; + assert_eq!(mt.get_row(0), Some(expected.clone())); + assert_eq!(mt.iter_rows().collect::>(), vec![expected]); + assert_eq!(mt.get_row(1), None); + } +} diff --git a/nodedb-columnar/src/pk_index.rs b/nodedb-columnar/src/pk_index.rs index f5014f58f..3bb691f32 100644 --- a/nodedb-columnar/src/pk_index.rs +++ b/nodedb-columnar/src/pk_index.rs @@ -178,7 +178,7 @@ pub fn encode_pk(value: &nodedb_types::value::Value) -> Vec { Value::String(s) => s.as_bytes().to_vec(), Value::Uuid(s) => s.as_bytes().to_vec(), Value::Decimal(d) => d.serialize().to_vec(), - Value::DateTime(dt) => { + Value::DateTime(dt) | Value::NaiveDateTime(dt) => { let sortable = (dt.micros as u64) ^ (1u64 << 63); sortable.to_be_bytes().to_vec() } diff --git a/nodedb-columnar/src/reader/types.rs b/nodedb-columnar/src/reader/types.rs index cf8ebd34f..8f0c605ed 100644 --- a/nodedb-columnar/src/reader/types.rs +++ b/nodedb-columnar/src/reader/types.rs @@ -1,8 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 /// Decoded column data from a segment scan. +/// +/// The variant set is the closed set of physical column encodings, so a +/// reader matches it exhaustively and the compiler names every site an added +/// encoding must handle. #[derive(Debug)] -#[non_exhaustive] pub enum DecodedColumn { Int64 { values: Vec, diff --git a/nodedb-query/src/expr/binary.rs b/nodedb-query/src/expr/binary.rs index 281d96f10..91b55c9d8 100644 --- a/nodedb-query/src/expr/binary.rs +++ b/nodedb-query/src/expr/binary.rs @@ -7,7 +7,8 @@ use rust_decimal::Decimal; use nodedb_types::Value; use crate::value_ops::{ - coerced_eq, compare_values, is_truthy, to_value_number, value_to_display_string, value_to_f64, + coerced_eq, is_truthy, partial_compare_values, to_value_number, value_to_display_string, + value_to_f64, }; use super::eval::EvalError; @@ -123,24 +124,22 @@ pub(super) fn eval_binary_op( } BinaryOp::Eq => Ok(Value::Bool(coerced_eq(left, right))), BinaryOp::NotEq => Ok(Value::Bool(!coerced_eq(left, right))), - BinaryOp::Gt => Ok(Value::Bool( - compare_values(left, right) == std::cmp::Ordering::Greater, - )), - BinaryOp::GtEq => { - let c = compare_values(left, right); - Ok(Value::Bool( - c == std::cmp::Ordering::Greater || c == std::cmp::Ordering::Equal, - )) - } - BinaryOp::Lt => Ok(Value::Bool( - compare_values(left, right) == std::cmp::Ordering::Less, - )), - BinaryOp::LtEq => { - let c = compare_values(left, right); - Ok(Value::Bool( - c == std::cmp::Ordering::Less || c == std::cmp::Ordering::Equal, - )) - } + BinaryOp::Gt => Ok(Value::Bool(matches!( + partial_compare_values(left, right), + Some(std::cmp::Ordering::Greater) + ))), + BinaryOp::GtEq => Ok(Value::Bool(matches!( + partial_compare_values(left, right), + Some(std::cmp::Ordering::Greater | std::cmp::Ordering::Equal) + ))), + BinaryOp::Lt => Ok(Value::Bool(matches!( + partial_compare_values(left, right), + Some(std::cmp::Ordering::Less) + ))), + BinaryOp::LtEq => Ok(Value::Bool(matches!( + partial_compare_values(left, right), + Some(std::cmp::Ordering::Less | std::cmp::Ordering::Equal) + ))), BinaryOp::And => Ok(Value::Bool(is_truthy(left) && is_truthy(right))), BinaryOp::Or => Ok(Value::Bool(is_truthy(left) || is_truthy(right))), } diff --git a/nodedb-query/src/functions/datetime.rs b/nodedb-query/src/functions/datetime.rs index ef245a4b1..6837eba0a 100644 --- a/nodedb-query/src/functions/datetime.rs +++ b/nodedb-query/src/functions/datetime.rs @@ -1,52 +1,61 @@ // SPDX-License-Identifier: Apache-2.0 //! DateTime and duration scalar functions. +//! +//! Instant-typed arguments (`Value::DateTime`, `Value::NaiveDateTime`, or a +//! parseable ISO 8601 string) are accepted through [`instant_arg`]. A +//! function that returns an instant preserves the kind of its input: +//! `Utc` in, `DateTime` out; `Naive` in, `NaiveDateTime` out. use crate::value_ops::{to_value_number, value_to_display_string}; -use nodedb_types::Value; +use nodedb_types::{InstantKind, NdbDateTime, Value}; + +/// Resolve an argument to a typed instant. +/// +/// `Value::DateTime`/`Value::NaiveDateTime` pass through their own kind. A +/// `Value::String` that parses via [`NdbDateTime::parse`] resolves to +/// `InstantKind::Utc` when it ends in `Z`/`z` (the only zone marker +/// `NdbDateTime::parse` recognizes) and `InstantKind::Naive` otherwise. +/// Anything else, or a string that fails to parse, is `None`. +fn instant_arg(v: &Value) -> Option<(InstantKind, NdbDateTime)> { + if let Some(pair) = v.as_instant() { + return Some(pair); + } + let s = v.as_str()?; + let dt = NdbDateTime::parse(s)?; + let trimmed = s.trim(); + let kind = if trimmed.ends_with('Z') || trimmed.ends_with('z') { + InstantKind::Utc + } else { + InstantKind::Naive + }; + Some((kind, dt)) +} pub(super) fn try_eval(name: &str, args: &[Value]) -> Option { let v = match name { "now" | "current_timestamp" => { - let dt = nodedb_types::NdbDateTime::now(); + let dt = NdbDateTime::now(); Value::DateTime(dt) } - "datetime" | "to_datetime" => args - .first() - .and_then(|v| match v { - Value::String(s) => { - nodedb_types::NdbDateTime::parse(s).map(|dt| Value::String(dt.to_iso8601())) - } - Value::Integer(micros) => Some(Value::String( - nodedb_types::NdbDateTime::from_micros(*micros).to_iso8601(), - )), - Value::Float(f) => Some(Value::String( - nodedb_types::NdbDateTime::from_micros(*f as i64).to_iso8601(), - )), - Value::DateTime(dt) | Value::NaiveDateTime(dt) => { - Some(Value::String(dt.to_iso8601())) - } - _ => None, - }) - .unwrap_or(Value::Null), + "datetime" | "to_datetime" => args.first().map_or(Value::Null, |v| match v { + Value::Integer(micros) => InstantKind::Utc.from_micros(*micros), + Value::Float(f) => InstantKind::Utc.from_micros(*f as i64), + _ => instant_arg(v).map_or(Value::Null, |(kind, dt)| kind.value(dt)), + }), "unix_secs" | "epoch_secs" => args .first() - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse) - .map_or(Value::Null, |dt| Value::Integer(dt.unix_secs())), + .and_then(instant_arg) + .map_or(Value::Null, |(_, dt)| Value::Integer(dt.unix_secs())), "unix_millis" | "epoch_millis" => args .first() - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse) - .map_or(Value::Null, |dt| Value::Integer(dt.unix_millis())), + .and_then(instant_arg) + .map_or(Value::Null, |(_, dt)| Value::Integer(dt.unix_millis())), "extract" | "date_part" => { let part = args.first().and_then(|v| v.as_str()).unwrap_or(""); - let dt = args - .get(1) - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse); + let dt = args.get(1).and_then(instant_arg); match dt { - Some(dt) => { + Some((_, dt)) => { let c = dt.components(); let val: i64 = match part.to_lowercase().as_str() { "year" | "y" => c.year as i64, @@ -70,88 +79,70 @@ pub(super) fn try_eval(name: &str, args: &[Value]) -> Option { } "date_trunc" | "datetrunc" => { let part = args.first().and_then(|v| v.as_str()).unwrap_or(""); - let dt = args - .get(1) - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse); + let dt = args.get(1).and_then(instant_arg); match dt { - Some(dt) => { + Some((kind, dt)) => { let c = dt.components(); let truncated = match part.to_lowercase().as_str() { - "year" => nodedb_types::NdbDateTime::parse(&format!( - "{:04}-01-01T00:00:00Z", - c.year - )), - "month" => nodedb_types::NdbDateTime::parse(&format!( + "year" => NdbDateTime::parse(&format!("{:04}-01-01T00:00:00Z", c.year)), + "month" => NdbDateTime::parse(&format!( "{:04}-{:02}-01T00:00:00Z", c.year, c.month )), - "day" => nodedb_types::NdbDateTime::parse(&format!( + "day" => NdbDateTime::parse(&format!( "{:04}-{:02}-{:02}T00:00:00Z", c.year, c.month, c.day )), - "hour" => nodedb_types::NdbDateTime::parse(&format!( + "hour" => NdbDateTime::parse(&format!( "{:04}-{:02}-{:02}T{:02}:00:00Z", c.year, c.month, c.day, c.hour )), - "minute" => nodedb_types::NdbDateTime::parse(&format!( + "minute" => NdbDateTime::parse(&format!( "{:04}-{:02}-{:02}T{:02}:{:02}:00Z", c.year, c.month, c.day, c.hour, c.minute )), - "second" => nodedb_types::NdbDateTime::parse(&format!( + "second" => NdbDateTime::parse(&format!( "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", c.year, c.month, c.day, c.hour, c.minute, c.second )), _ => None, }; - truncated.map_or(Value::Null, |t| Value::String(t.to_iso8601())) + truncated.map_or(Value::Null, |t| kind.value(t)) } None => Value::Null, } } "date_add" | "datetime_add" => { - let dt = args - .first() - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse); + let dt = args.first().and_then(instant_arg); let dur = args .get(1) .and_then(|v| v.as_str()) .and_then(nodedb_types::NdbDuration::parse); match (dt, dur) { - (Some(dt), Some(dur)) => dt + (Some((kind, dt)), Some(dur)) => dt .add_duration(dur) - .map(|r| Value::String(r.to_iso8601())) + .map(|r| kind.value(r)) .unwrap_or(Value::Null), _ => Value::Null, } } "date_sub" | "datetime_sub" => { - let dt = args - .first() - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse); + let dt = args.first().and_then(instant_arg); let dur = args .get(1) .and_then(|v| v.as_str()) .and_then(nodedb_types::NdbDuration::parse); match (dt, dur) { - (Some(dt), Some(dur)) => dt + (Some((kind, dt)), Some(dur)) => dt .sub_duration(dur) - .map(|r| Value::String(r.to_iso8601())) + .map(|r| kind.value(r)) .unwrap_or(Value::Null), _ => Value::Null, } } "date_diff" | "datediff" => { - let dt1 = args - .first() - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse); - let dt2 = args - .get(1) - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse); + let dt1 = args.first().and_then(instant_arg).map(|(_, dt)| dt); + let dt2 = args.get(1).and_then(instant_arg).map(|(_, dt)| dt); match (dt1, dt2) { (Some(a), Some(b)) => a .duration_since(&b) @@ -178,39 +169,51 @@ pub(super) fn try_eval(name: &str, args: &[Value]) -> Option { Some(v) } -/// `time_bucket(interval, timestamp)` — truncate a millisecond timestamp -/// to the start of the given interval bucket. +/// `time_bucket(interval, timestamp)` — truncate a timestamp to the start +/// of the given interval bucket. /// /// Accepts two argument orders (both common in SQL): /// - `time_bucket('1 hour', timestamp_col)` — interval first /// - `time_bucket(timestamp_col, '1 hour')` — timestamp first /// -/// The interval is a string like `'1h'`, `'5m'`, `'1 hour'`, `'30 seconds'`. -/// The timestamp is an integer (epoch milliseconds). +/// The interval is a string like `'1h'`, `'5m'`, `'1 hour'`, `'30 seconds'`, +/// or an integer number of seconds. +/// +/// The timestamp is either: +/// - a typed instant (`Value::DateTime`, `Value::NaiveDateTime`, or a +/// parseable instant string) — bucketed on epoch microseconds, floored +/// toward negative infinity, and returned as the same instant kind +/// - an `Value::Integer`/`Value::Float` epoch-millisecond value — bucketed +/// on epoch milliseconds and returned as `Value::Integer` milliseconds, +/// truncated toward zero fn eval_time_bucket(args: &[Value]) -> Value { if args.len() < 2 { return Value::Null; } - // Detect which arg is the interval string and which is the timestamp. + let instant_and_interval = instant_arg(&args[0]) + .map(|pair| (pair, &args[1])) + .or_else(|| instant_arg(&args[1]).map(|pair| (pair, &args[0]))); + + if let Some(((kind, dt), interval)) = instant_and_interval { + let interval_us = interval_arg_ms(interval).and_then(|ms| ms.checked_mul(1000)); + return match interval_us { + Some(i) if i > 0 => { + let bucket = dt.micros.div_euclid(i) * i; + kind.value(NdbDateTime::from_micros(bucket)) + } + _ => Value::Null, + }; + } + + // Detect which arg is the interval and which is the timestamp. let (interval_ms, timestamp_ms) = match (&args[0], &args[1]) { // time_bucket('1 hour', timestamp) - (Value::String(s), ts_val) => { - let interval = parse_interval_to_ms(s); - let ts = value_to_timestamp_ms(ts_val); - (interval, ts) - } + (Value::String(_), ts_val) => (interval_arg_ms(&args[0]), value_to_timestamp_ms(ts_val)), // time_bucket(timestamp, '1 hour') - (ts_val, Value::String(s)) => { - let interval = parse_interval_to_ms(s); - let ts = value_to_timestamp_ms(ts_val); - (interval, ts) - } + (ts_val, Value::String(_)) => (interval_arg_ms(&args[1]), value_to_timestamp_ms(ts_val)), // time_bucket(3600, timestamp) — interval as integer seconds - (Value::Integer(interval_secs), ts_val) => { - let ts = value_to_timestamp_ms(ts_val); - (Some((*interval_secs) * 1000), ts) - } + (Value::Integer(_), ts_val) => (interval_arg_ms(&args[0]), value_to_timestamp_ms(ts_val)), _ => return Value::Null, }; @@ -220,6 +223,16 @@ fn eval_time_bucket(args: &[Value]) -> Value { } } +/// The bucket interval in milliseconds: an interval string such as +/// `'1 hour'`, or an integer count of seconds. +fn interval_arg_ms(v: &Value) -> Option { + match v { + Value::String(s) => parse_interval_to_ms(s), + Value::Integer(secs) => secs.checked_mul(1000), + _ => None, + } +} + fn value_to_timestamp_ms(v: &Value) -> Option { match v { Value::Integer(n) => Some(*n), @@ -237,3 +250,117 @@ fn parse_interval_to_ms(s: &str) -> Option { .map(|ms| ms as i64) .filter(|&ms| ms > 0) } + +#[cfg(test)] +mod tests { + use super::*; + + fn naive(s: &str) -> Value { + Value::NaiveDateTime(NdbDateTime::parse(s).expect("valid test timestamp")) + } + + fn utc(s: &str) -> Value { + Value::DateTime(NdbDateTime::parse(s).expect("valid test timestamp")) + } + + #[test] + fn date_part_year_accepts_naive_and_utc() { + let naive_year = try_eval( + "date_part", + &[Value::String("year".into()), naive("2020-03-05T10:00:00")], + ); + assert_eq!(naive_year, Some(Value::Integer(2020))); + + let utc_year = try_eval( + "date_part", + &[Value::String("year".into()), utc("2020-03-05T10:00:00Z")], + ); + assert_eq!(utc_year, Some(Value::Integer(2020))); + } + + #[test] + fn date_trunc_preserves_utc_kind() { + let truncated = try_eval( + "date_trunc", + &[Value::String("hour".into()), utc("2020-03-05T10:42:17Z")], + ); + assert_eq!( + truncated, + Some(Value::DateTime( + NdbDateTime::parse("2020-03-05T10:00:00Z").expect("valid test timestamp") + )) + ); + } + + #[test] + fn time_bucket_naive_instant_floors_to_hour() { + let bucketed = try_eval( + "time_bucket", + &[Value::String("1 hour".into()), naive("2020-03-05T10:00:00")], + ); + assert_eq!(bucketed, Some(naive("2020-03-05T10:00:00"))); + + let bucketed = try_eval( + "time_bucket", + &[Value::String("1 hour".into()), naive("2020-03-05T10:59:59")], + ); + assert_eq!(bucketed, Some(naive("2020-03-05T10:00:00"))); + } + + #[test] + fn time_bucket_negative_micros_floors_toward_negative_infinity() { + let before_epoch = Value::NaiveDateTime(NdbDateTime::from_micros(-1_800_000_000)); + let bucketed = try_eval( + "time_bucket", + &[Value::String("1 hour".into()), before_epoch], + ); + assert_eq!( + bucketed, + Some(Value::NaiveDateTime(NdbDateTime::from_micros( + -3_600_000_000 + ))) + ); + } + + #[test] + fn time_bucket_integer_ms_unchanged() { + let bucketed = try_eval( + "time_bucket", + &[Value::String("1 hour".into()), Value::Integer(3_661_000)], + ); + assert_eq!(bucketed, Some(Value::Integer(3_600_000))); + } + + #[test] + fn datetime_of_naive_instant_keeps_kind() { + let result = try_eval("datetime", &[naive("2020-03-05T10:00:00")]); + assert_eq!(result, Some(naive("2020-03-05T10:00:00"))); + } + + #[test] + fn extract_on_integer_returns_null() { + let result = try_eval( + "extract", + &[Value::String("year".into()), Value::Integer(1_583_400_000)], + ); + assert_eq!(result, Some(Value::Null)); + } + + #[test] + fn time_bucket_integer_seconds_interval_accepts_an_instant() { + let bucketed = try_eval( + "time_bucket", + &[Value::Integer(3600), naive("2020-03-05T10:59:59")], + ); + assert_eq!(bucketed, Some(naive("2020-03-05T10:00:00"))); + } + + #[test] + fn date_diff_accepts_typed_instants() { + let diff = try_eval( + "date_diff", + &[naive("2020-03-05T10:00:00"), naive("2020-03-05T09:00:00")], + ); + assert_eq!(diff, Some(Value::Integer(3600))); + } +} diff --git a/nodedb-query/src/msgpack_scan/compare.rs b/nodedb-query/src/msgpack_scan/compare.rs index dc0769592..b48122876 100644 --- a/nodedb-query/src/msgpack_scan/compare.rs +++ b/nodedb-query/src/msgpack_scan/compare.rs @@ -8,6 +8,8 @@ use std::cmp::Ordering; use std::hash::{BuildHasher, Hasher}; +use nodedb_types::read_instant; + use crate::msgpack_scan::reader::{read_f64, read_i64, read_null, str_bounds}; /// Hash the raw bytes of a MessagePack value at `range` within `buf`. @@ -46,11 +48,12 @@ pub fn hash_field_bytes_with( /// Compare two MessagePack values by their decoded content. /// /// Comparison order: -/// 1. Null < Bool < Number < String < Binary < Array < Map +/// 1. Null < Bool < Number < Instant < String < Binary < Array < Map < Ext /// 2. Within numbers: compare as f64 -/// 3. Within strings: lexicographic on raw bytes (valid UTF-8 guarantees +/// 3. Within instants: by kind (UTC before naive), then signed epoch micros +/// 4. Within strings: lexicographic on raw bytes (valid UTF-8 guarantees /// byte order = Unicode code-point order for ASCII/Latin-1) -/// 4. Fallback: raw byte comparison +/// 5. Fallback: raw byte comparison pub fn compare_field_bytes( a_buf: &[u8], a_range: (usize, usize), @@ -69,23 +72,22 @@ pub fn compare_field_bytes( None => return Ordering::Greater, }; - let a_type = type_rank(a_tag); - let b_type = type_rank(b_tag); + let a_type = type_rank(a_buf, a_off, a_tag); + let b_type = type_rank(b_buf, b_off, b_tag); if a_type != b_type { return a_type.cmp(&b_type); } match a_type { - 0 => Ordering::Equal, // both null - 1 => { - // bool + RANK_NULL => Ordering::Equal, + RANK_BOOL => { let a_val = a_tag == 0xc3; // true let b_val = b_tag == 0xc3; a_val.cmp(&b_val) } - 2 => { - // number — compare as f64 + RANK_NUMBER => { + // compare as f64 match (read_f64(a_buf, a_off), read_f64(b_buf, b_off)) { (Some(a), Some(b)) => a.partial_cmp(&b).unwrap_or(Ordering::Equal), (Some(_), None) => Ordering::Greater, @@ -93,7 +95,20 @@ pub fn compare_field_bytes( (None, None) => Ordering::Equal, } } - 3 => { + RANK_INSTANT => { + // Both sides decoded as instants by `type_rank`. Kind first, then + // signed micros: the raw payload is not byte-comparable below 0. + match (read_instant(a_buf, a_off), read_instant(b_buf, b_off)) { + (Some((a_kind, a_us)), Some((b_kind, b_us))) => a_kind + .ext_type() + .cmp(&b_kind.ext_type()) + .then(a_us.cmp(&b_us)), + (Some(_), None) => Ordering::Greater, + (None, Some(_)) => Ordering::Less, + (None, None) => Ordering::Equal, + } + } + RANK_STRING => { // string — compare raw bytes match (str_bounds(a_buf, a_off), str_bounds(b_buf, b_off)) { (Some((a_s, a_l)), Some((b_s, b_l))) => { @@ -148,20 +163,41 @@ pub fn is_field_null(buf: &[u8], range: (usize, usize)) -> bool { read_null(buf, range.0) } +const RANK_NULL: u8 = 0; +const RANK_BOOL: u8 = 1; +const RANK_NUMBER: u8 = 2; +const RANK_INSTANT: u8 = 3; +const RANK_STRING: u8 = 4; +const RANK_BINARY: u8 = 5; +const RANK_ARRAY: u8 = 6; +const RANK_MAP: u8 = 7; +const RANK_EXT: u8 = 8; +const RANK_UNKNOWN: u8 = 9; + /// Type rank for cross-type ordering. Lower rank = sorts first. -/// Null(0) < Bool(1) < Number(2) < String(3) < Binary(4) < Array(5) < Map(6) < Ext(7) -fn type_rank(tag: u8) -> u8 { +/// Null < Bool < Number < Instant < String < Binary < Array < Map < Ext +/// +/// An instant is a complete `fixext8` of type 1 / 2 at `off`. A `fixext8` +/// of any other type, or a truncated one, ranks as ext. +fn type_rank(buf: &[u8], off: usize, tag: u8) -> u8 { match tag { - 0xc0 => 0, // nil - 0xc2 | 0xc3 => 1, // bool - 0x00..=0x7f | 0xe0..=0xff => 2, // fixint - 0xca..=0xd3 => 2, // float/uint/int - 0xa0..=0xbf | 0xd9..=0xdb => 3, // string - 0xc4..=0xc6 => 4, // binary - 0x90..=0x9f | 0xdc | 0xdd => 5, // array - 0x80..=0x8f | 0xde | 0xdf => 6, // map - 0xc7..=0xc9 | 0xd4..=0xd8 => 7, // ext - _ => 8, // unknown + 0xc0 => RANK_NULL, + 0xc2 | 0xc3 => RANK_BOOL, + 0x00..=0x7f | 0xe0..=0xff => RANK_NUMBER, // fixint + 0xca..=0xd3 => RANK_NUMBER, // float/uint/int + 0xa0..=0xbf | 0xd9..=0xdb => RANK_STRING, + 0xc4..=0xc6 => RANK_BINARY, + 0x90..=0x9f | 0xdc | 0xdd => RANK_ARRAY, + 0x80..=0x8f | 0xde | 0xdf => RANK_MAP, + 0xd7 => { + if read_instant(buf, off).is_some() { + RANK_INSTANT + } else { + RANK_EXT + } + } + 0xc7..=0xc9 | 0xd4..=0xd6 | 0xd8 => RANK_EXT, + _ => RANK_UNKNOWN, } } @@ -292,6 +328,77 @@ mod tests { ); } + #[test] + fn compare_instants_negative_micros() { + use nodedb_types::{InstantKind, write_instant}; + let mut a = Vec::new(); + write_instant(&mut a, InstantKind::Utc, -10); + let mut b = Vec::new(); + write_instant(&mut b, InstantKind::Utc, -5); + let mut c = Vec::new(); + write_instant(&mut c, InstantKind::Utc, 1); + assert_eq!( + compare_field_bytes(&a, val_range(&a), &b, val_range(&b)), + Ordering::Less + ); + assert_eq!( + compare_field_bytes(&a, val_range(&a), &c, val_range(&c)), + Ordering::Less + ); + assert_eq!( + compare_field_bytes(&c, val_range(&c), &b, val_range(&b)), + Ordering::Greater + ); + assert_eq!( + compare_field_bytes(&a, val_range(&a), &a, val_range(&a)), + Ordering::Equal + ); + } + + #[test] + fn compare_instant_kinds_order_utc_first() { + use nodedb_types::{InstantKind, write_instant}; + let mut utc = Vec::new(); + write_instant(&mut utc, InstantKind::Utc, 100); + let mut naive = Vec::new(); + write_instant(&mut naive, InstantKind::Naive, 1); + assert_eq!( + compare_field_bytes(&utc, val_range(&utc), &naive, val_range(&naive)), + Ordering::Less + ); + } + + #[test] + fn compare_instant_vs_integer_uses_rank() { + use nodedb_types::{InstantKind, write_instant}; + // A negative instant has a payload starting 0xff, above any fixint + // byte. Rank places it after every number and before every string. + let mut inst = Vec::new(); + write_instant(&mut inst, InstantKind::Naive, -1); + let int = encode(&json!(i64::MAX)); + let s = encode(&json!("")); + assert_eq!( + compare_field_bytes(&int, val_range(&int), &inst, val_range(&inst)), + Ordering::Less + ); + assert_eq!( + compare_field_bytes(&inst, val_range(&inst), &s, val_range(&s)), + Ordering::Less + ); + } + + #[test] + fn unknown_fixext8_ranks_as_ext() { + use nodedb_types::{InstantKind, write_instant}; + let mut inst = Vec::new(); + write_instant(&mut inst, InstantKind::Utc, 0); + let other = [0xd7, 0x09, 0, 0, 0, 0, 0, 0, 0, 0]; + assert_eq!( + compare_field_bytes(&inst, val_range(&inst), &other, val_range(&other)), + Ordering::Less + ); + } + #[test] fn hash_from_extracted_field() { let buf = encode(&json!({"id": 42})); diff --git a/nodedb-query/src/msgpack_scan/filter.rs b/nodedb-query/src/msgpack_scan/filter.rs index f05f93381..9ec23d83a 100644 --- a/nodedb-query/src/msgpack_scan/filter.rs +++ b/nodedb-query/src/msgpack_scan/filter.rs @@ -4,7 +4,7 @@ //! //! `ScanFilter::matches_binary(doc: &[u8])` evaluates a filter predicate //! directly on msgpack bytes without decoding to `serde_json::Value`. -//! Uses `Value::eq_coerced`/`cmp_coerced` for type coercion — single +//! Uses `Value::eq_coerced`/`partial_cmp_coerced` for type coercion — single //! source of truth shared with the JSON filter path. use std::cmp::Ordering; @@ -16,6 +16,7 @@ use crate::msgpack_scan::reader::{ array_header, map_header, read_null, read_str, read_value, skip_value, }; use crate::scan_filter::like::sql_like_match; +use crate::scan_filter::types::column_compare; use crate::scan_filter::{FilterOp, ScanFilter}; impl ScanFilter { @@ -79,7 +80,30 @@ impl ScanFilter { _ => Ok(false), }; } - _ => {} + FilterOp::Eq + | FilterOp::Ne + | FilterOp::Gt + | FilterOp::Gte + | FilterOp::Lt + | FilterOp::Lte + | FilterOp::Contains + | FilterOp::Like + | FilterOp::NotLike + | FilterOp::Ilike + | FilterOp::NotIlike + | FilterOp::In + | FilterOp::NotIn + | FilterOp::IsNull + | FilterOp::IsNotNull + | FilterOp::ArrayContains + | FilterOp::ArrayContainsAll + | FilterOp::ArrayOverlap + | FilterOp::GtColumn + | FilterOp::GteColumn + | FilterOp::LtColumn + | FilterOp::LteColumn + | FilterOp::EqColumn + | FilterOp::NeColumn => {} } let (start, end) = match extract_field(doc, 0, &self.field) { @@ -117,7 +141,30 @@ impl ScanFilter { _ => Ok(false), }; } - _ => {} + FilterOp::Eq + | FilterOp::Ne + | FilterOp::Gt + | FilterOp::Gte + | FilterOp::Lt + | FilterOp::Lte + | FilterOp::Contains + | FilterOp::Like + | FilterOp::NotLike + | FilterOp::Ilike + | FilterOp::NotIlike + | FilterOp::In + | FilterOp::NotIn + | FilterOp::IsNull + | FilterOp::IsNotNull + | FilterOp::ArrayContains + | FilterOp::ArrayContainsAll + | FilterOp::ArrayOverlap + | FilterOp::GtColumn + | FilterOp::GteColumn + | FilterOp::LtColumn + | FilterOp::LteColumn + | FilterOp::EqColumn + | FilterOp::NeColumn => {} } let (start, end) = match idx.get(&self.field) { @@ -136,16 +183,16 @@ fn eval_op(filter: &ScanFilter, doc: &[u8], start: usize, _end: usize) -> bool { FilterOp::IsNotNull => !read_null(doc, start), FilterOp::Eq => eq_value(doc, start, &filter.value), FilterOp::Ne => !eq_value(doc, start, &filter.value), - FilterOp::Gt => cmp_value(doc, start, &filter.value) == Ordering::Greater, - FilterOp::Gte => { - let c = cmp_value(doc, start, &filter.value); - c == Ordering::Greater || c == Ordering::Equal - } - FilterOp::Lt => cmp_value(doc, start, &filter.value) == Ordering::Less, - FilterOp::Lte => { - let c = cmp_value(doc, start, &filter.value); - c == Ordering::Less || c == Ordering::Equal - } + FilterOp::Gt => cmp_value(doc, start, &filter.value) == Some(Ordering::Greater), + FilterOp::Gte => matches!( + cmp_value(doc, start, &filter.value), + Some(Ordering::Greater | Ordering::Equal) + ), + FilterOp::Lt => cmp_value(doc, start, &filter.value) == Some(Ordering::Less), + FilterOp::Lte => matches!( + cmp_value(doc, start, &filter.value), + Some(Ordering::Less | Ordering::Equal) + ), FilterOp::Contains => { if let (Some(s), Some(pattern)) = (read_str(doc, start), filter.value.as_str()) { s.contains(pattern) @@ -214,17 +261,11 @@ fn eval_op(filter: &ScanFilter, doc: &[u8], start: usize, _end: usize) -> bool { // Read both sides as Value and compare. let left = read_value(doc, start).unwrap_or(nodedb_types::Value::Null); let right = read_value(doc, other_start).unwrap_or(nodedb_types::Value::Null); - match filter.op { - FilterOp::GtColumn => left.cmp_coerced(&right) == Ordering::Greater, - FilterOp::GteColumn => left.cmp_coerced(&right) != Ordering::Less, - FilterOp::LtColumn => left.cmp_coerced(&right) == Ordering::Less, - FilterOp::LteColumn => left.cmp_coerced(&right) != Ordering::Greater, - FilterOp::EqColumn => left.eq_coerced(&right), - FilterOp::NeColumn => !left.eq_coerced(&right), - _ => false, - } + column_compare(filter.op, &left, &right) } - _ => false, + // Handled before field extraction by both `matches_binary` paths. + FilterOp::MatchAll | FilterOp::Exists | FilterOp::NotExists => true, + FilterOp::Or | FilterOp::Expr => false, } } @@ -262,15 +303,14 @@ fn eq_value(buf: &[u8], offset: usize, filter_val: &nodedb_types::Value) -> bool } /// Coerced ordering: read msgpack value at offset → compare with `Value`. -/// Uses `Value::cmp_coerced` — single source of truth for ordering. +/// Uses `Value::partial_cmp_coerced` — single source of truth for ordering. /// -/// Returns ordering of field_val relative to filter_val (field <=> filter). +/// Returns ordering of field_val relative to filter_val (field <=> filter), +/// or `None` when the cell cannot be read or the pair has no defined order, +/// so a range predicate over it matches nothing. #[inline] -fn cmp_value(buf: &[u8], offset: usize, filter_val: &nodedb_types::Value) -> Ordering { - match read_value(buf, offset) { - Some(field_val) => field_val.cmp_coerced(filter_val), - None => Ordering::Equal, - } +fn cmp_value(buf: &[u8], offset: usize, filter_val: &nodedb_types::Value) -> Option { + read_value(buf, offset)?.partial_cmp_coerced(filter_val) } /// LIKE/ILIKE/NOT LIKE/NOT ILIKE helper. @@ -319,7 +359,7 @@ mod tests { fn filter(field: &str, op: &str, value: nodedb_types::Value) -> ScanFilter { ScanFilter { field: field.into(), - op: op.into(), + op: FilterOp::parse(op).expect(op), value, clauses: vec![], expr: None, @@ -341,6 +381,52 @@ mod tests { ); } + /// An instant cell compares by epoch microseconds against a typed + /// instant literal and against an ISO-8601 string literal. + #[test] + fn instant_cell_compares_by_micros() { + use nodedb_types::{InstantKind, NdbDateTime, Value}; + const MICROS: i64 = 1_583_402_400_000_000; + let mut doc = Vec::new(); + crate::msgpack_scan::write_map_header(&mut doc, 1); + crate::msgpack_scan::write_kv_instant(&mut doc, "ts", InstantKind::Naive, MICROS); + + let same = Value::NaiveDateTime(NdbDateTime::from_micros(MICROS)); + let later = Value::NaiveDateTime(NdbDateTime::from_micros(MICROS + 1)); + assert!( + filter("ts", "eq", same.clone()) + .matches_binary(&doc) + .unwrap() + ); + assert!( + !filter("ts", "eq", later.clone()) + .matches_binary(&doc) + .unwrap() + ); + assert!(filter("ts", "lt", later).matches_binary(&doc).unwrap()); + assert!(filter("ts", "gte", same).matches_binary(&doc).unwrap()); + assert!( + filter("ts", "eq", Value::String("2020-03-05T10:00:00Z".into())) + .matches_binary(&doc) + .unwrap() + ); + assert!( + filter("ts", "gt", Value::String("2020-03-05 09:00:00".into())) + .matches_binary(&doc) + .unwrap() + ); + let idx = FieldIndex::build(&doc, 0).unwrap_or_else(FieldIndex::empty); + assert!( + filter( + "ts", + "eq", + Value::NaiveDateTime(NdbDateTime::from_micros(MICROS)) + ) + .matches_binary_indexed(&doc, &idx) + .unwrap() + ); + } + #[test] fn eq_coerces_string_to_integer() { let doc = encode(&json!({"age": 25})); @@ -539,7 +625,7 @@ mod tests { assert!( ScanFilter { field: "status".into(), - op: "in".into(), + op: FilterOp::In, value: vals.clone(), clauses: vec![], expr: None @@ -552,7 +638,7 @@ mod tests { assert!( ScanFilter { field: "status".into(), - op: "not_in".into(), + op: FilterOp::NotIn, value: vals, clauses: vec![], expr: None @@ -595,7 +681,7 @@ mod tests { assert!( ScanFilter { field: "tags".into(), - op: "array_contains_all".into(), + op: FilterOp::ArrayContainsAll, value: needles, clauses: vec![], expr: None @@ -615,7 +701,7 @@ mod tests { assert!( ScanFilter { field: "tags".into(), - op: "array_overlap".into(), + op: FilterOp::ArrayOverlap, value: needles, clauses: vec![], expr: None @@ -630,7 +716,7 @@ mod tests { let doc = encode(&json!({"x": 5})); let f = ScanFilter { field: String::new(), - op: "or".into(), + op: FilterOp::Or, value: nodedb_types::Value::Null, clauses: vec![ vec![filter("x", "eq", nodedb_types::Value::Integer(10))], diff --git a/nodedb-query/src/msgpack_scan/group_key.rs b/nodedb-query/src/msgpack_scan/group_key.rs index e3af3408c..916d22a0d 100644 --- a/nodedb-query/src/msgpack_scan/group_key.rs +++ b/nodedb-query/src/msgpack_scan/group_key.rs @@ -5,6 +5,8 @@ //! Builds a deterministic string key from field values extracted directly //! from msgpack bytes, avoiding full document decode. +use nodedb_types::{NdbDateTime, read_instant}; + use crate::expr::{EvalError, GroupKeySpec, SqlExpr}; use crate::msgpack_scan::field::extract_field; use crate::msgpack_scan::index::FieldIndex; @@ -94,7 +96,12 @@ pub fn build_group_key_indexed( } /// Append the msgpack value at `doc[start..end]` to the key buffer as a JSON -/// literal (string quoted, numbers/null verbatim, complex values hex-encoded). +/// literal (string quoted, numbers/null verbatim, an instant as its quoted +/// ISO-8601 form, complex values hex-encoded). +/// +/// An instant keys by its epoch microseconds rendered as ISO-8601: the same +/// text the JSON transcoder gives the cell, so a group column parsed back +/// from the key reads as the cell would. fn append_value_at(buf: &mut String, doc: &[u8], start: usize, end: usize) { if read_null(doc, start) { buf.push_str("null"); @@ -102,6 +109,10 @@ fn append_value_at(buf: &mut String, doc: &[u8], start: usize, end: usize) { buf.push('"'); buf.push_str(s); buf.push('"'); + } else if let Some((_, micros)) = read_instant(doc, start) { + buf.push('"'); + buf.push_str(&NdbDateTime::from_micros(micros).to_iso8601()); + buf.push('"'); } else if let Some(n) = read_i64(doc, start) { use std::fmt::Write; let _ = write!(buf, "{n}"); @@ -219,6 +230,30 @@ mod tests { assert_eq!(key, "[36.6]"); } + /// An instant cell keys by its ISO-8601 form, a scalar JSON string, and + /// the indexed builder produces the same key. + #[test] + fn instant_field_keys_as_iso8601() { + use nodedb_types::InstantKind; + let mut doc = Vec::new(); + crate::msgpack_scan::write_map_header(&mut doc, 2); + crate::msgpack_scan::write_kv_instant( + &mut doc, + "ts", + InstantKind::Naive, + 1_583_402_400_000_000, + ); + crate::msgpack_scan::write_kv_i64(&mut doc, "n", 1); + + let key = build_group_key(&doc, &keys(&["ts", "n"])).unwrap(); + assert_eq!(key, r#"["2020-03-05T10:00:00.000000Z",1]"#); + let idx = FieldIndex::build(&doc, 0).unwrap_or_else(FieldIndex::empty); + assert_eq!( + build_group_key_indexed(&doc, &keys(&["ts", "n"]), &idx).unwrap(), + key + ); + } + /// A computed group key evaluates its expression and folds the result into /// the key slot; `alpha` and `ALPHA` collapse under `UPPER(label)`. fn upper_label_spec() -> GroupKeySpec { diff --git a/nodedb-query/src/msgpack_scan/mod.rs b/nodedb-query/src/msgpack_scan/mod.rs index e306b6b44..62c227ee0 100644 --- a/nodedb-query/src/msgpack_scan/mod.rs +++ b/nodedb-query/src/msgpack_scan/mod.rs @@ -34,6 +34,7 @@ pub use sidecar::{ }; pub use writer::{ build_str_map, inject_str_field, merge_fields, write_array_header, write_bin, write_bool, - write_f64, write_i64, write_kv_bool, write_kv_f64, write_kv_i64, write_kv_null, write_kv_raw, - write_kv_str, write_map_header, write_null, write_str, + write_f64, write_i64, write_instant, write_kv_bool, write_kv_f64, write_kv_i64, + write_kv_instant, write_kv_null, write_kv_raw, write_kv_str, write_map_header, write_null, + write_str, }; diff --git a/nodedb-query/src/msgpack_scan/reader.rs b/nodedb-query/src/msgpack_scan/reader.rs deleted file mode 100644 index afaeaca5f..000000000 --- a/nodedb-query/src/msgpack_scan/reader.rs +++ /dev/null @@ -1,1145 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Low-level MessagePack binary reader: tag parsing, value skipping, and typed reads. -//! -//! All functions operate on `&[u8]` with explicit offsets. Zero allocation, -//! zero copy. Returns `None` on truncated/invalid data — never panics. - -use std::str; - -// ── Tag constants ────────────────────────────────────────────────────── - -const NIL: u8 = 0xc0; -const FALSE: u8 = 0xc2; -const TRUE: u8 = 0xc3; -const BIN8: u8 = 0xc4; -const BIN16: u8 = 0xc5; -const BIN32: u8 = 0xc6; -const EXT8: u8 = 0xc7; -const EXT16: u8 = 0xc8; -const EXT32: u8 = 0xc9; -const FLOAT32: u8 = 0xca; -const FLOAT64: u8 = 0xcb; -const UINT8: u8 = 0xcc; -const UINT16: u8 = 0xcd; -const UINT32: u8 = 0xce; -const UINT64: u8 = 0xcf; -const INT8: u8 = 0xd0; -const INT16: u8 = 0xd1; -const INT32: u8 = 0xd2; -const INT64: u8 = 0xd3; -const FIXEXT1: u8 = 0xd4; -const FIXEXT2: u8 = 0xd5; -const FIXEXT4: u8 = 0xd6; -const FIXEXT8: u8 = 0xd7; -const FIXEXT16: u8 = 0xd8; -const STR8: u8 = 0xd9; -const STR16: u8 = 0xda; -const STR32: u8 = 0xdb; -const ARRAY16: u8 = 0xdc; -const ARRAY32: u8 = 0xdd; -const MAP16: u8 = 0xde; -const MAP32: u8 = 0xdf; - -/// Maximum nesting depth to prevent stack overflow on malicious payloads. -const MAX_DEPTH: u16 = 128; - -// ── Inline helpers ───────────────────────────────────────────────────── - -#[inline(always)] -fn get(buf: &[u8], pos: usize) -> Option { - buf.get(pos).copied() -} - -#[inline(always)] -fn read_u16_be(buf: &[u8], pos: usize) -> Option { - let bytes = buf.get(pos..pos + 2)?; - Some(u16::from_be_bytes([bytes[0], bytes[1]])) -} - -#[inline(always)] -fn read_u32_be(buf: &[u8], pos: usize) -> Option { - let bytes = buf.get(pos..pos + 4)?; - Some(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) -} - -#[inline(always)] -fn read_u64_be(buf: &[u8], pos: usize) -> Option { - let bytes = buf.get(pos..pos + 8)?; - Some(u64::from_be_bytes([ - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], - ])) -} - -/// Return `Some(offset + size)` only if the buffer has enough bytes. -#[inline(always)] -fn checked_advance(buf: &[u8], offset: usize, size: usize) -> Option { - let end = offset + size; - if end <= buf.len() { Some(end) } else { None } -} - -// ── skip_value ───────────────────────────────────────────────────────── - -/// Advance past the MessagePack value starting at `offset`, returning the -/// offset of the next value. Returns `None` if the buffer is truncated or -/// nesting exceeds `MAX_DEPTH`. -/// -/// This is the performance-critical primitive. It never allocates. -pub fn skip_value(buf: &[u8], offset: usize) -> Option { - skip_value_depth(buf, offset, 0) -} - -fn skip_value_depth(buf: &[u8], offset: usize, depth: u16) -> Option { - if depth > MAX_DEPTH { - return None; - } - let tag = get(buf, offset)?; - match tag { - // positive fixint (0x00..=0x7f) - 0x00..=0x7f => Some(offset + 1), - // negative fixint (0xe0..=0xff) - 0xe0..=0xff => Some(offset + 1), - // nil, false, true - NIL | FALSE | TRUE => Some(offset + 1), - - // fixmap (0x80..=0x8f) - 0x80..=0x8f => { - let count = (tag & 0x0f) as usize; - skip_n_pairs(buf, offset + 1, count, depth) - } - MAP16 => { - let count = read_u16_be(buf, offset + 1)? as usize; - skip_n_pairs(buf, offset + 3, count, depth) - } - MAP32 => { - let count = read_u32_be(buf, offset + 1)? as usize; - skip_n_pairs(buf, offset + 5, count, depth) - } - - // fixarray (0x90..=0x9f) - 0x90..=0x9f => { - let count = (tag & 0x0f) as usize; - skip_n_values(buf, offset + 1, count, depth) - } - ARRAY16 => { - let count = read_u16_be(buf, offset + 1)? as usize; - skip_n_values(buf, offset + 3, count, depth) - } - ARRAY32 => { - let count = read_u32_be(buf, offset + 1)? as usize; - skip_n_values(buf, offset + 5, count, depth) - } - - // fixstr (0xa0..=0xbf) - 0xa0..=0xbf => { - let len = (tag & 0x1f) as usize; - checked_advance(buf, offset, 1 + len) - } - STR8 => { - let len = get(buf, offset + 1)? as usize; - checked_advance(buf, offset, 2 + len) - } - STR16 => { - let len = read_u16_be(buf, offset + 1)? as usize; - checked_advance(buf, offset, 3 + len) - } - STR32 => { - let len = read_u32_be(buf, offset + 1)? as usize; - checked_advance(buf, offset, 5 + len) - } - - // bin - BIN8 => { - let len = get(buf, offset + 1)? as usize; - checked_advance(buf, offset, 2 + len) - } - BIN16 => { - let len = read_u16_be(buf, offset + 1)? as usize; - checked_advance(buf, offset, 3 + len) - } - BIN32 => { - let len = read_u32_be(buf, offset + 1)? as usize; - checked_advance(buf, offset, 5 + len) - } - - // fixed-width numerics (bounds-check against buffer length) - FLOAT32 => checked_advance(buf, offset, 5), - FLOAT64 => checked_advance(buf, offset, 9), - UINT8 | INT8 => checked_advance(buf, offset, 2), - UINT16 | INT16 => checked_advance(buf, offset, 3), - UINT32 | INT32 => checked_advance(buf, offset, 5), - UINT64 | INT64 => checked_advance(buf, offset, 9), - - // ext - FIXEXT1 => checked_advance(buf, offset, 3), - FIXEXT2 => checked_advance(buf, offset, 4), - FIXEXT4 => checked_advance(buf, offset, 6), - FIXEXT8 => checked_advance(buf, offset, 10), - FIXEXT16 => checked_advance(buf, offset, 18), - EXT8 => { - let len = get(buf, offset + 1)? as usize; - checked_advance(buf, offset, 3 + len) - } - EXT16 => { - let len = read_u16_be(buf, offset + 1)? as usize; - checked_advance(buf, offset, 4 + len) - } - EXT32 => { - let len = read_u32_be(buf, offset + 1)? as usize; - checked_advance(buf, offset, 6 + len) - } - - // 0xc1 is never used in the spec - _ => None, - } -} - -fn skip_n_values(buf: &[u8], mut pos: usize, count: usize, depth: u16) -> Option { - for _ in 0..count { - pos = skip_value_depth(buf, pos, depth + 1)?; - } - Some(pos) -} - -fn skip_n_pairs(buf: &[u8], mut pos: usize, count: usize, depth: u16) -> Option { - for _ in 0..count { - pos = skip_value_depth(buf, pos, depth + 1)?; // key - pos = skip_value_depth(buf, pos, depth + 1)?; // value - } - Some(pos) -} - -// ── Typed reads ──────────────────────────────────────────────────────── - -/// Read an f64 from the value at `offset`. Handles float32, float64, -/// and all integer types (coerced to f64). -pub fn read_f64(buf: &[u8], offset: usize) -> Option { - let tag = get(buf, offset)?; - match tag { - // positive fixint - 0x00..=0x7f => Some(tag as f64), - // negative fixint - 0xe0..=0xff => Some((tag as i8) as f64), - FLOAT64 => { - let bits = read_u64_be(buf, offset + 1)?; - Some(f64::from_bits(bits)) - } - FLOAT32 => { - let bits = read_u32_be(buf, offset + 1)?; - Some(f32::from_bits(bits) as f64) - } - UINT8 => Some(get(buf, offset + 1)? as f64), - UINT16 => Some(read_u16_be(buf, offset + 1)? as f64), - UINT32 => Some(read_u32_be(buf, offset + 1)? as f64), - UINT64 => Some(read_u64_be(buf, offset + 1)? as f64), - INT8 => Some(get(buf, offset + 1)? as i8 as f64), - INT16 => Some(read_u16_be(buf, offset + 1)? as i16 as f64), - INT32 => Some(read_u32_be(buf, offset + 1)? as i32 as f64), - INT64 => Some(read_u64_be(buf, offset + 1)? as i64 as f64), - _ => None, - } -} - -/// Read an i64 from the value at `offset`. Handles all integer types. -/// Floats return `None` — use `read_f64` for those. -pub fn read_i64(buf: &[u8], offset: usize) -> Option { - let tag = get(buf, offset)?; - match tag { - 0x00..=0x7f => Some(tag as i64), - 0xe0..=0xff => Some((tag as i8) as i64), - UINT8 => Some(get(buf, offset + 1)? as i64), - UINT16 => Some(read_u16_be(buf, offset + 1)? as i64), - UINT32 => Some(read_u32_be(buf, offset + 1)? as i64), - UINT64 => { - let v = read_u64_be(buf, offset + 1)?; - Some(v as i64) - } - INT8 => Some(get(buf, offset + 1)? as i8 as i64), - INT16 => Some(read_u16_be(buf, offset + 1)? as i16 as i64), - INT32 => Some(read_u32_be(buf, offset + 1)? as i32 as i64), - INT64 => { - let v = read_u64_be(buf, offset + 1)?; - Some(v as i64) - } - _ => None, - } -} - -/// Read a string slice from the value at `offset`. Zero-copy — borrows -/// directly from the input buffer. Returns `None` for non-string types -/// or invalid UTF-8. -pub fn read_str(buf: &[u8], offset: usize) -> Option<&str> { - let (start, len) = str_bounds(buf, offset)?; - let bytes = buf.get(start..start + len)?; - str::from_utf8(bytes).ok() -} - -/// Read a string slice at `*off`, advancing `*off` past it. Zero-copy. -/// Returns `None` for non-string types, invalid UTF-8, or truncated input. -pub fn read_str_advance<'a>(buf: &'a [u8], off: &mut usize) -> Option<&'a str> { - let (start, len) = str_bounds(buf, *off)?; - let bytes = buf.get(start..start + len)?; - let s = str::from_utf8(bytes).ok()?; - *off = start + len; - Some(s) -} - -/// Read a `bin` value at `*off`, advancing `*off` past it. Zero-copy — -/// the returned slice borrows from `buf`. Returns `None` for non-bin tags -/// or truncated input. -pub fn read_bin_advance<'a>(buf: &'a [u8], off: &mut usize) -> Option<&'a [u8]> { - let tag = get(buf, *off)?; - let (len, header) = match tag { - BIN8 => (get(buf, *off + 1)? as usize, 2), - BIN16 => (read_u16_be(buf, *off + 1)? as usize, 3), - BIN32 => (read_u32_be(buf, *off + 1)? as usize, 5), - _ => return None, - }; - let start = *off + header; - let end = start + len; - let data = buf.get(start..end)?; - *off = end; - Some(data) -} - -/// Read an unsigned integer that fits in a `u32` at `*off`, advancing `*off` -/// past it. Accepts positive fixint, uint8, uint16, uint32. Returns `None` -/// for negative, signed-typed, oversized (uint64), or non-integer values. -pub fn read_u32_advance(buf: &[u8], off: &mut usize) -> Option { - let tag = get(buf, *off)?; - match tag { - 0x00..=0x7f => { - *off += 1; - Some(tag as u32) - } - UINT8 => { - let v = get(buf, *off + 1)? as u32; - *off += 2; - Some(v) - } - UINT16 => { - let v = read_u16_be(buf, *off + 1)? as u32; - *off += 3; - Some(v) - } - UINT32 => { - let v = read_u32_be(buf, *off + 1)?; - *off += 5; - Some(v) - } - _ => None, - } -} - -/// Return `(data_start, byte_len)` for the string at `offset` without -/// validating UTF-8. Used internally for key comparison. -pub(crate) fn str_bounds(buf: &[u8], offset: usize) -> Option<(usize, usize)> { - let tag = get(buf, offset)?; - match tag { - 0xa0..=0xbf => { - let len = (tag & 0x1f) as usize; - Some((offset + 1, len)) - } - STR8 => { - let len = get(buf, offset + 1)? as usize; - Some((offset + 2, len)) - } - STR16 => { - let len = read_u16_be(buf, offset + 1)? as usize; - Some((offset + 3, len)) - } - STR32 => { - let len = read_u32_be(buf, offset + 1)? as usize; - Some((offset + 5, len)) - } - _ => None, - } -} - -/// Read a boolean from the value at `offset`. -pub fn read_bool(buf: &[u8], offset: usize) -> Option { - match get(buf, offset)? { - TRUE => Some(true), - FALSE => Some(false), - _ => None, - } -} - -/// Check if the value at `offset` is nil. -pub fn read_null(buf: &[u8], offset: usize) -> bool { - get(buf, offset) == Some(NIL) -} - -/// Read a scalar msgpack value at `offset` into `nodedb_types::Value`. -/// -/// Handles null, bool, integers, floats, and strings. For complex types -/// (array, map, bin, ext), returns `None` — caller should use -/// `json_from_msgpack` for those. -pub fn read_value(buf: &[u8], offset: usize) -> Option { - let tag = get(buf, offset)?; - match tag { - NIL => Some(nodedb_types::Value::Null), - TRUE => Some(nodedb_types::Value::Bool(true)), - FALSE => Some(nodedb_types::Value::Bool(false)), - // Integers - 0x00..=0x7f => Some(nodedb_types::Value::Integer(tag as i64)), - 0xe0..=0xff => Some(nodedb_types::Value::Integer((tag as i8) as i64)), - UINT8 => Some(nodedb_types::Value::Integer(get(buf, offset + 1)? as i64)), - UINT16 => Some(nodedb_types::Value::Integer( - read_u16_be(buf, offset + 1)? as i64 - )), - UINT32 => Some(nodedb_types::Value::Integer( - read_u32_be(buf, offset + 1)? as i64 - )), - UINT64 => Some(nodedb_types::Value::Integer( - read_u64_be(buf, offset + 1)? as i64 - )), - INT8 => Some(nodedb_types::Value::Integer( - get(buf, offset + 1)? as i8 as i64 - )), - INT16 => Some(nodedb_types::Value::Integer( - read_u16_be(buf, offset + 1)? as i16 as i64, - )), - INT32 => Some(nodedb_types::Value::Integer( - read_u32_be(buf, offset + 1)? as i32 as i64, - )), - INT64 => Some(nodedb_types::Value::Integer( - read_u64_be(buf, offset + 1)? as i64 - )), - // Floats - FLOAT32 => { - let bits = read_u32_be(buf, offset + 1)?; - Some(nodedb_types::Value::Float(f32::from_bits(bits) as f64)) - } - FLOAT64 => { - let bits = read_u64_be(buf, offset + 1)?; - Some(nodedb_types::Value::Float(f64::from_bits(bits))) - } - // Strings - 0xa0..=0xbf | STR8 | STR16 | STR32 => { - read_str(buf, offset).map(|s| nodedb_types::Value::String(s.to_string())) - } - _ => None, - } -} - -/// Return the number of key-value pairs and the offset of the first pair, -/// for the map starting at `offset`. Returns `None` if not a map. -pub fn map_header(buf: &[u8], offset: usize) -> Option<(usize, usize)> { - let tag = get(buf, offset)?; - match tag { - 0x80..=0x8f => Some(((tag & 0x0f) as usize, offset + 1)), - MAP16 => Some((read_u16_be(buf, offset + 1)? as usize, offset + 3)), - MAP32 => Some((read_u32_be(buf, offset + 1)? as usize, offset + 5)), - _ => None, - } -} - -/// Return the number of elements and the offset of the first element, -/// for the array starting at `offset`. Returns `None` if not an array. -pub fn array_header(buf: &[u8], offset: usize) -> Option<(usize, usize)> { - let tag = get(buf, offset)?; - match tag { - 0x90..=0x9f => Some(((tag & 0x0f) as usize, offset + 1)), - ARRAY16 => Some((read_u16_be(buf, offset + 1)? as usize, offset + 3)), - ARRAY32 => Some((read_u32_be(buf, offset + 1)? as usize, offset + 5)), - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - use serde_json::json; - - /// Helper: encode a serde_json::Value to MessagePack bytes. - fn encode(v: &serde_json::Value) -> Vec { - nodedb_types::json_msgpack::json_to_msgpack(v).expect("encode") - } - - #[test] - fn skip_positive_fixint() { - let buf = [0x05, 0xff]; - assert_eq!(skip_value(&buf, 0), Some(1)); - } - - #[test] - fn skip_negative_fixint() { - let buf = [0xe0, 0x00]; - assert_eq!(skip_value(&buf, 0), Some(1)); - } - - #[test] - fn skip_nil_bool() { - assert_eq!(skip_value(&[NIL], 0), Some(1)); - assert_eq!(skip_value(&[TRUE], 0), Some(1)); - assert_eq!(skip_value(&[FALSE], 0), Some(1)); - } - - #[test] - fn skip_float64() { - let buf = encode(&json!(9.81)); - assert_eq!(skip_value(&buf, 0), Some(buf.len())); - } - - #[test] - fn skip_string() { - let buf = encode(&json!("hello")); - assert_eq!(skip_value(&buf, 0), Some(buf.len())); - } - - #[test] - fn skip_map() { - let buf = encode(&json!({"a": 1, "b": 2})); - assert_eq!(skip_value(&buf, 0), Some(buf.len())); - } - - #[test] - fn skip_nested_array() { - let buf = encode(&json!([[1, 2], [3, 4, 5]])); - assert_eq!(skip_value(&buf, 0), Some(buf.len())); - } - - #[test] - fn skip_truncated_returns_none() { - let buf = [FLOAT64, 0x40]; // truncated float64 - assert_eq!(skip_value(&buf, 0), None); - } - - #[test] - fn read_f64_fixint() { - assert_eq!(read_f64(&[42u8], 0), Some(42.0)); - } - - #[test] - fn read_f64_negative_fixint() { - assert_eq!(read_f64(&[0xffu8], 0), Some(-1.0)); - } - - #[test] - fn read_f64_float64() { - let buf = encode(&json!(std::f64::consts::PI)); - assert_eq!(read_f64(&buf, 0), Some(std::f64::consts::PI)); - } - - #[test] - fn read_f64_uint16() { - let buf = encode(&json!(1000)); - assert_eq!(read_f64(&buf, 0), Some(1000.0)); - } - - #[test] - fn read_i64_values() { - assert_eq!(read_i64(&[42u8], 0), Some(42)); - assert_eq!(read_i64(&[0xffu8], 0), Some(-1)); - - let buf = encode(&json!(300)); - assert_eq!(read_i64(&buf, 0), Some(300)); - - let buf = encode(&json!(-500)); - assert_eq!(read_i64(&buf, 0), Some(-500)); - } - - #[test] - fn read_str_fixstr() { - let buf = encode(&json!("hi")); - assert_eq!(read_str(&buf, 0), Some("hi")); - } - - #[test] - fn read_str_str8() { - let long = "a".repeat(40); - let buf = encode(&json!(long)); - assert_eq!(read_str(&buf, 0), Some(long.as_str())); - } - - #[test] - fn read_bool_values() { - assert_eq!(read_bool(&[TRUE], 0), Some(true)); - assert_eq!(read_bool(&[FALSE], 0), Some(false)); - assert_eq!(read_bool(&[NIL], 0), None); - } - - #[test] - fn read_null_check() { - assert!(read_null(&[NIL], 0)); - assert!(!read_null(&[TRUE], 0)); - } - - #[test] - fn map_header_fixmap() { - let buf = encode(&json!({"x": 1})); - let (count, _data_offset) = map_header(&buf, 0).unwrap(); - assert_eq!(count, 1); - } - - #[test] - fn skip_bin() { - // bin8: 0xc4, len=3, 3 bytes of data - let buf = [BIN8, 3, 0xde, 0xad, 0xbe, 0xff]; - assert_eq!(skip_value(&buf, 0), Some(5)); - } - - #[test] - fn skip_ext() { - // fixext1: 0xd4, type byte, 1 data byte - let buf = [FIXEXT1, 0x01, 0xab, 0xff]; - assert_eq!(skip_value(&buf, 0), Some(3)); - } - - #[test] - fn read_f64_float32() { - // json! always produces f64, so test float32 with raw bytes - // float32 tag (0xca) + 1.5 in IEEE 754 big-endian - let buf = [0xca, 0x3f, 0xc0, 0x00, 0x00]; - let val = read_f64(&buf, 0).unwrap(); - assert!((val - 1.5).abs() < 1e-6); - } - - #[test] - fn skip_empty_containers() { - // empty fixmap - assert_eq!(skip_value(&[0x80], 0), Some(1)); - // empty fixarray - assert_eq!(skip_value(&[0x90], 0), Some(1)); - } - - #[test] - fn array_header_fixarray() { - let buf = encode(&json!([10, 20, 30])); - let (count, data_offset) = array_header(&buf, 0).unwrap(); - assert_eq!(count, 3); - assert_eq!(read_i64(&buf, data_offset), Some(10)); - } - - // ── Canonical encoding guarantee tests ───────────────────────────── - - #[test] - fn canonical_integer_smallest_representation() { - // fixint (0-127): single byte - let buf = encode(&json!(42)); - assert_eq!(buf.len(), 1); - assert_eq!(buf[0], 42); - - // 0 as fixint - let buf = encode(&json!(0)); - assert_eq!(buf.len(), 1); - assert_eq!(buf[0], 0); - - // 127 as fixint - let buf = encode(&json!(127)); - assert_eq!(buf.len(), 1); - assert_eq!(buf[0], 127); - - // 128 should NOT be fixint. JSON parses as i64, so zerompk uses - // int16 (0xd1) since 128 > i8::MAX. This is canonical for signed path. - let buf = encode(&json!(128)); - assert_eq!(buf[0], 0xd1); // int16 tag - assert_eq!(buf.len(), 3); // tag + 2 bytes - - // negative fixint (-32 to -1) - let buf = encode(&json!(-1)); - assert_eq!(buf.len(), 1); - assert_eq!(buf[0], 0xff); // -1 as negative fixint - - let buf = encode(&json!(-32)); - assert_eq!(buf.len(), 1); - assert_eq!(buf[0], 0xe0); // -32 as negative fixint - } - - #[test] - fn canonical_map_keys_sorted() { - // Keys should be lexicographically sorted in msgpack output. - // Encode with keys in non-sorted order in JSON source. - let buf = encode(&json!({"z": 1, "a": 2, "m": 3})); - - // Parse map and verify keys come out sorted - let (count, mut pos) = map_header(&buf, 0).unwrap(); - assert_eq!(count, 3); - - let mut keys = Vec::new(); - for _ in 0..count { - let key = read_str(&buf, pos).unwrap(); - keys.push(key.to_string()); - pos = skip_value(&buf, pos).unwrap(); // skip key - pos = skip_value(&buf, pos).unwrap(); // skip value - } - assert_eq!(keys, vec!["a", "m", "z"]); - } - - #[test] - fn canonical_deterministic_bytes() { - // Same logical document encoded twice must produce identical bytes. - let doc1 = encode(&json!({"name": "alice", "age": 30, "active": true})); - let doc2 = encode(&json!({"age": 30, "active": true, "name": "alice"})); - assert_eq!( - doc1, doc2, - "same logical doc must produce identical msgpack bytes" - ); - } - - #[test] - fn canonical_nested_map_keys_sorted() { - let buf = encode(&json!({"outer": {"z": 1, "a": 2}})); - // Extract the inner map - let (start, _end) = crate::msgpack_scan::field::extract_field(&buf, 0, "outer").unwrap(); - - let (count, mut pos) = map_header(&buf, start).unwrap(); - assert_eq!(count, 2); - - let key1 = read_str(&buf, pos).unwrap(); - pos = skip_value(&buf, pos).unwrap(); - pos = skip_value(&buf, pos).unwrap(); - let key2 = read_str(&buf, pos).unwrap(); - - assert_eq!(key1, "a"); - assert_eq!(key2, "z"); - } - - // ── Fuzz-style tests ─────────────────────────────────────────────────── - - /// Feed every single-byte sequence through all reader functions. None may - /// panic — they must return `None` or a valid result. - #[test] - fn fuzz_all_single_byte_sequences() { - for byte in 0u8..=255 { - let buf = [byte]; - // None of these must panic - let _ = skip_value(&buf, 0); - let _ = read_f64(&buf, 0); - let _ = read_i64(&buf, 0); - let _ = read_str(&buf, 0); - let _ = read_bool(&buf, 0); - let _ = read_null(&buf, 0); - let _ = map_header(&buf, 0); - let _ = array_header(&buf, 0); - let _ = read_value(&buf, 0); - } - } - - /// Feed two-byte patterns to cover tag + partial payload (truncated). - #[test] - fn fuzz_two_byte_patterns() { - // Tags that expect more bytes than we provide - let tags_need_extra: &[u8] = &[ - 0xca, // FLOAT32 needs 4 more - 0xcb, // FLOAT64 needs 8 more - 0xcc, // UINT8 needs 1 more - 0xcd, // UINT16 needs 2 more - 0xce, // UINT32 needs 4 more - 0xcf, // UINT64 needs 8 more - 0xd0, // INT8 needs 1 more - 0xd1, // INT16 needs 2 more - 0xd2, // INT32 needs 4 more - 0xd3, // INT64 needs 8 more - 0xd9, // STR8 length byte then data - 0xda, // STR16 2-byte length then data - 0xdb, // STR32 4-byte length then data - 0xdc, // ARRAY16 2-byte count then elements - 0xdd, // ARRAY32 4-byte count then elements - 0xde, // MAP16 2-byte count then pairs - 0xdf, // MAP32 4-byte count then pairs - 0xc4, // BIN8 - 0xc5, // BIN16 - 0xc6, // BIN32 - 0xd4, // FIXEXT1 - 0xd5, // FIXEXT2 - 0xd6, // FIXEXT4 - 0xd7, // FIXEXT8 - 0xd8, // FIXEXT16 - ]; - for &tag in tags_need_extra { - // Single byte (completely truncated payload) - let buf = [tag]; - let _ = skip_value(&buf, 0); - let _ = read_f64(&buf, 0); - let _ = read_i64(&buf, 0); - let _ = read_value(&buf, 0); - - // Tag + one garbage byte - for second in [0x00u8, 0x01, 0x7f, 0x80, 0xff] { - let buf = [tag, second]; - let _ = skip_value(&buf, 0); - let _ = read_f64(&buf, 0); - let _ = read_i64(&buf, 0); - let _ = read_value(&buf, 0); - } - } - } - - /// Deterministic pseudo-random byte sequences must not cause panics. - #[test] - fn fuzz_deterministic_random_payloads() { - // Generate deterministic sequences without external crates using a - // simple LCG (Knuth multiplicative hash). - let mut state: u64 = 0xdeadbeef_cafebabe; - let next = |s: &mut u64| -> u8 { - *s = s - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); - (*s >> 33) as u8 - }; - - let mut buf = vec![0u8; 256]; - for _ in 0..2000 { - // Randomize buffer length (1..=256) and contents - let len = (next(&mut state) as usize % 256) + 1; - for b in buf[..len].iter_mut() { - *b = next(&mut state); - } - let slice = &buf[..len]; - - // Try reading from multiple offsets - for offset in [0, 1, len / 2, len.saturating_sub(1)] { - let _ = skip_value(slice, offset); - let _ = read_f64(slice, offset); - let _ = read_i64(slice, offset); - let _ = read_str(slice, offset); - let _ = read_bool(slice, offset); - let _ = read_null(slice, offset); - let _ = map_header(slice, offset); - let _ = array_header(slice, offset); - let _ = read_value(slice, offset); - } - } - } - - /// Truncate a valid msgpack buffer at every byte position. - /// All reader functions must return `None` — never panic. - #[test] - fn fuzz_truncated_valid_payloads() { - let docs = [ - json!({"key": "value", "num": 42, "flag": true}), - json!({"nested": {"a": 1, "b": [1, 2, 3]}}), - json!([1, "two", 3.0, null, false]), - json!({"large": 9999999999_i64}), - json!({"float": 1.23456789}), - ]; - - for doc in &docs { - let full = encode(doc); - // Truncate at every position from 0 to full.len()-1 - for truncate_at in 0..full.len() { - let slice = &full[..truncate_at]; - // None of these may panic; result doesn't matter - let _ = skip_value(slice, 0); - let _ = read_f64(slice, 0); - let _ = read_i64(slice, 0); - let _ = read_str(slice, 0); - let _ = read_bool(slice, 0); - let _ = map_header(slice, 0); - let _ = array_header(slice, 0); - let _ = read_value(slice, 0); - } - } - } - - /// The never-used 0xc1 tag must return `None` for all functions. - #[test] - fn fuzz_never_used_tag_c1() { - // 0xc1 is explicitly "never used" in the msgpack spec - let buf = [0xc1u8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; - assert_eq!( - skip_value(&buf, 0), - None, - "0xc1 must return None from skip_value" - ); - assert_eq!(read_f64(&buf, 0), None); - assert_eq!(read_i64(&buf, 0), None); - assert_eq!(read_str(&buf, 0), None); - assert_eq!(read_bool(&buf, 0), None); - assert_eq!(map_header(&buf, 0), None); - assert_eq!(array_header(&buf, 0), None); - assert_eq!(read_value(&buf, 0), None); - } - - /// All tag boundary bytes — test transitions at fixint/fixmap/fixarray/fixstr edges. - #[test] - fn fuzz_tag_boundaries() { - // Each entry: (tag, expected_skip_result) - // For tags that are self-contained single bytes, skip returns Some(1). - // For tags requiring more data we just verify no panic with empty tail. - let boundary_tags: &[(u8, bool)] = &[ - (0x00, true), // positive fixint 0 - (0x7f, true), // positive fixint 127 - (0x80, true), // fixmap length 0 (empty map) - (0x8f, false), // fixmap length 15 — needs 15 pairs - (0x90, true), // fixarray length 0 (empty array) - (0x9f, false), // fixarray length 15 — needs 15 elements - (0xa0, true), // fixstr length 0 (empty string) - (0xbf, false), // fixstr length 31 — needs 31 bytes after - (0xc0, true), // nil - (0xc1, false), // never used — must return None - (0xc2, true), // false - (0xc3, true), // true - (0xe0, true), // negative fixint -32 - (0xff, true), // negative fixint -1 - ]; - for &(tag, self_contained) in boundary_tags { - let buf = [tag; 64]; // fill with the same tag as padding - let result = skip_value(&buf, 0); - if self_contained { - assert!(result.is_some(), "tag 0x{tag:02x} should skip OK"); - } else if tag == 0xc1 { - assert_eq!(result, None, "0xc1 must always return None"); - } - // For non-self-contained tags with valid padding we just verify no panic. - } - } - - /// Buffers where length fields claim enormous sizes but the buffer is tiny. - #[test] - fn fuzz_adversarial_length_fields() { - // STR32: tag 0xdb + 4-byte big-endian length claiming 0xffffffff bytes - let buf = [0xdbu8, 0xff, 0xff, 0xff, 0xff, b'x', b'y']; - assert_eq!(skip_value(&buf, 0), None); - assert_eq!(read_str(&buf, 0), None); - - // STR16: tag 0xda + 2-byte length claiming 0xffff bytes - let buf = [0xdau8, 0xff, 0xff, b'x']; - assert_eq!(skip_value(&buf, 0), None); - - // ARRAY32: claims 0xffffffff elements but buffer is empty after header - let buf = [0xddu8, 0xff, 0xff, 0xff, 0xff]; - assert_eq!(skip_value(&buf, 0), None); - - // MAP32: claims 0xffffffff pairs but buffer is empty after header - let buf = [0xdfu8, 0xff, 0xff, 0xff, 0xff]; - assert_eq!(skip_value(&buf, 0), None); - - // ARRAY16: claims 0xffff elements - let buf = [0xdcu8, 0xff, 0xff]; - assert_eq!(skip_value(&buf, 0), None); - - // MAP16: claims 0xffff pairs - let buf = [0xdeu8, 0xff, 0xff]; - assert_eq!(skip_value(&buf, 0), None); - - // BIN32: claims max length - let buf = [0xc6u8, 0xff, 0xff, 0xff, 0xff, 0x00]; - assert_eq!(skip_value(&buf, 0), None); - - // EXT32: claims max length - let buf = [0xc9u8, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00]; - assert_eq!(skip_value(&buf, 0), None); - } - - /// Deeply nested maps/arrays must cause `skip_value` to return `None` - /// once nesting exceeds MAX_DEPTH (128). - #[test] - fn fuzz_malicious_nesting_depth() { - // Build a buffer with 200 levels of fixarray (each containing 1 element) - // fixarray tag for 1 element = 0x91 - let depth = 200usize; - let mut buf = vec![0x91u8; depth]; // fixarray(1) — opens 1-element array - buf.push(0xc0u8); // nil at the innermost leaf - - // skip_value must return None because nesting > MAX_DEPTH - assert_eq!( - skip_value(&buf, 0), - None, - "deeply nested arrays must return None to guard against stack overflow" - ); - - // Same with maps: fixmap(1) = 0x81, then a fixstr(1) key + value - // Build 200 levels of fixmap(1) — each pair is (fixstr key, next map) - let mut map_buf: Vec = Vec::new(); - for i in 0..(depth as u8) { - map_buf.push(0x81); // fixmap(1) - map_buf.push(0xa1); // fixstr(1) key - map_buf.push(b'a'.wrapping_add(i % 26)); - // value = next map (already pushed in next iteration), or nil at end - } - map_buf.push(0xc0); // nil leaf - - assert_eq!( - skip_value(&map_buf, 0), - None, - "deeply nested maps must return None" - ); - } - - /// Verify skip_value correctly consumes exactly the right number of bytes - /// for all fixed-width numeric types and returns the correct next offset. - #[test] - fn fuzz_fixed_width_numeric_skip_offsets() { - // (tag, expected_total_bytes_consumed) - let cases: &[(u8, usize)] = &[ - (0xca, 5), // FLOAT32: 1 tag + 4 data - (0xcb, 9), // FLOAT64: 1 tag + 8 data - (0xcc, 2), // UINT8 - (0xcd, 3), // UINT16 - (0xce, 5), // UINT32 - (0xcf, 9), // UINT64 - (0xd0, 2), // INT8 - (0xd1, 3), // INT16 - (0xd2, 5), // INT32 - (0xd3, 9), // INT64 - ]; - for &(tag, size) in cases { - let mut buf = vec![0u8; size + 4]; // extra padding - buf[0] = tag; - let result = skip_value(&buf, 0); - assert_eq!( - result, - Some(size), - "tag 0x{tag:02x} should advance by {size} bytes" - ); - } - } - - /// Verify all fixext types consume the correct byte count. - #[test] - fn fuzz_fixext_skip_offsets() { - // (tag, expected_bytes_consumed) - let cases: &[(u8, usize)] = &[ - (0xd4, 3), // FIXEXT1: 1+1+1 - (0xd5, 4), // FIXEXT2: 1+1+2 - (0xd6, 6), // FIXEXT4: 1+1+4 - (0xd7, 10), // FIXEXT8: 1+1+8 - (0xd8, 18), // FIXEXT16: 1+1+16 - ]; - for &(tag, size) in cases { - let mut buf = vec![0u8; size + 4]; - buf[0] = tag; - let result = skip_value(&buf, 0); - assert_eq!( - result, - Some(size), - "fixext tag 0x{tag:02x} should advance by {size} bytes" - ); - } - } - - /// Out-of-bounds offset must return `None` — not panic. - #[test] - fn fuzz_out_of_bounds_offset() { - let buf = encode(&json!({"x": 1})); - let way_out = buf.len() + 1000; - assert_eq!(skip_value(&buf, way_out), None); - assert_eq!(read_f64(&buf, way_out), None); - assert_eq!(read_i64(&buf, way_out), None); - assert_eq!(read_str(&buf, way_out), None); - assert_eq!(read_bool(&buf, way_out), None); - assert_eq!(map_header(&buf, way_out), None); - assert_eq!(array_header(&buf, way_out), None); - assert_eq!(read_value(&buf, way_out), None); - } - - #[test] - fn read_bin_advance_all_widths() { - // bin8: 0xc4, len=3 - let mut off = 0; - let buf = [BIN8, 3, 0xde, 0xad, 0xbe, 0xff]; - assert_eq!( - read_bin_advance(&buf, &mut off), - Some(&[0xde, 0xad, 0xbe][..]) - ); - assert_eq!(off, 5); - - // bin16: 0xc5, big-endian len=4 - let mut off = 0; - let buf = [BIN16, 0x00, 0x04, 0x01, 0x02, 0x03, 0x04]; - assert_eq!( - read_bin_advance(&buf, &mut off), - Some(&[0x01, 0x02, 0x03, 0x04][..]) - ); - assert_eq!(off, 7); - - // bin32: 0xc6, big-endian len=2 - let mut off = 0; - let buf = [BIN32, 0x00, 0x00, 0x00, 0x02, 0xaa, 0xbb]; - assert_eq!(read_bin_advance(&buf, &mut off), Some(&[0xaa, 0xbb][..])); - assert_eq!(off, 7); - - // Non-bin tag returns None and does not advance. - let mut off = 0; - let buf = [0xc0u8]; // nil - assert_eq!(read_bin_advance(&buf, &mut off), None); - assert_eq!(off, 0); - - // Truncated returns None. - let mut off = 0; - let buf = [BIN8, 5, 0x01]; // claims 5 bytes, only 1 present - assert_eq!(read_bin_advance(&buf, &mut off), None); - } - - #[test] - fn read_u32_advance_all_widths() { - // positive fixint - let mut off = 0; - assert_eq!(read_u32_advance(&[42u8], &mut off), Some(42)); - assert_eq!(off, 1); - - // uint8 - let mut off = 0; - assert_eq!(read_u32_advance(&[UINT8, 200], &mut off), Some(200)); - assert_eq!(off, 2); - - // uint16 - let mut off = 0; - let buf = [UINT16, 0x12, 0x34]; - assert_eq!(read_u32_advance(&buf, &mut off), Some(0x1234)); - assert_eq!(off, 3); - - // uint32 - let mut off = 0; - let buf = [UINT32, 0xde, 0xad, 0xbe, 0xef]; - assert_eq!(read_u32_advance(&buf, &mut off), Some(0xdeadbeef)); - assert_eq!(off, 5); - - // negative fixint, int*, uint64, float, etc. all rejected - let mut off = 0; - assert_eq!(read_u32_advance(&[0xffu8], &mut off), None); // negative fixint - assert_eq!(off, 0); - let mut off = 0; - assert_eq!(read_u32_advance(&[INT8, 5], &mut off), None); - let mut off = 0; - assert_eq!( - read_u32_advance(&[UINT64, 0, 0, 0, 0, 0, 0, 0, 1], &mut off), - None - ); - - // Truncated returns None. - let mut off = 0; - assert_eq!(read_u32_advance(&[UINT16, 0x12], &mut off), None); - } - - #[test] - fn read_str_advance_basic() { - // fixstr "hi" - let mut off = 0; - let buf = encode(&json!("hi")); - assert_eq!(read_str_advance(&buf, &mut off), Some("hi")); - assert_eq!(off, buf.len()); - - // Sequential reads - let buf = encode(&json!(["one", "two"])); - let (count, mut off) = array_header(&buf, 0).unwrap(); - assert_eq!(count, 2); - assert_eq!(read_str_advance(&buf, &mut off), Some("one")); - assert_eq!(read_str_advance(&buf, &mut off), Some("two")); - assert_eq!(off, buf.len()); - - // Non-string returns None. - let mut off = 0; - assert_eq!(read_str_advance(&[NIL], &mut off), None); - assert_eq!(off, 0); - } - - /// Empty buffer must return `None` for all functions that can. - #[test] - fn fuzz_empty_buffer() { - let buf: &[u8] = &[]; - assert_eq!(skip_value(buf, 0), None); - assert_eq!(read_f64(buf, 0), None); - assert_eq!(read_i64(buf, 0), None); - assert_eq!(read_str(buf, 0), None); - assert_eq!(read_bool(buf, 0), None); - assert!(!read_null(buf, 0)); // returns bool, not Option - assert_eq!(map_header(buf, 0), None); - assert_eq!(array_header(buf, 0), None); - assert_eq!(read_value(buf, 0), None); - } -} diff --git a/nodedb-query/src/msgpack_scan/reader/mod.rs b/nodedb-query/src/msgpack_scan/reader/mod.rs new file mode 100644 index 000000000..1a19f8c94 --- /dev/null +++ b/nodedb-query/src/msgpack_scan/reader/mod.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Low-level MessagePack binary reader: tag parsing, value skipping, and typed reads. +//! +//! All functions operate on `&[u8]` with explicit offsets. Zero allocation, +//! zero copy. Returns `None` on truncated/invalid data — never panics. + +pub mod scalar; +pub mod skip; +pub mod tags; +pub mod value; + +pub(crate) use scalar::str_bounds; +pub use scalar::{ + array_header, map_header, read_bin_advance, read_bool, read_f64, read_i64, read_null, read_str, + read_str_advance, read_u32_advance, +}; +pub use skip::skip_value; +pub use value::read_value; diff --git a/nodedb-query/src/msgpack_scan/reader/scalar.rs b/nodedb-query/src/msgpack_scan/reader/scalar.rs new file mode 100644 index 000000000..1067b7e04 --- /dev/null +++ b/nodedb-query/src/msgpack_scan/reader/scalar.rs @@ -0,0 +1,470 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Typed scalar reads and container headers. + +use std::str; + +use super::tags::*; + +/// Read an f64 from the value at `offset`. Handles float32, float64, +/// and all integer types (coerced to f64). +pub fn read_f64(buf: &[u8], offset: usize) -> Option { + let tag = get(buf, offset)?; + match tag { + // positive fixint + 0x00..=0x7f => Some(tag as f64), + // negative fixint + 0xe0..=0xff => Some((tag as i8) as f64), + FLOAT64 => { + let bits = read_u64_be(buf, offset + 1)?; + Some(f64::from_bits(bits)) + } + FLOAT32 => { + let bits = read_u32_be(buf, offset + 1)?; + Some(f32::from_bits(bits) as f64) + } + UINT8 => Some(get(buf, offset + 1)? as f64), + UINT16 => Some(read_u16_be(buf, offset + 1)? as f64), + UINT32 => Some(read_u32_be(buf, offset + 1)? as f64), + UINT64 => Some(read_u64_be(buf, offset + 1)? as f64), + INT8 => Some(get(buf, offset + 1)? as i8 as f64), + INT16 => Some(read_u16_be(buf, offset + 1)? as i16 as f64), + INT32 => Some(read_u32_be(buf, offset + 1)? as i32 as f64), + INT64 => Some(read_u64_be(buf, offset + 1)? as i64 as f64), + _ => None, + } +} + +/// Read an i64 from the value at `offset`. Handles all integer types. +/// Floats return `None` — use `read_f64` for those. +pub fn read_i64(buf: &[u8], offset: usize) -> Option { + let tag = get(buf, offset)?; + match tag { + 0x00..=0x7f => Some(tag as i64), + 0xe0..=0xff => Some((tag as i8) as i64), + UINT8 => Some(get(buf, offset + 1)? as i64), + UINT16 => Some(read_u16_be(buf, offset + 1)? as i64), + UINT32 => Some(read_u32_be(buf, offset + 1)? as i64), + UINT64 => { + let v = read_u64_be(buf, offset + 1)?; + Some(v as i64) + } + INT8 => Some(get(buf, offset + 1)? as i8 as i64), + INT16 => Some(read_u16_be(buf, offset + 1)? as i16 as i64), + INT32 => Some(read_u32_be(buf, offset + 1)? as i32 as i64), + INT64 => { + let v = read_u64_be(buf, offset + 1)?; + Some(v as i64) + } + _ => None, + } +} + +/// Read a string slice from the value at `offset`. Zero-copy — borrows +/// directly from the input buffer. Returns `None` for non-string types +/// or invalid UTF-8. +pub fn read_str(buf: &[u8], offset: usize) -> Option<&str> { + let (start, len) = str_bounds(buf, offset)?; + let bytes = buf.get(start..start + len)?; + str::from_utf8(bytes).ok() +} + +/// Read a string slice at `*off`, advancing `*off` past it. Zero-copy. +/// Returns `None` for non-string types, invalid UTF-8, or truncated input. +pub fn read_str_advance<'a>(buf: &'a [u8], off: &mut usize) -> Option<&'a str> { + let (start, len) = str_bounds(buf, *off)?; + let bytes = buf.get(start..start + len)?; + let s = str::from_utf8(bytes).ok()?; + *off = start + len; + Some(s) +} + +/// Read a `bin` value at `*off`, advancing `*off` past it. Zero-copy — +/// the returned slice borrows from `buf`. Returns `None` for non-bin tags +/// or truncated input. +pub fn read_bin_advance<'a>(buf: &'a [u8], off: &mut usize) -> Option<&'a [u8]> { + let tag = get(buf, *off)?; + let (len, header) = match tag { + BIN8 => (get(buf, *off + 1)? as usize, 2), + BIN16 => (read_u16_be(buf, *off + 1)? as usize, 3), + BIN32 => (read_u32_be(buf, *off + 1)? as usize, 5), + _ => return None, + }; + let start = *off + header; + let end = start + len; + let data = buf.get(start..end)?; + *off = end; + Some(data) +} + +/// Read an unsigned integer that fits in a `u32` at `*off`, advancing `*off` +/// past it. Accepts positive fixint, uint8, uint16, uint32. Returns `None` +/// for negative, signed-typed, oversized (uint64), or non-integer values. +pub fn read_u32_advance(buf: &[u8], off: &mut usize) -> Option { + let tag = get(buf, *off)?; + match tag { + 0x00..=0x7f => { + *off += 1; + Some(tag as u32) + } + UINT8 => { + let v = get(buf, *off + 1)? as u32; + *off += 2; + Some(v) + } + UINT16 => { + let v = read_u16_be(buf, *off + 1)? as u32; + *off += 3; + Some(v) + } + UINT32 => { + let v = read_u32_be(buf, *off + 1)?; + *off += 5; + Some(v) + } + _ => None, + } +} + +/// Return `(data_start, byte_len)` for the string at `offset` without +/// validating UTF-8. Used internally for key comparison. +pub(crate) fn str_bounds(buf: &[u8], offset: usize) -> Option<(usize, usize)> { + let tag = get(buf, offset)?; + match tag { + 0xa0..=0xbf => { + let len = (tag & 0x1f) as usize; + Some((offset + 1, len)) + } + STR8 => { + let len = get(buf, offset + 1)? as usize; + Some((offset + 2, len)) + } + STR16 => { + let len = read_u16_be(buf, offset + 1)? as usize; + Some((offset + 3, len)) + } + STR32 => { + let len = read_u32_be(buf, offset + 1)? as usize; + Some((offset + 5, len)) + } + _ => None, + } +} + +/// Read a boolean from the value at `offset`. +pub fn read_bool(buf: &[u8], offset: usize) -> Option { + match get(buf, offset)? { + TRUE => Some(true), + FALSE => Some(false), + _ => None, + } +} + +/// Check if the value at `offset` is nil. +pub fn read_null(buf: &[u8], offset: usize) -> bool { + get(buf, offset) == Some(NIL) +} + +/// Return the number of key-value pairs and the offset of the first pair, +/// for the map starting at `offset`. Returns `None` if not a map. +pub fn map_header(buf: &[u8], offset: usize) -> Option<(usize, usize)> { + let tag = get(buf, offset)?; + match tag { + 0x80..=0x8f => Some(((tag & 0x0f) as usize, offset + 1)), + MAP16 => Some((read_u16_be(buf, offset + 1)? as usize, offset + 3)), + MAP32 => Some((read_u32_be(buf, offset + 1)? as usize, offset + 5)), + _ => None, + } +} + +/// Return the number of elements and the offset of the first element, +/// for the array starting at `offset`. Returns `None` if not an array. +pub fn array_header(buf: &[u8], offset: usize) -> Option<(usize, usize)> { + let tag = get(buf, offset)?; + match tag { + 0x90..=0x9f => Some(((tag & 0x0f) as usize, offset + 1)), + ARRAY16 => Some((read_u16_be(buf, offset + 1)? as usize, offset + 3)), + ARRAY32 => Some((read_u32_be(buf, offset + 1)? as usize, offset + 5)), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::msgpack_scan::reader::skip_value; + + use serde_json::json; + + /// Helper: encode a serde_json::Value to MessagePack bytes. + fn encode(v: &serde_json::Value) -> Vec { + nodedb_types::json_msgpack::json_to_msgpack(v).expect("encode") + } + + #[test] + fn read_f64_fixint() { + assert_eq!(read_f64(&[42u8], 0), Some(42.0)); + } + + #[test] + fn read_f64_negative_fixint() { + assert_eq!(read_f64(&[0xffu8], 0), Some(-1.0)); + } + + #[test] + fn read_f64_float64() { + let buf = encode(&json!(std::f64::consts::PI)); + assert_eq!(read_f64(&buf, 0), Some(std::f64::consts::PI)); + } + + #[test] + fn read_f64_uint16() { + let buf = encode(&json!(1000)); + assert_eq!(read_f64(&buf, 0), Some(1000.0)); + } + + #[test] + fn read_f64_float32() { + // json! always produces f64, so test float32 with raw bytes + // float32 tag (0xca) + 1.5 in IEEE 754 big-endian + let buf = [0xca, 0x3f, 0xc0, 0x00, 0x00]; + let val = read_f64(&buf, 0).unwrap(); + assert!((val - 1.5).abs() < 1e-6); + } + + #[test] + fn read_i64_values() { + assert_eq!(read_i64(&[42u8], 0), Some(42)); + assert_eq!(read_i64(&[0xffu8], 0), Some(-1)); + + let buf = encode(&json!(300)); + assert_eq!(read_i64(&buf, 0), Some(300)); + + let buf = encode(&json!(-500)); + assert_eq!(read_i64(&buf, 0), Some(-500)); + } + + #[test] + fn read_str_fixstr() { + let buf = encode(&json!("hi")); + assert_eq!(read_str(&buf, 0), Some("hi")); + } + + #[test] + fn read_str_str8() { + let long = "a".repeat(40); + let buf = encode(&json!(long)); + assert_eq!(read_str(&buf, 0), Some(long.as_str())); + } + + #[test] + fn read_bool_values() { + assert_eq!(read_bool(&[TRUE], 0), Some(true)); + assert_eq!(read_bool(&[FALSE], 0), Some(false)); + assert_eq!(read_bool(&[NIL], 0), None); + } + + #[test] + fn read_null_check() { + assert!(read_null(&[NIL], 0)); + assert!(!read_null(&[TRUE], 0)); + } + + #[test] + fn map_header_fixmap() { + let buf = encode(&json!({"x": 1})); + let (count, _data_offset) = map_header(&buf, 0).unwrap(); + assert_eq!(count, 1); + } + + #[test] + fn array_header_fixarray() { + let buf = encode(&json!([10, 20, 30])); + let (count, data_offset) = array_header(&buf, 0).unwrap(); + assert_eq!(count, 3); + assert_eq!(read_i64(&buf, data_offset), Some(10)); + } + + #[test] + fn canonical_integer_smallest_representation() { + // fixint (0-127): single byte + let buf = encode(&json!(42)); + assert_eq!(buf.len(), 1); + assert_eq!(buf[0], 42); + + // 0 as fixint + let buf = encode(&json!(0)); + assert_eq!(buf.len(), 1); + assert_eq!(buf[0], 0); + + // 127 as fixint + let buf = encode(&json!(127)); + assert_eq!(buf.len(), 1); + assert_eq!(buf[0], 127); + + // 128 should NOT be fixint. JSON parses as i64, so zerompk uses + // int16 (0xd1) since 128 > i8::MAX. This is canonical for signed path. + let buf = encode(&json!(128)); + assert_eq!(buf[0], 0xd1); // int16 tag + assert_eq!(buf.len(), 3); // tag + 2 bytes + + // negative fixint (-32 to -1) + let buf = encode(&json!(-1)); + assert_eq!(buf.len(), 1); + assert_eq!(buf[0], 0xff); // -1 as negative fixint + + let buf = encode(&json!(-32)); + assert_eq!(buf.len(), 1); + assert_eq!(buf[0], 0xe0); // -32 as negative fixint + } + + #[test] + fn canonical_map_keys_sorted() { + // Keys should be lexicographically sorted in msgpack output. + // Encode with keys in non-sorted order in JSON source. + let buf = encode(&json!({"z": 1, "a": 2, "m": 3})); + + // Parse map and verify keys come out sorted + let (count, mut pos) = map_header(&buf, 0).unwrap(); + assert_eq!(count, 3); + + let mut keys = Vec::new(); + for _ in 0..count { + let key = read_str(&buf, pos).unwrap(); + keys.push(key.to_string()); + pos = skip_value(&buf, pos).unwrap(); // skip key + pos = skip_value(&buf, pos).unwrap(); // skip value + } + assert_eq!(keys, vec!["a", "m", "z"]); + } + + #[test] + fn canonical_deterministic_bytes() { + // Same logical document encoded twice must produce identical bytes. + let doc1 = encode(&json!({"name": "alice", "age": 30, "active": true})); + let doc2 = encode(&json!({"age": 30, "active": true, "name": "alice"})); + assert_eq!( + doc1, doc2, + "same logical doc must produce identical msgpack bytes" + ); + } + + #[test] + fn canonical_nested_map_keys_sorted() { + let buf = encode(&json!({"outer": {"z": 1, "a": 2}})); + // Extract the inner map + let (start, _end) = crate::msgpack_scan::field::extract_field(&buf, 0, "outer").unwrap(); + + let (count, mut pos) = map_header(&buf, start).unwrap(); + assert_eq!(count, 2); + + let key1 = read_str(&buf, pos).unwrap(); + pos = skip_value(&buf, pos).unwrap(); + pos = skip_value(&buf, pos).unwrap(); + let key2 = read_str(&buf, pos).unwrap(); + + assert_eq!(key1, "a"); + assert_eq!(key2, "z"); + } + + #[test] + fn read_bin_advance_all_widths() { + // bin8: 0xc4, len=3 + let mut off = 0; + let buf = [BIN8, 3, 0xde, 0xad, 0xbe, 0xff]; + assert_eq!( + read_bin_advance(&buf, &mut off), + Some(&[0xde, 0xad, 0xbe][..]) + ); + assert_eq!(off, 5); + + // bin16: 0xc5, big-endian len=4 + let mut off = 0; + let buf = [BIN16, 0x00, 0x04, 0x01, 0x02, 0x03, 0x04]; + assert_eq!( + read_bin_advance(&buf, &mut off), + Some(&[0x01, 0x02, 0x03, 0x04][..]) + ); + assert_eq!(off, 7); + + // bin32: 0xc6, big-endian len=2 + let mut off = 0; + let buf = [BIN32, 0x00, 0x00, 0x00, 0x02, 0xaa, 0xbb]; + assert_eq!(read_bin_advance(&buf, &mut off), Some(&[0xaa, 0xbb][..])); + assert_eq!(off, 7); + + // Non-bin tag returns None and does not advance. + let mut off = 0; + let buf = [0xc0u8]; // nil + assert_eq!(read_bin_advance(&buf, &mut off), None); + assert_eq!(off, 0); + + // Truncated returns None. + let mut off = 0; + let buf = [BIN8, 5, 0x01]; // claims 5 bytes, only 1 present + assert_eq!(read_bin_advance(&buf, &mut off), None); + } + + #[test] + fn read_u32_advance_all_widths() { + // positive fixint + let mut off = 0; + assert_eq!(read_u32_advance(&[42u8], &mut off), Some(42)); + assert_eq!(off, 1); + + // uint8 + let mut off = 0; + assert_eq!(read_u32_advance(&[UINT8, 200], &mut off), Some(200)); + assert_eq!(off, 2); + + // uint16 + let mut off = 0; + let buf = [UINT16, 0x12, 0x34]; + assert_eq!(read_u32_advance(&buf, &mut off), Some(0x1234)); + assert_eq!(off, 3); + + // uint32 + let mut off = 0; + let buf = [UINT32, 0xde, 0xad, 0xbe, 0xef]; + assert_eq!(read_u32_advance(&buf, &mut off), Some(0xdeadbeef)); + assert_eq!(off, 5); + + // negative fixint, int*, uint64, float, etc. all rejected + let mut off = 0; + assert_eq!(read_u32_advance(&[0xffu8], &mut off), None); // negative fixint + assert_eq!(off, 0); + let mut off = 0; + assert_eq!(read_u32_advance(&[INT8, 5], &mut off), None); + let mut off = 0; + assert_eq!( + read_u32_advance(&[UINT64, 0, 0, 0, 0, 0, 0, 0, 1], &mut off), + None + ); + + // Truncated returns None. + let mut off = 0; + assert_eq!(read_u32_advance(&[UINT16, 0x12], &mut off), None); + } + + #[test] + fn read_str_advance_basic() { + // fixstr "hi" + let mut off = 0; + let buf = encode(&json!("hi")); + assert_eq!(read_str_advance(&buf, &mut off), Some("hi")); + assert_eq!(off, buf.len()); + + // Sequential reads + let buf = encode(&json!(["one", "two"])); + let (count, mut off) = array_header(&buf, 0).unwrap(); + assert_eq!(count, 2); + assert_eq!(read_str_advance(&buf, &mut off), Some("one")); + assert_eq!(read_str_advance(&buf, &mut off), Some("two")); + assert_eq!(off, buf.len()); + + // Non-string returns None. + let mut off = 0; + assert_eq!(read_str_advance(&[NIL], &mut off), None); + assert_eq!(off, 0); + } +} diff --git a/nodedb-query/src/msgpack_scan/reader/skip.rs b/nodedb-query/src/msgpack_scan/reader/skip.rs new file mode 100644 index 000000000..9f63ee35b --- /dev/null +++ b/nodedb-query/src/msgpack_scan/reader/skip.rs @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! `skip_value`: advance past one MessagePack value without decoding it. + +use super::tags::*; + +/// Advance past the MessagePack value starting at `offset`, returning the +/// offset of the next value. Returns `None` if the buffer is truncated or +/// nesting exceeds `MAX_DEPTH`. +/// +/// This is the performance-critical primitive. It never allocates. +pub fn skip_value(buf: &[u8], offset: usize) -> Option { + skip_value_depth(buf, offset, 0) +} + +fn skip_value_depth(buf: &[u8], offset: usize, depth: u16) -> Option { + if depth > MAX_DEPTH { + return None; + } + let tag = get(buf, offset)?; + match tag { + // positive fixint (0x00..=0x7f) + 0x00..=0x7f => Some(offset + 1), + // negative fixint (0xe0..=0xff) + 0xe0..=0xff => Some(offset + 1), + // nil, false, true + NIL | FALSE | TRUE => Some(offset + 1), + + // fixmap (0x80..=0x8f) + 0x80..=0x8f => { + let count = (tag & 0x0f) as usize; + skip_n_pairs(buf, offset + 1, count, depth) + } + MAP16 => { + let count = read_u16_be(buf, offset + 1)? as usize; + skip_n_pairs(buf, offset + 3, count, depth) + } + MAP32 => { + let count = read_u32_be(buf, offset + 1)? as usize; + skip_n_pairs(buf, offset + 5, count, depth) + } + + // fixarray (0x90..=0x9f) + 0x90..=0x9f => { + let count = (tag & 0x0f) as usize; + skip_n_values(buf, offset + 1, count, depth) + } + ARRAY16 => { + let count = read_u16_be(buf, offset + 1)? as usize; + skip_n_values(buf, offset + 3, count, depth) + } + ARRAY32 => { + let count = read_u32_be(buf, offset + 1)? as usize; + skip_n_values(buf, offset + 5, count, depth) + } + + // fixstr (0xa0..=0xbf) + 0xa0..=0xbf => { + let len = (tag & 0x1f) as usize; + checked_advance(buf, offset, 1 + len) + } + STR8 => { + let len = get(buf, offset + 1)? as usize; + checked_advance(buf, offset, 2 + len) + } + STR16 => { + let len = read_u16_be(buf, offset + 1)? as usize; + checked_advance(buf, offset, 3 + len) + } + STR32 => { + let len = read_u32_be(buf, offset + 1)? as usize; + checked_advance(buf, offset, 5 + len) + } + + // bin + BIN8 => { + let len = get(buf, offset + 1)? as usize; + checked_advance(buf, offset, 2 + len) + } + BIN16 => { + let len = read_u16_be(buf, offset + 1)? as usize; + checked_advance(buf, offset, 3 + len) + } + BIN32 => { + let len = read_u32_be(buf, offset + 1)? as usize; + checked_advance(buf, offset, 5 + len) + } + + // fixed-width numerics (bounds-check against buffer length) + FLOAT32 => checked_advance(buf, offset, 5), + FLOAT64 => checked_advance(buf, offset, 9), + UINT8 | INT8 => checked_advance(buf, offset, 2), + UINT16 | INT16 => checked_advance(buf, offset, 3), + UINT32 | INT32 => checked_advance(buf, offset, 5), + UINT64 | INT64 => checked_advance(buf, offset, 9), + + // ext + FIXEXT1 => checked_advance(buf, offset, 3), + FIXEXT2 => checked_advance(buf, offset, 4), + FIXEXT4 => checked_advance(buf, offset, 6), + FIXEXT8 => checked_advance(buf, offset, 10), + FIXEXT16 => checked_advance(buf, offset, 18), + EXT8 => { + let len = get(buf, offset + 1)? as usize; + checked_advance(buf, offset, 3 + len) + } + EXT16 => { + let len = read_u16_be(buf, offset + 1)? as usize; + checked_advance(buf, offset, 4 + len) + } + EXT32 => { + let len = read_u32_be(buf, offset + 1)? as usize; + checked_advance(buf, offset, 6 + len) + } + + // 0xc1 is never used in the spec + _ => None, + } +} + +fn skip_n_values(buf: &[u8], mut pos: usize, count: usize, depth: u16) -> Option { + for _ in 0..count { + pos = skip_value_depth(buf, pos, depth + 1)?; + } + Some(pos) +} + +fn skip_n_pairs(buf: &[u8], mut pos: usize, count: usize, depth: u16) -> Option { + for _ in 0..count { + pos = skip_value_depth(buf, pos, depth + 1)?; // key + pos = skip_value_depth(buf, pos, depth + 1)?; // value + } + Some(pos) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::msgpack_scan::reader::read_str; + + use serde_json::json; + + /// Helper: encode a serde_json::Value to MessagePack bytes. + fn encode(v: &serde_json::Value) -> Vec { + nodedb_types::json_msgpack::json_to_msgpack(v).expect("encode") + } + + #[test] + fn skip_positive_fixint() { + let buf = [0x05, 0xff]; + assert_eq!(skip_value(&buf, 0), Some(1)); + } + + #[test] + fn skip_negative_fixint() { + let buf = [0xe0, 0x00]; + assert_eq!(skip_value(&buf, 0), Some(1)); + } + + #[test] + fn skip_nil_bool() { + assert_eq!(skip_value(&[NIL], 0), Some(1)); + assert_eq!(skip_value(&[TRUE], 0), Some(1)); + assert_eq!(skip_value(&[FALSE], 0), Some(1)); + } + + #[test] + fn skip_float64() { + let buf = encode(&json!(9.81)); + assert_eq!(skip_value(&buf, 0), Some(buf.len())); + } + + #[test] + fn skip_string() { + let buf = encode(&json!("hello")); + assert_eq!(skip_value(&buf, 0), Some(buf.len())); + } + + #[test] + fn skip_map() { + let buf = encode(&json!({"a": 1, "b": 2})); + assert_eq!(skip_value(&buf, 0), Some(buf.len())); + } + + #[test] + fn skip_nested_array() { + let buf = encode(&json!([[1, 2], [3, 4, 5]])); + assert_eq!(skip_value(&buf, 0), Some(buf.len())); + } + + #[test] + fn skip_truncated_returns_none() { + let buf = [FLOAT64, 0x40]; // truncated float64 + assert_eq!(skip_value(&buf, 0), None); + } + + #[test] + fn skip_bin() { + // bin8: 0xc4, len=3, 3 bytes of data + let buf = [BIN8, 3, 0xde, 0xad, 0xbe, 0xff]; + assert_eq!(skip_value(&buf, 0), Some(5)); + } + + #[test] + fn skip_ext() { + // fixext1: 0xd4, type byte, 1 data byte + let buf = [FIXEXT1, 0x01, 0xab, 0xff]; + assert_eq!(skip_value(&buf, 0), Some(3)); + } + + #[test] + fn skip_empty_containers() { + // empty fixmap + assert_eq!(skip_value(&[0x80], 0), Some(1)); + // empty fixarray + assert_eq!(skip_value(&[0x90], 0), Some(1)); + } + + /// All tag boundary bytes — test transitions at fixint/fixmap/fixarray/fixstr edges. + #[test] + fn fuzz_tag_boundaries() { + // Each entry: (tag, expected_skip_result) + // For tags that are self-contained single bytes, skip returns Some(1). + // For tags requiring more data we just verify no panic with empty tail. + let boundary_tags: &[(u8, bool)] = &[ + (0x00, true), // positive fixint 0 + (0x7f, true), // positive fixint 127 + (0x80, true), // fixmap length 0 (empty map) + (0x8f, false), // fixmap length 15 — needs 15 pairs + (0x90, true), // fixarray length 0 (empty array) + (0x9f, false), // fixarray length 15 — needs 15 elements + (0xa0, true), // fixstr length 0 (empty string) + (0xbf, false), // fixstr length 31 — needs 31 bytes after + (0xc0, true), // nil + (0xc1, false), // never used — must return None + (0xc2, true), // false + (0xc3, true), // true + (0xe0, true), // negative fixint -32 + (0xff, true), // negative fixint -1 + ]; + for &(tag, self_contained) in boundary_tags { + let buf = [tag; 64]; // fill with the same tag as padding + let result = skip_value(&buf, 0); + if self_contained { + assert!(result.is_some(), "tag 0x{tag:02x} should skip OK"); + } else if tag == 0xc1 { + assert_eq!(result, None, "0xc1 must always return None"); + } + // For non-self-contained tags with valid padding we just verify no panic. + } + } + + /// Buffers where length fields claim enormous sizes but the buffer is tiny. + #[test] + fn fuzz_adversarial_length_fields() { + // STR32: tag 0xdb + 4-byte big-endian length claiming 0xffffffff bytes + let buf = [0xdbu8, 0xff, 0xff, 0xff, 0xff, b'x', b'y']; + assert_eq!(skip_value(&buf, 0), None); + assert_eq!(read_str(&buf, 0), None); + + // STR16: tag 0xda + 2-byte length claiming 0xffff bytes + let buf = [0xdau8, 0xff, 0xff, b'x']; + assert_eq!(skip_value(&buf, 0), None); + + // ARRAY32: claims 0xffffffff elements but buffer is empty after header + let buf = [0xddu8, 0xff, 0xff, 0xff, 0xff]; + assert_eq!(skip_value(&buf, 0), None); + + // MAP32: claims 0xffffffff pairs but buffer is empty after header + let buf = [0xdfu8, 0xff, 0xff, 0xff, 0xff]; + assert_eq!(skip_value(&buf, 0), None); + + // ARRAY16: claims 0xffff elements + let buf = [0xdcu8, 0xff, 0xff]; + assert_eq!(skip_value(&buf, 0), None); + + // MAP16: claims 0xffff pairs + let buf = [0xdeu8, 0xff, 0xff]; + assert_eq!(skip_value(&buf, 0), None); + + // BIN32: claims max length + let buf = [0xc6u8, 0xff, 0xff, 0xff, 0xff, 0x00]; + assert_eq!(skip_value(&buf, 0), None); + + // EXT32: claims max length + let buf = [0xc9u8, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00]; + assert_eq!(skip_value(&buf, 0), None); + } + + /// Deeply nested maps/arrays must cause `skip_value` to return `None` + /// once nesting exceeds MAX_DEPTH (128). + #[test] + fn fuzz_malicious_nesting_depth() { + // Build a buffer with 200 levels of fixarray (each containing 1 element) + // fixarray tag for 1 element = 0x91 + let depth = 200usize; + let mut buf = vec![0x91u8; depth]; // fixarray(1) — opens 1-element array + buf.push(0xc0u8); // nil at the innermost leaf + + // skip_value must return None because nesting > MAX_DEPTH + assert_eq!( + skip_value(&buf, 0), + None, + "deeply nested arrays must return None to guard against stack overflow" + ); + + // Same with maps: fixmap(1) = 0x81, then a fixstr(1) key + value + // Build 200 levels of fixmap(1) — each pair is (fixstr key, next map) + let mut map_buf: Vec = Vec::new(); + for i in 0..(depth as u8) { + map_buf.push(0x81); // fixmap(1) + map_buf.push(0xa1); // fixstr(1) key + map_buf.push(b'a'.wrapping_add(i % 26)); + // value = next map (already pushed in next iteration), or nil at end + } + map_buf.push(0xc0); // nil leaf + + assert_eq!( + skip_value(&map_buf, 0), + None, + "deeply nested maps must return None" + ); + } + + /// Verify skip_value correctly consumes exactly the right number of bytes + /// for all fixed-width numeric types and returns the correct next offset. + #[test] + fn fuzz_fixed_width_numeric_skip_offsets() { + // (tag, expected_total_bytes_consumed) + let cases: &[(u8, usize)] = &[ + (0xca, 5), // FLOAT32: 1 tag + 4 data + (0xcb, 9), // FLOAT64: 1 tag + 8 data + (0xcc, 2), // UINT8 + (0xcd, 3), // UINT16 + (0xce, 5), // UINT32 + (0xcf, 9), // UINT64 + (0xd0, 2), // INT8 + (0xd1, 3), // INT16 + (0xd2, 5), // INT32 + (0xd3, 9), // INT64 + ]; + for &(tag, size) in cases { + let mut buf = vec![0u8; size + 4]; // extra padding + buf[0] = tag; + let result = skip_value(&buf, 0); + assert_eq!( + result, + Some(size), + "tag 0x{tag:02x} should advance by {size} bytes" + ); + } + } + + /// Verify all fixext types consume the correct byte count. + #[test] + fn fuzz_fixext_skip_offsets() { + // (tag, expected_bytes_consumed) + let cases: &[(u8, usize)] = &[ + (0xd4, 3), // FIXEXT1: 1+1+1 + (0xd5, 4), // FIXEXT2: 1+1+2 + (0xd6, 6), // FIXEXT4: 1+1+4 + (0xd7, 10), // FIXEXT8: 1+1+8 + (0xd8, 18), // FIXEXT16: 1+1+16 + ]; + for &(tag, size) in cases { + let mut buf = vec![0u8; size + 4]; + buf[0] = tag; + let result = skip_value(&buf, 0); + assert_eq!( + result, + Some(size), + "fixext tag 0x{tag:02x} should advance by {size} bytes" + ); + } + } +} diff --git a/nodedb-query/src/msgpack_scan/reader/tags.rs b/nodedb-query/src/msgpack_scan/reader/tags.rs new file mode 100644 index 000000000..4b1daf45c --- /dev/null +++ b/nodedb-query/src/msgpack_scan/reader/tags.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! MessagePack tag constants and bounds-checked byte helpers. + +pub(super) const NIL: u8 = 0xc0; +pub(super) const FALSE: u8 = 0xc2; +pub(super) const TRUE: u8 = 0xc3; +pub(super) const BIN8: u8 = 0xc4; +pub(super) const BIN16: u8 = 0xc5; +pub(super) const BIN32: u8 = 0xc6; +pub(super) const EXT8: u8 = 0xc7; +pub(super) const EXT16: u8 = 0xc8; +pub(super) const EXT32: u8 = 0xc9; +pub(super) const FLOAT32: u8 = 0xca; +pub(super) const FLOAT64: u8 = 0xcb; +pub(super) const UINT8: u8 = 0xcc; +pub(super) const UINT16: u8 = 0xcd; +pub(super) const UINT32: u8 = 0xce; +pub(super) const UINT64: u8 = 0xcf; +pub(super) const INT8: u8 = 0xd0; +pub(super) const INT16: u8 = 0xd1; +pub(super) const INT32: u8 = 0xd2; +pub(super) const INT64: u8 = 0xd3; +pub(super) const FIXEXT1: u8 = 0xd4; +pub(super) const FIXEXT2: u8 = 0xd5; +pub(super) const FIXEXT4: u8 = 0xd6; +pub(super) const FIXEXT8: u8 = 0xd7; +pub(super) const FIXEXT16: u8 = 0xd8; +pub(super) const STR8: u8 = 0xd9; +pub(super) const STR16: u8 = 0xda; +pub(super) const STR32: u8 = 0xdb; +pub(super) const ARRAY16: u8 = 0xdc; +pub(super) const ARRAY32: u8 = 0xdd; +pub(super) const MAP16: u8 = 0xde; +pub(super) const MAP32: u8 = 0xdf; + +/// Maximum nesting depth to prevent stack overflow on malicious payloads. +pub(super) const MAX_DEPTH: u16 = 128; + +#[inline(always)] +pub(super) fn get(buf: &[u8], pos: usize) -> Option { + buf.get(pos).copied() +} + +#[inline(always)] +pub(super) fn read_u16_be(buf: &[u8], pos: usize) -> Option { + let bytes = buf.get(pos..pos + 2)?; + Some(u16::from_be_bytes([bytes[0], bytes[1]])) +} + +#[inline(always)] +pub(super) fn read_u32_be(buf: &[u8], pos: usize) -> Option { + let bytes = buf.get(pos..pos + 4)?; + Some(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) +} + +#[inline(always)] +pub(super) fn read_u64_be(buf: &[u8], pos: usize) -> Option { + let bytes = buf.get(pos..pos + 8)?; + Some(u64::from_be_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ])) +} + +/// Return `Some(offset + size)` only if the buffer has enough bytes. +#[inline(always)] +pub(super) fn checked_advance(buf: &[u8], offset: usize, size: usize) -> Option { + let end = offset + size; + if end <= buf.len() { Some(end) } else { None } +} diff --git a/nodedb-query/src/msgpack_scan/reader/value.rs b/nodedb-query/src/msgpack_scan/reader/value.rs new file mode 100644 index 000000000..74b830d59 --- /dev/null +++ b/nodedb-query/src/msgpack_scan/reader/value.rs @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! `read_value`: decode one scalar into `nodedb_types::Value`. + +use nodedb_types::read_instant; + +use super::scalar::read_str; +use super::tags::*; + +/// Read a scalar msgpack value at `offset` into `nodedb_types::Value`. +/// +/// Handles null, bool, integers, floats, strings, and instant ext +/// (`fixext8` type 1 / 2 → `Value::DateTime` / `Value::NaiveDateTime`). +/// For complex types (array, map, bin, other ext), returns `None` — caller +/// should use `json_from_msgpack` for those. +pub fn read_value(buf: &[u8], offset: usize) -> Option { + let tag = get(buf, offset)?; + match tag { + NIL => Some(nodedb_types::Value::Null), + TRUE => Some(nodedb_types::Value::Bool(true)), + FALSE => Some(nodedb_types::Value::Bool(false)), + // Integers + 0x00..=0x7f => Some(nodedb_types::Value::Integer(tag as i64)), + 0xe0..=0xff => Some(nodedb_types::Value::Integer((tag as i8) as i64)), + UINT8 => Some(nodedb_types::Value::Integer(get(buf, offset + 1)? as i64)), + UINT16 => Some(nodedb_types::Value::Integer( + read_u16_be(buf, offset + 1)? as i64 + )), + UINT32 => Some(nodedb_types::Value::Integer( + read_u32_be(buf, offset + 1)? as i64 + )), + UINT64 => Some(nodedb_types::Value::Integer( + read_u64_be(buf, offset + 1)? as i64 + )), + INT8 => Some(nodedb_types::Value::Integer( + get(buf, offset + 1)? as i8 as i64 + )), + INT16 => Some(nodedb_types::Value::Integer( + read_u16_be(buf, offset + 1)? as i16 as i64, + )), + INT32 => Some(nodedb_types::Value::Integer( + read_u32_be(buf, offset + 1)? as i32 as i64, + )), + INT64 => Some(nodedb_types::Value::Integer( + read_u64_be(buf, offset + 1)? as i64 + )), + // Floats + FLOAT32 => { + let bits = read_u32_be(buf, offset + 1)?; + Some(nodedb_types::Value::Float(f32::from_bits(bits) as f64)) + } + FLOAT64 => { + let bits = read_u64_be(buf, offset + 1)?; + Some(nodedb_types::Value::Float(f64::from_bits(bits))) + } + // Strings + 0xa0..=0xbf | STR8 | STR16 | STR32 => { + read_str(buf, offset).map(|s| nodedb_types::Value::String(s.to_string())) + } + // Instants + FIXEXT8 => read_instant(buf, offset).map(|(kind, micros)| kind.from_micros(micros)), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::msgpack_scan::reader::{ + array_header, map_header, read_bool, read_f64, read_i64, read_null, skip_value, + }; + use nodedb_types::{InstantKind, NdbDateTime, Value, write_instant}; + + use serde_json::json; + + /// Helper: encode a serde_json::Value to MessagePack bytes. + fn encode(v: &serde_json::Value) -> Vec { + nodedb_types::json_msgpack::json_to_msgpack(v).expect("encode") + } + + #[test] + fn read_value_instants() { + let mut buf = vec![0xc0]; + write_instant(&mut buf, InstantKind::Utc, 1_710_498_600_000_000); + write_instant(&mut buf, InstantKind::Naive, -5); + assert_eq!( + read_value(&buf, 1), + Some(Value::DateTime(NdbDateTime::from_micros( + 1_710_498_600_000_000 + ))) + ); + assert_eq!( + read_value(&buf, 11), + Some(Value::NaiveDateTime(NdbDateTime::from_micros(-5))) + ); + assert_eq!(skip_value(&buf, 1), Some(11)); + } + + #[test] + fn read_value_unknown_ext_is_none() { + let buf = [FIXEXT8, 0x09, 0, 0, 0, 0, 0, 0, 0, 1]; + assert_eq!(read_value(&buf, 0), None); + let buf = [FIXEXT1, 0x01, 0xab]; + assert_eq!(read_value(&buf, 0), None); + } + + #[test] + fn read_value_truncated_instant_is_none() { + let mut buf = Vec::new(); + write_instant(&mut buf, InstantKind::Utc, 42); + buf.truncate(9); + assert_eq!(read_value(&buf, 0), None); + } + + /// Feed every single-byte sequence through all reader functions. None may + /// panic — they must return `None` or a valid result. + #[test] + fn fuzz_all_single_byte_sequences() { + for byte in 0u8..=255 { + let buf = [byte]; + // None of these must panic + let _ = skip_value(&buf, 0); + let _ = read_f64(&buf, 0); + let _ = read_i64(&buf, 0); + let _ = read_str(&buf, 0); + let _ = read_bool(&buf, 0); + let _ = read_null(&buf, 0); + let _ = map_header(&buf, 0); + let _ = array_header(&buf, 0); + let _ = read_value(&buf, 0); + } + } + + /// Feed two-byte patterns to cover tag + partial payload (truncated). + #[test] + fn fuzz_two_byte_patterns() { + // Tags that expect more bytes than we provide + let tags_need_extra: &[u8] = &[ + 0xca, // FLOAT32 needs 4 more + 0xcb, // FLOAT64 needs 8 more + 0xcc, // UINT8 needs 1 more + 0xcd, // UINT16 needs 2 more + 0xce, // UINT32 needs 4 more + 0xcf, // UINT64 needs 8 more + 0xd0, // INT8 needs 1 more + 0xd1, // INT16 needs 2 more + 0xd2, // INT32 needs 4 more + 0xd3, // INT64 needs 8 more + 0xd9, // STR8 length byte then data + 0xda, // STR16 2-byte length then data + 0xdb, // STR32 4-byte length then data + 0xdc, // ARRAY16 2-byte count then elements + 0xdd, // ARRAY32 4-byte count then elements + 0xde, // MAP16 2-byte count then pairs + 0xdf, // MAP32 4-byte count then pairs + 0xc4, // BIN8 + 0xc5, // BIN16 + 0xc6, // BIN32 + 0xd4, // FIXEXT1 + 0xd5, // FIXEXT2 + 0xd6, // FIXEXT4 + 0xd7, // FIXEXT8 + 0xd8, // FIXEXT16 + ]; + for &tag in tags_need_extra { + // Single byte (completely truncated payload) + let buf = [tag]; + let _ = skip_value(&buf, 0); + let _ = read_f64(&buf, 0); + let _ = read_i64(&buf, 0); + let _ = read_value(&buf, 0); + + // Tag + one garbage byte + for second in [0x00u8, 0x01, 0x7f, 0x80, 0xff] { + let buf = [tag, second]; + let _ = skip_value(&buf, 0); + let _ = read_f64(&buf, 0); + let _ = read_i64(&buf, 0); + let _ = read_value(&buf, 0); + } + } + } + + /// Deterministic pseudo-random byte sequences must not cause panics. + #[test] + fn fuzz_deterministic_random_payloads() { + // Generate deterministic sequences without external crates using a + // simple LCG (Knuth multiplicative hash). + let mut state: u64 = 0xdeadbeef_cafebabe; + let next = |s: &mut u64| -> u8 { + *s = s + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + (*s >> 33) as u8 + }; + + let mut buf = vec![0u8; 256]; + for _ in 0..2000 { + // Randomize buffer length (1..=256) and contents + let len = (next(&mut state) as usize % 256) + 1; + for b in buf[..len].iter_mut() { + *b = next(&mut state); + } + let slice = &buf[..len]; + + // Try reading from multiple offsets + for offset in [0, 1, len / 2, len.saturating_sub(1)] { + let _ = skip_value(slice, offset); + let _ = read_f64(slice, offset); + let _ = read_i64(slice, offset); + let _ = read_str(slice, offset); + let _ = read_bool(slice, offset); + let _ = read_null(slice, offset); + let _ = map_header(slice, offset); + let _ = array_header(slice, offset); + let _ = read_value(slice, offset); + } + } + } + + /// Truncate a valid msgpack buffer at every byte position. + /// All reader functions must return `None` — never panic. + #[test] + fn fuzz_truncated_valid_payloads() { + let docs = [ + json!({"key": "value", "num": 42, "flag": true}), + json!({"nested": {"a": 1, "b": [1, 2, 3]}}), + json!([1, "two", 3.0, null, false]), + json!({"large": 9999999999_i64}), + json!({"float": 1.23456789}), + ]; + + for doc in &docs { + let full = encode(doc); + // Truncate at every position from 0 to full.len()-1 + for truncate_at in 0..full.len() { + let slice = &full[..truncate_at]; + // None of these may panic; result doesn't matter + let _ = skip_value(slice, 0); + let _ = read_f64(slice, 0); + let _ = read_i64(slice, 0); + let _ = read_str(slice, 0); + let _ = read_bool(slice, 0); + let _ = map_header(slice, 0); + let _ = array_header(slice, 0); + let _ = read_value(slice, 0); + } + } + } + + /// The never-used 0xc1 tag must return `None` for all functions. + #[test] + fn fuzz_never_used_tag_c1() { + // 0xc1 is explicitly "never used" in the msgpack spec + let buf = [0xc1u8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; + assert_eq!( + skip_value(&buf, 0), + None, + "0xc1 must return None from skip_value" + ); + assert_eq!(read_f64(&buf, 0), None); + assert_eq!(read_i64(&buf, 0), None); + assert_eq!(read_str(&buf, 0), None); + assert_eq!(read_bool(&buf, 0), None); + assert_eq!(map_header(&buf, 0), None); + assert_eq!(array_header(&buf, 0), None); + assert_eq!(read_value(&buf, 0), None); + } + + /// Out-of-bounds offset must return `None` — not panic. + #[test] + fn fuzz_out_of_bounds_offset() { + let buf = encode(&json!({"x": 1})); + let way_out = buf.len() + 1000; + assert_eq!(skip_value(&buf, way_out), None); + assert_eq!(read_f64(&buf, way_out), None); + assert_eq!(read_i64(&buf, way_out), None); + assert_eq!(read_str(&buf, way_out), None); + assert_eq!(read_bool(&buf, way_out), None); + assert_eq!(map_header(&buf, way_out), None); + assert_eq!(array_header(&buf, way_out), None); + assert_eq!(read_value(&buf, way_out), None); + } + + /// Empty buffer must return `None` for all functions that can. + #[test] + fn fuzz_empty_buffer() { + let buf: &[u8] = &[]; + assert_eq!(skip_value(buf, 0), None); + assert_eq!(read_f64(buf, 0), None); + assert_eq!(read_i64(buf, 0), None); + assert_eq!(read_str(buf, 0), None); + assert_eq!(read_bool(buf, 0), None); + assert!(!read_null(buf, 0)); // returns bool, not Option + assert_eq!(map_header(buf, 0), None); + assert_eq!(array_header(buf, 0), None); + assert_eq!(read_value(buf, 0), None); + } +} diff --git a/nodedb-query/src/msgpack_scan/writer.rs b/nodedb-query/src/msgpack_scan/writer.rs index 876eee582..622f88c12 100644 --- a/nodedb-query/src/msgpack_scan/writer.rs +++ b/nodedb-query/src/msgpack_scan/writer.rs @@ -7,6 +7,8 @@ //! Follow the timeseries ingest pattern: `SqlValue → row_to_msgpack()` at ingress, //! raw msgpack throughout, `msgpack_to_json_string()` at outermost pgwire/HTTP layer. +use nodedb_types::InstantKind; + /// Write a msgpack map header. #[inline] pub fn write_map_header(buf: &mut Vec, len: usize) { @@ -96,6 +98,12 @@ pub fn write_null(buf: &mut Vec) { buf.push(0xC0); } +/// Write a typed instant as the ten-byte instant ext (`fixext8` type 1 / 2). +#[inline] +pub fn write_instant(buf: &mut Vec, kind: InstantKind, micros: i64) { + nodedb_types::write_instant(buf, kind, micros); +} + /// Write a msgpack binary blob (bin 8/16/32). #[inline] pub fn write_bin(buf: &mut Vec, data: &[u8]) { @@ -161,6 +169,13 @@ pub fn write_kv_null(buf: &mut Vec, key: &str) { write_null(buf); } +/// Write a key-value pair (string key, instant value). +#[inline] +pub fn write_kv_instant(buf: &mut Vec, key: &str, kind: InstantKind, micros: i64) { + write_str(buf, key); + write_instant(buf, kind, micros); +} + /// Inject a string field into a msgpack map without full decode. /// /// If `value` is a valid msgpack map that already contains `field_name`, @@ -352,4 +367,20 @@ mod tests { write_i64(&mut buf, -100); assert_eq!(buf, vec![0xD0, (-100i8) as u8]); } + + #[test] + fn instant_cell_reads_back() { + let mut buf = Vec::new(); + write_map_header(&mut buf, 1); + write_kv_instant(&mut buf, "ts", InstantKind::Naive, -42); + + let field = crate::msgpack_scan::field::extract_field(&buf, 0, "ts").unwrap(); + assert_eq!(field.1 - field.0, 10); + assert_eq!( + crate::msgpack_scan::reader::read_value(&buf, field.0), + Some(nodedb_types::Value::NaiveDateTime( + nodedb_types::NdbDateTime::from_micros(-42) + )) + ); + } } diff --git a/nodedb-query/src/scan_filter/mod.rs b/nodedb-query/src/scan_filter/mod.rs index 39c261f14..066462534 100644 --- a/nodedb-query/src/scan_filter/mod.rs +++ b/nodedb-query/src/scan_filter/mod.rs @@ -9,11 +9,9 @@ pub mod like; pub mod op; pub mod parse; -pub mod timestamp; pub mod types; pub use like::sql_like_match; -pub use op::FilterOp; +pub use op::{FilterOp, UnknownFilterOp}; pub use parse::parse_simple_predicates; -pub use timestamp::value_as_timestamp_ms; pub use types::ScanFilter; diff --git a/nodedb-query/src/scan_filter/op.rs b/nodedb-query/src/scan_filter/op.rs index 6f79bec41..3c4db3400 100644 --- a/nodedb-query/src/scan_filter/op.rs +++ b/nodedb-query/src/scan_filter/op.rs @@ -4,10 +4,25 @@ //! //! `FilterOp` is an O(1)-dispatch discriminant used by the scan filter //! evaluator. On-wire it travels as a lowercase string tag so physical -//! plans remain debuggable by hand. +//! plans remain debuggable by hand. Decoding a tag is fallible: a name +//! `FilterOp::parse` does not know is an error, never a default operator. + +/// An operator name no `FilterOp` variant carries. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("unknown filter operator `{op}`")] +pub struct UnknownFilterOp { + /// The operator text as it arrived. + pub op: String, +} + +impl From for zerompk::Error { + fn from(e: UnknownFilterOp) -> Self { + zerompk::Error::IoError(std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + } +} /// Filter operator enum for O(1) dispatch instead of string comparison. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FilterOp { Eq, Ne, @@ -27,7 +42,8 @@ pub enum FilterOp { ArrayContains, ArrayContainsAll, ArrayOverlap, - #[default] + /// Every row passes. Emitted by policy lowering for a predicate decided + /// true at plan time and by the planner for a filter it cannot lower. MatchAll, Exists, NotExists, @@ -50,8 +66,10 @@ pub enum FilterOp { } impl FilterOp { - pub fn parse_op(s: &str) -> Self { - match s { + /// Parse a wire operator name. Every name `as_str` emits parses back to + /// the same variant. `ne`/`neq`, `gte`/`ge`, and `lte`/`le` are aliases. + pub fn parse(name: &str) -> Result { + Ok(match name { "eq" => Self::Eq, "ne" | "neq" => Self::Ne, "gt" => Self::Gt, @@ -81,8 +99,12 @@ impl FilterOp { "lte_col" => Self::LteColumn, "eq_col" => Self::EqColumn, "ne_col" => Self::NeColumn, - _ => Self::MatchAll, - } + other => { + return Err(UnknownFilterOp { + op: other.to_string(), + }); + } + }) } pub fn as_str(&self) -> &'static str { @@ -120,18 +142,6 @@ impl FilterOp { } } -impl From<&str> for FilterOp { - fn from(s: &str) -> Self { - Self::parse_op(s) - } -} - -impl From for FilterOp { - fn from(s: String) -> Self { - Self::parse_op(&s) - } -} - impl serde::Serialize for FilterOp { fn serialize(&self, serializer: S) -> Result { serializer.serialize_str(self.as_str()) @@ -141,6 +151,74 @@ impl serde::Serialize for FilterOp { impl<'de> serde::Deserialize<'de> for FilterOp { fn deserialize>(deserializer: D) -> Result { let s = String::deserialize(deserializer)?; - Ok(FilterOp::parse_op(&s)) + FilterOp::parse(&s).map_err(serde::de::Error::custom) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ALL: [FilterOp; 29] = [ + FilterOp::Eq, + FilterOp::Ne, + FilterOp::Gt, + FilterOp::Gte, + FilterOp::Lt, + FilterOp::Lte, + FilterOp::Contains, + FilterOp::Like, + FilterOp::NotLike, + FilterOp::Ilike, + FilterOp::NotIlike, + FilterOp::In, + FilterOp::NotIn, + FilterOp::IsNull, + FilterOp::IsNotNull, + FilterOp::ArrayContains, + FilterOp::ArrayContainsAll, + FilterOp::ArrayOverlap, + FilterOp::MatchAll, + FilterOp::Exists, + FilterOp::NotExists, + FilterOp::Or, + FilterOp::Expr, + FilterOp::GtColumn, + FilterOp::GteColumn, + FilterOp::LtColumn, + FilterOp::LteColumn, + FilterOp::EqColumn, + FilterOp::NeColumn, + ]; + + #[test] + fn every_name_round_trips() { + for op in ALL { + assert_eq!(FilterOp::parse(op.as_str()), Ok(op), "{op:?}"); + } + } + + #[test] + fn aliases_parse_to_their_canonical_variant() { + assert_eq!(FilterOp::parse("neq"), Ok(FilterOp::Ne)); + assert_eq!(FilterOp::parse("ge"), Ok(FilterOp::Gte)); + assert_eq!(FilterOp::parse("le"), Ok(FilterOp::Lte)); + } + + #[test] + fn unknown_name_is_an_error_naming_the_text() { + let err = FilterOp::parse("any_in").expect_err("unknown"); + assert_eq!(err.op, "any_in"); + assert_eq!(err.to_string(), "unknown filter operator `any_in`"); + assert!(FilterOp::parse("").is_err()); + assert!(FilterOp::parse("EQ").is_err()); + } + + #[test] + fn serde_rejects_an_unknown_name() { + let ok: FilterOp = serde_json::from_str("\"array_overlap\"").expect("known"); + assert_eq!(ok, FilterOp::ArrayOverlap); + let err = serde_json::from_str::("\"any_in\"").expect_err("unknown"); + assert!(err.to_string().contains("any_in"), "{err}"); } } diff --git a/nodedb-query/src/scan_filter/parse.rs b/nodedb-query/src/scan_filter/parse.rs index 9020e442c..238e9822e 100644 --- a/nodedb-query/src/scan_filter/parse.rs +++ b/nodedb-query/src/scan_filter/parse.rs @@ -2,7 +2,7 @@ use nodedb_types::find_ascii_case_insensitive; -use super::ScanFilter; +use super::{FilterOp, ScanFilter}; /// Parse simple SQL predicates into `ScanFilter` values. /// @@ -33,17 +33,17 @@ fn parse_single_predicate(clause: &str) -> Option { let field = clause[..pos].trim().to_string(); let raw_value = clause[pos + op_str.len()..].trim(); let op = match *op_str { - "=" => "eq", - "!=" | "<>" => "ne", - ">" => "gt", - ">=" => "gte", - "<" => "lt", - "<=" => "lte", + "=" => FilterOp::Eq, + "!=" | "<>" => FilterOp::Ne, + ">" => FilterOp::Gt, + ">=" => FilterOp::Gte, + "<" => FilterOp::Lt, + "<=" => FilterOp::Lte, _ => return None, }; return Some(ScanFilter { field, - op: super::FilterOp::parse_op(op), + op, value: nodedb_types::Value::from(parse_predicate_value(raw_value)), clauses: Vec::new(), expr: None, @@ -56,7 +56,7 @@ fn parse_single_predicate(clause: &str) -> Option { let raw_value = clause[pos + 6..].trim(); return Some(ScanFilter { field, - op: super::FilterOp::Like, + op: FilterOp::Like, value: nodedb_types::Value::from(parse_predicate_value(raw_value)), clauses: Vec::new(), expr: None, @@ -67,7 +67,7 @@ fn parse_single_predicate(clause: &str) -> Option { let raw_value = clause[pos + 7..].trim(); return Some(ScanFilter { field, - op: super::FilterOp::Ilike, + op: FilterOp::Ilike, value: nodedb_types::Value::from(parse_predicate_value(raw_value)), clauses: Vec::new(), expr: None, diff --git a/nodedb-query/src/scan_filter/timestamp.rs b/nodedb-query/src/scan_filter/timestamp.rs deleted file mode 100644 index 58d875a1f..000000000 --- a/nodedb-query/src/scan_filter/timestamp.rs +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Coercion of a filter's comparison value to epoch milliseconds. -//! -//! A predicate against a timestamp column can arrive carrying any of the -//! shapes SQL accepts for an instant: a datetime literal (`ts < '2020-03-05 -//! 10:00:00'`), an epoch-millisecond integer (`ts < 1583402400000`), or an -//! already-typed datetime value. Timestamp columns store epoch milliseconds, -//! so every one of those has to reduce to the same scalar before comparison — -//! otherwise a perfectly ordinary predicate silently matches nothing. - -use nodedb_types::Value; - -/// Reduce a filter value to epoch milliseconds, or `None` when it cannot -/// denote an instant. -pub fn value_as_timestamp_ms(value: &Value) -> Option { - match value { - Value::Integer(ms) => Some(*ms), - Value::Float(ms) => Some(*ms as i64), - Value::DateTime(dt) | Value::NaiveDateTime(dt) => Some(dt.unix_millis()), - Value::String(text) => nodedb_types::datetime::NdbDateTime::parse(text) - .map(|dt| dt.unix_millis()) - .or_else(|| text.trim().parse::().ok()), - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// 2020-03-05T10:00:00Z in epoch milliseconds. - const MARCH_5_2020: i64 = 1_583_402_400_000; - - #[test] - fn epoch_millis_pass_through() { - assert_eq!( - value_as_timestamp_ms(&Value::Integer(MARCH_5_2020)), - Some(MARCH_5_2020) - ); - } - - #[test] - fn datetime_literals_are_parsed() { - assert_eq!( - value_as_timestamp_ms(&Value::String("2020-03-05 10:00:00".into())), - Some(MARCH_5_2020) - ); - assert_eq!( - value_as_timestamp_ms(&Value::String("2020-03-05T10:00:00Z".into())), - Some(MARCH_5_2020) - ); - } - - #[test] - fn numeric_strings_are_read_as_epoch_millis() { - assert_eq!( - value_as_timestamp_ms(&Value::String(MARCH_5_2020.to_string())), - Some(MARCH_5_2020) - ); - } - - #[test] - fn typed_datetimes_are_accepted() { - let dt = nodedb_types::NdbDateTime::from_millis(MARCH_5_2020).unwrap(); - assert_eq!( - value_as_timestamp_ms(&Value::NaiveDateTime(dt)), - Some(MARCH_5_2020) - ); - assert_eq!( - value_as_timestamp_ms(&Value::DateTime(dt)), - Some(MARCH_5_2020) - ); - } - - #[test] - fn non_instants_are_rejected() { - assert_eq!(value_as_timestamp_ms(&Value::Null), None); - assert_eq!(value_as_timestamp_ms(&Value::Bool(true)), None); - assert_eq!( - value_as_timestamp_ms(&Value::String("not a date".into())), - None - ); - } -} diff --git a/nodedb-query/src/scan_filter/types.rs b/nodedb-query/src/scan_filter/types.rs index 29fb1aec8..60432e81a 100644 --- a/nodedb-query/src/scan_filter/types.rs +++ b/nodedb-query/src/scan_filter/types.rs @@ -19,7 +19,15 @@ use super::op::FilterOp; /// OR representation: `{"op": "or", "clauses": [[filter1, filter2], [filter3]]}` /// means `(filter1 AND filter2) OR filter3`. Each clause is an AND-group; /// the document matches if ANY clause group fully matches. -#[derive(Clone, serde::Serialize, serde::Deserialize, Default)] +/// +/// Wire form (zerompk): `[field, op_name, value, clauses, expr]`. `op_name` +/// is the `FilterOp::as_str` tag and decodes through `FilterOp::parse`, so +/// an unknown name is a decode error. `value` is the tagged +/// `nodedb_types::Value` encoding, the same form `SqlExpr::Literal` uses: +/// every variant, including `DateTime` / `NaiveDateTime` / `Bytes` / +/// `Decimal`, arrives typed. The serde derive is a JSON rendering; it never +/// crosses the bridge. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct ScanFilter { #[serde(default)] pub field: String, @@ -41,9 +49,7 @@ impl zerompk::ToMessagePack for ScanFilter { writer.write_array_len(5)?; self.field.write(writer)?; writer.write_string(self.op.as_str())?; - // Convert nodedb_types::Value → serde_json::Value for wire compat. - let json_val: serde_json::Value = self.value.clone().into(); - nodedb_types::JsonValue(json_val).write(writer)?; + self.value.write(writer)?; self.clauses.write(writer)?; self.expr.write(writer) } @@ -53,15 +59,14 @@ impl<'a> zerompk::FromMessagePack<'a> for ScanFilter { fn read>(reader: &mut R) -> zerompk::Result { reader.check_array_len(5)?; let field = String::read(reader)?; - let op_str = String::read(reader)?; - let jv = nodedb_types::JsonValue::read(reader)?; + let op = FilterOp::parse(&reader.read_string()?)?; + let value = nodedb_types::Value::read(reader)?; let clauses = Vec::>::read(reader)?; let expr = Option::::read(reader)?; Ok(Self { field, - op: FilterOp::parse_op(&op_str), - // Convert serde_json::Value → nodedb_types::Value at wire boundary. - value: nodedb_types::Value::from(jv.0), + op, + value, clauses, expr, }) @@ -117,7 +122,30 @@ impl ScanFilter { None => Ok(false), }; } - _ => {} + FilterOp::Eq + | FilterOp::Ne + | FilterOp::Gt + | FilterOp::Gte + | FilterOp::Lt + | FilterOp::Lte + | FilterOp::Contains + | FilterOp::Like + | FilterOp::NotLike + | FilterOp::Ilike + | FilterOp::NotIlike + | FilterOp::In + | FilterOp::NotIn + | FilterOp::IsNull + | FilterOp::IsNotNull + | FilterOp::ArrayContains + | FilterOp::ArrayContainsAll + | FilterOp::ArrayOverlap + | FilterOp::GtColumn + | FilterOp::GteColumn + | FilterOp::LtColumn + | FilterOp::LteColumn + | FilterOp::EqColumn + | FilterOp::NeColumn => {} } let field_val = match doc.get(&self.field) { @@ -126,18 +154,24 @@ impl ScanFilter { }; Ok(match self.op { + FilterOp::MatchAll | FilterOp::Exists | FilterOp::NotExists => true, + FilterOp::Or | FilterOp::Expr => false, FilterOp::Eq => self.value.eq_coerced(field_val), FilterOp::Ne => !self.value.eq_coerced(field_val), - FilterOp::Gt => self.value.cmp_coerced(field_val) == std::cmp::Ordering::Less, - FilterOp::Gte => { - let cmp = self.value.cmp_coerced(field_val); - cmp == std::cmp::Ordering::Less || cmp == std::cmp::Ordering::Equal + FilterOp::Gt => { + field_val.partial_cmp_coerced(&self.value) == Some(std::cmp::Ordering::Greater) } - FilterOp::Lt => self.value.cmp_coerced(field_val) == std::cmp::Ordering::Greater, - FilterOp::Lte => { - let cmp = self.value.cmp_coerced(field_val); - cmp == std::cmp::Ordering::Greater || cmp == std::cmp::Ordering::Equal + FilterOp::Gte => matches!( + field_val.partial_cmp_coerced(&self.value), + Some(std::cmp::Ordering::Greater | std::cmp::Ordering::Equal) + ), + FilterOp::Lt => { + field_val.partial_cmp_coerced(&self.value) == Some(std::cmp::Ordering::Less) } + FilterOp::Lte => matches!( + field_val.partial_cmp_coerced(&self.value), + Some(std::cmp::Ordering::Less | std::cmp::Ordering::Equal) + ), FilterOp::Contains => { if let (Some(s), Some(pattern)) = (field_val.as_str(), self.value.as_str()) { s.contains(pattern) @@ -228,25 +262,222 @@ impl ScanFilter { Some(v) => v, None => return Ok(false), }; - match self.op { - FilterOp::GtColumn => { - field_val.cmp_coerced(other_val) == std::cmp::Ordering::Greater - } - FilterOp::GteColumn => { - field_val.cmp_coerced(other_val) != std::cmp::Ordering::Less - } - FilterOp::LtColumn => { - field_val.cmp_coerced(other_val) == std::cmp::Ordering::Less - } - FilterOp::LteColumn => { - field_val.cmp_coerced(other_val) != std::cmp::Ordering::Greater - } - FilterOp::EqColumn => field_val.eq_coerced(other_val), - FilterOp::NeColumn => !field_val.eq_coerced(other_val), - _ => false, - } + column_compare(self.op, field_val, other_val) } - _ => false, }) } } + +/// Evaluate a column-vs-column operator over two cells of the same row. +/// Any other operator has no column form and matches nothing. +pub(crate) fn column_compare( + op: FilterOp, + left: &nodedb_types::Value, + right: &nodedb_types::Value, +) -> bool { + use std::cmp::Ordering; + match op { + FilterOp::GtColumn => left.partial_cmp_coerced(right) == Some(Ordering::Greater), + FilterOp::GteColumn => matches!( + left.partial_cmp_coerced(right), + Some(Ordering::Greater | Ordering::Equal) + ), + FilterOp::LtColumn => left.partial_cmp_coerced(right) == Some(Ordering::Less), + FilterOp::LteColumn => matches!( + left.partial_cmp_coerced(right), + Some(Ordering::Less | Ordering::Equal) + ), + FilterOp::EqColumn => left.eq_coerced(right), + FilterOp::NeColumn => !left.eq_coerced(right), + FilterOp::Eq + | FilterOp::Ne + | FilterOp::Gt + | FilterOp::Gte + | FilterOp::Lt + | FilterOp::Lte + | FilterOp::Contains + | FilterOp::Like + | FilterOp::NotLike + | FilterOp::Ilike + | FilterOp::NotIlike + | FilterOp::In + | FilterOp::NotIn + | FilterOp::IsNull + | FilterOp::IsNotNull + | FilterOp::ArrayContains + | FilterOp::ArrayContainsAll + | FilterOp::ArrayOverlap + | FilterOp::MatchAll + | FilterOp::Exists + | FilterOp::NotExists + | FilterOp::Or + | FilterOp::Expr => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_types::Value; + use nodedb_types::datetime::NdbDateTime; + + fn round_trip(filter: &ScanFilter) -> ScanFilter { + let bytes = zerompk::to_msgpack_vec(filter).expect("encode"); + zerompk::from_msgpack(&bytes).expect("decode") + } + + fn compare(op: FilterOp, value: Value) -> ScanFilter { + ScanFilter { + field: "captured_at".into(), + op, + value, + clauses: Vec::new(), + expr: None, + } + } + + #[test] + fn naive_instant_crosses_the_bridge_typed() { + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + let back = round_trip(&compare(FilterOp::Gte, Value::NaiveDateTime(at))); + assert_eq!(back.op, FilterOp::Gte); + assert_eq!(back.field, "captured_at"); + assert_eq!(back.value, Value::NaiveDateTime(at)); + } + + #[test] + fn utc_instant_crosses_the_bridge_typed() { + let at = NdbDateTime::from_micros(-86_400_000_000); + let back = round_trip(&compare(FilterOp::Lt, Value::DateTime(at))); + assert_eq!(back.value, Value::DateTime(at)); + } + + #[test] + fn bytes_cross_the_bridge_as_bytes() { + let back = round_trip(&compare(FilterOp::Eq, Value::Bytes(vec![0, 159, 146, 150]))); + assert_eq!(back.value, Value::Bytes(vec![0, 159, 146, 150])); + } + + #[test] + fn nested_clauses_keep_typed_values() { + let at = NdbDateTime::from_micros(1_000_000); + let filter = ScanFilter { + field: String::new(), + op: FilterOp::Or, + value: Value::Null, + clauses: vec![ + vec![compare(FilterOp::Gt, Value::NaiveDateTime(at))], + vec![compare( + FilterOp::In, + Value::Array(vec![Value::DateTime(at)]), + )], + ], + expr: None, + }; + let back = round_trip(&filter); + assert_eq!(back.op, FilterOp::Or); + assert_eq!(back.clauses[0][0].value, Value::NaiveDateTime(at)); + assert_eq!( + back.clauses[1][0].value, + Value::Array(vec![Value::DateTime(at)]) + ); + } + + #[test] + fn every_operator_name_round_trips_through_the_wire() { + for name in [ + "eq", + "ne", + "gt", + "gte", + "lt", + "lte", + "contains", + "like", + "not_like", + "ilike", + "not_ilike", + "in", + "not_in", + "is_null", + "is_not_null", + "array_contains", + "array_contains_all", + "array_overlap", + "match_all", + "exists", + "not_exists", + "or", + "expr", + "gt_col", + "gte_col", + "lt_col", + "lte_col", + "eq_col", + "ne_col", + ] { + let op = FilterOp::parse(name).expect(name); + let back = round_trip(&compare(op, Value::Null)); + assert_eq!(back.op, op, "{name}"); + assert_eq!(back.op.as_str(), name); + } + } + + /// Msgpack `fixarray` header for `len` elements (`len < 16`). + fn fixarray(len: u8) -> u8 { + 0x90 | len + } + + /// The wire bytes of a filter carrying operator name `op`, built from + /// the same fragments `ScanFilter::write` emits so the layout matches. + fn wire_with_op(op: &str) -> Vec { + let mut buf = vec![fixarray(5)]; + buf.extend(zerompk::to_msgpack_vec(&"tags".to_string()).expect("field")); + buf.extend(zerompk::to_msgpack_vec(&op.to_string()).expect("op")); + buf.extend(zerompk::to_msgpack_vec(&Value::Null).expect("value")); + buf.push(fixarray(0)); + buf.extend(zerompk::to_msgpack_vec(&None::).expect("expr")); + buf + } + + #[test] + fn unknown_operator_fails_the_decode_naming_the_operator() { + let err = + zerompk::from_msgpack::(&wire_with_op("any_in")).expect_err("unknown"); + let text = err.to_string(); + assert!(text.contains("any_in"), "{text}"); + assert!(text.contains("unknown filter operator"), "{text}"); + + let ok: ScanFilter = zerompk::from_msgpack(&wire_with_op("array_overlap")).expect("known"); + assert_eq!(ok.op, FilterOp::ArrayOverlap); + } + + #[test] + fn unknown_operator_inside_a_clause_fails_the_whole_set() { + let mut buf = vec![fixarray(1), fixarray(5)]; + buf.extend(zerompk::to_msgpack_vec(&String::new()).expect("field")); + buf.extend(zerompk::to_msgpack_vec(&"or".to_string()).expect("op")); + buf.extend(zerompk::to_msgpack_vec(&Value::Null).expect("value")); + buf.push(fixarray(1)); + buf.push(fixarray(1)); + buf.extend_from_slice(&wire_with_op("any_in")); + buf.extend(zerompk::to_msgpack_vec(&None::).expect("expr")); + let err = zerompk::from_msgpack::>(&buf).expect_err("unknown"); + assert!(err.to_string().contains("any_in"), "{err}"); + } + + /// The hand-built bytes decode when the operator is known, which pins + /// the layout the malformed cases rely on. + #[test] + fn hand_built_wire_matches_the_codec_layout() { + let filter = ScanFilter { + field: "tags".into(), + op: FilterOp::Eq, + value: Value::Null, + clauses: Vec::new(), + expr: None, + }; + let encoded = zerompk::to_msgpack_vec(&filter).expect("encode"); + assert_eq!(encoded, wire_with_op("eq")); + } +} diff --git a/nodedb-query/src/value_ops.rs b/nodedb-query/src/value_ops.rs index a2e13622d..4d2226e58 100644 --- a/nodedb-query/src/value_ops.rs +++ b/nodedb-query/src/value_ops.rs @@ -30,27 +30,54 @@ pub fn value_to_f64(v: &Value, coerce_bool: bool) -> Option { } } -/// Compare two Values with type coercion. +/// Whether either side is a typed instant, so the pair compares by epoch +/// microseconds through `Value::partial_cmp_coerced` (an ISO string on the +/// other side is parsed). +fn involves_instant(a: &Value, b: &Value) -> bool { + a.as_instant().is_some() || b.as_instant().is_some() +} + +/// Compare two Values with type coercion, for a predicate. /// -/// Tries numeric comparison first (with bool coercion), then falls -/// back to string comparison. -pub fn compare_values(a: &Value, b: &Value) -> Ordering { +/// A typed instant compares by epoch microseconds against another instant +/// or an ISO-8601 string, and against anything else has no order: `None`, +/// so `WHERE at >= 5` over an instant matches nothing. Otherwise numeric +/// comparison first (with bool coercion; `None` for a NaN), then string +/// comparison of the display forms. +pub fn partial_compare_values(a: &Value, b: &Value) -> Option { + if involves_instant(a, b) { + return a.partial_cmp_coerced(b); + } if let (Some(na), Some(nb)) = (value_to_f64(a, true), value_to_f64(b, true)) { - return na.partial_cmp(&nb).unwrap_or(Ordering::Equal); + return na.partial_cmp(&nb); } let sa = value_to_display_string(a); let sb = value_to_display_string(b); - sa.cmp(&sb) + Some(sa.cmp(&sb)) +} + +/// Compare two Values with type coercion, for sorting and extremum +/// selection (ORDER BY, MIN / MAX, GREATEST / LEAST, window frames). +/// +/// [`partial_compare_values`] with an unordered pair placed as `Equal`, so a +/// stable sort keeps such rows in their input order. A predicate uses +/// `partial_compare_values` instead. +pub fn compare_values(a: &Value, b: &Value) -> Ordering { + partial_compare_values(a, b).unwrap_or(Ordering::Equal) } /// Check equality with type coercion. /// -/// Handles `"5" == 5` by coercing both sides to f64 when one is a -/// number and the other is a numeric string. +/// A typed instant equals another instant or an ISO-8601 string with the +/// same epoch microseconds. Handles `"5" == 5` by coercing both sides to +/// f64 when one is a number and the other is a numeric string. pub fn coerced_eq(a: &Value, b: &Value) -> bool { if a == b { return true; } + if involves_instant(a, b) { + return a.eq_coerced(b); + } if let (Some(af), Some(bf)) = (value_to_f64(a, true), value_to_f64(b, true)) { return (af - bf).abs() < f64::EPSILON; } @@ -138,6 +165,38 @@ mod tests { ); } + #[test] + fn instants_compare_by_micros_against_instants_and_iso_strings() { + let earlier = Value::NaiveDateTime(nodedb_types::NdbDateTime::from_micros( + 1_583_402_400_000_000, + )); + let later = Value::DateTime(nodedb_types::NdbDateTime::from_micros( + 1_583_406_000_000_000, + )); + assert_eq!(compare_values(&earlier, &later), Ordering::Less); + assert_eq!( + compare_values(&later, &Value::String("2020-03-05 10:00:00".into())), + Ordering::Greater + ); + assert!(coerced_eq( + &earlier, + &Value::String("2020-03-05T10:00:00Z".into()) + )); + assert!(!coerced_eq(&earlier, &later)); + } + + /// An instant against a bare integer has no order on the predicate path + /// and sorts as equal on the extremum path. + #[test] + fn an_instant_against_an_integer_is_unordered_for_predicates() { + let at = Value::NaiveDateTime(nodedb_types::NdbDateTime::from_micros( + 1_583_402_400_000_000, + )); + assert_eq!(partial_compare_values(&at, &Value::Integer(5)), None); + assert_eq!(partial_compare_values(&Value::Integer(5), &at), None); + assert_eq!(compare_values(&at, &Value::Integer(5)), Ordering::Equal); + } + #[test] fn truthiness() { assert!(is_truthy(&Value::Bool(true))); diff --git a/nodedb-sql/src/engine_rules/array.rs b/nodedb-sql/src/engine_rules/array.rs index 1e9b7e8f7..523a9c3cd 100644 --- a/nodedb-sql/src/engine_rules/array.rs +++ b/nodedb-sql/src/engine_rules/array.rs @@ -96,7 +96,7 @@ mod tests { collection: "g".into(), columns: vec![], rows: vec![], - column_defaults: vec![], + volatile_defaults: false, if_absent: false, column_schema: vec![], primary_key: None, @@ -115,7 +115,7 @@ mod tests { collection: "g".into(), columns: vec![], rows: vec![], - column_defaults: vec![], + volatile_defaults: false, on_conflict_updates: vec![], column_schema: vec![], primary_key: None, diff --git a/nodedb-sql/src/engine_rules/columnar.rs b/nodedb-sql/src/engine_rules/columnar.rs index 7decfbd69..e6939dcb6 100644 --- a/nodedb-sql/src/engine_rules/columnar.rs +++ b/nodedb-sql/src/engine_rules/columnar.rs @@ -15,7 +15,7 @@ impl EngineRules for ColumnarRules { engine: EngineType::Columnar, route: WriteRoute::ColumnarFamily, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, if_absent: p.if_absent, column_schema: p.column_schema, primary_key: p.primary_key, @@ -32,7 +32,7 @@ impl EngineRules for ColumnarRules { engine: EngineType::Columnar, route: WriteRoute::ColumnarFamily, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, on_conflict_updates: p.on_conflict_updates, column_schema: p.column_schema, primary_key: p.primary_key, diff --git a/nodedb-sql/src/engine_rules/document_schemaless.rs b/nodedb-sql/src/engine_rules/document_schemaless.rs index 17937962f..bdad8b51f 100644 --- a/nodedb-sql/src/engine_rules/document_schemaless.rs +++ b/nodedb-sql/src/engine_rules/document_schemaless.rs @@ -15,7 +15,7 @@ impl EngineRules for SchemalessRules { engine: EngineType::DocumentSchemaless, route: WriteRoute::Document, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, if_absent: p.if_absent, column_schema: vec![], primary_key: p.primary_key, @@ -28,7 +28,7 @@ impl EngineRules for SchemalessRules { engine: EngineType::DocumentSchemaless, route: WriteRoute::Document, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, on_conflict_updates: p.on_conflict_updates, column_schema: vec![], primary_key: p.primary_key, diff --git a/nodedb-sql/src/engine_rules/document_strict.rs b/nodedb-sql/src/engine_rules/document_strict.rs index 0aaca86ac..3e8d25327 100644 --- a/nodedb-sql/src/engine_rules/document_strict.rs +++ b/nodedb-sql/src/engine_rules/document_strict.rs @@ -15,7 +15,7 @@ impl EngineRules for StrictRules { engine: EngineType::DocumentStrict, route: WriteRoute::Document, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, if_absent: p.if_absent, column_schema: vec![], primary_key: p.primary_key, @@ -28,7 +28,7 @@ impl EngineRules for StrictRules { engine: EngineType::DocumentStrict, route: WriteRoute::Document, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, on_conflict_updates: p.on_conflict_updates, column_schema: vec![], primary_key: p.primary_key, diff --git a/nodedb-sql/src/engine_rules/params.rs b/nodedb-sql/src/engine_rules/params.rs index 6483fe75b..e4e98a890 100644 --- a/nodedb-sql/src/engine_rules/params.rs +++ b/nodedb-sql/src/engine_rules/params.rs @@ -8,8 +8,13 @@ use crate::types::*; pub struct InsertParams { pub collection: String, pub columns: Vec, + /// Every declared DEFAULT already materialized and every literal already + /// coerced to its declared column type. An engine stores these as they + /// are; none re-reads the catalog's DEFAULT text. pub rows: Vec>, - pub column_defaults: Vec<(String, String)>, + /// Whether a DEFAULT materialized into `rows` was volatile. A plan that + /// carries one is never admitted to the plan cache. + pub volatile_defaults: bool, /// `ON CONFLICT DO NOTHING` semantics: duplicate-PK rows are skipped /// silently. `false` for plain `INSERT` (raises `unique_violation`). pub if_absent: bool, @@ -109,8 +114,11 @@ pub struct MergeParams { pub struct UpsertParams { pub collection: String, pub columns: Vec, + /// Defaults materialized and literals coerced, as in + /// `InsertParams::rows`. pub rows: Vec>, - pub column_defaults: Vec<(String, String)>, + /// Mirrors `InsertParams::volatile_defaults`. + pub volatile_defaults: bool, /// `ON CONFLICT (...) DO UPDATE SET` assignments. Empty for plain /// `UPSERT INTO ...`; populated when the caller is /// `INSERT ... ON CONFLICT ... DO UPDATE SET`. diff --git a/nodedb-sql/src/engine_rules/spatial.rs b/nodedb-sql/src/engine_rules/spatial.rs index afe708e52..6373f11e9 100644 --- a/nodedb-sql/src/engine_rules/spatial.rs +++ b/nodedb-sql/src/engine_rules/spatial.rs @@ -15,7 +15,7 @@ impl EngineRules for SpatialRules { engine: EngineType::Spatial, route: WriteRoute::ColumnarFamily, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, if_absent: p.if_absent, column_schema: p.column_schema, primary_key: p.primary_key, @@ -31,7 +31,7 @@ impl EngineRules for SpatialRules { engine: EngineType::Spatial, route: WriteRoute::ColumnarFamily, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, on_conflict_updates: p.on_conflict_updates, column_schema: p.column_schema, primary_key: p.primary_key, diff --git a/nodedb-sql/src/engine_rules/timeseries.rs b/nodedb-sql/src/engine_rules/timeseries.rs index ab6ab210d..4c4122bed 100644 --- a/nodedb-sql/src/engine_rules/timeseries.rs +++ b/nodedb-sql/src/engine_rules/timeseries.rs @@ -18,6 +18,7 @@ impl EngineRules for TimeseriesRules { Ok(vec![SqlPlan::TimeseriesIngest { collection: p.collection, rows: p.rows, + volatile_defaults: p.volatile_defaults, }]) } diff --git a/nodedb-sql/src/planner/declared_type_coerce.rs b/nodedb-sql/src/planner/declared_type_coerce.rs index cffdad297..d098ab358 100644 --- a/nodedb-sql/src/planner/declared_type_coerce.rs +++ b/nodedb-sql/src/planner/declared_type_coerce.rs @@ -31,26 +31,64 @@ //! //! # Scope //! -//! Only [`SqlDataType::Int64`] and [`SqlDataType::Float64`] columns are -//! coerced, and both are coerced symmetrically — this is not a float special -//! case. They are the two declared types whose stored representation is -//! decided by the declaration rather than by the value: nodedb keeps every -//! integer as an `i64` and every float as an `f64`, and the declared width -//! that drives the wire OID (see `ColumnInfo::int_width` / -//! `ColumnInfo::float_width`) is only honest if the stored cell is a number of -//! that family to begin with. Every other declared type either has one -//! unambiguous literal form already or (like `DECIMAL`) is deliberately -//! carried as text for exactness, and passes through untouched. +//! [`SqlDataType::Int64`] and [`SqlDataType::Float64`] columns are coerced +//! symmetrically — this is not a float special case. They are two declared +//! types whose stored representation is decided by the declaration rather +//! than by the value: nodedb keeps every integer as an `i64` and every float +//! as an `f64`, and the declared width that drives the wire OID (see +//! `ColumnInfo::int_width` / `ColumnInfo::float_width`) is only honest if the +//! stored cell is a number of that family to begin with. //! -//! The conversions mirror the strict document encoder's `coerce_value`, so the -//! two engines accept and reject exactly the same literals for a given -//! declared type. +//! [`SqlDataType::Timestamp`] and [`SqlDataType::Timestamptz`] columns are +//! coerced to a typed instant, once, here, for every engine. An integer time +//! literal carries no unit of its own, and the engines that re-type on write +//! disagree on one (the strict encoder reads an integer as microseconds, the +//! columnar family as milliseconds), while the engines that store the +//! planner's value verbatim would persist the bare integer and leave the read +//! side to guess. Resolving the literal here fixes the unit in one place: +//! SQL reads an integer time literal as epoch milliseconds, the unit every +//! engine already reads it as through its own ingest paths. A text literal is +//! parsed here for the same reason, so an unparseable spelling is refused at +//! the statement instead of being stored as text that no read can render. +//! +//! Every other declared type either has one unambiguous literal form already +//! or (like `DECIMAL`) is deliberately carried as text for exactness, and +//! passes through untouched. +//! +//! The numeric conversions mirror the strict document encoder's +//! `coerce_value`, so the two engines accept and reject exactly the same +//! literals for a given declared type. +use nodedb_types::datetime::NdbDateTime; use rust_decimal::prelude::ToPrimitive; +use super::dml_helpers::{check_declared_float_ranges, check_declared_int_ranges}; use crate::error::{Result, SqlError}; use crate::types::{ColumnInfo, SqlDataType, SqlExpr, SqlValue}; +/// Coerce one literal bound for `column` by the write-side rule, then +/// range-check it against the column's declared width. +/// +/// The one entry a caller outside the planner uses for a literal that will +/// reach storage under `column`: a DDL gate checks a column `DEFAULT` through +/// it, so a default is accepted or refused exactly as the same literal in a +/// `VALUES` clause is. The primary-key column keeps its literal as written, +/// the same exemption every `VALUES` and `SET` path carries — see +/// [`coerce_rows_to_declared_types`]. +/// +/// Errors name the column and the literal. +pub fn coerce_write_literal(column: &ColumnInfo, value: SqlValue) -> Result { + if column.is_primary_key { + return Ok(value); + } + let coerced = coerce_value(&column.name, value, &column.data_type)?; + let row = [vec![(column.name.clone(), coerced)]]; + check_declared_int_ranges(std::slice::from_ref(column), &row)?; + check_declared_float_ranges(std::slice::from_ref(column), &row)?; + let [mut row] = row; + Ok(row.swap_remove(0).1) +} + /// Coerce every value in one `(column, value)` row to its declared column /// type, in place. /// @@ -142,15 +180,25 @@ fn is_exempt(exempt_column: Option<&str>, column: &str) -> bool { /// Coerce one literal to `declared`, returning it unchanged when the declared /// type imposes no representation of its own. -fn coerce_value(column: &str, value: SqlValue, declared: &SqlDataType) -> Result { +/// +/// The one rule for a literal bound for a declared column, on the write side +/// (`VALUES`, `SET`) and on the read side (`predicate_coerce`, for a literal +/// compared against the column), so a row is found by the same literal that +/// stored it. +pub(crate) fn coerce_value( + column: &str, + value: SqlValue, + declared: &SqlDataType, +) -> Result { match declared { SqlDataType::Int64 => coerce_to_int(column, value), SqlDataType::Float64 => coerce_to_float(column, value), + SqlDataType::Timestamp | SqlDataType::Timestamptz => { + coerce_to_instant(column, value, declared) + } SqlDataType::String | SqlDataType::Bool | SqlDataType::Bytes - | SqlDataType::Timestamp - | SqlDataType::Timestamptz | SqlDataType::Decimal | SqlDataType::Uuid | SqlDataType::Vector(_) @@ -205,6 +253,98 @@ fn coerce_to_float(column: &str, value: SqlValue) -> Result { } } +/// Timestamp column: every accepted literal becomes the one typed instant +/// the column stores, tagged as the declared kind. +/// +/// A typed literal keeps its instant and takes the declared tag, so a +/// `TIMESTAMP` column holds `SqlValue::Timestamp` and a `TIMESTAMPTZ` column +/// holds `SqlValue::Timestamptz` whichever spelling wrote it. Text is parsed +/// as ISO-8601 (`Z`, a `±HH:MM` offset, and fractional seconds are all +/// accepted). A numeric literal is epoch milliseconds — the unit every engine +/// reads an integer time literal as — and a fractional one contributes its +/// integer part, mirroring the columnar engine. `NULL` is left alone: +/// nullability is enforced elsewhere. Any other literal kind is refused +/// naming the column and the kind, because no instant can be read from it. +fn coerce_to_instant(column: &str, value: SqlValue, declared: &SqlDataType) -> Result { + let declared_name = instant_declared_name(declared); + let tag = |at: NdbDateTime| match declared { + SqlDataType::Timestamptz => SqlValue::Timestamptz(at), + SqlDataType::Int64 + | SqlDataType::Float64 + | SqlDataType::String + | SqlDataType::Bool + | SqlDataType::Bytes + | SqlDataType::Timestamp + | SqlDataType::Decimal + | SqlDataType::Uuid + | SqlDataType::Vector(_) + | SqlDataType::Geometry + | SqlDataType::Unknown => SqlValue::Timestamp(at), + }; + match value { + SqlValue::Null => Ok(SqlValue::Null), + SqlValue::Timestamp(at) | SqlValue::Timestamptz(at) => Ok(tag(at)), + SqlValue::String(s) => NdbDateTime::parse(&s) + .map(tag) + .ok_or_else(|| not_representable(column, &s, declared_name)), + SqlValue::Int(millis) => NdbDateTime::from_millis(millis) + .map(tag) + .map_err(|_| not_representable(column, &millis.to_string(), declared_name)), + SqlValue::Float(f) => whole_f64(f.trunc()) + .and_then(|millis| NdbDateTime::from_millis(millis).ok()) + .map(tag) + .ok_or_else(|| not_representable(column, &f.to_string(), declared_name)), + SqlValue::Decimal(d) => d + .trunc() + .to_i64() + .and_then(|millis| NdbDateTime::from_millis(millis).ok()) + .map(tag) + .ok_or_else(|| not_representable(column, &d.to_string(), declared_name)), + SqlValue::Bool(_) | SqlValue::Bytes(_) | SqlValue::Array(_) => { + Err(SqlError::TypeMismatch { + detail: format!( + "column '{column}': {} is not representable as {declared_name}", + literal_kind(&value) + ), + }) + } + } +} + +/// The declared type name an instant coercion error reports. +fn instant_declared_name(declared: &SqlDataType) -> &'static str { + match declared { + SqlDataType::Timestamptz => "TIMESTAMPTZ", + SqlDataType::Int64 + | SqlDataType::Float64 + | SqlDataType::String + | SqlDataType::Bool + | SqlDataType::Bytes + | SqlDataType::Timestamp + | SqlDataType::Decimal + | SqlDataType::Uuid + | SqlDataType::Vector(_) + | SqlDataType::Geometry + | SqlDataType::Unknown => "TIMESTAMP", + } +} + +/// The article-prefixed kind name an error names a refused literal by. +fn literal_kind(value: &SqlValue) -> &'static str { + match value { + SqlValue::Null => "null", + SqlValue::Bool(_) => "a boolean", + SqlValue::Int(_) => "an integer", + SqlValue::Float(_) => "a float", + SqlValue::Decimal(_) => "a decimal", + SqlValue::String(_) => "text", + SqlValue::Bytes(_) => "bytes", + SqlValue::Array(_) => "an array", + SqlValue::Timestamp(_) => "a timestamp", + SqlValue::Timestamptz(_) => "a timestamptz", + } +} + /// `Some(n)` when `f` is a whole number inside `i64`'s range. fn whole_f64(f: f64) -> Option { (f.fract() == 0.0 && f >= i64::MIN as f64 && f <= i64::MAX as f64).then_some(f as i64) @@ -212,7 +352,7 @@ fn whole_f64(f: f64) -> Option { fn not_representable(column: &str, value: &str, declared: &str) -> SqlError { SqlError::TypeMismatch { - detail: format!("column '{column}': cannot store '{value}' as {declared}"), + detail: format!("column '{column}': '{value}' is not representable as {declared}"), } } @@ -361,6 +501,152 @@ mod tests { ); } + /// `2020-03-05T10:00:00Z` as microseconds since the Unix epoch. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + fn early() -> NdbDateTime { + NdbDateTime::from_micros(EARLY_MICROS) + } + + /// A typed literal keeps its instant and takes the declared tag, in both + /// directions. + #[test] + fn typed_instant_literals_are_retagged_to_the_declared_kind() { + let columns = [ + column("at", SqlDataType::Timestamp), + column("at_tz", SqlDataType::Timestamptz), + ]; + assert_eq!( + coerced(&columns, "at", SqlValue::Timestamptz(early())) + .expect("a typed instant fits a TIMESTAMP column"), + SqlValue::Timestamp(early()) + ); + assert_eq!( + coerced(&columns, "at_tz", SqlValue::Timestamp(early())) + .expect("a typed instant fits a TIMESTAMPTZ column"), + SqlValue::Timestamptz(early()) + ); + } + + /// Text is parsed as ISO-8601: a `Z` suffix, an offset, and fractional + /// seconds all resolve to the one instant, and the offset shifts it. + #[test] + fn text_literals_are_parsed_to_the_instant_they_spell() { + let columns = [column("at", SqlDataType::Timestamp)]; + for spelling in [ + "2020-03-05 10:00:00", + "2020-03-05T10:00:00Z", + "2020-03-05T10:00:00.000000Z", + "2020-03-05T15:30:00+05:30", + ] { + assert_eq!( + coerced(&columns, "at", SqlValue::String(spelling.into())) + .unwrap_or_else(|e| panic!("{spelling} parses: {e}")), + SqlValue::Timestamp(early()), + "{spelling}" + ); + } + } + + /// Text that spells no instant is refused, and the error names the + /// column and the literal. + #[test] + fn non_datetime_text_is_refused_naming_the_column_and_literal() { + let columns = [column("created_at", SqlDataType::Timestamp)]; + let err = coerced( + &columns, + "created_at", + SqlValue::String("not a date".into()), + ) + .expect_err("non-datetime text must be refused"); + let detail = err.to_string(); + assert!( + detail.contains("created_at") && detail.contains("not a date"), + "error must name the column and the literal: {detail}" + ); + } + + /// An integer literal is epoch milliseconds; a fractional literal + /// contributes its integer part. + #[test] + fn numeric_literals_are_epoch_milliseconds() { + let columns = [ + column("at", SqlDataType::Timestamp), + column("at_tz", SqlDataType::Timestamptz), + ]; + assert_eq!( + coerced(&columns, "at", SqlValue::Int(1_583_402_400_000)) + .expect("an integer is epoch milliseconds"), + SqlValue::Timestamp(early()) + ); + assert_eq!( + coerced(&columns, "at_tz", SqlValue::Int(1_583_402_400_000)) + .expect("an integer is epoch milliseconds"), + SqlValue::Timestamptz(early()) + ); + assert_eq!( + coerced(&columns, "at", SqlValue::Float(1_583_402_400_000.7)) + .expect("a float contributes its integer part"), + SqlValue::Timestamp(early()) + ); + assert_eq!( + coerced(&columns, "at", decimal("1583402400000.9")) + .expect("a decimal contributes its integer part"), + SqlValue::Timestamp(early()) + ); + } + + /// A numeric literal whose milliseconds overflow the instant's range, or + /// that is not a finite number, is refused rather than wrapped. + #[test] + fn out_of_range_numeric_literals_are_refused() { + let columns = [column("at", SqlDataType::Timestamp)]; + assert!(coerced(&columns, "at", SqlValue::Int(i64::MAX)).is_err()); + assert!(coerced(&columns, "at", SqlValue::Float(f64::NAN)).is_err()); + assert!(coerced(&columns, "at", SqlValue::Float(f64::INFINITY)).is_err()); + assert!(coerced(&columns, "at", decimal("99999999999999999999999")).is_err()); + } + + /// NULL is untouched, and every literal kind that carries no instant is + /// refused with an error naming the column and the kind. + #[test] + fn null_passes_and_non_instant_kinds_are_refused() { + let columns = [column("created_at", SqlDataType::Timestamptz)]; + assert_eq!( + coerced(&columns, "created_at", SqlValue::Null).expect("null is not an error"), + SqlValue::Null + ); + for (literal, kind) in [ + (SqlValue::Bool(true), "a boolean"), + (SqlValue::Bytes(vec![1, 2]), "bytes"), + (SqlValue::Array(vec![SqlValue::Int(1)]), "an array"), + ] { + let err = coerced(&columns, "created_at", literal) + .expect_err("a literal with no instant must be refused"); + let detail = err.to_string(); + assert!( + detail.contains("created_at") && detail.contains(kind), + "error must name the column and the kind: {detail}" + ); + } + } + + /// `SET at = ` runs the same instant coercion as `VALUES`. + #[test] + fn assignments_coerce_instants() { + let columns = [column("at", SqlDataType::Timestamp)]; + let mut assignments = vec![( + "at".to_string(), + SqlExpr::Literal(SqlValue::Int(1_583_402_400_000)), + )]; + coerce_assignments_to_declared_types(&columns, &mut assignments, None) + .expect("an integer assignment is epoch milliseconds"); + match &assignments[0].1 { + SqlExpr::Literal(value) => assert_eq!(*value, SqlValue::Timestamp(early())), + other => panic!("expected a literal assignment, got {other:?}"), + } + } + /// The primary key keeps its literal exactly as written even when its /// declared type would otherwise re-type it: the engines derive a row's /// identity from the literal's own rendering on both the write and the @@ -407,6 +693,48 @@ mod tests { assert_eq!(literal(&assignments[1].1), SqlValue::Float(1.5)); } + /// The public write-side entry coerces, then range-checks: a numeric + /// literal on a TIMESTAMP column becomes the instant, text that spells + /// no instant is refused naming the column, and an integer past the + /// declared SMALLINT width is refused naming the column. + #[test] + fn write_literal_coerces_then_range_checks() { + let at = column("at", SqlDataType::Timestamp); + assert_eq!( + coerce_write_literal(&at, SqlValue::Int(1_583_402_400_000)) + .expect("an integer is epoch milliseconds"), + SqlValue::Timestamp(early()) + ); + let err = coerce_write_literal(&at, SqlValue::String("not a date".into())) + .expect_err("text with no instant is refused"); + assert!(err.to_string().contains("'at'"), "{err}"); + + let mut small = column("s", SqlDataType::Int64); + small.int_width = Some(nodedb_types::columnar::IntWidth::I16); + assert_eq!( + coerce_write_literal(&small, SqlValue::Int(7)).expect("7 fits SMALLINT"), + SqlValue::Int(7) + ); + let err = coerce_write_literal(&small, SqlValue::Int(999_999)) + .expect_err("999999 does not fit SMALLINT"); + assert!( + matches!(err, SqlError::IntegerOutOfRange { ref column, .. } if column == "s"), + "{err}" + ); + } + + /// The write-side entry keeps the primary key as written, like every + /// `VALUES` and `SET` path. + #[test] + fn write_literal_exempts_the_primary_key() { + let mut id = column("id", SqlDataType::Int64); + id.is_primary_key = true; + assert_eq!( + coerce_write_literal(&id, decimal("5.0")).expect("an exempt key is never re-typed"), + decimal("5.0") + ); + } + /// A column the catalog does not declare (the KV `key`/`value` /// convention, a `ttl` pseudo-column) is left exactly as written. #[test] diff --git a/nodedb-sql/src/planner/defaults/compiled.rs b/nodedb-sql/src/planner/defaults/compiled.rs index 9905e3e23..69e25e460 100644 --- a/nodedb-sql/src/planner/defaults/compiled.rs +++ b/nodedb-sql/src/planner/defaults/compiled.rs @@ -63,6 +63,19 @@ impl CompiledDefault { &self.column } + /// The constant this DEFAULT spells, when it is a bare literal. + /// + /// A generator or a parsed expression has no value until evaluation, so + /// it yields `None`. The DDL gate checks a literal against the declared + /// column type through this, so a DEFAULT the column cannot hold is + /// refused where it is declared. + pub fn literal(&self) -> Option<&nodedb_types::Value> { + match &self.kind { + DefaultKind::Literal(value) => Some(value), + DefaultKind::Generator(_) | DefaultKind::Expr(_) => None, + } + } + /// Whether this DEFAULT produces a fresh value on every evaluation. /// /// A plan carrying one is never admitted to the plan cache, or the cache diff --git a/nodedb-sql/src/planner/defaults/mod.rs b/nodedb-sql/src/planner/defaults/mod.rs index 98b05a51b..2e19ca84c 100644 --- a/nodedb-sql/src/planner/defaults/mod.rs +++ b/nodedb-sql/src/planner/defaults/mod.rs @@ -14,13 +14,10 @@ //! The column is never omitted: an omitted column stores NULL where the //! declaration promised a value, and nothing reports it. //! -//! Lives in the SQL crate rather than beside one engine's converter because -//! every engine that materializes a DEFAULT has to produce the SAME value for -//! the same expression — a `DEFAULT now()` that means one thing on a document -//! collection and another on a key-value one would be a difference nobody -//! declared. The key-value planner also needs it BEFORE its declared-type -//! coercion and range checks run, so a materialized default is validated -//! exactly like a supplied one. +//! Lives in the planner because every engine's rows materialize their +//! DEFAULTs there, before declared-type coercion and range checks run, so a +//! materialized default is validated exactly like a supplied one and a +//! `DEFAULT now()` means the same thing on every engine. mod compiled; mod convert; diff --git a/nodedb-sql/src/planner/dml.rs b/nodedb-sql/src/planner/dml.rs deleted file mode 100644 index 350e078fd..000000000 --- a/nodedb-sql/src/planner/dml.rs +++ /dev/null @@ -1,422 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! INSERT, UPDATE, DELETE planning. - -use nodedb_types::DatabaseId; -use sqlparser::ast::{self}; - -use super::dml_helpers::{ - KvInsertParams, bind_insert_select_columns, build_kv_insert_plan, - build_vector_primary_insert_plan, check_declared_float_ranges_in_assignments, - check_declared_int_ranges_in_assignments, coerce_and_check_rows, convert_value_rows, - materialize_defaults_in_rows, resolve_insert_columns, -}; -use crate::engine_rules::{self, InsertParams}; -use crate::error::{Result, SqlError}; -use crate::parser::normalize::{normalize_insert_column, normalize_object_name_checked}; -use crate::planner::declared_type_coerce::coerce_assignments_to_declared_types; -use crate::resolver::ColumnScope; -use crate::resolver::columns::{ResolvedTable, TableScope}; -use crate::resolver::expr::convert_expr; -use crate::types::*; - -pub use dml_update_delete::{plan_delete, plan_truncate_stmt, plan_update}; - -#[path = "dml_update_delete.rs"] -mod dml_update_delete; - -/// The column namespace of an INSERT target. -fn target_scope(table_name: &str, info: &CollectionInfo) -> Result { - let mut scope = TableScope::single(ResolvedTable { - name: table_name.to_string(), - alias: None, - info: info.clone(), - })?; - // `ON CONFLICT DO UPDATE` addresses the proposed row as `excluded`. It - // carries the target's columns and is qualified-only, so a bare name in - // the SET clause names the stored row. - scope.add_qualified_only(ResolvedTable { - name: EXCLUDED_RELATION.to_string(), - alias: None, - info: info.clone(), - })?; - Ok(scope) -} - -/// The pseudo-relation `ON CONFLICT DO UPDATE` uses for the proposed row. -const EXCLUDED_RELATION: &str = "excluded"; - -/// Normalize an INSERT column list and reject a name the target does not have. -fn insert_columns(columns: &[ast::ObjectName], scope: &TableScope) -> Result> { - columns - .iter() - .map(|c| { - let col = normalize_insert_column(c)?; - scope.check_name(None, &col)?; - Ok(col) - }) - .collect() -} - -/// Classification of an `ON CONFLICT` clause attached to an INSERT. -enum OnConflict { - /// No `ON CONFLICT` clause — plain INSERT (error on duplicate PK). - None, - /// `ON CONFLICT DO NOTHING` — skip rows that would conflict, no error. - DoNothing, - /// `ON CONFLICT (...) DO UPDATE SET ...` — apply the assignments against - /// the existing row on conflict. - DoUpdate(Vec<(String, SqlExpr)>), -} - -fn classify_on_conflict(ins: &ast::Insert, scope: &TableScope) -> Result { - let Some(on) = ins.on.as_ref() else { - return Ok(OnConflict::None); - }; - let ast::OnInsert::OnConflict(oc) = on else { - return Ok(OnConflict::None); - }; - match &oc.action { - ast::OnConflictAction::DoNothing => Ok(OnConflict::DoNothing), - ast::OnConflictAction::DoUpdate(do_update) => { - let mut pairs = Vec::with_capacity(do_update.assignments.len()); - for a in &do_update.assignments { - let name = match &a.target { - ast::AssignmentTarget::ColumnName(obj) => normalize_object_name_checked(obj)?, - _ => { - return Err(SqlError::Unsupported { - detail: "ON CONFLICT DO UPDATE SET target must be a column name".into(), - }); - } - }; - scope.check_name(None, &name)?; - let expr = convert_expr(&a.value, &ColumnScope::Relations(scope))?; - pairs.push((name, expr)); - } - Ok(OnConflict::DoUpdate(pairs)) - } - } -} - -/// Plan an INSERT statement. -pub fn plan_insert(ins: &ast::Insert, catalog: &dyn SqlCatalog) -> Result> { - let table_name = match &ins.table { - ast::TableObject::TableName(name) => normalize_object_name_checked(name)?, - ast::TableObject::TableFunction(_) => { - return Err(SqlError::Unsupported { - detail: "INSERT INTO table function not supported".into(), - }); - } - // Oracle's `INSERT INTO (SELECT ...)`: the target is a subquery, so - // there is no collection to resolve or route to an engine. - ast::TableObject::TableQuery(_) => { - return Err(SqlError::Unsupported { - detail: "INSERT INTO a subquery target is not supported".into(), - }); - } - }; - let info = catalog - .get_collection(DatabaseId::DEFAULT, &table_name)? - .ok_or_else(|| SqlError::UnknownTable { - name: table_name.clone(), - })?; - let target_scope = target_scope(&table_name, &info)?; - - // `INSERT ... ON CONFLICT DO UPDATE SET` reroutes to the upsert path - // with the assignments carried through. `DO NOTHING` stays on the - // INSERT path with `if_absent=true`. - let if_absent = match classify_on_conflict(ins, &target_scope)? { - OnConflict::None => false, - OnConflict::DoNothing => true, - OnConflict::DoUpdate(updates) => { - return plan_upsert_with_on_conflict(ins, catalog, updates); - } - }; - - let columns = insert_columns(&ins.columns, &target_scope)?; - - // Check for INSERT...SELECT. - if let Some(source) = &ins.source - && let ast::SetExpr::Select(select) = &*source.body - { - let column_map = bind_insert_select_columns(catalog, &columns, select, &info)?; - let source_plan = super::select::plan_query( - source, - catalog, - &crate::functions::registry::FunctionRegistry::new(), - crate::TemporalScope::default(), - )?; - return Ok(vec![SqlPlan::InsertSelect { - target: table_name, - source: Box::new(source_plan), - limit: 0, - column_map, - }]); - } - - // VALUES clause. - let source = ins.source.as_ref().ok_or_else(|| SqlError::Parse { - detail: "INSERT requires VALUES or SELECT".into(), - })?; - - let rows_ast = match &*source.body { - ast::SetExpr::Values(values) => &values.rows, - _ => { - return Err(SqlError::Unsupported { - detail: "INSERT source must be VALUES or SELECT".into(), - }); - } - }; - - // KV engine: key and value are fundamentally separate — handle directly. - // Positional column binding (below) does not apply here: the KV path - // matches columns by name against `pk_col`/`"key"`/`"ttl"`, which is - // orthogonal to declared column order. - if info.engine == EngineType::KeyValue { - let intent = if if_absent { - KvInsertIntent::InsertIfAbsent - } else { - KvInsertIntent::Insert - }; - return build_kv_insert_plan(KvInsertParams { - collection: table_name, - columns: &columns, - rows_ast, - intent, - on_conflict_updates: Vec::new(), - pk_col: info.primary_key.as_deref(), - declared_columns: &info.columns, - catalog, - }); - } - - // Positional INSERT (no column list): bind values to the collection's - // declared column order so named projections/predicates can find them. - // No-op for named inserts and schemaless collections. - let columns = resolve_insert_columns(columns, &info, rows_ast)?; - - // Vector-primary collection: bypass document encoding. - // - // The vector path never reaches `EngineRules::plan_insert`, which is where - // every other engine hands its declared DEFAULTs on for expansion. It - // materializes them here instead, through the same helper the key-value - // path uses, and before coercion so a default is range-checked exactly like - // a supplied literal. - if info.primary == nodedb_types::PrimaryEngine::Vector - && let Some(ref vpc) = info.vector_primary - { - let mut rows_parsed = convert_value_rows(&columns, rows_ast)?; - let volatile_defaults = - materialize_defaults_in_rows(&info.columns, &mut rows_parsed, catalog)?; - coerce_and_check_rows(&info, &mut rows_parsed)?; - return build_vector_primary_insert_plan( - &table_name, - vpc, - &columns, - rows_parsed, - volatile_defaults, - ); - } - - // All other engines: delegate to engine rules. - let mut rows = convert_value_rows(&columns, rows_ast)?; - coerce_and_check_rows(&info, &mut rows)?; - let column_defaults: Vec<(String, String)> = info - .columns - .iter() - .filter_map(|c| c.default.as_ref().map(|d| (c.name.clone(), d.clone()))) - .collect(); - let column_schema: Vec<(String, String)> = info - .columns - .iter() - .filter_map(|c| c.raw_type.as_ref().map(|t| (c.name.clone(), t.clone()))) - .collect(); - let rules = engine_rules::resolve_engine_rules(info.engine); - rules.plan_insert(InsertParams { - collection: table_name, - columns, - rows, - column_defaults, - if_absent, - column_schema, - primary_key: info.primary_key.clone(), - }) -} - -/// Plan an UPSERT statement (pre-processed from `UPSERT INTO` to `INSERT INTO`). -/// -/// Same parsing as INSERT but routes through `engine_rules.plan_upsert()`. -pub fn plan_upsert(ins: &ast::Insert, catalog: &dyn SqlCatalog) -> Result> { - let table_name = match &ins.table { - ast::TableObject::TableName(name) => normalize_object_name_checked(name)?, - ast::TableObject::TableFunction(_) => { - return Err(SqlError::Unsupported { - detail: "UPSERT INTO table function not supported".into(), - }); - } - // Oracle's `INSERT INTO (SELECT ...)`: the target is a subquery, so - // there is no collection to resolve or route to an engine. - ast::TableObject::TableQuery(_) => { - return Err(SqlError::Unsupported { - detail: "UPSERT INTO a subquery target is not supported".into(), - }); - } - }; - let info = catalog - .get_collection(DatabaseId::DEFAULT, &table_name)? - .ok_or_else(|| SqlError::UnknownTable { - name: table_name.clone(), - })?; - - let columns = insert_columns(&ins.columns, &target_scope(&table_name, &info)?)?; - - let source = ins.source.as_ref().ok_or_else(|| SqlError::Parse { - detail: "UPSERT requires VALUES".into(), - })?; - - let rows_ast = match &*source.body { - ast::SetExpr::Values(values) => &values.rows, - _ => { - return Err(SqlError::Unsupported { - detail: "UPSERT source must be VALUES".into(), - }); - } - }; - - // KV: upsert is just a PUT (natural overwrite). Positional column - // binding (below) does not apply here — see `plan_insert`. - if info.engine == EngineType::KeyValue { - return build_kv_insert_plan(KvInsertParams { - collection: table_name, - columns: &columns, - rows_ast, - intent: KvInsertIntent::Put, - on_conflict_updates: Vec::new(), - pk_col: info.primary_key.as_deref(), - declared_columns: &info.columns, - catalog, - }); - } - - // Positional UPSERT (no column list): bind to the collection's declared - // column order — see `plan_insert` for the full rationale. - let columns = resolve_insert_columns(columns, &info, rows_ast)?; - - let mut rows = convert_value_rows(&columns, rows_ast)?; - coerce_and_check_rows(&info, &mut rows)?; - let column_defaults: Vec<(String, String)> = info - .columns - .iter() - .filter_map(|c| c.default.as_ref().map(|d| (c.name.clone(), d.clone()))) - .collect(); - let column_schema: Vec<(String, String)> = info - .columns - .iter() - .filter_map(|c| c.raw_type.as_ref().map(|t| (c.name.clone(), t.clone()))) - .collect(); - let rules = engine_rules::resolve_engine_rules(info.engine); - rules.plan_upsert(engine_rules::UpsertParams { - collection: table_name, - columns, - rows, - column_defaults, - on_conflict_updates: Vec::new(), - column_schema, - primary_key: info.primary_key.clone(), - }) -} - -/// Plan an `INSERT ... ON CONFLICT DO UPDATE SET` statement. -fn plan_upsert_with_on_conflict( - ins: &ast::Insert, - catalog: &dyn SqlCatalog, - mut on_conflict_updates: Vec<(String, SqlExpr)>, -) -> Result> { - let table_name = match &ins.table { - ast::TableObject::TableName(name) => normalize_object_name_checked(name)?, - ast::TableObject::TableFunction(_) => { - return Err(SqlError::Unsupported { - detail: "INSERT ... ON CONFLICT on a table function is not supported".into(), - }); - } - // Oracle's `INSERT INTO (SELECT ...)`: the target is a subquery, so - // there is no collection to resolve or route to an engine. - ast::TableObject::TableQuery(_) => { - return Err(SqlError::Unsupported { - detail: "INSERT ... ON CONFLICT on a subquery target is not supported".into(), - }); - } - }; - let info = catalog - .get_collection(DatabaseId::DEFAULT, &table_name)? - .ok_or_else(|| SqlError::UnknownTable { - name: table_name.clone(), - })?; - - let columns = insert_columns(&ins.columns, &target_scope(&table_name, &info)?)?; - - let source = ins.source.as_ref().ok_or_else(|| SqlError::Parse { - detail: "INSERT ... ON CONFLICT requires VALUES".into(), - })?; - let rows_ast = match &*source.body { - ast::SetExpr::Values(values) => &values.rows, - _ => { - return Err(SqlError::Unsupported { - detail: "INSERT ... ON CONFLICT source must be VALUES".into(), - }); - } - }; - - // KV: `INSERT ... ON CONFLICT (key) DO UPDATE SET ...` is an opt-in - // overwrite — same physical semantics as UPSERT, with the optional - // per-row assignments carried through for the Data Plane to apply - // against the existing row. - if info.engine == EngineType::KeyValue { - return build_kv_insert_plan(KvInsertParams { - collection: table_name, - columns: &columns, - rows_ast, - intent: KvInsertIntent::Put, - on_conflict_updates, - pk_col: info.primary_key.as_deref(), - declared_columns: &info.columns, - catalog, - }); - } - - // Positional UPSERT (no column list): bind to the collection's declared - // column order — see `plan_insert` for the full rationale. - let columns = resolve_insert_columns(columns, &info, rows_ast)?; - - let mut rows = convert_value_rows(&columns, rows_ast)?; - coerce_and_check_rows(&info, &mut rows)?; - // `DO UPDATE SET col = ` writes through the same path as the - // inserted row, so its literals carry the same declared-type contract. - coerce_assignments_to_declared_types( - &info.columns, - &mut on_conflict_updates, - info.primary_key.as_deref(), - )?; - check_declared_int_ranges_in_assignments(&info.columns, &on_conflict_updates)?; - check_declared_float_ranges_in_assignments(&info.columns, &on_conflict_updates)?; - let column_defaults: Vec<(String, String)> = info - .columns - .iter() - .filter_map(|c| c.default.as_ref().map(|d| (c.name.clone(), d.clone()))) - .collect(); - let column_schema: Vec<(String, String)> = info - .columns - .iter() - .filter_map(|c| c.raw_type.as_ref().map(|t| (c.name.clone(), t.clone()))) - .collect(); - let rules = engine_rules::resolve_engine_rules(info.engine); - rules.plan_upsert(engine_rules::UpsertParams { - collection: table_name, - columns, - rows, - column_defaults, - on_conflict_updates, - column_schema, - primary_key: info.primary_key.clone(), - }) -} diff --git a/nodedb-sql/src/planner/dml/insert.rs b/nodedb-sql/src/planner/dml/insert.rs new file mode 100644 index 000000000..bbbe1ce1b --- /dev/null +++ b/nodedb-sql/src/planner/dml/insert.rs @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! INSERT planning. + +use sqlparser::ast; + +use super::super::dml_helpers::{ + KvInsertParams, bind_insert_select_columns, build_kv_insert_plan, + build_vector_primary_insert_plan, resolve_insert_columns, +}; +use super::target::{ + OnConflict, classify_on_conflict, column_schema, insert_columns, resolve_target, target_scope, + typed_rows, values_rows, +}; +use super::upsert::plan_upsert_with_on_conflict; +use crate::engine_rules::{self, InsertParams}; +use crate::error::Result; +use crate::types::*; + +/// Plan an INSERT statement. +pub fn plan_insert(ins: &ast::Insert, catalog: &dyn SqlCatalog) -> Result> { + let (table_name, info) = resolve_target(ins, "INSERT", catalog)?; + let target_scope = target_scope(&table_name, &info)?; + + // `INSERT ... ON CONFLICT DO UPDATE SET` reroutes to the upsert path + // with the assignments carried through. `DO NOTHING` stays on the + // INSERT path with `if_absent=true`. + let if_absent = match classify_on_conflict(ins, &target_scope)? { + OnConflict::None => false, + OnConflict::DoNothing => true, + OnConflict::DoUpdate(updates) => { + return plan_upsert_with_on_conflict(ins, catalog, updates); + } + }; + + let columns = insert_columns(&ins.columns, &target_scope)?; + + // Check for INSERT...SELECT. + if let Some(source) = &ins.source + && let ast::SetExpr::Select(select) = &*source.body + { + let column_map = bind_insert_select_columns(catalog, &columns, select, &info)?; + let source_plan = super::super::select::plan_query( + source, + catalog, + &crate::functions::registry::FunctionRegistry::new(), + crate::TemporalScope::default(), + )?; + return Ok(vec![SqlPlan::InsertSelect { + target: table_name, + source: Box::new(source_plan), + limit: 0, + column_map, + }]); + } + + let rows_ast = values_rows(ins, "INSERT")?; + + // KV engine: key and value are fundamentally separate — handle directly. + // Positional column binding (below) does not apply here: the KV path + // matches columns by name against `pk_col`/`"key"`/`"ttl"`, which is + // orthogonal to declared column order. + if info.engine == EngineType::KeyValue { + let intent = if if_absent { + KvInsertIntent::InsertIfAbsent + } else { + KvInsertIntent::Insert + }; + return build_kv_insert_plan(KvInsertParams { + collection: table_name, + columns: &columns, + rows_ast, + intent, + on_conflict_updates: Vec::new(), + pk_col: info.primary_key.as_deref(), + declared_columns: &info.columns, + catalog, + }); + } + + // Positional INSERT (no column list): bind values to the collection's + // declared column order so named projections/predicates can find them. + // No-op for named inserts and schemaless collections. + let columns = resolve_insert_columns(columns, &info, rows_ast)?; + + // One typing pass for every remaining engine: defaults materialized, + // then every cell coerced and range-checked against its declared type. + let typed = typed_rows(&info, &columns, rows_ast, catalog)?; + + // Vector-primary collection: bypass document encoding. + if info.primary == nodedb_types::PrimaryEngine::Vector + && let Some(ref vpc) = info.vector_primary + { + return build_vector_primary_insert_plan( + &table_name, + vpc, + &columns, + typed.rows, + typed.volatile_defaults, + ); + } + + // All other engines: delegate to engine rules. + let column_schema = column_schema(&info); + let rules = engine_rules::resolve_engine_rules(info.engine); + rules.plan_insert(InsertParams { + collection: table_name, + columns, + rows: typed.rows, + volatile_defaults: typed.volatile_defaults, + if_absent, + column_schema, + primary_key: info.primary_key.clone(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::catalog::{SqlCatalog, SqlCatalogError}; + use crate::parser::statement::parse_sql; + use nodedb_types::columnar::IntWidth; + use nodedb_types::datetime::NdbDateTime; + + fn column(name: &str, data_type: SqlDataType, default: Option<&str>) -> ColumnInfo { + ColumnInfo { + name: name.to_string(), + data_type, + nullable: true, + is_primary_key: false, + default: default.map(str::to_string), + raw_type: None, + int_width: None, + float_width: None, + } + } + + fn collection(engine: EngineType, columns: Vec) -> CollectionInfo { + let primary_key = columns + .iter() + .find(|c| c.is_primary_key) + .map(|c| c.name.clone()); + CollectionInfo { + name: "t".into(), + engine, + columns, + primary_key, + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(engine), + } + } + + /// A catalog with one collection named `t`. + struct OneCollection(CollectionInfo); + + impl SqlCatalog for OneCollection { + fn get_collection( + &self, + _: nodedb_types::DatabaseId, + name: &str, + ) -> std::result::Result, SqlCatalogError> { + Ok((name == "t").then(|| self.0.clone())) + } + } + + fn plan(sql: &str, catalog: &dyn SqlCatalog) -> Result { + let statements = parse_sql(sql)?; + let sqlparser::ast::Statement::Insert(ins) = &statements[0] else { + panic!("expected an INSERT statement"); + }; + let mut plans = plan_insert(ins, catalog)?; + Ok(plans.remove(0)) + } + + /// `2020-03-05T10:00:00Z` as microseconds since the Unix epoch. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + fn cell<'a>(row: &'a [(String, SqlValue)], name: &str) -> &'a SqlValue { + &row.iter() + .find(|(n, _)| n == name) + .unwrap_or_else(|| panic!("row carries {name}")) + .1 + } + + /// A numeric DEFAULT on a TIMESTAMP column is materialized, then coerced + /// like a supplied literal: the plan carries the typed instant. + #[test] + fn a_numeric_timestamp_default_is_materialized_then_coerced() { + let mut id = column("id", SqlDataType::String, None); + id.is_primary_key = true; + let catalog = OneCollection(collection( + EngineType::DocumentSchemaless, + vec![ + id, + column("at", SqlDataType::Timestamp, Some("1583402400000")), + ], + )); + let plan = plan("INSERT INTO t (id) VALUES ('r1')", &catalog).expect("plans"); + let SqlPlan::Insert { + rows, + volatile_defaults, + .. + } = plan + else { + panic!("expected SqlPlan::Insert, got {plan:?}"); + }; + assert!(!volatile_defaults, "a literal DEFAULT is not volatile"); + assert_eq!( + cell(&rows[0], "at"), + &SqlValue::Timestamp(NdbDateTime::from_micros(EARLY_MICROS)) + ); + } + + /// A DEFAULT the column cannot hold is refused at the insert, naming the + /// column, exactly as the same literal in VALUES is. + #[test] + fn a_default_out_of_declared_range_is_refused() { + let mut id = column("id", SqlDataType::String, None); + id.is_primary_key = true; + let mut small = column("s", SqlDataType::Int64, Some("999999")); + small.int_width = Some(IntWidth::I16); + let catalog = OneCollection(collection(EngineType::DocumentStrict, vec![id, small])); + let err = plan("INSERT INTO t (id) VALUES ('r1')", &catalog) + .expect_err("DEFAULT 999999 does not fit SMALLINT"); + assert!( + matches!(err, crate::SqlError::IntegerOutOfRange { ref column, .. } if column == "s"), + "{err}" + ); + } + + /// A timeseries plan carries the declared default of an omitted column + /// in its rows, the TIME_KEY column included. + #[test] + fn a_timeseries_plan_carries_defaults_in_its_rows() { + let catalog = OneCollection(collection( + EngineType::Timeseries, + vec![ + column("at", SqlDataType::Timestamp, Some("1583402400000")), + column("host", SqlDataType::String, Some("'h0'")), + column("v", SqlDataType::Float64, None), + ], + )); + let plan = plan("INSERT INTO t (v) VALUES (1.0)", &catalog).expect("plans"); + let SqlPlan::TimeseriesIngest { + rows, + volatile_defaults, + .. + } = plan + else { + panic!("expected SqlPlan::TimeseriesIngest, got {plan:?}"); + }; + assert!(!volatile_defaults); + assert_eq!( + cell(&rows[0], "at"), + &SqlValue::Timestamp(NdbDateTime::from_micros(EARLY_MICROS)) + ); + assert_eq!(cell(&rows[0], "host"), &SqlValue::String("h0".into())); + assert_eq!(cell(&rows[0], "v"), &SqlValue::Float(1.0)); + } + + /// A volatile DEFAULT marks the plan so the plan cache never replays it. + #[test] + fn a_volatile_default_marks_the_plan() { + let mut id = column("id", SqlDataType::String, Some("UUID_V7")); + id.is_primary_key = true; + let catalog = OneCollection(collection( + EngineType::DocumentStrict, + vec![id, column("n", SqlDataType::Int64, None)], + )); + let plan = plan("INSERT INTO t (n) VALUES (1)", &catalog).expect("plans"); + assert!( + !plan.cache_eligibility().is_cacheable(), + "a plan carrying a volatile DEFAULT is never cached" + ); + } +} diff --git a/nodedb-sql/src/planner/dml/mod.rs b/nodedb-sql/src/planner/dml/mod.rs new file mode 100644 index 000000000..2e815dfc8 --- /dev/null +++ b/nodedb-sql/src/planner/dml/mod.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! INSERT, UPSERT, UPDATE, DELETE, and TRUNCATE planning. + +mod insert; +mod target; +mod update_delete; +mod upsert; + +pub use insert::plan_insert; +pub use update_delete::{plan_delete, plan_truncate_stmt, plan_update}; +pub use upsert::plan_upsert; diff --git a/nodedb-sql/src/planner/dml/target.rs b/nodedb-sql/src/planner/dml/target.rs new file mode 100644 index 000000000..0f1a46b56 --- /dev/null +++ b/nodedb-sql/src/planner/dml/target.rs @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The parts of an INSERT-shaped statement every row-writing planner shares: +//! target resolution, the column namespace, `ON CONFLICT` classification, +//! and the one row-typing pass. + +use nodedb_types::DatabaseId; +use sqlparser::ast; + +use super::super::dml_helpers::{ + coerce_and_check_rows, convert_value_rows, materialize_defaults_in_rows, +}; +use crate::error::{Result, SqlError}; +use crate::parser::normalize::{normalize_insert_column, normalize_object_name_checked}; +use crate::resolver::ColumnScope; +use crate::resolver::columns::{ResolvedTable, TableScope}; +use crate::resolver::expr::convert_expr; +use crate::types::*; + +/// The pseudo-relation `ON CONFLICT DO UPDATE` uses for the proposed row. +const EXCLUDED_RELATION: &str = "excluded"; + +/// The statement's target collection, resolved through the catalog. +/// +/// `verb` names the statement in the refusal a non-collection target gets. +pub(super) fn resolve_target( + ins: &ast::Insert, + verb: &str, + catalog: &dyn SqlCatalog, +) -> Result<(String, CollectionInfo)> { + let table_name = match &ins.table { + ast::TableObject::TableName(name) => normalize_object_name_checked(name)?, + ast::TableObject::TableFunction(_) => { + return Err(SqlError::Unsupported { + detail: format!("{verb} INTO a table function is not supported"), + }); + } + // Oracle's `INSERT INTO (SELECT ...)`: the target is a subquery, so + // there is no collection to resolve or route to an engine. + ast::TableObject::TableQuery(_) => { + return Err(SqlError::Unsupported { + detail: format!("{verb} INTO a subquery target is not supported"), + }); + } + }; + let info = catalog + .get_collection(DatabaseId::DEFAULT, &table_name)? + .ok_or_else(|| SqlError::UnknownTable { + name: table_name.clone(), + })?; + Ok((table_name, info)) +} + +/// The statement's `VALUES` rows. +/// +/// `verb` names the statement in the refusal a non-`VALUES` source gets. +pub(super) fn values_rows<'a>( + ins: &'a ast::Insert, + verb: &str, +) -> Result<&'a [ast::Parens>]> { + let source = ins.source.as_ref().ok_or_else(|| SqlError::Parse { + detail: format!("{verb} requires VALUES"), + })?; + match &*source.body { + ast::SetExpr::Values(values) => Ok(&values.rows), + _ => Err(SqlError::Unsupported { + detail: format!("{verb} source must be VALUES"), + }), + } +} + +/// The column namespace of an INSERT target. +pub(super) fn target_scope(table_name: &str, info: &CollectionInfo) -> Result { + let mut scope = TableScope::single(ResolvedTable { + name: table_name.to_string(), + alias: None, + info: info.clone(), + })?; + // `ON CONFLICT DO UPDATE` addresses the proposed row as `excluded`. It + // carries the target's columns and is qualified-only, so a bare name in + // the SET clause names the stored row. + scope.add_qualified_only(ResolvedTable { + name: EXCLUDED_RELATION.to_string(), + alias: None, + info: info.clone(), + })?; + Ok(scope) +} + +/// Normalize an INSERT column list and reject a name the target does not have. +pub(super) fn insert_columns( + columns: &[ast::ObjectName], + scope: &TableScope, +) -> Result> { + columns + .iter() + .map(|c| { + let col = normalize_insert_column(c)?; + scope.check_name(None, &col)?; + Ok(col) + }) + .collect() +} + +/// Classification of an `ON CONFLICT` clause attached to an INSERT. +pub(super) enum OnConflict { + /// No `ON CONFLICT` clause — plain INSERT (error on duplicate PK). + None, + /// `ON CONFLICT DO NOTHING` — skip rows that would conflict, no error. + DoNothing, + /// `ON CONFLICT (...) DO UPDATE SET ...` — apply the assignments against + /// the existing row on conflict. + DoUpdate(Vec<(String, SqlExpr)>), +} + +pub(super) fn classify_on_conflict(ins: &ast::Insert, scope: &TableScope) -> Result { + let Some(on) = ins.on.as_ref() else { + return Ok(OnConflict::None); + }; + let ast::OnInsert::OnConflict(oc) = on else { + return Ok(OnConflict::None); + }; + match &oc.action { + ast::OnConflictAction::DoNothing => Ok(OnConflict::DoNothing), + ast::OnConflictAction::DoUpdate(do_update) => { + let mut pairs = Vec::with_capacity(do_update.assignments.len()); + for a in &do_update.assignments { + let name = match &a.target { + ast::AssignmentTarget::ColumnName(obj) => normalize_object_name_checked(obj)?, + ast::AssignmentTarget::Tuple(_) => { + return Err(SqlError::Unsupported { + detail: "ON CONFLICT DO UPDATE SET target must be a column name".into(), + }); + } + }; + scope.check_name(None, &name)?; + let expr = convert_expr(&a.value, &ColumnScope::Relations(scope))?; + pairs.push((name, expr)); + } + Ok(OnConflict::DoUpdate(pairs)) + } + } +} + +/// A `VALUES` row set with every declared DEFAULT materialized and every +/// value coerced to its declared column type. +pub(super) struct TypedRows { + pub rows: Vec>, + /// Whether a materialized DEFAULT was volatile. A plan carrying one is + /// never admitted to the plan cache. + pub volatile_defaults: bool, +} + +/// Resolve `VALUES` literals, fill in declared DEFAULTs, then coerce and +/// range-check every cell against its declared column type. +/// +/// This is the one row-typing pass for every engine. Defaults are +/// materialized BEFORE coercion so a defaulted value is checked exactly like +/// a supplied literal: `DEFAULT 999999` on a `SMALLINT` column is refused +/// where `VALUES (999999)` is, and `DEFAULT 1583402400000` on a `TIMESTAMP` +/// column becomes the same instant `VALUES (1583402400000)` does. Nothing +/// downstream reads the catalog's DEFAULT text again. +pub(super) fn typed_rows( + info: &CollectionInfo, + columns: &[String], + rows_ast: &[ast::Parens>], + catalog: &dyn SqlCatalog, +) -> Result { + let mut rows = convert_value_rows(columns, rows_ast)?; + let volatile_defaults = materialize_defaults_in_rows(&info.columns, &mut rows, catalog)?; + coerce_and_check_rows(info, &mut rows)?; + Ok(TypedRows { + rows, + volatile_defaults, + }) +} + +/// Raw column type strings from the catalog: `(column_name, type_str)`. +/// +/// Columnar converters read these to reconstruct the exact `ColumnType` for +/// columns whose `SqlDataType` is ambiguous. +pub(super) fn column_schema(info: &CollectionInfo) -> Vec<(String, String)> { + info.columns + .iter() + .filter_map(|c| c.raw_type.as_ref().map(|t| (c.name.clone(), t.clone()))) + .collect() +} diff --git a/nodedb-sql/src/planner/dml_update_delete.rs b/nodedb-sql/src/planner/dml/update_delete.rs similarity index 97% rename from nodedb-sql/src/planner/dml_update_delete.rs rename to nodedb-sql/src/planner/dml/update_delete.rs index 5973a88b8..b49470263 100644 --- a/nodedb-sql/src/planner/dml_update_delete.rs +++ b/nodedb-sql/src/planner/dml/update_delete.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -//! UPDATE, DELETE, and TRUNCATE planning — extracted from `dml.rs`. +//! UPDATE, DELETE, and TRUNCATE planning. use nodedb_types::DatabaseId; use sqlparser::ast; @@ -38,8 +38,8 @@ pub fn plan_update(stmt: &ast::Statement, catalog: &dyn SqlCatalog) -> Result Result Result> { +fn plan_update_from( + update: &ast::Update, + from_kind: &ast::UpdateTableFromKind, + catalog: &dyn SqlCatalog, +) -> Result> { let target_name = extract_table_name_from_table_with_joins(&update.table)?; // Extract alias for the target table if present. @@ -101,7 +105,6 @@ fn plan_update_from(update: &ast::Update, catalog: &dyn SqlCatalog) -> Result = match from_kind { ast::UpdateTableFromKind::AfterSet(tables) | ast::UpdateTableFromKind::BeforeSet(tables) => tables, diff --git a/nodedb-sql/src/planner/dml/upsert.rs b/nodedb-sql/src/planner/dml/upsert.rs new file mode 100644 index 000000000..f8d75d3ce --- /dev/null +++ b/nodedb-sql/src/planner/dml/upsert.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! UPSERT and `INSERT ... ON CONFLICT DO UPDATE` planning. + +use sqlparser::ast; + +use super::super::dml_helpers::{ + KvInsertParams, build_kv_insert_plan, check_declared_float_ranges_in_assignments, + check_declared_int_ranges_in_assignments, resolve_insert_columns, +}; +use super::target::{ + column_schema, insert_columns, resolve_target, target_scope, typed_rows, values_rows, +}; +use crate::engine_rules::{self, UpsertParams}; +use crate::error::Result; +use crate::planner::declared_type_coerce::coerce_assignments_to_declared_types; +use crate::types::*; + +/// Plan an UPSERT statement (pre-processed from `UPSERT INTO` to `INSERT INTO`). +/// +/// Same parsing as INSERT but routes through `engine_rules.plan_upsert()`. +pub fn plan_upsert(ins: &ast::Insert, catalog: &dyn SqlCatalog) -> Result> { + plan_upsert_rows(ins, catalog, "UPSERT", Vec::new()) +} + +/// Plan an `INSERT ... ON CONFLICT DO UPDATE SET` statement. +pub(super) fn plan_upsert_with_on_conflict( + ins: &ast::Insert, + catalog: &dyn SqlCatalog, + on_conflict_updates: Vec<(String, SqlExpr)>, +) -> Result> { + plan_upsert_rows(ins, catalog, "INSERT ... ON CONFLICT", on_conflict_updates) +} + +/// The shared body of `UPSERT` and `INSERT ... ON CONFLICT DO UPDATE`. +/// +/// The two differ only in `verb` (for error messages) and in whether +/// `on_conflict_updates` carries per-row assignments for the Data Plane to +/// apply against the existing row. +fn plan_upsert_rows( + ins: &ast::Insert, + catalog: &dyn SqlCatalog, + verb: &str, + mut on_conflict_updates: Vec<(String, SqlExpr)>, +) -> Result> { + let (table_name, info) = resolve_target(ins, verb, catalog)?; + let columns = insert_columns(&ins.columns, &target_scope(&table_name, &info)?)?; + let rows_ast = values_rows(ins, verb)?; + + // KV: upsert is a PUT (natural overwrite), and `INSERT ... ON CONFLICT + // (key) DO UPDATE SET ...` is the same physical write with the optional + // per-row assignments carried through. Positional column binding + // (below) does not apply here — see `plan_insert`. + if info.engine == EngineType::KeyValue { + return build_kv_insert_plan(KvInsertParams { + collection: table_name, + columns: &columns, + rows_ast, + intent: KvInsertIntent::Put, + on_conflict_updates, + pk_col: info.primary_key.as_deref(), + declared_columns: &info.columns, + catalog, + }); + } + + // Positional UPSERT (no column list): bind to the collection's declared + // column order — see `plan_insert` for the full rationale. + let columns = resolve_insert_columns(columns, &info, rows_ast)?; + + let typed = typed_rows(&info, &columns, rows_ast, catalog)?; + // `DO UPDATE SET col = ` writes through the same path as the + // inserted row, so its literals carry the same declared-type contract. + coerce_assignments_to_declared_types( + &info.columns, + &mut on_conflict_updates, + info.primary_key.as_deref(), + )?; + check_declared_int_ranges_in_assignments(&info.columns, &on_conflict_updates)?; + check_declared_float_ranges_in_assignments(&info.columns, &on_conflict_updates)?; + let column_schema = column_schema(&info); + let rules = engine_rules::resolve_engine_rules(info.engine); + rules.plan_upsert(UpsertParams { + collection: table_name, + columns, + rows: typed.rows, + volatile_defaults: typed.volatile_defaults, + on_conflict_updates, + column_schema, + primary_key: info.primary_key.clone(), + }) +} diff --git a/nodedb-sql/src/planner/dml_helpers/declared_defaults.rs b/nodedb-sql/src/planner/dml_helpers/declared_defaults.rs index 5c5e6e5ab..0a6bee325 100644 --- a/nodedb-sql/src/planner/dml_helpers/declared_defaults.rs +++ b/nodedb-sql/src/planner/dml_helpers/declared_defaults.rs @@ -9,11 +9,10 @@ use crate::types::*; /// Materialize declared DEFAULTs across a whole `VALUES` row set. /// -/// The key-value and vector-primary engines store the values they are handed -/// and have no typed write path, so a DEFAULT that is not materialized HERE is -/// materialized nowhere: the catalog would keep the declaration and every read -/// return nothing for it. Documents and columnar rows expand theirs through the -/// same [`ColumnDefaults`], so one expression yields one value on every engine. +/// This is the one place a declared DEFAULT becomes a value, for every +/// engine. The plan an engine receives carries the materialized cell like a +/// supplied one, and nothing downstream reads the catalog's DEFAULT text +/// again, so one expression yields one value on every engine. /// /// Two rules the ordering encodes: /// diff --git a/nodedb-sql/src/planner/dml_helpers/mod.rs b/nodedb-sql/src/planner/dml_helpers/mod.rs index 242266de8..23a2d3446 100644 --- a/nodedb-sql/src/planner/dml_helpers/mod.rs +++ b/nodedb-sql/src/planner/dml_helpers/mod.rs @@ -29,8 +29,8 @@ pub(super) use insert_select_bind::bind_insert_select_columns; pub(super) use kv_insert::build_kv_insert_plan; pub(super) use params::KvInsertParams; pub(super) use range_check::{ - check_declared_float_ranges_in_assignments, check_declared_int_ranges_in_assignments, - coerce_and_check_rows, + check_declared_float_ranges, check_declared_float_ranges_in_assignments, + check_declared_int_ranges, check_declared_int_ranges_in_assignments, coerce_and_check_rows, }; pub(super) use value_convert::convert_value_rows; pub(super) use vector_primary_insert::build_vector_primary_insert_plan; diff --git a/nodedb-sql/src/planner/join/constraint.rs b/nodedb-sql/src/planner/join/constraint.rs index 7b7c9a999..df26a5da2 100644 --- a/nodedb-sql/src/planner/join/constraint.rs +++ b/nodedb-sql/src/planner/join/constraint.rs @@ -6,6 +6,7 @@ use sqlparser::ast; use crate::error::{Result, SqlError}; use crate::parser::normalize::normalize_ident; +use crate::planner::predicate_coerce::coerce_predicate_literals; use crate::resolver::ColumnScope; use crate::resolver::columns::TableScope; use crate::resolver::expr::convert_expr; @@ -74,6 +75,10 @@ fn extract_join_constraint( right: Box::new(convert_expr(pred, &ColumnScope::Relations(scope))?), }; } + // A non-equi ON predicate is evaluated per candidate pair by + // the same comparison the WHERE path uses, so its literals + // follow the same declared-instant rule. + coerce_predicate_literals(&mut combined, scope)?; Some(combined) }; Ok((keys, cond)) diff --git a/nodedb-sql/src/planner/merge/actions.rs b/nodedb-sql/src/planner/merge/actions.rs new file mode 100644 index 000000000..e8fdf0730 --- /dev/null +++ b/nodedb-sql/src/planner/merge/actions.rs @@ -0,0 +1,401 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! MERGE `WHEN ... THEN` clause conversion. +//! +//! Every literal an action writes to the target passes the declared-type +//! coercion here, once, the same pass an ordinary `INSERT VALUES` or +//! `UPDATE SET` literal gets. The document-schemaless engine stores the +//! planner's value verbatim, so a literal left uncoerced would reach storage +//! as whatever the author typed: a bare integer under a `TIMESTAMP` column, +//! text under an `INT` column. +//! +//! Only literals are coerced. A source column reference or a function call +//! is evaluated on the Data Plane against the source row and carries no +//! value at plan time. + +use sqlparser::ast::{self, MergeAction, MergeClauseKind as AstMergeClauseKind, MergeInsertKind}; + +use super::super::ast_helpers::strip_and_convert_filters; +use super::super::declared_type_coerce::coerce_assignments_to_declared_types; +use super::super::dml_helpers::{ + check_declared_float_ranges_in_assignments, check_declared_int_ranges_in_assignments, + coerce_and_check_rows, +}; +use crate::error::{Result, SqlError}; +use crate::parser::normalize::normalize_object_name_checked; +use crate::resolver::ColumnScope; +use crate::resolver::columns::TableScope; +use crate::resolver::expr::convert_expr; +use crate::types::*; + +/// Convert every `WHEN` clause of a MERGE statement. +/// +/// `scope` addresses both target and source, `target_scope` the target +/// alone, and `target` carries the declared columns the action literals are +/// coerced against. +pub(super) fn convert_merge_clauses( + clauses: &[ast::MergeClause], + target_ref: &str, + scope: &TableScope, + target_scope: &TableScope, + target: &CollectionInfo, +) -> Result> { + clauses + .iter() + .map(|c| convert_one_clause(c, target_ref, scope, target_scope, target)) + .collect() +} + +fn convert_one_clause( + clause: &ast::MergeClause, + target_ref: &str, + scope: &TableScope, + target_scope: &TableScope, + target: &CollectionInfo, +) -> Result { + let kind = match clause.clause_kind { + AstMergeClauseKind::Matched => MergeClauseKind::Matched, + AstMergeClauseKind::NotMatched | AstMergeClauseKind::NotMatchedByTarget => { + MergeClauseKind::NotMatched + } + AstMergeClauseKind::NotMatchedBySource => MergeClauseKind::NotMatchedBySource, + }; + + let extra_predicate = match &clause.predicate { + Some(expr) => strip_and_convert_filters(vec![expr.clone()], target_ref, scope)?, + None => Vec::new(), + }; + + let action = convert_merge_action(&clause.action, scope, target_scope, target)?; + + Ok(MergePlanClause { + kind, + extra_predicate, + action, + }) +} + +/// Convert one `THEN` action, coercing every literal it writes to the +/// declared type of the target column it lands in. +pub(super) fn convert_merge_action( + action: &MergeAction, + scope: &TableScope, + target_scope: &TableScope, + target: &CollectionInfo, +) -> Result { + match action { + MergeAction::Update(update_expr) => { + let mut assignments = update_expr + .assignments + .iter() + .map(|a| { + let col = match &a.target { + ast::AssignmentTarget::ColumnName(name) => { + normalize_object_name_checked(name) + } + ast::AssignmentTarget::Tuple(_) => Err(SqlError::Unsupported { + detail: "tuple assignment target in MERGE UPDATE is not supported" + .into(), + }), + }?; + target_scope.check_name(None, &col)?; + let val = convert_expr(&a.value, &ColumnScope::Relations(scope))?; + Ok((col, val)) + }) + .collect::>>()?; + // `SET col = ` rewrites the stored row through the same + // path as `UPDATE ... SET`, so it carries the same declared-type + // contract, primary-key exemption included. + coerce_assignments_to_declared_types( + &target.columns, + &mut assignments, + target.primary_key.as_deref(), + )?; + check_declared_int_ranges_in_assignments(&target.columns, &assignments)?; + check_declared_float_ranges_in_assignments(&target.columns, &assignments)?; + Ok(MergePlanAction::Update { assignments }) + } + MergeAction::Delete { .. } => Ok(MergePlanAction::Delete), + MergeAction::Insert(insert_expr) => { + let columns: Vec = insert_expr + .columns + .iter() + .map(|c| { + let col = normalize_object_name_checked(c)?; + target_scope.check_name(None, &col)?; + Ok(col) + }) + .collect::>>()?; + + let mut values: Vec = match &insert_expr.kind { + MergeInsertKind::Values(vals) => { + if vals.rows.len() != 1 { + return Err(SqlError::Unsupported { + detail: format!( + "MERGE INSERT VALUES must have exactly one row; got {}", + vals.rows.len() + ), + }); + } + vals.rows[0] + .iter() + .map(|e| convert_expr(e, &ColumnScope::Relations(scope))) + .collect::>>()? + } + MergeInsertKind::Row => { + return Err(SqlError::Unsupported { + detail: "MERGE INSERT ROW is not supported; use explicit VALUES".into(), + }); + } + }; + + if !columns.is_empty() && columns.len() != values.len() { + return Err(SqlError::Parse { + detail: format!( + "MERGE INSERT column list ({}) and VALUES ({}) lengths do not match", + columns.len(), + values.len() + ), + }); + } + + let columns = bind_positional_columns(columns, values.len(), target)?; + coerce_insert_literals(&columns, &mut values, target)?; + Ok(MergePlanAction::Insert { columns, values }) + } + } +} + +/// Bind a column-less `INSERT VALUES (...)` arm to the target's declared +/// column order, the way a positional `INSERT INTO t VALUES (...)` binds. +/// +/// A target with no declared columns has no order to bind to and keeps the +/// empty list. More values than declared columns is refused: there is no +/// column name for the overflow to land under. +fn bind_positional_columns( + columns: Vec, + value_count: usize, + target: &CollectionInfo, +) -> Result> { + if !columns.is_empty() || target.columns.is_empty() { + return Ok(columns); + } + if value_count > target.columns.len() { + return Err(SqlError::InsertColumnArityMismatch { + collection: target.name.clone(), + given: value_count, + declared: target.columns.len(), + }); + } + Ok(target + .columns + .iter() + .take(value_count) + .map(|c| c.name.clone()) + .collect()) +} + +/// Coerce every literal of an INSERT arm to the declared type of the target +/// column it is bound to, and range-check it, through the one row-typing +/// pass every `VALUES` clause takes. +/// +/// Non-literal expressions are left in place: they are evaluated against the +/// source row on the Data Plane and carry no value here. +fn coerce_insert_literals( + columns: &[String], + values: &mut [SqlExpr], + target: &CollectionInfo, +) -> Result<()> { + let mut literal_row: Vec<(String, SqlValue)> = Vec::new(); + let mut literal_slots: Vec = Vec::new(); + for (slot, (column, expr)) in columns.iter().zip(values.iter()).enumerate() { + if let SqlExpr::Literal(value) = expr { + literal_row.push((column.clone(), value.clone())); + literal_slots.push(slot); + } + } + if literal_row.is_empty() { + return Ok(()); + } + let mut rows = [literal_row]; + coerce_and_check_rows(target, &mut rows)?; + let [literal_row] = rows; + for (slot, (_, value)) in literal_slots.into_iter().zip(literal_row) { + values[slot] = SqlExpr::Literal(value); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::statement::parse_sql; + use crate::resolver::columns::ResolvedTable; + use nodedb_types::columnar::IntWidth; + use nodedb_types::datetime::NdbDateTime; + + /// `2020-03-05T10:00:00Z` as microseconds since the Unix epoch. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + fn column(name: &str, data_type: SqlDataType) -> ColumnInfo { + ColumnInfo { + name: name.to_string(), + data_type, + nullable: true, + is_primary_key: false, + default: None, + raw_type: None, + int_width: None, + float_width: None, + } + } + + /// `(id INT PRIMARY KEY, n INT, at TIMESTAMP, s SMALLINT)` on the + /// schemaless engine, the engine that stores the planner's value verbatim. + fn target() -> CollectionInfo { + let mut id = column("id", SqlDataType::Int64); + id.is_primary_key = true; + let mut small = column("s", SqlDataType::Int64); + small.int_width = Some(IntWidth::I16); + CollectionInfo { + name: "t".into(), + engine: EngineType::DocumentSchemaless, + columns: vec![ + id, + column("n", SqlDataType::Int64), + column("at", SqlDataType::Timestamp), + small, + ], + primary_key: Some("id".into()), + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentSchemaless), + } + } + + /// Plan the first WHEN clause of `MERGE INTO t t USING t s ON t.id = s.id + /// `. + fn action(when: &str) -> Result { + let sql = format!("MERGE INTO t t USING t s ON t.id = s.id {when}"); + let statements = parse_sql(&sql)?; + let ast::Statement::Merge(merge) = &statements[0] else { + panic!("expected a MERGE statement"); + }; + let target = target(); + let relation = |alias: &str| ResolvedTable { + name: "t".into(), + alias: Some(alias.into()), + info: target.clone(), + }; + let target_scope = TableScope::single(relation("t"))?; + let mut scope = TableScope::new(); + scope.add(relation("t"))?; + scope.add_qualified_only(relation("s"))?; + convert_merge_action(&merge.clauses[0].action, &scope, &target_scope, &target) + } + + fn literal(expr: &SqlExpr) -> SqlValue { + match expr { + SqlExpr::Literal(value) => value.clone(), + other => panic!("expected a literal, got {other:?}"), + } + } + + /// The INSERT arm coerces each literal to its bound column's declared + /// type and leaves a source column reference untouched. + #[test] + fn insert_arm_literals_take_the_declared_column_types() { + let action = + action("WHEN NOT MATCHED THEN INSERT (id, n, at) VALUES (s.id, '1', 1583402400000)") + .expect("plans"); + let MergePlanAction::Insert { columns, values } = action else { + panic!("expected an INSERT action, got {action:?}"); + }; + assert_eq!(columns, vec!["id", "n", "at"]); + assert!( + matches!(values[0], SqlExpr::Column { .. }), + "a source reference stays an expression: {:?}", + values[0] + ); + assert_eq!(literal(&values[1]), SqlValue::Int(1)); + assert_eq!( + literal(&values[2]), + SqlValue::Timestamp(NdbDateTime::from_micros(EARLY_MICROS)) + ); + } + + /// The UPDATE arm coerces its literals the same way. + #[test] + fn update_arm_literals_take_the_declared_column_types() { + let action = + action("WHEN MATCHED THEN UPDATE SET n = '2', at = 1583402400000").expect("plans"); + let MergePlanAction::Update { assignments } = action else { + panic!("expected an UPDATE action, got {action:?}"); + }; + assert_eq!(literal(&assignments[0].1), SqlValue::Int(2)); + assert_eq!( + literal(&assignments[1].1), + SqlValue::Timestamp(NdbDateTime::from_micros(EARLY_MICROS)) + ); + } + + /// A literal the column cannot hold is refused naming the column, on + /// both arms: text into INT, a boolean into TIMESTAMP, and an integer + /// past the declared SMALLINT width. + #[test] + fn a_literal_the_column_cannot_hold_is_refused_naming_the_column() { + let err = action("WHEN NOT MATCHED THEN INSERT (id, n) VALUES (s.id, 'abc')") + .expect_err("'abc' does not fit INT"); + assert!(err.to_string().contains("'n'"), "{err}"); + + let err = + action("WHEN MATCHED THEN UPDATE SET at = true").expect_err("true is not an instant"); + assert!(err.to_string().contains("'at'"), "{err}"); + + let err = action("WHEN NOT MATCHED THEN INSERT (id, s) VALUES (s.id, 999999)") + .expect_err("999999 does not fit SMALLINT"); + assert!( + matches!(err, SqlError::IntegerOutOfRange { ref column, .. } if column == "s"), + "{err}" + ); + } + + /// The primary key keeps its literal as written on both arms, like every + /// `VALUES` and `SET` path. + #[test] + fn the_primary_key_literal_is_exempt() { + let action = action("WHEN NOT MATCHED THEN INSERT (id, n) VALUES ('7', 1)").expect("plans"); + let MergePlanAction::Insert { values, .. } = action else { + panic!("expected an INSERT action, got {action:?}"); + }; + assert_eq!(literal(&values[0]), SqlValue::String("7".into())); + } + + /// A column-less INSERT arm binds to the declared column order, so its + /// literals still find their declared types. + #[test] + fn a_column_less_insert_arm_binds_positionally() { + let planned = action("WHEN NOT MATCHED THEN INSERT VALUES (s.id, '3', 1583402400000)") + .expect("plans"); + let MergePlanAction::Insert { columns, values } = planned else { + panic!("expected an INSERT action, got {planned:?}"); + }; + assert_eq!(columns, vec!["id", "n", "at"]); + assert_eq!(literal(&values[1]), SqlValue::Int(3)); + assert_eq!( + literal(&values[2]), + SqlValue::Timestamp(NdbDateTime::from_micros(EARLY_MICROS)) + ); + + let err = action("WHEN NOT MATCHED THEN INSERT VALUES (1, 2, 3, 4, 5)") + .expect_err("more values than declared columns"); + assert!( + matches!(err, SqlError::InsertColumnArityMismatch { .. }), + "{err}" + ); + } +} diff --git a/nodedb-sql/src/planner/merge/mod.rs b/nodedb-sql/src/planner/merge/mod.rs new file mode 100644 index 000000000..25ff6b2dc --- /dev/null +++ b/nodedb-sql/src/planner/merge/mod.rs @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! MERGE statement planning. + +mod actions; +mod plan; + +pub use plan::plan_merge; diff --git a/nodedb-sql/src/planner/merge.rs b/nodedb-sql/src/planner/merge/plan.rs similarity index 70% rename from nodedb-sql/src/planner/merge.rs rename to nodedb-sql/src/planner/merge/plan.rs index 24087614f..1886ffe38 100644 --- a/nodedb-sql/src/planner/merge.rs +++ b/nodedb-sql/src/planner/merge/plan.rs @@ -1,24 +1,23 @@ // SPDX-License-Identifier: Apache-2.0 -//! MERGE statement planning. +//! MERGE statement planning: target and source resolution, the ON clause, +//! and dispatch to the engine rules. //! //! Translates `sqlparser::ast::Statement::Merge` into `SqlPlan::Merge`. //! Supported engines: `document_schemaless`, `document_strict`. //! All other engines return `SqlError::Unsupported`. use nodedb_types::DatabaseId; -use sqlparser::ast::{self, MergeAction, MergeClauseKind as AstMergeClauseKind, MergeInsertKind}; +use sqlparser::ast; -use super::ast_helpers::{qualified_ident_pair, strip_and_convert_filters}; +use super::super::ast_helpers::qualified_ident_pair; +use super::actions::convert_merge_clauses; use crate::engine_rules::{self, MergeParams, ScanParams}; use crate::error::{Result, SqlError}; use crate::parser::normalize::{normalize_ident, normalize_object_name_checked}; -use crate::resolver::ColumnScope; use crate::resolver::columns::{ResolvedTable, TableScope}; -use crate::resolver::expr::convert_expr; use crate::temporal::TemporalScope; use crate::types::*; -use crate::types::{MergeClauseKind, MergePlanAction, MergePlanClause, SqlPlan}; /// Plan a `MERGE INTO target USING source ON ... WHEN ... THEN ...` statement. pub fn plan_merge(stmt: &ast::Statement, catalog: &dyn SqlCatalog) -> Result> { @@ -73,9 +72,9 @@ pub fn plan_merge(stmt: &ast::Statement, catalog: &dyn SqlCatalog) -> Result Result> { - clauses - .iter() - .map(|c| convert_one_clause(c, target_ref, source_ref, scope, target_scope)) - .collect() -} - -fn convert_one_clause( - clause: &ast::MergeClause, - target_ref: &str, - source_ref: &str, - scope: &TableScope, - target_scope: &TableScope, -) -> Result { - let kind = match clause.clause_kind { - AstMergeClauseKind::Matched => MergeClauseKind::Matched, - AstMergeClauseKind::NotMatched | AstMergeClauseKind::NotMatchedByTarget => { - MergeClauseKind::NotMatched - } - AstMergeClauseKind::NotMatchedBySource => MergeClauseKind::NotMatchedBySource, - }; - - let extra_predicate = match &clause.predicate { - Some(expr) => strip_and_convert_filters(vec![expr.clone()], target_ref, scope)?, - None => Vec::new(), - }; - - let action = convert_merge_action(&clause.action, source_ref, scope, target_scope)?; - - Ok(MergePlanClause { - kind, - extra_predicate, - action, - }) -} - -fn convert_merge_action( - action: &MergeAction, - source_ref: &str, - scope: &TableScope, - target_scope: &TableScope, -) -> Result { - match action { - MergeAction::Update(update_expr) => { - let assignments = update_expr - .assignments - .iter() - .map(|a| { - let col = match &a.target { - ast::AssignmentTarget::ColumnName(name) => { - normalize_object_name_checked(name) - } - ast::AssignmentTarget::Tuple(_) => Err(SqlError::Unsupported { - detail: "tuple assignment target in MERGE UPDATE is not supported" - .into(), - }), - }?; - target_scope.check_name(None, &col)?; - let val = convert_expr(&a.value, &ColumnScope::Relations(scope))?; - Ok((col, val)) - }) - .collect::>>()?; - Ok(MergePlanAction::Update { assignments }) - } - MergeAction::Delete { .. } => Ok(MergePlanAction::Delete), - MergeAction::Insert(insert_expr) => { - let columns: Vec = insert_expr - .columns - .iter() - .map(|c| { - let col = normalize_object_name_checked(c)?; - target_scope.check_name(None, &col)?; - Ok(col) - }) - .collect::>>()?; - - let values: Vec = match &insert_expr.kind { - MergeInsertKind::Values(vals) => { - if vals.rows.len() != 1 { - return Err(SqlError::Unsupported { - detail: format!( - "MERGE INSERT VALUES must have exactly one row; got {}", - vals.rows.len() - ), - }); - } - vals.rows[0] - .iter() - .map(|e| convert_expr(e, &ColumnScope::Relations(scope))) - .collect::>>()? - } - MergeInsertKind::Row => { - return Err(SqlError::Unsupported { - detail: "MERGE INSERT ROW is not supported; use explicit VALUES".into(), - }); - } - }; - - if !columns.is_empty() && columns.len() != values.len() { - return Err(SqlError::Parse { - detail: format!( - "MERGE INSERT column list ({}) and VALUES ({}) lengths do not match", - columns.len(), - values.len() - ), - }); - } - - let _ = source_ref; // for future multi-row insert support - Ok(MergePlanAction::Insert { columns, values }) - } - } -} - // ── Helpers ──────────────────────────────────────────────────────────────── -pub(super) fn extract_table_factor_name_alias( - factor: &ast::TableFactor, -) -> Result<(String, Option)> { +fn extract_table_factor_name_alias(factor: &ast::TableFactor) -> Result<(String, Option)> { match factor { ast::TableFactor::Table { name, alias, .. } => { let table_name = normalize_object_name_checked(name)?; diff --git a/nodedb-sql/src/planner/mod.rs b/nodedb-sql/src/planner/mod.rs index 6c6062d5e..e222cbc53 100644 --- a/nodedb-sql/src/planner/mod.rs +++ b/nodedb-sql/src/planner/mod.rs @@ -27,6 +27,7 @@ pub mod index_ddl; pub mod join; pub mod lateral; pub mod merge; +pub mod predicate_coerce; pub mod select; pub use select::qualified_name; diff --git a/nodedb-sql/src/planner/predicate_coerce.rs b/nodedb-sql/src/planner/predicate_coerce.rs new file mode 100644 index 000000000..085b58b6e --- /dev/null +++ b/nodedb-sql/src/planner/predicate_coerce.rs @@ -0,0 +1,459 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Coerce the literals a predicate compares against a declared `TIMESTAMP` / +//! `TIMESTAMPTZ` column into typed instants, once, at plan time. +//! +//! # Why this exists +//! +//! A time literal in a `WHERE`, `ON`, or `WHEN` clause reaches the planner as +//! whatever the user typed: `at > 1583402400000`, `at > '2020-03-05 +//! 10:00:00'`, `at > TIMESTAMP '...'`. The write side already resolves the +//! same three spellings to one typed instant through +//! [`super::declared_type_coerce::coerce_value`], so every engine stores an +//! instant. Left as typed, a numeric read literal is an `Integer` the Data +//! Plane compares against an instant: the pair has no defined equality or +//! order, so `=` never matches and `>=` either matches nothing or, on a path +//! that guessed a unit, matches by accident. +//! +//! Resolving the literal here, against the declared column type the scope +//! already carries, means the Data Plane only ever compares instants with +//! instants. It also puts the type error where PostgreSQL puts it: `WHERE at +//! = true` fails the statement naming the column, instead of scanning every +//! row to match none. +//! +//! # Scope +//! +//! Only `TIMESTAMP` / `TIMESTAMPTZ` columns are coerced. Every other declared +//! type is left as written: the Data Plane's coerced comparison already +//! reads `"5"` against `5` and `1.5` against `1`, so no read literal for +//! those types is lost the way a bare integer against an instant is. +//! +//! The primary-key column is exempt, mirroring the write side: the engines +//! derive a row's identity from the literal's own rendering on both the +//! write and the read side, and re-typing one side would make the row +//! unfindable by the other. +//! +//! A column the scope does not declare (a schemaless field, a computed +//! alias, an aggregate output) is left untouched: there is no declared type +//! to coerce to. + +use crate::error::Result; +use crate::resolver::columns::TableScope; +use crate::types::{BinaryOp, ColumnInfo, SqlDataType, SqlExpr, SqlValue, UnaryOp}; + +use super::declared_type_coerce::coerce_value; + +/// Coerce one literal compared against `column` by the read-side rule. +/// +/// The one rule for a literal a predicate compares against a declared +/// column: an instant column (`TIMESTAMP` / `TIMESTAMPTZ`) types the literal +/// through [`coerce_value`], the primary-key column and every other declared +/// type keep the literal as written. `WHERE` clauses reach it through +/// [`coerce_predicate_literals`]; row-level-security policy compilation calls +/// it directly, so a policy literal and a query literal on the same column +/// resolve to the same typed instant. +/// +/// Errors name the column and the literal when the literal carries no +/// instant. +pub fn coerce_read_literal(column: &ColumnInfo, value: SqlValue) -> Result { + if column.is_primary_key || !is_instant(&column.data_type) { + return Ok(value); + } + coerce_value(&column.name, value, &column.data_type) +} + +/// Coerce, in place, every literal `expr` compares against a declared +/// instant column. +/// +/// Walks `AND` / `OR` / `NOT` down to each predicate and handles the shapes +/// that pair one column with literals: `column literal` (either +/// orientation), `column BETWEEN literal AND literal`, and `column IN +/// (literal, …)`. A literal of a kind that carries no instant (a boolean, +/// bytes, an array) is a typed error naming the column. +pub(crate) fn coerce_predicate_literals(expr: &mut SqlExpr, scope: &TableScope) -> Result<()> { + match expr { + SqlExpr::BinaryOp { left, op, right } => match op { + BinaryOp::And | BinaryOp::Or => { + coerce_predicate_literals(left, scope)?; + coerce_predicate_literals(right, scope) + } + BinaryOp::Eq + | BinaryOp::Ne + | BinaryOp::Gt + | BinaryOp::Ge + | BinaryOp::Lt + | BinaryOp::Le => match (left.as_mut(), right.as_mut()) { + (SqlExpr::Column { table, name }, SqlExpr::Literal(value)) + | (SqlExpr::Literal(value), SqlExpr::Column { table, name }) => { + coerce_literal(scope, table.as_deref(), name, value) + } + // Column-vs-column, computed operands, subqueries: no literal + // to coerce. + _ => Ok(()), + }, + BinaryOp::Add + | BinaryOp::Sub + | BinaryOp::Mul + | BinaryOp::Div + | BinaryOp::Mod + | BinaryOp::Concat => Ok(()), + }, + SqlExpr::UnaryOp { + op: UnaryOp::Not, + expr, + } => coerce_predicate_literals(expr, scope), + SqlExpr::Between { + expr, low, high, .. + } => { + let SqlExpr::Column { table, name } = expr.as_ref() else { + return Ok(()); + }; + for bound in [low.as_mut(), high.as_mut()] { + if let SqlExpr::Literal(value) = bound { + coerce_literal(scope, table.as_deref(), name, value)?; + } + } + Ok(()) + } + SqlExpr::InList { expr, list, .. } => { + let SqlExpr::Column { table, name } = expr.as_ref() else { + return Ok(()); + }; + for element in list.iter_mut() { + if let SqlExpr::Literal(value) = element { + coerce_literal(scope, table.as_deref(), name, value)?; + } + } + Ok(()) + } + SqlExpr::UnaryOp { + op: UnaryOp::Neg, .. + } + | SqlExpr::Column { .. } + | SqlExpr::Literal(_) + | SqlExpr::Function { .. } + | SqlExpr::Case { .. } + | SqlExpr::Cast { .. } + | SqlExpr::Subquery(_) + | SqlExpr::Wildcard + | SqlExpr::IsNull { .. } + | SqlExpr::Like { .. } + | SqlExpr::ArrayLiteral(_) => Ok(()), + } +} + +/// Coerce one literal compared against the column `table.name` names, when +/// the scope declares that column as an instant. +fn coerce_literal( + scope: &TableScope, + table: Option<&str>, + name: &str, + value: &mut SqlValue, +) -> Result<()> { + let Some(column) = scope.declared_column(table, name) else { + return Ok(()); + }; + let taken = std::mem::replace(value, SqlValue::Null); + *value = coerce_read_literal(column, taken)?; + Ok(()) +} + +/// Whether a declared type stores a typed instant. +fn is_instant(declared: &SqlDataType) -> bool { + match declared { + SqlDataType::Timestamp | SqlDataType::Timestamptz => true, + SqlDataType::Int64 + | SqlDataType::Float64 + | SqlDataType::String + | SqlDataType::Bool + | SqlDataType::Bytes + | SqlDataType::Decimal + | SqlDataType::Uuid + | SqlDataType::Vector(_) + | SqlDataType::Geometry + | SqlDataType::Unknown => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::resolver::columns::ResolvedTable; + use crate::types::{CollectionInfo, ColumnInfo, EngineType}; + use nodedb_types::datetime::NdbDateTime; + + /// `2020-03-05T10:00:00Z` as epoch milliseconds. + const EARLY_MS: i64 = 1_583_402_400_000; + + fn early() -> NdbDateTime { + NdbDateTime::from_micros(EARLY_MS * 1_000) + } + + fn column(name: &str, data_type: SqlDataType, is_primary_key: bool) -> ColumnInfo { + ColumnInfo { + name: name.to_string(), + data_type, + nullable: true, + is_primary_key, + default: None, + raw_type: None, + int_width: None, + float_width: None, + } + } + + /// `t (id TEXT PRIMARY KEY, at TIMESTAMP, at_tz TIMESTAMPTZ, n BIGINT, + /// pk_at TIMESTAMP PRIMARY KEY)` under alias `a`. + fn scope() -> TableScope { + let info = CollectionInfo { + name: "t".into(), + engine: EngineType::DocumentStrict, + columns: vec![ + column("id", SqlDataType::String, true), + column("at", SqlDataType::Timestamp, false), + column("at_tz", SqlDataType::Timestamptz, false), + column("n", SqlDataType::Int64, false), + column("pk_at", SqlDataType::Timestamp, true), + ], + primary_key: Some("id".into()), + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentStrict), + }; + TableScope::single(ResolvedTable { + name: info.name.clone(), + alias: Some("a".into()), + info, + }) + .expect("single-relation scope") + } + + fn col(table: Option<&str>, name: &str) -> SqlExpr { + SqlExpr::Column { + table: table.map(str::to_string), + name: name.into(), + } + } + + fn lit(value: SqlValue) -> SqlExpr { + SqlExpr::Literal(value) + } + + fn cmp(left: SqlExpr, op: BinaryOp, right: SqlExpr) -> SqlExpr { + SqlExpr::BinaryOp { + left: Box::new(left), + op, + right: Box::new(right), + } + } + + fn literal_of(expr: &SqlExpr) -> &SqlValue { + match expr { + SqlExpr::Literal(v) => v, + other => panic!("expected a literal, got {other:?}"), + } + } + + fn coerced(mut expr: SqlExpr) -> SqlExpr { + coerce_predicate_literals(&mut expr, &scope()).expect("predicate coerces"); + expr + } + + #[test] + fn column_op_literal_coerces_a_numeric_literal_to_the_declared_instant() { + let expr = coerced(cmp( + col(None, "at"), + BinaryOp::Gt, + lit(SqlValue::Int(EARLY_MS)), + )); + let SqlExpr::BinaryOp { right, .. } = &expr else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(right), &SqlValue::Timestamp(early())); + } + + #[test] + fn literal_op_column_is_coerced_in_the_mirrored_orientation() { + let expr = coerced(cmp( + lit(SqlValue::String("2020-03-05 10:00:00".into())), + BinaryOp::Lt, + col(Some("a"), "at_tz"), + )); + let SqlExpr::BinaryOp { left, .. } = &expr else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(left), &SqlValue::Timestamptz(early())); + } + + #[test] + fn between_coerces_both_bounds() { + let expr = coerced(SqlExpr::Between { + expr: Box::new(col(None, "at")), + low: Box::new(lit(SqlValue::Int(EARLY_MS))), + high: Box::new(lit(SqlValue::String("2020-03-05T11:00:00Z".into()))), + negated: false, + }); + let SqlExpr::Between { low, high, .. } = &expr else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(low), &SqlValue::Timestamp(early())); + assert_eq!( + literal_of(high), + &SqlValue::Timestamp(NdbDateTime::from_micros((EARLY_MS + 3_600_000) * 1_000)) + ); + } + + #[test] + fn in_list_coerces_every_literal_element() { + let expr = coerced(SqlExpr::InList { + expr: Box::new(col(None, "at")), + list: vec![ + lit(SqlValue::Int(EARLY_MS)), + lit(SqlValue::String("2020-03-05 10:00:00".into())), + ], + negated: true, + }); + let SqlExpr::InList { list, .. } = &expr else { + panic!("shape preserved"); + }; + for element in list { + assert_eq!(literal_of(element), &SqlValue::Timestamp(early())); + } + } + + #[test] + fn nested_and_or_not_reach_every_predicate() { + let expr = coerced(SqlExpr::UnaryOp { + op: UnaryOp::Not, + expr: Box::new(cmp( + cmp(col(None, "at"), BinaryOp::Ge, lit(SqlValue::Int(EARLY_MS))), + BinaryOp::Or, + cmp( + cmp(col(None, "n"), BinaryOp::Eq, lit(SqlValue::Int(1))), + BinaryOp::And, + cmp( + col(None, "at_tz"), + BinaryOp::Le, + lit(SqlValue::Int(EARLY_MS)), + ), + ), + )), + }); + let SqlExpr::UnaryOp { expr, .. } = &expr else { + panic!("shape preserved"); + }; + let SqlExpr::BinaryOp { left, right, .. } = expr.as_ref() else { + panic!("shape preserved"); + }; + let SqlExpr::BinaryOp { right: at_lit, .. } = left.as_ref() else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(at_lit), &SqlValue::Timestamp(early())); + let SqlExpr::BinaryOp { + left: n_pred, + right: tz_pred, + .. + } = right.as_ref() + else { + panic!("shape preserved"); + }; + let SqlExpr::BinaryOp { right: n_lit, .. } = n_pred.as_ref() else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(n_lit), &SqlValue::Int(1), "BIGINT is untouched"); + let SqlExpr::BinaryOp { right: tz_lit, .. } = tz_pred.as_ref() else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(tz_lit), &SqlValue::Timestamptz(early())); + } + + #[test] + fn a_column_the_scope_does_not_declare_is_left_untouched() { + let expr = coerced(cmp( + col(None, "created"), + BinaryOp::Eq, + lit(SqlValue::Int(EARLY_MS)), + )); + let SqlExpr::BinaryOp { right, .. } = &expr else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(right), &SqlValue::Int(EARLY_MS)); + } + + #[test] + fn a_qualifier_naming_another_relation_is_left_untouched() { + let expr = coerced(cmp( + col(Some("other"), "at"), + BinaryOp::Eq, + lit(SqlValue::Int(EARLY_MS)), + )); + let SqlExpr::BinaryOp { right, .. } = &expr else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(right), &SqlValue::Int(EARLY_MS)); + } + + #[test] + fn the_primary_key_column_is_exempt() { + let expr = coerced(cmp( + col(None, "pk_at"), + BinaryOp::Eq, + lit(SqlValue::Int(EARLY_MS)), + )); + let SqlExpr::BinaryOp { right, .. } = &expr else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(right), &SqlValue::Int(EARLY_MS)); + } + + #[test] + fn column_vs_column_and_null_are_left_alone() { + let expr = coerced(cmp( + cmp(col(None, "at"), BinaryOp::Lt, col(None, "at_tz")), + BinaryOp::And, + cmp(col(None, "at"), BinaryOp::Eq, lit(SqlValue::Null)), + )); + let SqlExpr::BinaryOp { right, .. } = &expr else { + panic!("shape preserved"); + }; + let SqlExpr::BinaryOp { + right: null_lit, .. + } = right.as_ref() + else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(null_lit), &SqlValue::Null); + } + + #[test] + fn a_boolean_literal_is_a_typed_error_naming_the_column() { + let mut expr = cmp(col(None, "at"), BinaryOp::Eq, lit(SqlValue::Bool(true))); + let err = coerce_predicate_literals(&mut expr, &scope()) + .expect_err("a boolean carries no instant"); + let detail = err.to_string(); + assert!( + detail.contains("at") && detail.contains("a boolean"), + "error must name the column and the kind: {detail}" + ); + } + + #[test] + fn unparseable_text_is_a_typed_error_naming_the_column_and_literal() { + let mut expr = SqlExpr::InList { + expr: Box::new(col(None, "at")), + list: vec![lit(SqlValue::String("not a date".into()))], + negated: false, + }; + let err = coerce_predicate_literals(&mut expr, &scope()) + .expect_err("text that spells no instant is refused"); + let detail = err.to_string(); + assert!( + detail.contains("at") && detail.contains("not a date"), + "error must name the column and the literal: {detail}" + ); + } +} diff --git a/nodedb-sql/src/planner/select/helpers.rs b/nodedb-sql/src/planner/select/helpers.rs index d8bcc2f2c..c46421920 100644 --- a/nodedb-sql/src/planner/select/helpers.rs +++ b/nodedb-sql/src/planner/select/helpers.rs @@ -7,6 +7,7 @@ use sqlparser::ast; use crate::error::{Result, SqlError}; use crate::parser::normalize::{SCHEMA_QUALIFIED_MSG, normalize_ident}; +use crate::planner::predicate_coerce::coerce_predicate_literals; use crate::resolver::ColumnScope; use crate::resolver::columns::TableScope; use crate::resolver::expr::convert_expr; @@ -95,8 +96,15 @@ pub fn qualified_name(table: Option<&str>, name: &str) -> String { } /// Convert a WHERE expression into a list of Filter. +/// +/// The one choke point every predicate passes through: `SELECT ... WHERE`, +/// `UPDATE` / `DELETE ... WHERE`, a join's post-filter, `HAVING`, a LATERAL +/// body, and a MERGE `WHEN ... AND` predicate. Literals compared against a +/// declared instant column are coerced here, so no downstream path sees a +/// bare integer where an instant is stored. pub fn convert_where_to_filters(expr: &ast::Expr, scope: &TableScope) -> Result> { - let sql_expr = canonicalize_predicate(convert_expr(expr, &ColumnScope::Relations(scope))?); + let mut sql_expr = canonicalize_predicate(convert_expr(expr, &ColumnScope::Relations(scope))?); + coerce_predicate_literals(&mut sql_expr, scope)?; Ok(vec![Filter { expr: FilterExpr::Expr(sql_expr), }]) diff --git a/nodedb-sql/src/types/plan/cacheability.rs b/nodedb-sql/src/types/plan/cacheability.rs index c03d0b509..1c2b392c8 100644 --- a/nodedb-sql/src/types/plan/cacheability.rs +++ b/nodedb-sql/src/types/plan/cacheability.rs @@ -45,12 +45,18 @@ impl SqlPlan { match self { Self::ConstantResult { volatile: true, .. } => DataDependent, Self::Insert { - column_defaults, .. + volatile_defaults: true, + .. } | Self::Upsert { - column_defaults, .. - } if super::volatility_scan::defaults_are_volatile(column_defaults) => DataDependent, - Self::KvInsert { + volatile_defaults: true, + .. + } + | Self::TimeseriesIngest { + volatile_defaults: true, + .. + } + | Self::KvInsert { volatile_defaults: true, .. } diff --git a/nodedb-sql/src/types/plan/mod.rs b/nodedb-sql/src/types/plan/mod.rs index ad6020b61..371c9a13e 100644 --- a/nodedb-sql/src/types/plan/mod.rs +++ b/nodedb-sql/src/types/plan/mod.rs @@ -15,4 +15,4 @@ pub use merge_types::{MergeClauseKind, MergePlanAction, MergePlanClause}; pub use row_types::{KvInsertIntent, VectorPrimaryRow, WriteRoute}; pub use variants::{DistanceMetric, SqlPlan}; pub use vector_opts::{ArrayPrefilter, VectorAnnOptions, VectorQuantization}; -pub use volatility_scan::{default_expr_is_volatile, defaults_are_volatile, expr_is_volatile}; +pub use volatility_scan::expr_is_volatile; diff --git a/nodedb-sql/src/types/plan/variants.rs b/nodedb-sql/src/types/plan/variants.rs index f5ad2b736..7439de518 100644 --- a/nodedb-sql/src/types/plan/variants.rs +++ b/nodedb-sql/src/types/plan/variants.rs @@ -107,10 +107,14 @@ pub enum SqlPlan { /// The lowering these rows take, chosen by the engine's `EngineRules`. /// The conversion layer reads it instead of re-deciding from `engine`. route: WriteRoute, + /// Every declared DEFAULT already materialized and every literal + /// coerced to its declared column type by the planner. rows: Vec>, - /// Column defaults from schema: `(column_name, default_expr)`. - /// Used to auto-generate values for missing columns (e.g. `id` with `UUID_V7`). - column_defaults: Vec<(String, String)>, + /// Whether a DEFAULT materialized into `rows` was volatile. The + /// planner evaluates declared defaults while building this plan, so a + /// cached plan would replay one execution's value; a volatile plan is + /// never cached. + volatile_defaults: bool, /// `ON CONFLICT DO NOTHING` semantics: when true, duplicate-PK rows /// are silently skipped instead of raising `unique_violation`. Plain /// `INSERT` (no `ON CONFLICT` clause) sets this to `false`. @@ -157,8 +161,10 @@ pub enum SqlPlan { engine: EngineType, /// The lowering these rows take. Mirrors `Insert::route`. route: WriteRoute, + /// Defaults materialized and literals coerced, as in `Insert::rows`. rows: Vec>, - column_defaults: Vec<(String, String)>, + /// Mirrors `Insert::volatile_defaults`. + volatile_defaults: bool, /// `ON CONFLICT (...) DO UPDATE SET field = expr` assignments. /// When empty, upsert is a plain merge: new columns overwrite existing. /// When non-empty, the engine applies these per-row against the @@ -296,7 +302,13 @@ pub enum SqlPlan { }, TimeseriesIngest { collection: String, + /// Defaults materialized and literals coerced, as in `Insert::rows`. + /// A row that omits the `TIME_KEY` column carries its declared + /// default here when one exists; only a row with no time value at + /// all takes the ingest clock. rows: Vec>, + /// Mirrors `Insert::volatile_defaults`. + volatile_defaults: bool, }, // ── Search (first-class) ── diff --git a/nodedb-sql/src/types/plan/volatility_scan.rs b/nodedb-sql/src/types/plan/volatility_scan.rs index b72965c36..60d518d1f 100644 --- a/nodedb-sql/src/types/plan/volatility_scan.rs +++ b/nodedb-sql/src/types/plan/volatility_scan.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -//! Volatility scanning over plan expressions and column DEFAULT strings. +//! Volatility scanning over plan expressions. //! //! A plan holding a volatile call must re-plan per execution, so the plan //! cache never replays a frozen value. @@ -42,39 +42,3 @@ pub fn expr_is_volatile(expr: &SqlExpr) -> bool { } } } - -/// Column DEFAULT spellings that generate a fresh value per row but name no -/// registered function. Kept in step with the generator arms of -/// `crate::planner::defaults`. -const VOLATILE_DEFAULT_ALIASES: &[&str] = - &["uuidv7", "uuidv4", "gen_uuid_v7", "gen_uuid_v4", "gen_ulid"]; - -/// Whether a stored column DEFAULT expression re-evaluates per execution. -/// -/// The catalog stores a DEFAULT as text, in either a bare form (`UUID_V7`) or -/// a call form (`uuid_v7()`, `nextval('s')`), so the leading identifier is -/// checked before the string is parsed as an expression. -pub fn default_expr_is_volatile(expr: &str) -> bool { - let trimmed = expr.trim(); - let head: String = trimmed - .chars() - .take_while(|c| c.is_alphanumeric() || *c == '_') - .collect::() - .to_ascii_lowercase(); - if !head.is_empty() - && (default_registry().is_volatile(&head) - || VOLATILE_DEFAULT_ALIASES.contains(&head.as_str())) - { - return true; - } - crate::parse_expr_string(trimmed) - .ok() - .is_some_and(|parsed| expr_is_volatile(&parsed)) -} - -/// Whether any `(column, default_expr)` pair re-evaluates per execution. -pub fn defaults_are_volatile(defaults: &[(String, String)]) -> bool { - defaults - .iter() - .any(|(_, expr)| default_expr_is_volatile(expr)) -} diff --git a/nodedb-sql/src/visitor/plan_visitor/args.rs b/nodedb-sql/src/visitor/plan_visitor/args.rs index 81207a39a..f3678a436 100644 --- a/nodedb-sql/src/visitor/plan_visitor/args.rs +++ b/nodedb-sql/src/visitor/plan_visitor/args.rs @@ -68,7 +68,6 @@ pub struct InsertVisitArgs<'a> { /// The lowering these rows take, decided by the engine's `EngineRules`. pub route: WriteRoute, pub rows: &'a [Vec<(String, SqlValue)>], - pub column_defaults: &'a [(String, String)], pub if_absent: bool, pub column_schema: &'a [(String, String)], pub primary_key: Option<&'a str>, @@ -81,7 +80,6 @@ pub struct UpsertVisitArgs<'a> { /// The lowering these rows take, decided by the engine's `EngineRules`. pub route: WriteRoute, pub rows: &'a [Vec<(String, SqlValue)>], - pub column_defaults: &'a [(String, String)], pub on_conflict_updates: &'a [(String, SqlExpr)], pub column_schema: &'a [(String, String)], pub primary_key: Option<&'a str>, diff --git a/nodedb-sql/src/visitor/plan_visitor/dispatch.rs b/nodedb-sql/src/visitor/plan_visitor/dispatch.rs index dc8c98930..641295e96 100644 --- a/nodedb-sql/src/visitor/plan_visitor/dispatch.rs +++ b/nodedb-sql/src/visitor/plan_visitor/dispatch.rs @@ -97,7 +97,7 @@ pub fn dispatch(visitor: &mut V, plan: &SqlPlan) -> Result(visitor: &mut V, plan: &SqlPlan) -> Result(visitor: &mut V, plan: &SqlPlan) -> Result(visitor: &mut V, plan: &SqlPlan) -> Result(visitor: &mut V, plan: &SqlPlan) -> Result { - visitor.timeseries_ingest(collection, rows) - } + SqlPlan::TimeseriesIngest { + collection, + rows, + volatile_defaults: _, + } => visitor.timeseries_ingest(collection, rows), SqlPlan::VectorSearch { collection, field, diff --git a/nodedb-types/src/columnar/column_type.rs b/nodedb-types/src/columnar/column_type.rs index 262ef622a..7eb45829c 100644 --- a/nodedb-types/src/columnar/column_type.rs +++ b/nodedb-types/src/columnar/column_type.rs @@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize}; +use crate::InstantKind; use crate::value::Value; /// Typed column definition for strict document and columnar collections. @@ -118,7 +119,56 @@ impl ColumnType { /// `SystemTimestamp` is not an instant. It is engine-assigned from HLC at /// commit and the planner types it as text. pub const fn is_instant(&self) -> bool { - matches!(self, Self::Timestamp | Self::Timestamptz) + self.instant_kind().is_some() + } + + /// The instant kind a column of this type carries: `Naive` for + /// `Timestamp`, `Utc` for `Timestamptz`, `None` for every other type. + /// + /// A stored time cell is wrapped as `Value::NaiveDateTime` or + /// `Value::DateTime` by this kind, so a reader never has to decide which + /// variant a column means. `SystemTimestamp` and `Duration` share the + /// eight-byte time storage but are not instants: their cells read back as + /// the integer stored. + pub const fn instant_kind(&self) -> Option { + match self { + Self::Timestamp => Some(InstantKind::Naive), + Self::Timestamptz => Some(InstantKind::Utc), + Self::Int64 + | Self::Float64 + | Self::String + | Self::Bool + | Self::Bytes + | Self::SystemTimestamp + | Self::Decimal { .. } + | Self::Geometry + | Self::Vector(_) + | Self::SparseVector + | Self::Uuid + | Self::Json + | Self::Ulid + | Self::Duration + | Self::Array + | Self::Set + | Self::Regex + | Self::Range + | Self::Record => None, + } + } + + /// Type a stored time cell by this declared type. + /// + /// An instant column (`Timestamp`, `Timestamptz`) wraps `micros` as its + /// `instant_kind()` variant; every other declared type backed by + /// eight-byte time storage (`SystemTimestamp`, `Duration`) returns the + /// integer stored. The single mapping every segment-cell and memtable-cell + /// reader uses, so a row reads identically whether the reader inferred the + /// column's physical kind as `Int64` or `Timestamp`. + pub fn time_cell(&self, micros: i64) -> Value { + match self.instant_kind() { + Some(kind) => kind.from_micros(micros), + None => Value::Integer(micros), + } } /// Return the canonical PostgreSQL type OID for this column type. @@ -223,6 +273,23 @@ mod tests { crate::datetime::NdbDateTime::from_micros(0) } + #[test] + fn instant_kind_names_the_variant_a_time_column_reads_back_as() { + assert_eq!( + ColumnType::Timestamp.instant_kind(), + Some(InstantKind::Naive) + ); + assert_eq!( + ColumnType::Timestamptz.instant_kind(), + Some(InstantKind::Utc) + ); + assert_eq!(ColumnType::SystemTimestamp.instant_kind(), None); + assert_eq!(ColumnType::Duration.instant_kind(), None); + assert_eq!(ColumnType::Int64.instant_kind(), None); + assert!(ColumnType::Timestamp.is_instant()); + assert!(!ColumnType::SystemTimestamp.is_instant()); + } + #[test] fn to_pg_oid_stable() { assert_eq!(ColumnType::Bool.to_pg_oid(), 16); diff --git a/nodedb-types/src/datetime/timestamp.rs b/nodedb-types/src/datetime/timestamp.rs index d050924f7..c88daade6 100644 --- a/nodedb-types/src/datetime/timestamp.rs +++ b/nodedb-types/src/datetime/timestamp.rs @@ -133,53 +133,65 @@ impl NdbDateTime { ) } - /// Parse from ISO 8601 string (basic subset). + /// Parse from ISO 8601 text. /// - /// Supports: `"2024-03-15T10:30:00Z"`, `"2024-03-15T10:30:00.123456Z"`, - /// `"2024-03-15"` (midnight UTC). + /// Accepts `"2024-03-15T10:30:00Z"`, `"2024-03-15 10:30:00"`, + /// `"2024-03-15T10:30:00.123456Z"`, `"2024-03-15"` (midnight UTC), and + /// a trailing UTC offset in place of `Z` — `+05:30`, `-0800`, `+02` — + /// which shifts the result to UTC. Seconds are optional. Every component + /// must parse whole: trailing characters after the seconds, the fraction + /// or the offset are refused rather than dropped, so an unrecognised + /// spelling is `None`, never a silently different instant. pub fn parse(s: &str) -> Option { - let s = s.trim().trim_end_matches('Z').trim_end_matches('z'); + let (body, offset_secs) = split_utc_offset(s.trim())?; - if s.len() == 10 { + if body.len() == 10 { // Date only: "2024-03-15" → midnight UTC. - let parts: Vec<&str> = s.split('-').collect(); - if parts.len() != 3 { - return None; - } - let year: i32 = parts[0].parse().ok()?; - let month: u32 = parts[1].parse().ok()?; - let day: u32 = parts[2].parse().ok()?; - return Self::from_civil(year, month, day, 0, 0, 0, 0); + let (year, month, day) = parse_civil_date(body)?; + return Self::from_civil(year, month, day, 0, 0, 0, 0)?.shift_secs(-offset_secs); } // Full: "2024-03-15T10:30:00" or "2024-03-15T10:30:00.123456" - let (date_part, time_part) = s.split_once('T').or_else(|| s.split_once(' '))?; - let date_parts: Vec<&str> = date_part.split('-').collect(); - if date_parts.len() != 3 { - return None; - } - let year: i32 = date_parts[0].parse().ok()?; - let month: u32 = date_parts[1].parse().ok()?; - let day: u32 = date_parts[2].parse().ok()?; - - let (time_main, frac) = if let Some((t, f)) = time_part.split_once('.') { - (t, f) - } else { - (time_part, "0") + let (date_part, time_part) = body.split_once('T').or_else(|| body.split_once(' '))?; + let (year, month, day) = parse_civil_date(date_part)?; + + let (time_main, frac) = match time_part.split_once('.') { + Some((t, f)) => (t, Some(f)), + None => (time_part, None), }; let time_parts: Vec<&str> = time_main.split(':').collect(); - if time_parts.len() < 2 { + if time_parts.len() < 2 || time_parts.len() > 3 { return None; } let hour: u32 = time_parts[0].parse().ok()?; let minute: u32 = time_parts[1].parse().ok()?; - let second: u32 = time_parts.get(2).and_then(|s| s.parse().ok()).unwrap_or(0); + let second: u32 = match time_parts.get(2) { + Some(text) => text.parse().ok()?, + None => 0, + }; - // Parse fractional seconds (up to microseconds). - let frac_padded = format!("{frac:0<6}"); - let micros: u32 = frac_padded[..6].parse().unwrap_or(0); + // Fractional seconds: one to nine digits, read to microseconds. + let micros: u32 = match frac { + Some(digits) => { + if digits.is_empty() + || digits.len() > 9 + || !digits.bytes().all(|b| b.is_ascii_digit()) + { + return None; + } + let padded = format!("{digits:0<6}"); + padded[..6].parse().ok()? + } + None => 0, + }; - Self::from_civil(year, month, day, hour, minute, second, micros) + Self::from_civil(year, month, day, hour, minute, second, micros)?.shift_secs(-offset_secs) + } + + /// Shift by whole seconds, or `None` on overflow. + fn shift_secs(self, secs: i64) -> Option { + let micros = self.micros.checked_add(secs.checked_mul(1_000_000)?)?; + Some(Self { micros }) } /// Build from civil date components. @@ -262,6 +274,60 @@ impl NdbDateTime { } } +/// Split a trailing `Z` or `±HH[:MM]` / `±HHMM` UTC offset off `s`, returning +/// the remaining text and the offset in seconds east of UTC (`0` when there +/// is none). A `-` inside the date part is never read as an offset: only a +/// sign after the last `T` / ` ` time separator counts. +fn split_utc_offset(s: &str) -> Option<(&str, i64)> { + if let Some(body) = s.strip_suffix('Z').or_else(|| s.strip_suffix('z')) { + return Some((body, 0)); + } + // The offset can only follow the time part, or a bare `YYYY-MM-DD`. + let time_start = match s.rfind(['T', ' ']) { + Some(i) => i + 1, + None => { + let at = s.len().min(10); + if !s.is_char_boundary(at) { + return None; + } + at + } + }; + let Some(sign_at) = s[time_start..].rfind(['+', '-']).map(|i| time_start + i) else { + return Some((s, 0)); + }; + let sign: i64 = if s.as_bytes()[sign_at] == b'-' { -1 } else { 1 }; + let rest = &s[sign_at + 1..]; + let (hh, mm) = match (rest.len(), rest.as_bytes().get(2)) { + (2, None) => (rest, "0"), + (4, Some(b)) if b.is_ascii_digit() => (&rest[..2], &rest[2..]), + (5, Some(&b':')) => (&rest[..2], &rest[3..]), + _ => return None, + }; + if !hh.bytes().all(|b| b.is_ascii_digit()) || !mm.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let hours: i64 = hh.parse().ok()?; + let minutes: i64 = mm.parse().ok()?; + if hours > 23 || minutes > 59 { + return None; + } + Some((&s[..sign_at], sign * (hours * 3600 + minutes * 60))) +} + +/// Parse `YYYY-MM-DD` into its three components. +fn parse_civil_date(s: &str) -> Option<(i32, u32, u32)> { + let parts: Vec<&str> = s.split('-').collect(); + if parts.len() != 3 { + return None; + } + Some(( + parts[0].parse().ok()?, + parts[1].parse().ok()?, + parts[2].parse().ok()?, + )) +} + impl std::fmt::Display for NdbDateTime { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(&self.to_iso8601()) @@ -323,6 +389,53 @@ mod tests { assert_eq!(c.microsecond, 123456); } + /// A trailing offset shifts the instant to UTC; a space separator and + /// missing seconds are accepted. + #[test] + fn datetime_parse_applies_utc_offset() { + let utc = NdbDateTime::parse("2024-06-15T06:30:00Z").unwrap(); + assert_eq!( + NdbDateTime::parse("2024-06-15 12:00:00+05:30").unwrap(), + utc + ); + assert_eq!(NdbDateTime::parse("2024-06-15T12:00:00+0530").unwrap(), utc); + assert_eq!( + NdbDateTime::parse("2024-06-15T04:30:00-02:00").unwrap(), + utc + ); + assert_eq!(NdbDateTime::parse("2024-06-15T04:30-02").unwrap(), utc); + assert_eq!( + NdbDateTime::parse("2024-06-15T06:30:00.250+00:00").unwrap(), + NdbDateTime::from_micros(utc.micros + 250_000) + ); + assert_eq!( + NdbDateTime::parse("2024-06-16+05:30").unwrap(), + NdbDateTime::parse("2024-06-15T18:30:00Z").unwrap() + ); + } + + /// Text that is not a whole timestamp is refused, never read as a + /// different instant. + #[test] + fn datetime_parse_refuses_partial_spellings() { + for text in [ + "yesterday", + "1583402400000000", + "2024-06-15T12:00:00+05:30:00", + "2024-06-15T12:00:xx", + "2024-06-15T12:00:00.abc", + "2024-06-15T12:00:00.", + "2024-06-15T12:00:00.1234567890", + "2024-06-15T12", + "2024-06-15T12:00:00+25:00", + ] { + assert!( + NdbDateTime::parse(text).is_none(), + "{text:?} must not parse" + ); + } + } + #[test] fn datetime_date_only() { let dt = NdbDateTime::parse("2024-03-15").unwrap(); diff --git a/nodedb-types/src/error/sqlstate.rs b/nodedb-types/src/error/sqlstate.rs index 076dc0758..cc8a8899c 100644 --- a/nodedb-types/src/error/sqlstate.rs +++ b/nodedb-types/src/error/sqlstate.rs @@ -148,6 +148,10 @@ pub const UNDEFINED_COLUMN: &str = "42703"; /// more than one relation in scope. pub const AMBIGUOUS_COLUMN: &str = "42702"; +/// `42804` — `datatype_mismatch` (a literal the declared column type cannot +/// represent, such as a column `DEFAULT` refused at `CREATE`) +pub const DATATYPE_MISMATCH: &str = "42804"; + /// `42846` — `cannot_coerce` pub const CANNOT_COERCE: &str = "42846"; @@ -363,6 +367,7 @@ mod tests { UNDEFINED_COLUMN, AMBIGUOUS_COLUMN, UNDEFINED_OBJECT, + DATATYPE_MISMATCH, CANNOT_COERCE, UNDEFINED_TABLE, UNDEFINED_FUNCTION, diff --git a/nodedb-types/src/json_msgpack/instant_ext.rs b/nodedb-types/src/json_msgpack/instant_ext.rs new file mode 100644 index 000000000..3b5cf99bf --- /dev/null +++ b/nodedb-types/src/json_msgpack/instant_ext.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Msgpack ext encoding for typed instants. +//! +//! A typed instant is written to plain msgpack as `fixext8`: marker `0xD7`, +//! one ext type byte, then epoch microseconds as big-endian `i64`. Ten bytes +//! in total. Ext type `1` is a UTC instant (`Value::DateTime`), ext type `2` +//! is a naive instant (`Value::NaiveDateTime`). +//! +//! The payload is byte-comparable within one kind for non-negative micros +//! only. Comparators decode and compare `(kind, i64)` signed. + +use serde::{Deserialize, Serialize}; + +use crate::datetime::{NdbDateTime, NdbDateTimeError}; +use crate::value::Value; + +/// Ext type byte for a UTC instant. +pub const EXT_INSTANT_UTC: i8 = 1; +/// Ext type byte for a naive (timezone-less) instant. +pub const EXT_INSTANT_NAIVE: i8 = 2; +/// Encoded length of an instant: marker + ext type + `i64` payload. +pub const INSTANT_EXT_LEN: usize = 10; + +/// Msgpack `fixext8` marker. +const FIXEXT8: u8 = 0xD7; + +/// Which instant variant an ext payload carries. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum InstantKind { + /// Timezone-aware instant, `Value::DateTime`. + Utc, + /// Timezone-less instant, `Value::NaiveDateTime`. + Naive, +} + +impl InstantKind { + /// Ext type byte for this kind. + pub fn ext_type(self) -> i8 { + match self { + Self::Utc => EXT_INSTANT_UTC, + Self::Naive => EXT_INSTANT_NAIVE, + } + } + + /// Kind for an ext type byte. `None` when the byte is not an instant type. + pub fn from_ext_type(t: i8) -> Option { + match t { + EXT_INSTANT_UTC => Some(Self::Utc), + EXT_INSTANT_NAIVE => Some(Self::Naive), + _ => None, + } + } + + /// Wrap a timestamp in the `Value` variant for this kind. + pub fn value(self, dt: NdbDateTime) -> Value { + match self { + Self::Utc => Value::DateTime(dt), + Self::Naive => Value::NaiveDateTime(dt), + } + } + + /// Build the `Value` for this kind from epoch milliseconds. + pub fn from_millis(self, ms: i64) -> Result { + Ok(self.value(NdbDateTime::from_millis(ms)?)) + } + + /// Build the `Value` for this kind from epoch microseconds. + pub fn from_micros(self, micros: i64) -> Value { + self.value(NdbDateTime::from_micros(micros)) + } +} + +/// Append the ten-byte `fixext8` encoding of an instant to `buf`. +pub fn write_instant(buf: &mut Vec, kind: InstantKind, micros: i64) { + buf.push(FIXEXT8); + buf.push(kind.ext_type() as u8); + buf.extend_from_slice(µs.to_be_bytes()); +} + +/// Decode an instant from its ext type byte and eight-byte payload. +/// +/// `None` when the type byte is not an instant type or the payload is not +/// exactly eight bytes. +pub fn instant_from_ext(ext_type: i8, payload: &[u8]) -> Option<(InstantKind, i64)> { + let kind = InstantKind::from_ext_type(ext_type)?; + let bytes: [u8; 8] = payload.try_into().ok()?; + Some((kind, i64::from_be_bytes(bytes))) +} + +/// Decode the instant at `offset` in `bytes`. +/// +/// `None` when the bytes at `offset` are not a `fixext8` of an instant type +/// or the buffer is truncated. +pub fn read_instant(bytes: &[u8], offset: usize) -> Option<(InstantKind, i64)> { + let ext = bytes.get(offset..offset + INSTANT_EXT_LEN)?; + if ext[0] != FIXEXT8 { + return None; + } + instant_from_ext(ext[1] as i8, &ext[2..]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip_both_kinds() { + for kind in [InstantKind::Utc, InstantKind::Naive] { + let mut buf = Vec::new(); + write_instant(&mut buf, kind, 1_700_000_000_123_456); + assert_eq!(buf.len(), INSTANT_EXT_LEN); + assert_eq!(buf[0], 0xD7); + assert_eq!(buf[1] as i8, kind.ext_type()); + assert_eq!(read_instant(&buf, 0), Some((kind, 1_700_000_000_123_456))); + } + } + + #[test] + fn negative_micros_round_trip() { + let mut buf = vec![0xC0]; + write_instant(&mut buf, InstantKind::Naive, -86_400_000_000); + assert_eq!( + read_instant(&buf, 1), + Some((InstantKind::Naive, -86_400_000_000)) + ); + } + + #[test] + fn non_ext_byte_is_none() { + let buf = [0x2A, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + assert_eq!(read_instant(&buf, 0), None); + } + + #[test] + fn unknown_ext_type_is_none() { + let buf = [0xD7, 0x07, 0, 0, 0, 0, 0, 0, 0, 0]; + assert_eq!(read_instant(&buf, 0), None); + } + + #[test] + fn truncated_is_none() { + let mut buf = Vec::new(); + write_instant(&mut buf, InstantKind::Utc, 42); + buf.truncate(9); + assert_eq!(read_instant(&buf, 0), None); + assert_eq!(instant_from_ext(EXT_INSTANT_UTC, &buf[2..]), None); + } + + #[test] + fn kind_value_mapping() { + let dt = NdbDateTime::from_micros(5); + assert_eq!(InstantKind::Utc.value(dt), Value::DateTime(dt)); + assert_eq!(InstantKind::Naive.value(dt), Value::NaiveDateTime(dt)); + assert_eq!( + Value::DateTime(dt).as_instant(), + Some((InstantKind::Utc, dt)) + ); + assert_eq!( + Value::NaiveDateTime(dt).as_instant(), + Some((InstantKind::Naive, dt)) + ); + assert_eq!(Value::Integer(5).as_instant(), None); + assert_eq!( + InstantKind::Utc.from_millis(3).unwrap(), + Value::DateTime(NdbDateTime::from_micros(3_000)) + ); + assert!(InstantKind::Utc.from_millis(i64::MAX).is_err()); + } +} diff --git a/nodedb-types/src/json_msgpack/mod.rs b/nodedb-types/src/json_msgpack/mod.rs index 5f67b9789..8e76ce082 100644 --- a/nodedb-types/src/json_msgpack/mod.rs +++ b/nodedb-types/src/json_msgpack/mod.rs @@ -1,13 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 pub mod error; +pub mod instant_ext; pub mod json_value; +pub mod native_cell; pub mod reader; pub mod transcoder; pub mod writer; pub use error::{MsgpackError, MsgpackResult}; +pub use instant_ext::{ + EXT_INSTANT_NAIVE, EXT_INSTANT_UTC, INSTANT_EXT_LEN, InstantKind, instant_from_ext, + read_instant, write_instant, +}; pub use json_value::JsonValue; +pub use native_cell::NativeCell; pub use reader::{json_from_msgpack, value_from_msgpack}; pub use transcoder::msgpack_to_json_string; pub use writer::{json_to_msgpack, json_to_msgpack_or_empty, value_to_msgpack}; diff --git a/nodedb-types/src/json_msgpack/native_cell.rs b/nodedb-types/src/json_msgpack/native_cell.rs new file mode 100644 index 000000000..2a79279a0 --- /dev/null +++ b/nodedb-types/src/json_msgpack/native_cell.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! `NativeCell`: a `Value` carried in plain msgpack inside a zerompk struct. +//! +//! `Value`'s own zerompk impl writes the tagged `[tag, payload]` form. A +//! struct that derives `ToMessagePack` with a `Value` field therefore ships +//! `"alice"` as `[4, "alice"]`, which the JSON transcoder renders verbatim. +//! Wrapping the field in `NativeCell` writes the same bytes +//! `value_to_msgpack` produces, so the transcoder renders a plain cell and +//! `value_from_msgpack` reads it back typed. + +use zerompk::{FromMessagePack, Read, ToMessagePack, Write}; + +use super::reader::native::read_native_value; +use super::writer::write_native_value; +use crate::Value; + +/// A `Value` written and read as plain msgpack. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct NativeCell(pub Value); + +impl From for NativeCell { + #[inline] + fn from(v: Value) -> Self { + Self(v) + } +} + +impl From for Value { + #[inline] + fn from(c: NativeCell) -> Self { + c.0 + } +} + +impl ToMessagePack for NativeCell { + fn write(&self, writer: &mut W) -> zerompk::Result<()> { + write_native_value(writer, &self.0) + } +} + +impl<'a> FromMessagePack<'a> for NativeCell { + fn read>(reader: &mut R) -> zerompk::Result { + read_native_value(reader).map(NativeCell) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::NdbDateTime; + use crate::json_msgpack::{value_from_msgpack, value_to_msgpack}; + + #[derive(ToMessagePack, FromMessagePack, PartialEq, Debug)] + #[msgpack(map)] + struct Row { + cells: Vec, + } + + fn sample() -> Vec { + let mut obj = std::collections::HashMap::new(); + obj.insert("n".to_string(), Value::Integer(-200)); + obj.insert( + "arr".to_string(), + Value::Array(vec![Value::Float(1.5), Value::Null]), + ); + vec![ + Value::Null, + Value::Bool(true), + Value::Integer(300), + Value::Integer(-5), + Value::Float(2.25), + Value::String("alice".into()), + Value::Bytes(vec![1, 2, 3]), + Value::DateTime(NdbDateTime::from_micros(1_583_402_400_000_000)), + Value::NaiveDateTime(NdbDateTime::from_micros(-86_400_000_000)), + Value::Object(obj), + ] + } + + #[test] + fn cells_round_trip_through_a_derived_struct() { + let row = Row { + cells: sample().into_iter().map(NativeCell).collect(), + }; + let bytes = zerompk::to_msgpack_vec(&row).expect("encode"); + let back: Row = zerompk::from_msgpack(&bytes).expect("decode"); + assert_eq!(back, row); + } + + #[test] + fn cell_bytes_match_value_to_msgpack() { + for v in sample() { + let plain = value_to_msgpack(&v).expect("plain"); + let cell = zerompk::to_msgpack_vec(&NativeCell(v.clone())).expect("cell"); + assert_eq!(cell, plain, "{v:?}"); + assert_eq!(value_from_msgpack(&cell).expect("read"), v); + } + } +} diff --git a/nodedb-types/src/json_msgpack/reader.rs b/nodedb-types/src/json_msgpack/reader.rs deleted file mode 100644 index cdec98d6d..000000000 --- a/nodedb-types/src/json_msgpack/reader.rs +++ /dev/null @@ -1,682 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Cursor-based msgpack → `serde_json::Value` and `nodedb_types::Value` readers. -//! -//! Deterministic raw byte parser — the first byte of each msgpack value -//! unambiguously identifies its type per the msgpack specification. - -use super::error::{MsgpackError, MsgpackResult}; - -pub(crate) struct Cursor<'a> { - pub(crate) data: &'a [u8], - pub(crate) pos: usize, - pub(crate) depth: usize, -} - -impl<'a> Cursor<'a> { - pub(crate) fn new(data: &'a [u8]) -> Self { - Self { - data, - pos: 0, - depth: 0, - } - } - - #[inline] - pub(crate) fn peek(&self) -> zerompk::Result { - self.data - .get(self.pos) - .copied() - .ok_or(zerompk::Error::BufferTooSmall) - } - - #[inline] - pub(crate) fn take(&mut self) -> zerompk::Result { - let b = self.peek()?; - self.pos += 1; - Ok(b) - } - - #[inline] - pub(crate) fn take_n(&mut self, n: usize) -> zerompk::Result<&'a [u8]> { - if self.pos + n > self.data.len() { - return Err(zerompk::Error::BufferTooSmall); - } - let slice = &self.data[self.pos..self.pos + n]; - self.pos += n; - Ok(slice) - } - - pub(crate) fn read_u16_be(&mut self) -> zerompk::Result { - let b = self.take_n(2)?; - Ok(u16::from_be_bytes([b[0], b[1]])) - } - - pub(crate) fn read_u32_be(&mut self) -> zerompk::Result { - let b = self.take_n(4)?; - Ok(u32::from_be_bytes([b[0], b[1], b[2], b[3]])) - } - - /// Assert the whole input belonged to the value just read. - /// - /// A body carries exactly one top-level value, so anything left over means - /// the bytes are not the value they claim to be — a stray suffix, a - /// truncated body with another appended, or two values in one slot. - /// Returning the leading value and dropping the rest would report success - /// on corrupt bytes, so the remainder is an error. - pub(crate) fn finish(&self) -> MsgpackResult<()> { - if self.pos == self.data.len() { - Ok(()) - } else { - Err(MsgpackError::TrailingBytes { - consumed: self.pos, - total: self.data.len(), - }) - } - } -} - -/// Deserialize a `serde_json::Value` from MessagePack bytes. -/// -/// The input must contain exactly one top-level value and nothing else; -/// trailing bytes are rejected. -pub fn json_from_msgpack(bytes: &[u8]) -> MsgpackResult { - let mut cursor = Cursor::new(bytes); - let value = read_json_value(&mut cursor)?; - cursor.finish()?; - Ok(value) -} - -/// Deserialize a `nodedb_types::Value` from standard MessagePack bytes. -/// -/// The input must contain exactly one top-level value and nothing else; -/// trailing bytes are rejected. -pub fn value_from_msgpack(bytes: &[u8]) -> MsgpackResult { - let mut cursor = Cursor::new(bytes); - let value = read_native_value(&mut cursor)?; - cursor.finish()?; - Ok(value) -} - -// ── JSON value reader ── - -fn read_json_value(c: &mut Cursor<'_>) -> zerompk::Result { - if c.depth > 500 { - return Err(zerompk::Error::DepthLimitExceeded { max: 500 }); - } - - let marker = c.take()?; - match marker { - 0xC0 => Ok(serde_json::Value::Null), - 0xC2 => Ok(serde_json::Value::Bool(false)), - 0xC3 => Ok(serde_json::Value::Bool(true)), - - 0x00..=0x7F => Ok(serde_json::Value::Number((marker as i64).into())), - 0xE0..=0xFF => Ok(serde_json::Value::Number((marker as i8 as i64).into())), - - 0xCC => Ok(serde_json::Value::Number(c.take()?.into())), - 0xCD => Ok(serde_json::Value::Number(c.read_u16_be()?.into())), - 0xCE => Ok(serde_json::Value::Number(c.read_u32_be()?.into())), - 0xCF => { - let b = c.take_n(8)?; - let v = u64::from_be_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]); - Ok(serde_json::Value::Number(v.into())) - } - - 0xD0 => Ok(serde_json::Value::Number((c.take()? as i8 as i64).into())), - 0xD1 => { - let b = c.take_n(2)?; - Ok(serde_json::Value::Number( - (i16::from_be_bytes([b[0], b[1]]) as i64).into(), - )) - } - 0xD2 => { - let b = c.take_n(4)?; - Ok(serde_json::Value::Number( - (i32::from_be_bytes([b[0], b[1], b[2], b[3]]) as i64).into(), - )) - } - 0xD3 => { - let b = c.take_n(8)?; - Ok(serde_json::Value::Number( - i64::from_be_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]).into(), - )) - } - - 0xCA => { - let b = c.take_n(4)?; - Ok(serde_json::json!( - f32::from_be_bytes([b[0], b[1], b[2], b[3]]) as f64 - )) - } - 0xCB => { - let b = c.take_n(8)?; - Ok(serde_json::json!(f64::from_be_bytes([ - b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7] - ]))) - } - - m @ 0xA0..=0xBF => read_json_str(c, (m & 0x1F) as usize), - 0xD9 => { - let l = c.take()? as usize; - read_json_str(c, l) - } - 0xDA => { - let l = c.read_u16_be()? as usize; - read_json_str(c, l) - } - 0xDB => { - let l = c.read_u32_be()? as usize; - read_json_str(c, l) - } - - 0xC4 => { - let l = c.take()? as usize; - Ok(serde_json::Value::String(base64_encode(c.take_n(l)?))) - } - 0xC5 => { - let l = c.read_u16_be()? as usize; - Ok(serde_json::Value::String(base64_encode(c.take_n(l)?))) - } - 0xC6 => { - let l = c.read_u32_be()? as usize; - Ok(serde_json::Value::String(base64_encode(c.take_n(l)?))) - } - - m @ 0x90..=0x9F => read_json_array(c, (m & 0x0F) as usize), - 0xDC => { - let l = c.read_u16_be()? as usize; - read_json_array(c, l) - } - 0xDD => { - let l = c.read_u32_be()? as usize; - read_json_array(c, l) - } - - m @ 0x80..=0x8F => read_json_map(c, (m & 0x0F) as usize), - 0xDE => { - let l = c.read_u16_be()? as usize; - read_json_map(c, l) - } - 0xDF => { - let l = c.read_u32_be()? as usize; - read_json_map(c, l) - } - - // ext types — skip - 0xD4 => { - c.take_n(2)?; - Ok(serde_json::Value::Null) - } - 0xD5 => { - c.take_n(3)?; - Ok(serde_json::Value::Null) - } - 0xD6 => { - c.take_n(5)?; - Ok(serde_json::Value::Null) - } - 0xD7 => { - c.take_n(9)?; - Ok(serde_json::Value::Null) - } - 0xD8 => { - c.take_n(17)?; - Ok(serde_json::Value::Null) - } - 0xC7 => { - let l = c.take()? as usize; - c.take_n(1 + l)?; - Ok(serde_json::Value::Null) - } - 0xC8 => { - let l = c.read_u16_be()? as usize; - c.take_n(1 + l)?; - Ok(serde_json::Value::Null) - } - 0xC9 => { - let l = c.read_u32_be()? as usize; - c.take_n(1 + l)?; - Ok(serde_json::Value::Null) - } - - _ => Err(zerompk::Error::InvalidMarker(marker)), - } -} - -fn read_json_str(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - let bytes = c.take_n(len)?; - let s = String::from_utf8(bytes.to_vec()).map_err(|_| zerompk::Error::InvalidMarker(0))?; - Ok(serde_json::Value::String(s)) -} - -fn read_json_array(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - c.depth += 1; - let mut arr = Vec::with_capacity(len.min(4096)); - for _ in 0..len { - arr.push(read_json_value(c)?); - } - c.depth -= 1; - Ok(serde_json::Value::Array(arr)) -} - -fn read_json_map(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - c.depth += 1; - let mut map = serde_json::Map::with_capacity(len.min(4096)); - for _ in 0..len { - let key_marker = c.peek()?; - let key = if (0xA0..=0xBF).contains(&key_marker) - || key_marker == 0xD9 - || key_marker == 0xDA - || key_marker == 0xDB - { - match read_json_value(c)? { - serde_json::Value::String(s) => s, - other => other.to_string(), - } - } else { - read_json_value(c)?.to_string() - }; - let val = read_json_value(c)?; - map.insert(key, val); - } - c.depth -= 1; - Ok(serde_json::Value::Object(map)) -} - -// ── Native value reader ── - -fn read_native_value(c: &mut Cursor<'_>) -> zerompk::Result { - if c.depth > 500 { - return Err(zerompk::Error::DepthLimitExceeded { max: 500 }); - } - - let marker = c.take()?; - match marker { - 0xC0 => Ok(crate::Value::Null), - 0xC2 => Ok(crate::Value::Bool(false)), - 0xC3 => Ok(crate::Value::Bool(true)), - - 0x00..=0x7F => Ok(crate::Value::Integer(marker as i64)), - 0xE0..=0xFF => Ok(crate::Value::Integer(marker as i8 as i64)), - - 0xCC => Ok(crate::Value::Integer(c.take()? as i64)), - 0xCD => Ok(crate::Value::Integer(c.read_u16_be()? as i64)), - 0xCE => Ok(crate::Value::Integer(c.read_u32_be()? as i64)), - 0xCF => { - let b = c.take_n(8)?; - Ok(crate::Value::Integer(u64::from_be_bytes([ - b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], - ]) as i64)) - } - - 0xD0 => Ok(crate::Value::Integer(c.take()? as i8 as i64)), - 0xD1 => { - let b = c.take_n(2)?; - Ok(crate::Value::Integer( - i16::from_be_bytes([b[0], b[1]]) as i64 - )) - } - 0xD2 => { - let b = c.take_n(4)?; - Ok(crate::Value::Integer( - i32::from_be_bytes([b[0], b[1], b[2], b[3]]) as i64, - )) - } - 0xD3 => { - let b = c.take_n(8)?; - Ok(crate::Value::Integer(i64::from_be_bytes([ - b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], - ]))) - } - - 0xCA => { - let b = c.take_n(4)?; - Ok(crate::Value::Float( - f32::from_be_bytes([b[0], b[1], b[2], b[3]]) as f64, - )) - } - 0xCB => { - let b = c.take_n(8)?; - Ok(crate::Value::Float(f64::from_be_bytes([ - b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], - ]))) - } - - m @ 0xA0..=0xBF => read_native_str(c, (m & 0x1F) as usize), - 0xD9 => { - let l = c.take()? as usize; - read_native_str(c, l) - } - 0xDA => { - let l = c.read_u16_be()? as usize; - read_native_str(c, l) - } - 0xDB => { - let l = c.read_u32_be()? as usize; - read_native_str(c, l) - } - - 0xC4 => { - let l = c.take()? as usize; - Ok(crate::Value::Bytes(c.take_n(l)?.to_vec())) - } - 0xC5 => { - let l = c.read_u16_be()? as usize; - Ok(crate::Value::Bytes(c.take_n(l)?.to_vec())) - } - 0xC6 => { - let l = c.read_u32_be()? as usize; - Ok(crate::Value::Bytes(c.take_n(l)?.to_vec())) - } - - m @ 0x90..=0x9F => read_native_array(c, (m & 0x0F) as usize), - 0xDC => { - let l = c.read_u16_be()? as usize; - read_native_array(c, l) - } - 0xDD => { - let l = c.read_u32_be()? as usize; - read_native_array(c, l) - } - - m @ 0x80..=0x8F => read_native_map(c, (m & 0x0F) as usize), - 0xDE => { - let l = c.read_u16_be()? as usize; - read_native_map(c, l) - } - 0xDF => { - let l = c.read_u32_be()? as usize; - read_native_map(c, l) - } - - // ext types — skip - 0xD4 => { - c.take_n(2)?; - Ok(crate::Value::Null) - } - 0xD5 => { - c.take_n(3)?; - Ok(crate::Value::Null) - } - 0xD6 => { - c.take_n(5)?; - Ok(crate::Value::Null) - } - 0xD7 => { - c.take_n(9)?; - Ok(crate::Value::Null) - } - 0xD8 => { - c.take_n(17)?; - Ok(crate::Value::Null) - } - 0xC7 => { - let l = c.take()? as usize; - c.take_n(1 + l)?; - Ok(crate::Value::Null) - } - 0xC8 => { - let l = c.read_u16_be()? as usize; - c.take_n(1 + l)?; - Ok(crate::Value::Null) - } - 0xC9 => { - let l = c.read_u32_be()? as usize; - c.take_n(1 + l)?; - Ok(crate::Value::Null) - } - - _ => Err(zerompk::Error::InvalidMarker(marker)), - } -} - -fn read_native_str(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - let bytes = c.take_n(len)?; - let s = String::from_utf8(bytes.to_vec()).map_err(|_| zerompk::Error::InvalidMarker(0))?; - Ok(crate::Value::String(s)) -} - -fn read_native_array(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - c.depth += 1; - let mut arr = Vec::with_capacity(len.min(4096)); - for _ in 0..len { - arr.push(read_native_value(c)?); - } - c.depth -= 1; - Ok(crate::Value::Array(arr)) -} - -fn read_native_map(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - c.depth += 1; - let mut map = std::collections::HashMap::with_capacity(len.min(4096)); - for _ in 0..len { - let key_marker = c.peek()?; - let key = if (0xA0..=0xBF).contains(&key_marker) - || key_marker == 0xD9 - || key_marker == 0xDA - || key_marker == 0xDB - { - match read_native_value(c)? { - crate::Value::String(s) => s, - other => format!("{other:?}"), - } - } else { - let v = read_native_value(c)?; - format!("{v:?}") - }; - let val = read_native_value(c)?; - map.insert(key, val); - } - c.depth -= 1; - Ok(crate::Value::Object(map)) -} - -// ── Shared helpers ── - -pub(crate) fn base64_encode(data: &[u8]) -> String { - use std::fmt::Write; - const CHARS: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - let mut out = String::with_capacity(data.len().div_ceil(3) * 4); - for chunk in data.chunks(3) { - let b0 = chunk[0] as u32; - let b1 = chunk.get(1).copied().unwrap_or(0) as u32; - let b2 = chunk.get(2).copied().unwrap_or(0) as u32; - let triple = (b0 << 16) | (b1 << 8) | b2; - let _ = write!(out, "{}", CHARS[((triple >> 18) & 0x3F) as usize] as char); - let _ = write!(out, "{}", CHARS[((triple >> 12) & 0x3F) as usize] as char); - if chunk.len() > 1 { - let _ = write!(out, "{}", CHARS[((triple >> 6) & 0x3F) as usize] as char); - } - if chunk.len() > 2 { - let _ = write!(out, "{}", CHARS[(triple & 0x3F) as usize] as char); - } - } - out -} - -#[cfg(test)] -mod tests { - //! Roundtrip tests for json_msgpack reader/writer. - - use super::*; - use crate::json_msgpack::msgpack_to_json_string; - use crate::json_msgpack::writer::{json_to_msgpack, value_to_msgpack}; - use serde_json::json; - - #[test] - fn roundtrip_null() { - let val = json!(null); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn roundtrip_bool() { - for val in [json!(true), json!(false)] { - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - } - - #[test] - fn roundtrip_integers() { - for val in [ - json!(0), - json!(42), - json!(-1), - json!(i64::MAX), - json!(i64::MIN), - ] { - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - } - - #[test] - fn roundtrip_float() { - let val = json!(9.81); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn roundtrip_string() { - let val = json!("hello world"); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn roundtrip_array() { - let val = json!([1, "two", true, null, 2.72]); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn roundtrip_nested_object() { - let val = json!({"a": 1, "b": {"c": [2, 3]}, "d": null}); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn roundtrip_empty_map() { - let val = json!({}); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn roundtrip_empty_array() { - let val = json!([]); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn roundtrip_large_string() { - let s = "x".repeat(300); - let val = json!(s); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn native_value_roundtrip() { - let mut map = std::collections::HashMap::new(); - map.insert("id".to_string(), crate::Value::String("host1".into())); - map.insert("cpu".to_string(), crate::Value::Float(0.75)); - map.insert("mem".to_string(), crate::Value::Float(0.5)); - - let row = crate::Value::Object(map); - let arr = crate::Value::Array(vec![row]); - - let bytes = value_to_msgpack(&arr).unwrap(); - let decoded = value_from_msgpack(&bytes).unwrap(); - - match &decoded { - crate::Value::Array(items) => { - assert_eq!(items.len(), 1); - match &items[0] { - crate::Value::Object(m) => { - assert_eq!(m.len(), 3); - assert_eq!(m.get("id"), Some(&crate::Value::String("host1".into()))); - assert_eq!(m.get("cpu"), Some(&crate::Value::Float(0.75))); - assert_eq!(m.get("mem"), Some(&crate::Value::Float(0.5))); - } - other => panic!("expected Object, got {other:?}"), - } - } - other => panic!("expected Array, got {other:?}"), - } - } - - /// The readers decode exactly one top-level value. Bytes left over mean the - /// input is not the value it claims to be, so returning the leading value would - /// report success on corrupt input. - #[test] - fn trailing_byte_is_rejected() { - let val = json!({"a": 1, "b": "two"}); - let mut bytes = json_to_msgpack(&val).unwrap(); - assert_eq!(json_from_msgpack(&bytes).unwrap(), val); - - bytes.push(0xC0); - match json_from_msgpack(&bytes) { - Err(MsgpackError::TrailingBytes { consumed, total }) => { - assert_eq!(total, consumed + 1); - } - other => panic!("expected TrailingBytes, got {other:?}"), - } - assert!(value_from_msgpack(&bytes).is_err()); - assert!(msgpack_to_json_string(&bytes).is_err()); - } - - #[test] - fn two_concatenated_values_are_rejected() { - let first = json_to_msgpack(&json!({"a": 1})).unwrap(); - let second = json_to_msgpack(&json!({"b": 2})).unwrap(); - let mut joined = first.clone(); - joined.extend_from_slice(&second); - - assert!(json_from_msgpack(&first).is_ok()); - assert!(json_from_msgpack(&joined).is_err()); - assert!(value_from_msgpack(&joined).is_err()); - assert!(msgpack_to_json_string(&joined).is_err()); - } - - #[test] - fn empty_input_is_unchanged() { - // Readers still fail on empty input; the transcoder still yields "". - assert!(json_from_msgpack(&[]).is_err()); - assert!(value_from_msgpack(&[]).is_err()); - assert_eq!(msgpack_to_json_string(&[]).unwrap(), ""); - } - - #[test] - fn native_value_scalars() { - let cases: Vec = vec![ - crate::Value::Null, - crate::Value::Bool(true), - crate::Value::Integer(42), - crate::Value::Float(2.72), - crate::Value::String("hello".into()), - ]; - for val in cases { - let bytes = value_to_msgpack(&val).unwrap(); - let decoded = value_from_msgpack(&bytes).unwrap(); - assert_eq!(val, decoded); - } - } -} diff --git a/nodedb-types/src/json_msgpack/reader/cursor.rs b/nodedb-types/src/json_msgpack/reader/cursor.rs new file mode 100644 index 000000000..4ee11edb1 --- /dev/null +++ b/nodedb-types/src/json_msgpack/reader/cursor.rs @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Byte cursor shared by the msgpack readers and the JSON transcoder. + +use super::super::error::{MsgpackError, MsgpackResult}; + +pub(crate) struct Cursor<'a> { + pub(crate) data: &'a [u8], + pub(crate) pos: usize, + pub(crate) depth: usize, +} + +impl<'a> Cursor<'a> { + pub(crate) fn new(data: &'a [u8]) -> Self { + Self { + data, + pos: 0, + depth: 0, + } + } + + #[inline] + pub(crate) fn peek(&self) -> zerompk::Result { + self.data + .get(self.pos) + .copied() + .ok_or(zerompk::Error::BufferTooSmall) + } + + #[inline] + pub(crate) fn take(&mut self) -> zerompk::Result { + let b = self.peek()?; + self.pos += 1; + Ok(b) + } + + #[inline] + pub(crate) fn take_n(&mut self, n: usize) -> zerompk::Result<&'a [u8]> { + if self.pos + n > self.data.len() { + return Err(zerompk::Error::BufferTooSmall); + } + let slice = &self.data[self.pos..self.pos + n]; + self.pos += n; + Ok(slice) + } + + pub(crate) fn read_u16_be(&mut self) -> zerompk::Result { + let b = self.take_n(2)?; + Ok(u16::from_be_bytes([b[0], b[1]])) + } + + pub(crate) fn read_u32_be(&mut self) -> zerompk::Result { + let b = self.take_n(4)?; + Ok(u32::from_be_bytes([b[0], b[1], b[2], b[3]])) + } + + /// Read the ext type byte and eight-byte payload of a `fixext8` whose + /// marker is already consumed. + pub(crate) fn take_fixext8(&mut self) -> zerompk::Result<(i8, &'a [u8])> { + let ext_type = self.take()? as i8; + let payload = self.take_n(8)?; + Ok((ext_type, payload)) + } + + /// Assert the whole input belonged to the value just read. + /// + /// A body carries exactly one top-level value, so anything left over means + /// the bytes are not the value they claim to be — a stray suffix, a + /// truncated body with another appended, or two values in one slot. + /// Returning the leading value and dropping the rest would report success + /// on corrupt bytes, so the remainder is an error. + pub(crate) fn finish(&self) -> MsgpackResult<()> { + if self.pos == self.data.len() { + Ok(()) + } else { + Err(MsgpackError::TrailingBytes { + consumed: self.pos, + total: self.data.len(), + }) + } + } +} + +#[cfg(test)] +mod tests { + use crate::json_msgpack::error::MsgpackError; + use crate::json_msgpack::reader::{json_from_msgpack, value_from_msgpack}; + use crate::json_msgpack::transcoder::msgpack_to_json_string; + use crate::json_msgpack::writer::json_to_msgpack; + use serde_json::json; + + /// The readers decode exactly one top-level value. Bytes left over mean the + /// input is not the value it claims to be, so returning the leading value would + /// report success on corrupt input. + #[test] + fn trailing_byte_is_rejected() { + let val = json!({"a": 1, "b": "two"}); + let mut bytes = json_to_msgpack(&val).unwrap(); + assert_eq!(json_from_msgpack(&bytes).unwrap(), val); + + bytes.push(0xC0); + match json_from_msgpack(&bytes) { + Err(MsgpackError::TrailingBytes { consumed, total }) => { + assert_eq!(total, consumed + 1); + } + other => panic!("expected TrailingBytes, got {other:?}"), + } + assert!(value_from_msgpack(&bytes).is_err()); + assert!(msgpack_to_json_string(&bytes).is_err()); + } + + #[test] + fn two_concatenated_values_are_rejected() { + let first = json_to_msgpack(&json!({"a": 1})).unwrap(); + let second = json_to_msgpack(&json!({"b": 2})).unwrap(); + let mut joined = first.clone(); + joined.extend_from_slice(&second); + + assert!(json_from_msgpack(&first).is_ok()); + assert!(json_from_msgpack(&joined).is_err()); + assert!(value_from_msgpack(&joined).is_err()); + assert!(msgpack_to_json_string(&joined).is_err()); + } + + #[test] + fn empty_input_is_unchanged() { + // Readers fail on empty input; the transcoder yields "". + assert!(json_from_msgpack(&[]).is_err()); + assert!(value_from_msgpack(&[]).is_err()); + assert_eq!(msgpack_to_json_string(&[]).unwrap(), ""); + } +} diff --git a/nodedb-types/src/json_msgpack/reader/json.rs b/nodedb-types/src/json_msgpack/reader/json.rs new file mode 100644 index 000000000..61f449e60 --- /dev/null +++ b/nodedb-types/src/json_msgpack/reader/json.rs @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Msgpack → `serde_json::Value` reader. +//! +//! Instant ext values (fixext8 type 1 / 2) become ISO 8601 strings. Every +//! other ext type becomes `null`. + +use super::super::error::MsgpackResult; +use super::super::instant_ext::instant_from_ext; +use super::cursor::Cursor; +use crate::datetime::NdbDateTime; + +/// Deserialize a `serde_json::Value` from MessagePack bytes. +/// +/// The input must contain exactly one top-level value and nothing else; +/// trailing bytes are rejected. +pub fn json_from_msgpack(bytes: &[u8]) -> MsgpackResult { + let mut cursor = Cursor::new(bytes); + let value = read_json_value(&mut cursor)?; + cursor.finish()?; + Ok(value) +} + +fn read_json_value(c: &mut Cursor<'_>) -> zerompk::Result { + if c.depth > 500 { + return Err(zerompk::Error::DepthLimitExceeded { max: 500 }); + } + + let marker = c.take()?; + match marker { + 0xC0 => Ok(serde_json::Value::Null), + 0xC2 => Ok(serde_json::Value::Bool(false)), + 0xC3 => Ok(serde_json::Value::Bool(true)), + + 0x00..=0x7F => Ok(serde_json::Value::Number((marker as i64).into())), + 0xE0..=0xFF => Ok(serde_json::Value::Number((marker as i8 as i64).into())), + + 0xCC => Ok(serde_json::Value::Number(c.take()?.into())), + 0xCD => Ok(serde_json::Value::Number(c.read_u16_be()?.into())), + 0xCE => Ok(serde_json::Value::Number(c.read_u32_be()?.into())), + 0xCF => { + let b = c.take_n(8)?; + let v = u64::from_be_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]); + Ok(serde_json::Value::Number(v.into())) + } + + 0xD0 => Ok(serde_json::Value::Number((c.take()? as i8 as i64).into())), + 0xD1 => { + let b = c.take_n(2)?; + Ok(serde_json::Value::Number( + (i16::from_be_bytes([b[0], b[1]]) as i64).into(), + )) + } + 0xD2 => { + let b = c.take_n(4)?; + Ok(serde_json::Value::Number( + (i32::from_be_bytes([b[0], b[1], b[2], b[3]]) as i64).into(), + )) + } + 0xD3 => { + let b = c.take_n(8)?; + Ok(serde_json::Value::Number( + i64::from_be_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]).into(), + )) + } + + 0xCA => { + let b = c.take_n(4)?; + Ok(serde_json::json!( + f32::from_be_bytes([b[0], b[1], b[2], b[3]]) as f64 + )) + } + 0xCB => { + let b = c.take_n(8)?; + Ok(serde_json::json!(f64::from_be_bytes([ + b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7] + ]))) + } + + m @ 0xA0..=0xBF => read_json_str(c, (m & 0x1F) as usize), + 0xD9 => { + let l = c.take()? as usize; + read_json_str(c, l) + } + 0xDA => { + let l = c.read_u16_be()? as usize; + read_json_str(c, l) + } + 0xDB => { + let l = c.read_u32_be()? as usize; + read_json_str(c, l) + } + + 0xC4 => { + let l = c.take()? as usize; + Ok(serde_json::Value::String(base64_encode(c.take_n(l)?))) + } + 0xC5 => { + let l = c.read_u16_be()? as usize; + Ok(serde_json::Value::String(base64_encode(c.take_n(l)?))) + } + 0xC6 => { + let l = c.read_u32_be()? as usize; + Ok(serde_json::Value::String(base64_encode(c.take_n(l)?))) + } + + m @ 0x90..=0x9F => read_json_array(c, (m & 0x0F) as usize), + 0xDC => { + let l = c.read_u16_be()? as usize; + read_json_array(c, l) + } + 0xDD => { + let l = c.read_u32_be()? as usize; + read_json_array(c, l) + } + + m @ 0x80..=0x8F => read_json_map(c, (m & 0x0F) as usize), + 0xDE => { + let l = c.read_u16_be()? as usize; + read_json_map(c, l) + } + 0xDF => { + let l = c.read_u32_be()? as usize; + read_json_map(c, l) + } + + // fixext8: instants render as ISO 8601, other types as null + 0xD7 => { + let (ext_type, payload) = c.take_fixext8()?; + Ok(match instant_from_ext(ext_type, payload) { + Some((_, micros)) => { + serde_json::Value::String(NdbDateTime::from_micros(micros).to_iso8601()) + } + None => serde_json::Value::Null, + }) + } + + // other ext types — skip + 0xD4 => { + c.take_n(2)?; + Ok(serde_json::Value::Null) + } + 0xD5 => { + c.take_n(3)?; + Ok(serde_json::Value::Null) + } + 0xD6 => { + c.take_n(5)?; + Ok(serde_json::Value::Null) + } + 0xD8 => { + c.take_n(17)?; + Ok(serde_json::Value::Null) + } + 0xC7 => { + let l = c.take()? as usize; + c.take_n(1 + l)?; + Ok(serde_json::Value::Null) + } + 0xC8 => { + let l = c.read_u16_be()? as usize; + c.take_n(1 + l)?; + Ok(serde_json::Value::Null) + } + 0xC9 => { + let l = c.read_u32_be()? as usize; + c.take_n(1 + l)?; + Ok(serde_json::Value::Null) + } + + _ => Err(zerompk::Error::InvalidMarker(marker)), + } +} + +fn read_json_str(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { + let bytes = c.take_n(len)?; + let s = String::from_utf8(bytes.to_vec()).map_err(|_| zerompk::Error::InvalidMarker(0))?; + Ok(serde_json::Value::String(s)) +} + +fn read_json_array(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { + c.depth += 1; + let mut arr = Vec::with_capacity(len.min(4096)); + for _ in 0..len { + arr.push(read_json_value(c)?); + } + c.depth -= 1; + Ok(serde_json::Value::Array(arr)) +} + +fn read_json_map(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { + c.depth += 1; + let mut map = serde_json::Map::with_capacity(len.min(4096)); + for _ in 0..len { + let key_marker = c.peek()?; + let key = if (0xA0..=0xBF).contains(&key_marker) + || key_marker == 0xD9 + || key_marker == 0xDA + || key_marker == 0xDB + { + match read_json_value(c)? { + serde_json::Value::String(s) => s, + other => other.to_string(), + } + } else { + read_json_value(c)?.to_string() + }; + let val = read_json_value(c)?; + map.insert(key, val); + } + c.depth -= 1; + Ok(serde_json::Value::Object(map)) +} + +pub(crate) fn base64_encode(data: &[u8]) -> String { + use std::fmt::Write; + const CHARS: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::with_capacity(data.len().div_ceil(3) * 4); + for chunk in data.chunks(3) { + let b0 = chunk[0] as u32; + let b1 = chunk.get(1).copied().unwrap_or(0) as u32; + let b2 = chunk.get(2).copied().unwrap_or(0) as u32; + let triple = (b0 << 16) | (b1 << 8) | b2; + let _ = write!(out, "{}", CHARS[((triple >> 18) & 0x3F) as usize] as char); + let _ = write!(out, "{}", CHARS[((triple >> 12) & 0x3F) as usize] as char); + if chunk.len() > 1 { + let _ = write!(out, "{}", CHARS[((triple >> 6) & 0x3F) as usize] as char); + } + if chunk.len() > 2 { + let _ = write!(out, "{}", CHARS[(triple & 0x3F) as usize] as char); + } + } + out +} + +#[cfg(test)] +mod tests { + //! Roundtrip tests for the JSON reader against the JSON writer. + + use super::*; + use crate::json_msgpack::instant_ext::{InstantKind, write_instant}; + use crate::json_msgpack::writer::json_to_msgpack; + use serde_json::json; + + #[test] + fn roundtrip_null() { + let val = json!(null); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn roundtrip_bool() { + for val in [json!(true), json!(false)] { + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + } + + #[test] + fn roundtrip_integers() { + for val in [ + json!(0), + json!(42), + json!(-1), + json!(i64::MAX), + json!(i64::MIN), + ] { + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + } + + #[test] + fn roundtrip_float() { + let val = json!(9.81); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn roundtrip_string() { + let val = json!("hello world"); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn roundtrip_array() { + let val = json!([1, "two", true, null, 2.72]); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn roundtrip_nested_object() { + let val = json!({"a": 1, "b": {"c": [2, 3]}, "d": null}); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn roundtrip_empty_map() { + let val = json!({}); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn roundtrip_empty_array() { + let val = json!([]); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn roundtrip_large_string() { + let s = "x".repeat(300); + let val = json!(s); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn instant_ext_renders_iso8601() { + let mut bytes = vec![0x91]; + write_instant(&mut bytes, InstantKind::Utc, 1_710_498_600_000_000); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(restored, json!(["2024-03-15T10:30:00.000000Z"])); + } + + #[test] + fn unknown_ext_is_null() { + let bytes = [0xD7, 0x09, 0, 0, 0, 0, 0, 0, 0, 1]; + assert_eq!(json_from_msgpack(&bytes).unwrap(), json!(null)); + } + + #[test] + fn truncated_instant_ext_is_error() { + let mut bytes = Vec::new(); + write_instant(&mut bytes, InstantKind::Naive, 7); + bytes.truncate(6); + assert!(json_from_msgpack(&bytes).is_err()); + } +} diff --git a/nodedb-types/src/json_msgpack/reader/mod.rs b/nodedb-types/src/json_msgpack/reader/mod.rs new file mode 100644 index 000000000..afd926e62 --- /dev/null +++ b/nodedb-types/src/json_msgpack/reader/mod.rs @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Msgpack → `serde_json::Value` and `nodedb_types::Value` readers. +//! +//! The JSON reader walks the crate's own `Cursor`. The native reader is +//! generic over `zerompk::Read` and is shared with `NativeCell`. Both are +//! deterministic raw byte parsers — the first byte of each msgpack value +//! unambiguously identifies its type per the msgpack specification. + +pub mod cursor; +pub mod json; +pub mod native; + +pub(crate) use cursor::Cursor; +pub(crate) use json::base64_encode; +pub use json::json_from_msgpack; +pub use native::value_from_msgpack; diff --git a/nodedb-types/src/json_msgpack/reader/native.rs b/nodedb-types/src/json_msgpack/reader/native.rs new file mode 100644 index 000000000..ba0cc4386 --- /dev/null +++ b/nodedb-types/src/json_msgpack/reader/native.rs @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Msgpack → `nodedb_types::Value` reader. +//! +//! [`read_native_value`] is the one decoder of a plain (untagged) msgpack +//! value. It is generic over `zerompk::Read`, so `value_from_msgpack` (slice +//! entry point) and `NativeCell::read` (inside a zerompk derive) share it. +//! +//! Instant ext values (`fixext8` type 1 / 2) become `Value::DateTime` / +//! `Value::NaiveDateTime`. Every other ext marker, and a `fixext8` of any +//! other type, becomes `Value::Null`. Map keys that are not strings are +//! rendered with `Debug`. + +use zerompk::{Read, SliceReader}; + +use super::super::error::{MsgpackError, MsgpackResult}; +use super::super::instant_ext::instant_from_ext; +use crate::Value; + +const FIXEXT8: u8 = 0xD7; + +/// Deserialize a `nodedb_types::Value` from standard MessagePack bytes. +/// +/// The input must contain exactly one top-level value and nothing else; +/// trailing bytes are rejected. +pub fn value_from_msgpack(bytes: &[u8]) -> MsgpackResult { + let mut reader = SliceReader::new(bytes); + let value = read_native_value(&mut reader)?; + if reader.peek_marker().is_ok() { + return Err(MsgpackError::TrailingBytes { + consumed: consumed_len(bytes), + total: bytes.len(), + }); + } + Ok(value) +} + +/// Length of the top-level value at the start of `bytes`. +/// +/// `SliceReader` exposes no position, so the length is found by binary +/// search over prefix lengths: decoding a prefix shorter than the value fails +/// on the missing bytes, and decoding any prefix at least as long as the value +/// succeeds. Called only on the trailing-bytes error path, after a full decode +/// of `bytes` succeeded. +fn consumed_len(bytes: &[u8]) -> usize { + let (mut lo, mut hi) = (0usize, bytes.len()); + while lo < hi { + let mid = lo + (hi - lo) / 2; + match read_native_value(&mut SliceReader::new(&bytes[..mid])) { + Ok(_) => hi = mid, + Err(_) => lo = mid + 1, + } + } + lo +} + +/// Read one plain msgpack value, dispatching on the peeked marker. +pub(crate) fn read_native_value<'de, R: Read<'de>>(reader: &mut R) -> zerompk::Result { + let marker = reader.peek_marker()?; + match marker { + 0xC0 => { + reader.read_nil()?; + Ok(Value::Null) + } + 0xC2 | 0xC3 => Ok(Value::Bool(reader.read_boolean()?)), + 0x00..=0x7F | 0xE0..=0xFF | 0xD0..=0xD3 => Ok(Value::Integer(reader.read_i64()?)), + 0xCC..=0xCF => Ok(Value::Integer(reader.read_u64()? as i64)), + 0xCA => Ok(Value::Float(f64::from(reader.read_f32()?))), + 0xCB => Ok(Value::Float(reader.read_f64()?)), + 0xA0..=0xBF | 0xD9..=0xDB => Ok(Value::String(reader.read_string()?.into_owned())), + 0xC4..=0xC6 => Ok(Value::Bytes(reader.read_binary()?.into_owned())), + 0x90..=0x9F | 0xDC | 0xDD => { + let len = reader.read_array_len()?; + reader.increment_depth()?; + let mut arr = Vec::with_capacity(len.min(4096)); + for _ in 0..len { + arr.push(read_native_value(reader)?); + } + reader.decrement_depth(); + Ok(Value::Array(arr)) + } + 0x80..=0x8F | 0xDE | 0xDF => { + let len = reader.read_map_len()?; + reader.increment_depth()?; + let mut map = std::collections::HashMap::with_capacity(len.min(4096)); + for _ in 0..len { + let key = read_native_key(reader)?; + map.insert(key, read_native_value(reader)?); + } + reader.decrement_depth(); + Ok(Value::Object(map)) + } + 0xC7..=0xC9 | 0xD4..=0xD8 => { + let (ext_type, payload) = reader.read_ext()?; + let instant = if marker == FIXEXT8 { + instant_from_ext(ext_type, &payload) + } else { + None + }; + Ok(match instant { + Some((kind, micros)) => kind.from_micros(micros), + None => Value::Null, + }) + } + other => Err(zerompk::Error::InvalidMarker(other)), + } +} + +/// Read a map key. A string key is taken as-is; any other value is rendered +/// with `Debug`. +fn read_native_key<'de, R: Read<'de>>(reader: &mut R) -> zerompk::Result { + match reader.peek_marker()? { + 0xA0..=0xBF | 0xD9..=0xDB => Ok(reader.read_string()?.into_owned()), + _ => { + let v = read_native_value(reader)?; + Ok(format!("{v:?}")) + } + } +} + +#[cfg(test)] +mod tests { + //! Roundtrip tests for the native reader against the native writer, and + //! agreement between the `value_from_msgpack` and `NativeCell` entry points. + + use super::*; + use crate::NdbDateTime; + use crate::json_msgpack::NativeCell; + use crate::json_msgpack::transcoder::msgpack_to_json_string; + use crate::json_msgpack::writer::value_to_msgpack; + + fn through_cell(bytes: &[u8]) -> zerompk::Result { + zerompk::from_msgpack::(bytes).map(Value::from) + } + + #[test] + fn native_value_roundtrip() { + let mut map = std::collections::HashMap::new(); + map.insert("id".to_string(), Value::String("host1".into())); + map.insert("cpu".to_string(), Value::Float(0.75)); + map.insert("mem".to_string(), Value::Float(0.5)); + + let row = Value::Object(map); + let arr = Value::Array(vec![row]); + + let bytes = value_to_msgpack(&arr).unwrap(); + let decoded = value_from_msgpack(&bytes).unwrap(); + + match &decoded { + Value::Array(items) => { + assert_eq!(items.len(), 1); + match &items[0] { + Value::Object(m) => { + assert_eq!(m.len(), 3); + assert_eq!(m.get("id"), Some(&Value::String("host1".into()))); + assert_eq!(m.get("cpu"), Some(&Value::Float(0.75))); + assert_eq!(m.get("mem"), Some(&Value::Float(0.5))); + } + other => panic!("expected Object, got {other:?}"), + } + } + other => panic!("expected Array, got {other:?}"), + } + } + + #[test] + fn native_value_scalars() { + let cases: Vec = vec![ + Value::Null, + Value::Bool(true), + Value::Integer(42), + Value::Float(2.72), + Value::String("hello".into()), + ]; + for val in cases { + let bytes = value_to_msgpack(&val).unwrap(); + let decoded = value_from_msgpack(&bytes).unwrap(); + assert_eq!(val, decoded); + } + } + + #[test] + fn instant_roundtrip_both_kinds() { + let dt = NdbDateTime::from_micros(1_710_498_600_000_000); + for val in [ + Value::DateTime(dt), + Value::NaiveDateTime(dt), + Value::DateTime(NdbDateTime::from_micros(-1)), + ] { + let bytes = value_to_msgpack(&val).unwrap(); + assert_eq!(bytes.len(), 10); + assert_eq!(bytes[0], 0xD7); + assert_eq!(value_from_msgpack(&bytes).unwrap(), val); + } + + let bytes = value_to_msgpack(&Value::NaiveDateTime(dt)).unwrap(); + assert_eq!( + msgpack_to_json_string(&bytes).unwrap(), + "\"2024-03-15T10:30:00.000000Z\"" + ); + } + + #[test] + fn unknown_ext_is_null() { + let bytes = [0xD7, 0x09, 0, 0, 0, 0, 0, 0, 0, 1]; + assert_eq!(value_from_msgpack(&bytes).unwrap(), Value::Null); + } + + #[test] + fn unsigned_64_reads_as_integer() { + let bytes = [0xCF, 0, 0, 0, 0, 0, 0, 0x01, 0x00]; + assert_eq!(through_cell(&bytes).unwrap(), Value::Integer(256)); + assert_eq!(value_from_msgpack(&bytes).unwrap(), Value::Integer(256)); + } + + #[test] + fn unknown_ext_reads_as_null() { + let bytes = [0xD4, 0x07, 0x00]; + assert_eq!(through_cell(&bytes).unwrap(), Value::Null); + assert_eq!(value_from_msgpack(&bytes).unwrap(), Value::Null); + } + + #[test] + fn unused_marker_is_an_error() { + assert!(through_cell(&[0xC1]).is_err()); + assert!(value_from_msgpack(&[0xC1]).is_err()); + } + + #[test] + fn non_string_map_key_is_debug_rendered() { + // {1: "a"} + let bytes = [0x81, 0x01, 0xA1, b'a']; + let expected = Value::Object(std::collections::HashMap::from([( + "Integer(1)".to_string(), + Value::String("a".into()), + )])); + assert_eq!(value_from_msgpack(&bytes).unwrap(), expected); + assert_eq!(through_cell(&bytes).unwrap(), expected); + } + + #[test] + fn trailing_bytes_report_consumed_length() { + let mut bytes = value_to_msgpack(&Value::Array(vec![ + Value::Integer(1), + Value::String("ab".into()), + ])) + .unwrap(); + let len = bytes.len(); + bytes.extend_from_slice(&[0xC0, 0xC0]); + match value_from_msgpack(&bytes) { + Err(MsgpackError::TrailingBytes { consumed, total }) => { + assert_eq!(consumed, len); + assert_eq!(total, len + 2); + } + other => panic!("expected TrailingBytes, got {other:?}"), + } + } + + #[test] + fn both_entry_points_agree() { + let mut inner = std::collections::HashMap::new(); + inner.insert("k".to_string(), Value::Array(vec![Value::Integer(1)])); + let mut outer = std::collections::HashMap::new(); + outer.insert("nested".to_string(), Value::Object(inner)); + outer.insert("list".to_string(), Value::Array(vec![Value::Null])); + + let mut cases: Vec> = [ + Value::Object(outer), + Value::DateTime(NdbDateTime::from_micros(1_583_402_400_000_000)), + Value::NaiveDateTime(NdbDateTime::from_micros(-86_400_000_000)), + Value::Integer(-70_000), + Value::Bytes(vec![0, 255, 7]), + ] + .iter() + .map(|v| value_to_msgpack(v).unwrap()) + .collect(); + // u64 above i64::MAX + cases.push(vec![0xCF, 0x80, 0, 0, 0, 0, 0, 0, 0x01]); + // unknown ext (ext8, type 9, one payload byte) + cases.push(vec![0xC7, 0x01, 0x09, 0x2A]); + + for bytes in cases { + let a = value_from_msgpack(&bytes).unwrap(); + let b = through_cell(&bytes).unwrap(); + assert_eq!(a, b, "{bytes:02X?}"); + } + } +} diff --git a/nodedb-types/src/json_msgpack/transcoder.rs b/nodedb-types/src/json_msgpack/transcoder.rs index ddb4f099d..228669b59 100644 --- a/nodedb-types/src/json_msgpack/transcoder.rs +++ b/nodedb-types/src/json_msgpack/transcoder.rs @@ -5,11 +5,16 @@ //! Walks msgpack bytes and writes JSON text directly into a String. //! No intermediate `serde_json::Value` or `nodedb_types::Value`. //! Used ONLY at the outermost pgwire/HTTP layer for client compatibility. +//! +//! Instant ext values (fixext8 type 1 / 2) render as ISO 8601 strings. +//! Every other ext type renders as `null`. use std::fmt::Write as _; use super::error::MsgpackResult; +use super::instant_ext::instant_from_ext; use super::reader::{Cursor, base64_encode}; +use crate::datetime::NdbDateTime; /// Transcode raw msgpack bytes to a JSON string without intermediate types. /// @@ -139,7 +144,20 @@ fn transcode_value(c: &mut Cursor<'_>, out: &mut String) -> zerompk::Result<()> transcode_map(c, out, l)?; } - // ext types — render as null + // fixext8: instants render as ISO 8601, other types as null + 0xD7 => { + let (ext_type, payload) = c.take_fixext8()?; + match instant_from_ext(ext_type, payload) { + Some((_, micros)) => { + out.push('"'); + out.push_str(&NdbDateTime::from_micros(micros).to_iso8601()); + out.push('"'); + } + None => out.push_str("null"), + } + } + + // other ext types — render as null 0xD4 => { c.take_n(2)?; out.push_str("null"); @@ -152,10 +170,6 @@ fn transcode_value(c: &mut Cursor<'_>, out: &mut String) -> zerompk::Result<()> c.take_n(5)?; out.push_str("null"); } - 0xD7 => { - c.take_n(9)?; - out.push_str("null"); - } 0xD8 => { c.take_n(17)?; out.push_str("null"); @@ -310,6 +324,18 @@ mod tests { assert_eq!(msgpack_to_json_string(&[]).unwrap(), ""); } + #[test] + fn instant_ext_renders_iso8601() { + use crate::json_msgpack::instant_ext::{InstantKind, write_instant}; + let mut mp = vec![0x81, 0xA2, b't', b's']; + write_instant(&mut mp, InstantKind::Naive, 1_710_498_600_000_000); + let json_str = msgpack_to_json_string(&mp).unwrap(); + assert_eq!(json_str, "{\"ts\":\"2024-03-15T10:30:00.000000Z\"}"); + + let unknown = [0xD7, 0x09, 0, 0, 0, 0, 0, 0, 0, 1]; + assert_eq!(msgpack_to_json_string(&unknown).unwrap(), "null"); + } + #[test] fn nested() { let val = serde_json::json!({"a": {"b": [1, 2, {"c": 3}]}}); diff --git a/nodedb-types/src/json_msgpack/writer.rs b/nodedb-types/src/json_msgpack/writer.rs index e557b580d..2cd85ae50 100644 --- a/nodedb-types/src/json_msgpack/writer.rs +++ b/nodedb-types/src/json_msgpack/writer.rs @@ -1,7 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 //! Msgpack serialization for `serde_json::Value` and `nodedb_types::Value`. +//! +//! `Value::DateTime` and `Value::NaiveDateTime` are written as the instant +//! ext (`fixext8`, see `instant_ext`). `Duration`, `Decimal`, and `Geometry` +//! are written as strings. `Vector` is a float64 array, `ArrayCell` a map, +//! and `Range` / `Record` are `nil`. +use zerompk::Write; + +use super::instant_ext::InstantKind; use super::json_value::JsonValue; /// Serialize a `serde_json::Value` to MessagePack bytes. @@ -26,61 +34,68 @@ pub fn json_to_msgpack_or_empty(value: &serde_json::Value) -> Vec { /// Writes standard msgpack format (fixmap 0x80-0x8F, fixstr 0xA0-0xBF, etc.) /// directly from `Value` — no zerompk tagged encoding. pub fn value_to_msgpack(value: &crate::Value) -> zerompk::Result> { - let mut buf = Vec::with_capacity(128); - write_native_value(&mut buf, value); - Ok(buf) + zerompk::to_msgpack_vec(&NativeRef(value)) +} + +/// A borrowed `Value` written in its plain msgpack form. +struct NativeRef<'a>(&'a crate::Value); + +impl zerompk::ToMessagePack for NativeRef<'_> { + fn write(&self, writer: &mut W) -> zerompk::Result<()> { + write_native_value(writer, self.0) + } } -/// Write a `nodedb_types::Value` as standard msgpack bytes. -fn write_native_value(buf: &mut Vec, value: &crate::Value) { +/// Write a `nodedb_types::Value` as standard msgpack. +/// +/// `Duration`, `Decimal`, and `Geometry` are strings. `Vector` is a float64 +/// array, `ArrayCell` a map, and `Range` / `Record` are `nil`. +pub(crate) fn write_native_value( + writer: &mut W, + value: &crate::Value, +) -> zerompk::Result<()> { match value { - crate::Value::Null => buf.push(0xC0), - crate::Value::Bool(false) => buf.push(0xC2), - crate::Value::Bool(true) => buf.push(0xC3), - crate::Value::Integer(i) => write_native_int(buf, *i), - crate::Value::Float(f) => { - buf.push(0xCB); - buf.extend_from_slice(&f.to_be_bytes()); - } + crate::Value::Null => writer.write_nil(), + crate::Value::Bool(b) => writer.write_boolean(*b), + crate::Value::Integer(i) => writer.write_i64(*i), + crate::Value::Float(f) => writer.write_f64(*f), crate::Value::String(s) | crate::Value::Uuid(s) | crate::Value::Ulid(s) - | crate::Value::Regex(s) => write_native_str(buf, s), - crate::Value::Bytes(b) => write_native_bin(buf, b), + | crate::Value::Regex(s) => writer.write_string(s), + crate::Value::Bytes(b) => writer.write_binary(b), crate::Value::Array(arr) | crate::Value::Set(arr) => { - write_native_array_header(buf, arr.len()); + writer.write_array_len(arr.len())?; for v in arr { - write_native_value(buf, v); + write_native_value(writer, v)?; } + Ok(()) } crate::Value::Object(map) => { - write_native_map_header(buf, map.len()); + writer.write_map_len(map.len())?; for (k, v) in map { - write_native_str(buf, k); - write_native_value(buf, v); - } - } - crate::Value::DateTime(dt) | crate::Value::NaiveDateTime(dt) => { - write_native_str(buf, &dt.to_string()) - } - crate::Value::Duration(d) => write_native_str(buf, &d.to_string()), - crate::Value::Decimal(d) => write_native_str(buf, &d.to_string()), - crate::Value::Geometry(g) => { - if let Ok(s) = sonic_rs::to_string(g) { - write_native_str(buf, &s); - } else { - buf.push(0xC0); + writer.write_string(k)?; + write_native_value(writer, v)?; } + Ok(()) } - crate::Value::Range { .. } | crate::Value::Record { .. } => buf.push(0xC0), + crate::Value::DateTime(dt) => write_instant_ext(writer, InstantKind::Utc, dt.micros), + crate::Value::NaiveDateTime(dt) => write_instant_ext(writer, InstantKind::Naive, dt.micros), + crate::Value::Duration(d) => writer.write_string(&d.to_string()), + crate::Value::Decimal(d) => writer.write_string(&d.to_string()), + crate::Value::Geometry(g) => match sonic_rs::to_string(g) { + Ok(s) => writer.write_string(&s), + Err(_) => writer.write_nil(), + }, + crate::Value::Range { .. } | crate::Value::Record { .. } => writer.write_nil(), crate::Value::Vector(v) => { - // Encode as a standard msgpack array of float64 values so that - // pgwire clients receive a plain JSON number array. - write_native_array_header(buf, v.len()); + // A standard msgpack array of float64 values, so pgwire clients + // receive a plain JSON number array. + writer.write_array_len(v.len())?; for f in v.iter() { - buf.push(0xCB); - buf.extend_from_slice(&(*f as f64).to_be_bytes()); + writer.write_f64(f64::from(*f))?; } + Ok(()) } // ArrayCell is encoded as a `{coords:[...], attrs:[...]}` map so the // pgwire `msgpack_to_json_string` transcoder produces clean JSON for @@ -89,97 +104,31 @@ fn write_native_value(buf: &mut Vec, value: &crate::Value) { // `_ts_system` column (mirrors the document-engine audit-log shape). crate::Value::ArrayCell(cell) => { let map_len = if cell.system_time.is_some() { 3 } else { 2 }; - write_native_map_header(buf, map_len); - write_native_str(buf, "coords"); - write_native_array_header(buf, cell.coords.len()); + writer.write_map_len(map_len)?; + writer.write_string("coords")?; + writer.write_array_len(cell.coords.len())?; for v in &cell.coords { - write_native_value(buf, v); + write_native_value(writer, v)?; } - write_native_str(buf, "attrs"); - write_native_array_header(buf, cell.attrs.len()); + writer.write_string("attrs")?; + writer.write_array_len(cell.attrs.len())?; for v in &cell.attrs { - write_native_value(buf, v); + write_native_value(writer, v)?; } if let Some(ts) = cell.system_time { - write_native_str(buf, "_ts_system"); - write_native_int(buf, ts); + writer.write_string("_ts_system")?; + writer.write_i64(ts)?; } + Ok(()) } } } -fn write_native_int(buf: &mut Vec, i: i64) { - if (0..=0x7F).contains(&i) { - buf.push(i as u8); - } else if (-32..0).contains(&i) { - buf.push(i as u8); // negative fixint - } else if i >= i8::MIN as i64 && i <= i8::MAX as i64 { - buf.push(0xD0); - buf.push(i as i8 as u8); - } else if i >= i16::MIN as i64 && i <= i16::MAX as i64 { - buf.push(0xD1); - buf.extend_from_slice(&(i as i16).to_be_bytes()); - } else if i >= i32::MIN as i64 && i <= i32::MAX as i64 { - buf.push(0xD2); - buf.extend_from_slice(&(i as i32).to_be_bytes()); - } else { - buf.push(0xD3); - buf.extend_from_slice(&i.to_be_bytes()); - } -} - -fn write_native_str(buf: &mut Vec, s: &str) { - let len = s.len(); - if len < 32 { - buf.push(0xA0 | len as u8); - } else if len <= u8::MAX as usize { - buf.push(0xD9); - buf.push(len as u8); - } else if len <= u16::MAX as usize { - buf.push(0xDA); - buf.extend_from_slice(&(len as u16).to_be_bytes()); - } else { - buf.push(0xDB); - buf.extend_from_slice(&(len as u32).to_be_bytes()); - } - buf.extend_from_slice(s.as_bytes()); -} - -fn write_native_bin(buf: &mut Vec, b: &[u8]) { - let len = b.len(); - if len <= u8::MAX as usize { - buf.push(0xC4); - buf.push(len as u8); - } else if len <= u16::MAX as usize { - buf.push(0xC5); - buf.extend_from_slice(&(len as u16).to_be_bytes()); - } else { - buf.push(0xC6); - buf.extend_from_slice(&(len as u32).to_be_bytes()); - } - buf.extend_from_slice(b); -} - -fn write_native_array_header(buf: &mut Vec, len: usize) { - if len < 16 { - buf.push(0x90 | len as u8); - } else if len <= u16::MAX as usize { - buf.push(0xDC); - buf.extend_from_slice(&(len as u16).to_be_bytes()); - } else { - buf.push(0xDD); - buf.extend_from_slice(&(len as u32).to_be_bytes()); - } -} - -fn write_native_map_header(buf: &mut Vec, len: usize) { - if len < 16 { - buf.push(0x80 | len as u8); - } else if len <= u16::MAX as usize { - buf.push(0xDE); - buf.extend_from_slice(&(len as u16).to_be_bytes()); - } else { - buf.push(0xDF); - buf.extend_from_slice(&(len as u32).to_be_bytes()); - } +/// Write an instant as the `fixext8` ext `write_instant` produces. +fn write_instant_ext( + writer: &mut W, + kind: InstantKind, + micros: i64, +) -> zerompk::Result<()> { + writer.write_ext(kind.ext_type(), µs.to_be_bytes()) } diff --git a/nodedb-types/src/lib.rs b/nodedb-types/src/lib.rs index c7f9ec841..1342db40e 100644 --- a/nodedb-types/src/lib.rs +++ b/nodedb-types/src/lib.rs @@ -105,8 +105,9 @@ pub use id::{ }; pub use identity::KeyRepr; pub use json_msgpack::{ - JsonValue, MsgpackError, MsgpackResult, json_from_msgpack, json_to_msgpack, - json_to_msgpack_or_empty, msgpack_to_json_string, value_from_msgpack, value_to_msgpack, + InstantKind, JsonValue, MsgpackError, MsgpackResult, NativeCell, json_from_msgpack, + json_to_msgpack, json_to_msgpack_or_empty, msgpack_to_json_string, read_instant, + value_from_msgpack, value_to_msgpack, write_instant, }; pub use kv::{KV_DEFAULT_INLINE_THRESHOLD, KvConfig, KvTtlPolicy, is_valid_kv_key_type}; pub use lsn::Lsn; diff --git a/nodedb-types/src/value/coerce.rs b/nodedb-types/src/value/coerce.rs index 78a86aa3b..8ad93e798 100644 --- a/nodedb-types/src/value/coerce.rs +++ b/nodedb-types/src/value/coerce.rs @@ -39,6 +39,15 @@ impl Value { (Value::String(s), Value::Float(b)) => s.parse::().is_ok_and(|n| n == *b), // Structural equality on ND cells: same coords and same attrs. (Value::ArrayCell(a), Value::ArrayCell(b)) => a == b, + // Two exact decimals compare exactly; a decimal against any other + // number compares through f64 like the arms above. + (Value::Decimal(a), Value::Decimal(b)) => a == b, + (Value::Decimal(_), _) | (_, Value::Decimal(_)) => { + match (numeric_f64(self), numeric_f64(other)) { + (Some(x), Some(y)) => x == y, + _ => false, + } + } (a, b) => match (datetime_micros(a), datetime_micros(b)) { (Some(x), Some(y)) => x == y, _ => false, @@ -46,59 +55,71 @@ impl Value { } } - /// Coerced ordering: `Value` vs `Value` with numeric/string coercion. + /// Coerced partial ordering for predicate evaluation. /// - /// Single source of truth for ordering in filter/sort evaluation. - pub fn cmp_coerced(&self, other: &Value) -> std::cmp::Ordering { + /// Two numbers (or numeric strings) order numerically, two instants (or + /// ISO-8601 strings) by epoch microseconds, two other strings + /// lexicographically, and two ND cells coordinate-major. A pair with no + /// defined order — an integer against an instant, text against a number, + /// a NaN — is `None`, so a range predicate over it matches nothing rather + /// than every row: the row-level counterpart of PostgreSQL refusing to + /// compare the two types. + pub fn partial_cmp_coerced(&self, other: &Value) -> Option { use std::cmp::Ordering; - // ND cells: lexicographic on coords, then attrs. Matches array - // engine cell ordering (coordinate-major). if let (Value::ArrayCell(a), Value::ArrayCell(b)) = (self, other) { for (x, y) in a.coords.iter().zip(b.coords.iter()) { - match x.cmp_coerced(y) { + match x.partial_cmp_coerced(y)? { Ordering::Equal => continue, - non_eq => return non_eq, + non_eq => return Some(non_eq), } } match a.coords.len().cmp(&b.coords.len()) { Ordering::Equal => {} - non_eq => return non_eq, + non_eq => return Some(non_eq), } for (x, y) in a.attrs.iter().zip(b.attrs.iter()) { - match x.cmp_coerced(y) { + match x.partial_cmp_coerced(y)? { Ordering::Equal => continue, - non_eq => return non_eq, + non_eq => return Some(non_eq), } } - return a.attrs.len().cmp(&b.attrs.len()); + return Some(a.attrs.len().cmp(&b.attrs.len())); } - let self_f64 = match self { - Value::Integer(i) => Some(*i as f64), - Value::Float(f) => Some(*f), - Value::String(s) => s.parse::().ok(), - _ => None, - }; - let other_f64 = match other { - Value::Integer(i) => Some(*i as f64), - Value::Float(f) => Some(*f), - Value::String(s) => s.parse::().ok(), - _ => None, - }; - if let (Some(a), Some(b)) = (self_f64, other_f64) { - return a.partial_cmp(&b).unwrap_or(Ordering::Equal); + if let (Some(a), Some(b)) = (numeric_f64(self), numeric_f64(other)) { + return a.partial_cmp(&b); } if let (Some(a), Some(b)) = (datetime_micros(self), datetime_micros(other)) { - return a.cmp(&b); + return Some(a.cmp(&b)); + } + match (self, other) { + (Value::String(a), Value::String(b)) => Some(a.cmp(b)), + _ => None, } - let a_str = match self { - Value::String(s) => s.as_str(), - _ => return Ordering::Equal, - }; - let b_str = match other { - Value::String(s) => s.as_str(), - _ => return Ordering::Equal, - }; - a_str.cmp(b_str) + } + + /// Coerced total ordering for sorting. + /// + /// [`Value::partial_cmp_coerced`] with an unordered pair placed as + /// `Equal`, so a stable sort keeps such rows in their input order. Only + /// for ORDER BY / MIN / MAX style paths that need an `Ordering` for every + /// pair; a predicate uses `partial_cmp_coerced` so an unordered pair + /// matches nothing. + pub fn cmp_coerced(&self, other: &Value) -> std::cmp::Ordering { + self.partial_cmp_coerced(other) + .unwrap_or(std::cmp::Ordering::Equal) + } +} + +/// The number a value denotes for coerced ordering: an integer, a float, or +/// a string that parses as one. `None` for anything else. +fn numeric_f64(v: &Value) -> Option { + use rust_decimal::prelude::ToPrimitive; + match v { + Value::Integer(i) => Some(*i as f64), + Value::Float(f) => Some(*f), + Value::Decimal(d) => d.to_f64(), + Value::String(s) => s.parse::().ok(), + _ => None, } } @@ -117,6 +138,18 @@ fn datetime_micros(v: &Value) -> Option { mod tests { use super::*; + #[test] + fn a_decimal_literal_equals_the_float_it_denotes() { + let d = Value::Decimal(rust_decimal::Decimal::from_str_exact("2.5").expect("decimal")); + assert!(d.eq_coerced(&Value::Float(2.5))); + assert!(Value::Float(2.5).eq_coerced(&d)); + assert!(!d.eq_coerced(&Value::Float(2.25))); + assert_eq!( + d.partial_cmp_coerced(&Value::Integer(3)), + Some(std::cmp::Ordering::Less) + ); + } + #[test] fn eq_coerced_same_type() { assert!(Value::Null.eq_coerced(&Value::Null)); @@ -219,6 +252,59 @@ mod tests { ); } + #[test] + fn partial_cmp_coerced_orders_instants_against_instants_and_iso_text() { + use std::cmp::Ordering; + let earlier = Value::NaiveDateTime(crate::NdbDateTime::from_micros(1_583_402_400_000_000)); + let later = Value::DateTime(crate::NdbDateTime::from_micros(1_583_406_000_000_000)); + assert_eq!(earlier.partial_cmp_coerced(&later), Some(Ordering::Less)); + assert_eq!( + later.partial_cmp_coerced(&Value::String("2020-03-05 10:00:00".into())), + Some(Ordering::Greater) + ); + assert_eq!( + earlier.partial_cmp_coerced(&Value::String("2020-03-05T10:00:00Z".into())), + Some(Ordering::Equal) + ); + } + + /// An integer carries no unit, so it has no order against an instant: + /// `WHERE at >= 5` over an instant column matches nothing, in both + /// orientations. The same holds for text against a number and for NaN. + #[test] + fn partial_cmp_coerced_is_none_for_an_unordered_pair() { + let instant = Value::NaiveDateTime(crate::NdbDateTime::from_micros(1_583_402_400_000_000)); + assert_eq!(instant.partial_cmp_coerced(&Value::Integer(5)), None); + assert_eq!(Value::Integer(5).partial_cmp_coerced(&instant), None); + assert_eq!( + Value::Integer(5).partial_cmp_coerced(&Value::String("abc".into())), + None + ); + assert_eq!( + Value::Bool(true).partial_cmp_coerced(&Value::Integer(1)), + None + ); + assert_eq!(Value::Null.partial_cmp_coerced(&Value::Integer(0)), None); + assert_eq!( + Value::Float(f64::NAN).partial_cmp_coerced(&Value::Float(1.0)), + None + ); + } + + /// The sort order places an unordered pair as `Equal` so a stable sort + /// keeps its input order; an ordered pair sorts as the predicate orders + /// it. + #[test] + fn cmp_coerced_places_an_unordered_pair_as_equal_for_sorting() { + use std::cmp::Ordering; + let instant = Value::NaiveDateTime(crate::NdbDateTime::from_micros(1_583_402_400_000_000)); + assert_eq!(instant.cmp_coerced(&Value::Integer(5)), Ordering::Equal); + assert_eq!( + Value::Integer(5).cmp_coerced(&Value::Integer(7)), + Ordering::Less + ); + } + #[test] fn eq_coerced_symmetry() { let cases = [ diff --git a/nodedb-types/src/value/core.rs b/nodedb-types/src/value/core.rs index 32fd5528f..81ecab911 100644 --- a/nodedb-types/src/value/core.rs +++ b/nodedb-types/src/value/core.rs @@ -13,6 +13,7 @@ use serde::{Deserialize, Serialize}; use crate::array_cell::ArrayCell; use crate::datetime::{NdbDateTime, NdbDuration}; use crate::geometry::Geometry; +use crate::json_msgpack::InstantKind; /// A dynamic value that can represent any field type in a document /// or any parameter in a SQL query. @@ -195,6 +196,15 @@ impl Value { } } + /// Try to extract as an instant of either kind. + pub fn as_instant(&self) -> Option<(InstantKind, NdbDateTime)> { + match self { + Value::DateTime(dt) => Some((InstantKind::Utc, *dt)), + Value::NaiveDateTime(dt) => Some((InstantKind::Naive, *dt)), + _ => None, + } + } + /// Try to extract as Duration. pub fn as_duration(&self) -> Option<&NdbDuration> { match self { diff --git a/nodedb/src/control/backup/restore/orchestrate/restore.rs b/nodedb/src/control/backup/restore/orchestrate/restore.rs index 27f4b4460..0391e3f87 100644 --- a/nodedb/src/control/backup/restore/orchestrate/restore.rs +++ b/nodedb/src/control/backup/restore/orchestrate/restore.rs @@ -12,6 +12,7 @@ use nodedb_types::backup_envelope::{ use crate::Error; use crate::bridge::envelope::PhysicalPlan; +use crate::control::server::shared::ddl::neutral::collection::dispatch_register_from_stored; use crate::control::server::shared::ddl::sync_dispatch; use crate::control::state::SharedState; use crate::types::TenantId; @@ -84,7 +85,27 @@ pub async fn restore_tenant( }; if !dry_run { - apply_metadata_sections(state, tenant_id, &env)?; + let restored_collections = apply_metadata_sections(state, tenant_id, &env)?; + // Every restored collection's declaration reaches this node's Data + // Plane before any of its rows do. The catalog row alone leaves + // `doc_configs` empty for the collection, and the re-issue below + // would then ingest a timeseries collection's rows into an inferred + // shape: the declared time key becomes an integer field and the row + // is stamped with the restore-time clock. This is the same + // registration a committed DDL and the boot rehydration dispatch, + // and it replaces any registration already present, so a cluster + // applier's own register hook and a later boot seed are both + // idempotent with it. A registration failure fails the restore. + for coll in &restored_collections { + dispatch_register_from_stored(state, coll) + .await + .map_err(|e| Error::Internal { + detail: format!( + "restore: Data Plane registration of collection '{}' failed: {e}", + coll.name + ), + })?; + } } let mut merged = merge_sections(&env.sections)?; diff --git a/nodedb/src/control/backup/restore/sections.rs b/nodedb/src/control/backup/restore/sections.rs index 0153ac9ba..7614001b6 100644 --- a/nodedb/src/control/backup/restore/sections.rs +++ b/nodedb/src/control/backup/restore/sections.rs @@ -6,6 +6,7 @@ use nodedb_types::DatabaseId; use std::sync::Arc; use crate::Error; +use crate::control::security::catalog::StoredCollection; use crate::control::state::SharedState; use crate::types::{SurrogateBindEntry, TenantDataSnapshot}; @@ -81,16 +82,23 @@ pub(super) fn is_metadata_section(section: &nodedb_types::backup_envelope::Secti /// catalog-propose failure on this path is FATAL: returning the data /// restored but unqueryable on non-coordinator nodes is the /// silent-partial-success anti-pattern this codebase forbids. +/// +/// Returns every collection written to the catalog, in section order. The +/// caller registers each one with this node's Data Plane before any restored +/// row is installed or reissued: a catalog row alone leaves `doc_configs` +/// without the collection's declaration, and a reissued timeseries row would +/// then be ingested into an inferred shape. pub(super) fn apply_metadata_sections( state: &Arc, tenant_id: u64, env: &nodedb_types::backup_envelope::Envelope, -) -> Result<(), Error> { +) -> Result, Error> { use nodedb_types::backup_envelope::{ SECTION_ORIGIN_CATALOG_ROWS, SECTION_ORIGIN_SOURCE_TOMBSTONES, SourceTombstoneEntry, StoredCollectionBlob, }; let catalog = state.credentials.catalog(); + let mut restored: Vec = Vec::new(); for section in &env.sections { match section.origin_node_id { @@ -104,9 +112,7 @@ pub(super) fn apply_metadata_sections( continue; }; for blob in blobs { - let Ok(coll) = zerompk::from_msgpack::< - crate::control::security::catalog::StoredCollection, - >(&blob.bytes) else { + let Ok(coll) = zerompk::from_msgpack::(&blob.bytes) else { tracing::warn!( tenant_id, name = %blob.name, @@ -121,8 +127,9 @@ pub(super) fn apply_metadata_sections( // blocks on its local applied-index watcher, so on the // cluster path it has already applied the put via the // same applier — we must NOT also put locally (double-put). - let entry = - crate::control::catalog_entry::CatalogEntry::PutCollection(Box::new(coll)); + let entry = crate::control::catalog_entry::CatalogEntry::PutCollection( + Box::new(coll.clone()), + ); let outcome = crate::control::metadata_proposer::propose_catalog_entry(state, &entry)?; if outcome.needs_local_apply() { @@ -131,12 +138,9 @@ pub(super) fn apply_metadata_sections( // applier would have done on a clustered deployment. // A failure here is FATAL — the collection would be // unqueryable otherwise. - if let crate::control::catalog_entry::CatalogEntry::PutCollection(boxed) = - entry - { - catalog.put_collection(DatabaseId::DEFAULT, &boxed)?; - } + catalog.put_collection(DatabaseId::DEFAULT, &coll)?; } + restored.push(coll); } } SECTION_ORIGIN_SOURCE_TOMBSTONES => { @@ -184,5 +188,5 @@ pub(super) fn apply_metadata_sections( _ => {} } } - Ok(()) + Ok(restored) } diff --git a/nodedb/src/control/backup/restore/timeseries_reissue.rs b/nodedb/src/control/backup/restore/timeseries_reissue.rs index 64c9f3b34..3329ad0a9 100644 --- a/nodedb/src/control/backup/restore/timeseries_reissue.rs +++ b/nodedb/src/control/backup/restore/timeseries_reissue.rs @@ -14,6 +14,7 @@ use std::time::Duration; use nodedb_types::RlsWriteCheck; use nodedb_types::columnar::schema::TS_SYSTEM; +use nodedb_types::datetime::NdbDateTimeError; use nodedb_types::value::Value; use crate::Error; @@ -88,7 +89,8 @@ fn decode_memtable_rows( if name == TS_SYSTEM_COLUMN { continue; } - let cell = memtable_cell(&mt, *col_idx, *ty, idx); + let cell = memtable_cell(&mt, *col_idx, *ty, idx) + .map_err(|e| instant_cell_error(collection, name, e))?; insert_non_null(&mut map, name, cell); } rows.push(Value::Object(map)); @@ -96,10 +98,30 @@ fn decode_memtable_rows( Ok(()) } +/// A stored millisecond count that the column's instant kind cannot carry. +fn instant_cell_error(collection: &str, column: &str, e: NdbDateTimeError) -> Error { + Error::Storage { + engine: "timeseries".into(), + detail: format!("restore reissue: time column '{column}' of '{collection}': {e}"), + } +} + /// Extract one cell from a memtable column as a `Value`. -fn memtable_cell(mt: &ColumnarMemtable, col_idx: usize, ty: ColumnType, idx: usize) -> Value { - match ty { - ColumnType::Timestamp => Value::Integer(mt.column(col_idx).as_timestamps()[idx]), +/// +/// A time column yields the value its kind denotes — a typed instant for a +/// declared `TIMESTAMP` / `TIMESTAMPTZ` key, the integer stored for a +/// `BIGINT` key — so the reissued row carries the cell a client INSERT +/// would have sent. +fn memtable_cell( + mt: &ColumnarMemtable, + col_idx: usize, + ty: ColumnType, + idx: usize, +) -> Result { + let value = match ty { + ColumnType::Timestamp(kind) => { + return kind.cell_value(mt.column(col_idx).as_timestamps()[idx]); + } ColumnType::Int64 => Value::Integer(mt.column(col_idx).as_i64()[idx]), ColumnType::Float64 => { let v = mt.column(col_idx).as_f64()[idx]; @@ -132,7 +154,8 @@ fn memtable_cell(mt: &ColumnarMemtable, col_idx: usize, ty: ColumnType, idx: usi } _ => Value::Null, }, - } + }; + Ok(value) } /// Decode one flushed partition directory into row objects, appending to `rows`. @@ -190,7 +213,8 @@ fn decode_partition_rows( if name == TS_SYSTEM_COLUMN { continue; } - let cell = partition_cell(&col_data[col_i], *ty, col_i, &sym_dicts, idx); + let cell = partition_cell(&col_data[col_i], *ty, col_i, &sym_dicts, idx) + .map_err(|e| instant_cell_error(collection, name, e))?; insert_non_null(&mut map, name, cell); } rows.push(Value::Object(map)); @@ -198,16 +222,20 @@ fn decode_partition_rows( Ok(()) } -/// Extract one cell from a flushed-segment column as a `Value`. +/// Extract one cell from a flushed-segment column as a `Value`. The partition +/// schema carries each time column's kind, so the cell is typed the same way +/// a memtable cell is. fn partition_cell( data: &ColumnData, ty: ColumnType, col_idx: usize, sym_dicts: &HashMap, idx: usize, -) -> Value { - match ty { - ColumnType::Timestamp => Value::Integer(data.as_timestamps()[idx]), +) -> Result { + let value = match ty { + ColumnType::Timestamp(kind) => { + return kind.cell_value(data.as_timestamps()[idx]); + } ColumnType::Int64 => Value::Integer(data.as_i64()[idx]), ColumnType::Float64 => { let v = data.as_f64()[idx]; @@ -240,7 +268,8 @@ fn partition_cell( } _ => Value::Null, }, - } + }; + Ok(value) } /// Insert a cell, skipping nulls so a re-issued row carries only present fields @@ -330,3 +359,90 @@ pub async fn reissue_timeseries_durably( .await?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::engine::timeseries::columnar_memtable::{ColumnValue, ColumnarSchema, TimeKind}; + use nodedb_types::InstantKind; + use nodedb_types::datetime::NdbDateTime; + use nodedb_types::timeseries::SeriesId; + + /// `2020-03-05T10:00:00Z` in epoch milliseconds. + const EARLY_MS: i64 = 1_583_402_400_000; + + const NAIVE: ColumnType = ColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive)); + const UTC: ColumnType = ColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)); + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + + fn early() -> NdbDateTime { + NdbDateTime::from_micros(EARLY_MS * 1_000) + } + + /// A one-row memtable whose time column has the given type. + fn memtable_with_time_column(ty: ColumnType) -> ColumnarMemtable { + let schema = ColumnarSchema { + columns: vec![ + ("captured_at".into(), ty), + ("v".into(), ColumnType::Float64), + ], + timestamp_idx: 0, + codecs: vec![], + }; + let mut mt = ColumnarMemtable::new(schema, ColumnarMemtableConfig::default()); + let series: SeriesId = 1; + mt.ingest_row( + series, + &[ColumnValue::Timestamp(EARLY_MS), ColumnValue::Float64(1.5)], + ) + .expect("ingest one row"); + mt + } + + /// A memtable time cell is reissued as the value its kind denotes. + #[test] + fn a_memtable_time_cell_is_typed_by_its_kind() { + let mt = memtable_with_time_column(NAIVE); + assert_eq!( + memtable_cell(&mt, 0, NAIVE, 0).expect("in range"), + Value::NaiveDateTime(early()) + ); + let mt = memtable_with_time_column(UTC); + assert_eq!( + memtable_cell(&mt, 0, UTC, 0).expect("in range"), + Value::DateTime(early()) + ); + let mt = memtable_with_time_column(MILLIS); + assert_eq!( + memtable_cell(&mt, 0, MILLIS, 0).expect("an integer"), + Value::Integer(EARLY_MS) + ); + } + + /// A partition time cell is reissued as the value its kind denotes. + #[test] + fn a_partition_time_cell_is_typed_by_its_kind() { + let data = ColumnData::Timestamp(vec![EARLY_MS]); + let dicts = HashMap::new(); + assert_eq!( + partition_cell(&data, NAIVE, 0, &dicts, 0).expect("in range"), + Value::NaiveDateTime(early()) + ); + assert_eq!( + partition_cell(&data, UTC, 0, &dicts, 0).expect("in range"), + Value::DateTime(early()) + ); + assert_eq!( + partition_cell(&data, MILLIS, 0, &dicts, 0).expect("an integer"), + Value::Integer(EARLY_MS) + ); + } + + /// A millisecond count outside the instant range is an error, never a + /// silently wrong cell. + #[test] + fn an_out_of_range_instant_cell_is_an_error() { + let data = ColumnData::Timestamp(vec![i64::MAX]); + assert!(partition_cell(&data, NAIVE, 0, &HashMap::new(), 0).is_err()); + } +} diff --git a/nodedb/src/control/catalog_entry/post_apply/collection.rs b/nodedb/src/control/catalog_entry/post_apply/collection.rs index f7d28aa36..4eaabf5d4 100644 --- a/nodedb/src/control/catalog_entry/post_apply/collection.rs +++ b/nodedb/src/control/catalog_entry/post_apply/collection.rs @@ -25,6 +25,21 @@ pub fn put_owner_sync(stored: &StoredCollection, shared: Arc) { tenant_id: stored.tenant_id, owner_username: stored.owner.clone(), }); + // The collection's declared columns type its RLS policies' literals, so + // every node recompiles them against the schema this entry carries. + if let Err(e) = shared.rls.recompile_for_collection( + shared.credentials.catalog(), + stored.database_id, + stored.tenant_id, + &stored.name, + ) { + tracing::error!( + collection = %stored.name, + tenant = stored.tenant_id, + error = %e, + "post_apply: RLS policies could not be re-read for recompilation" + ); + } } /// Register-dispatch half: dispatch a `Register` request to this node's diff --git a/nodedb/src/control/catalog_entry/post_apply/rls.rs b/nodedb/src/control/catalog_entry/post_apply/rls.rs index 41ba46e74..140ef58c0 100644 --- a/nodedb/src/control/catalog_entry/post_apply/rls.rs +++ b/nodedb/src/control/catalog_entry/post_apply/rls.rs @@ -3,39 +3,27 @@ //! Post-apply side effects for RLS policy `CatalogEntry` variants. //! //! After the synchronous `apply::rls` step has written the redb row, -//! this rehydrates the runtime `RlsPolicy` (deserializing the -//! compiled predicate / deny mode JSON) and installs it into the -//! in-memory `RlsPolicyStore` on every node so the read/write -//! evaluators see the new policy on their next request. +//! this rehydrates the runtime `RlsPolicy` (compiling the stored `USING` +//! text against the collection's declared columns) and installs it into +//! the in-memory `RlsPolicyStore` on every node so the read/write +//! evaluators see the new policy on their next request. A row that cannot +//! be compiled installs as a restrictive deny-all +//! (`StoredRlsPolicy::rehydrate`). use std::sync::Arc; -use tracing::warn; - use crate::control::security::catalog::StoredRlsPolicy; use crate::control::state::SharedState; pub fn put(stored: StoredRlsPolicy, shared: Arc) { - match stored.to_runtime() { - Ok(runtime) => { - shared.rls.install_replicated_policy(runtime); - tracing::debug!( - policy = %stored.name, - collection = %stored.collection, - tenant = stored.tenant_id, - "post_apply: RLS policy replicated" - ); - } - Err(e) => { - warn!( - policy = %stored.name, - collection = %stored.collection, - tenant = stored.tenant_id, - error = %e, - "post_apply: RLS policy rehydration failed" - ); - } - } + let runtime = stored.rehydrate(shared.credentials.catalog()); + shared.rls.install_replicated_policy(runtime); + tracing::debug!( + policy = %stored.name, + collection = %stored.collection, + tenant = stored.tenant_id, + "post_apply: RLS policy replicated" + ); } pub fn delete(tenant_id: u64, collection: String, name: String, shared: Arc) { diff --git a/nodedb/src/control/planner/calvin/predicate.rs b/nodedb/src/control/planner/calvin/predicate.rs index 6f9f56c0b..52685298f 100644 --- a/nodedb/src/control/planner/calvin/predicate.rs +++ b/nodedb/src/control/planner/calvin/predicate.rs @@ -268,13 +268,15 @@ mod tests { field: "balance".into(), op: FilterOp::Gt, value: Value::Integer(1000), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let f2 = vec![ScanFilter { field: "balance".into(), op: FilterOp::Gt, value: Value::Integer(9999), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let h1 = predicate_class_for_filters(&encode_filters(&f1), "accounts"); let h2 = predicate_class_for_filters(&encode_filters(&f2), "accounts"); @@ -290,13 +292,15 @@ mod tests { field: "name".into(), op: FilterOp::Eq, value: Value::String("alice".into()), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let f2 = vec![ScanFilter { field: "name".into(), op: FilterOp::Eq, value: Value::String("bob".into()), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let h1 = predicate_class_for_filters(&encode_filters(&f1), "users"); let h2 = predicate_class_for_filters(&encode_filters(&f2), "users"); @@ -312,13 +316,15 @@ mod tests { field: "balance".into(), op: FilterOp::Gt, value: Value::Integer(100), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let f2 = vec![ScanFilter { field: "age".into(), op: FilterOp::Gt, value: Value::Integer(100), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let h1 = predicate_class_for_filters(&encode_filters(&f1), "accounts"); let h2 = predicate_class_for_filters(&encode_filters(&f2), "accounts"); @@ -331,13 +337,15 @@ mod tests { field: "score".into(), op: FilterOp::Gt, value: Value::Integer(5), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let f2 = vec![ScanFilter { field: "score".into(), op: FilterOp::Lt, value: Value::Integer(5), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let h1 = predicate_class_for_filters(&encode_filters(&f1), "items"); let h2 = predicate_class_for_filters(&encode_filters(&f2), "items"); @@ -350,7 +358,8 @@ mod tests { field: "x".into(), op: FilterOp::Eq, value: Value::Integer(1), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let bytes = encode_filters(&filters); let h1 = predicate_class_for_filters(&bytes, "col_a"); @@ -369,13 +378,15 @@ mod tests { field: "status".into(), op: FilterOp::Eq, value: Value::String("active".into()), - ..Default::default() + clauses: Vec::new(), + expr: None, }], vec![ScanFilter { field: "status".into(), op: FilterOp::Eq, value: Value::String("pending".into()), - ..Default::default() + clauses: Vec::new(), + expr: None, }], ], expr: None, diff --git a/nodedb/src/control/planner/catalog_adapter/mod.rs b/nodedb/src/control/planner/catalog_adapter/mod.rs index fe5c8beca..00ed1e9dc 100644 --- a/nodedb/src/control/planner/catalog_adapter/mod.rs +++ b/nodedb/src/control/planner/catalog_adapter/mod.rs @@ -36,3 +36,4 @@ mod sql_catalog_impl; mod type_convert; pub use adapter::OriginCatalog; +pub(crate) use type_convert::{convert_collection_type, declared_column_info}; diff --git a/nodedb/src/control/planner/catalog_adapter/type_convert.rs b/nodedb/src/control/planner/catalog_adapter/type_convert.rs index 7438b2666..2a21d4f31 100644 --- a/nodedb/src/control/planner/catalog_adapter/type_convert.rs +++ b/nodedb/src/control/planner/catalog_adapter/type_convert.rs @@ -6,7 +6,7 @@ use nodedb_sql::types::{ColumnInfo, EngineType, SqlDataType}; use nodedb_types::columnar::{FloatWidth, IntWidth}; /// Convert a StoredCollection to engine type, columns, and primary key. -pub(super) fn convert_collection_type( +pub(crate) fn convert_collection_type( stored: &crate::control::security::catalog::StoredCollection, ) -> (EngineType, Vec, Option) { use nodedb_types::CollectionType; @@ -84,16 +84,7 @@ pub(super) fn convert_collection_type( if name.eq_ignore_ascii_case(&pk_name) { continue; } - columns.push(ColumnInfo { - name: name.clone(), - data_type: parse_type_str(type_str), - nullable: true, - is_primary_key: false, - default: declared_default(type_str), - raw_type: None, - int_width: IntWidth::from_declared_type(type_str), - float_width: FloatWidth::from_declared_type(type_str), - }); + columns.push(declared_column_info(name, type_str)); } (EngineType::DocumentSchemaless, columns, Some(pk_name)) } @@ -174,16 +165,9 @@ pub(super) fn convert_collection_type( if !profile.is_timeseries() && name.eq_ignore_ascii_case(pk_name) { continue; } - columns.push(ColumnInfo { - name: name.clone(), - data_type: parse_type_str(type_str), - nullable: true, - is_primary_key: false, - default: declared_default(type_str), - raw_type: Some(type_str.clone()), - int_width: IntWidth::from_declared_type(type_str), - float_width: FloatWidth::from_declared_type(type_str), - }); + let mut column = declared_column_info(name, type_str); + column.raw_type = Some(type_str.clone()); + columns.push(column); } let pk = if profile.is_timeseries() { None @@ -195,6 +179,28 @@ pub(super) fn convert_collection_type( } } +/// The planner-facing column a raw DDL declaration (`name`, `type_str`) +/// resolves to: its SQL type, declared numeric width, and DEFAULT text. +/// +/// `type_str` is the text that followed the column name in the DDL, modifiers +/// included (`SMALLINT DEFAULT 5`, `TIMESTAMP TIME_KEY`). The schemaless and +/// columnar-family catalog arms read their tracked fields through this, and +/// the DDL gate checks a declared DEFAULT against the same resolution, so a +/// default is judged against exactly the type its column will carry at +/// INSERT time. +pub(crate) fn declared_column_info(name: &str, type_str: &str) -> ColumnInfo { + ColumnInfo { + name: name.to_string(), + data_type: parse_type_str(type_str), + nullable: true, + is_primary_key: false, + default: declared_default(type_str), + raw_type: None, + int_width: IntWidth::from_declared_type(type_str), + float_width: FloatWidth::from_declared_type(type_str), + } +} + /// Extract the `DEFAULT ` clause a columnar-family column declared. /// /// The columnar catalog stores each column as the raw DDL type string with its diff --git a/nodedb/src/control/planner/context/query/planning.rs b/nodedb/src/control/planner/context/query/planning.rs index 6d46558ef..3a600b923 100644 --- a/nodedb/src/control/planner/context/query/planning.rs +++ b/nodedb/src/control/planner/context/query/planning.rs @@ -141,7 +141,6 @@ impl QueryContext { .load(std::sync::atomic::Ordering::Relaxed), database_id, tenant_id, - sql_catalog: Some(Arc::clone(&catalog) as _), }; let output_schema = crate::control::planner::sql_plan_convert::output_schema::build_output_schema( @@ -423,7 +422,6 @@ impl QueryContext { .load(std::sync::atomic::Ordering::Relaxed), database_id, tenant_id, - sql_catalog: Some(Arc::clone(&catalog) as _), }; let output_schema = crate::control::planner::sql_plan_convert::output_schema::build_output_schema( diff --git a/nodedb/src/control/planner/rls_injection/filters.rs b/nodedb/src/control/planner/rls_injection/filters.rs index 2e4377948..721450a86 100644 --- a/nodedb/src/control/planner/rls_injection/filters.rs +++ b/nodedb/src/control/planner/rls_injection/filters.rs @@ -7,6 +7,9 @@ use crate::control::security::rls::RlsPolicyStore; use crate::types::TenantId; /// Fetch RLS bytes for a (tenant, collection) pair. +/// +/// An unresolvable `$auth.*` reference is the deny error; an unencodable +/// filter set is its own error. Both refuse the statement. pub(super) fn get_rls( rls_store: &RlsPolicyStore, tenant_id: u64, @@ -14,15 +17,16 @@ pub(super) fn get_rls( auth: &AuthContext, ) -> crate::Result> { rls_store - .combined_read_predicate_with_auth(tenant_id, collection, auth) + .combined_read_predicate_with_auth(tenant_id, collection, auth)? .ok_or_else(|| rls_deny_error(tenant_id, collection)) } /// Fetch the compiled write-policy bytes for a (tenant, collection) pair. /// /// Fails closed on an unresolvable `$auth.*` reference through the same deny -/// error the read fetch raises, so a write can never proceed on a predicate -/// that could not be resolved. +/// error the read fetch raises, and on an unencodable filter set through +/// its own error, so a write can never proceed on a predicate that could +/// not be resolved or encoded. pub(super) fn get_rls_write( rls_store: &RlsPolicyStore, tenant_id: u64, @@ -30,7 +34,7 @@ pub(super) fn get_rls_write( auth: &AuthContext, ) -> crate::Result> { rls_store - .combined_write_predicate_with_auth(tenant_id, collection, auth) + .combined_write_predicate_with_auth(tenant_id, collection, auth)? .ok_or_else(|| rls_deny_error(tenant_id, collection)) } diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs index c61cf62bc..d52fc535b 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs @@ -153,7 +153,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), } diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs index 845605a93..ed997b9f4 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs @@ -255,7 +255,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; diff --git a/nodedb/src/control/planner/sql_plan_convert/convert.rs b/nodedb/src/control/planner/sql_plan_convert/convert.rs index 91f6ae37b..1b38821ea 100644 --- a/nodedb/src/control/planner/sql_plan_convert/convert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/convert.rs @@ -129,12 +129,6 @@ pub struct ConvertContext { /// `DEFAULT_SHUFFLE_AGG_THRESHOLD`; overridable per-session via /// `nodedb.shuffle_agg_threshold` for operator control and test determinism. pub shuffle_agg_threshold: usize, - /// The catalog the plan was built against. INSERT/UPSERT conversion - /// evaluates each declared column DEFAULT here, and a sequence-backed - /// DEFAULT (`nextval`, `currval`) reads its value through this handle. - /// `None` for converters built by sub-planners that hold no catalog; a - /// catalog-reading DEFAULT then raises instead of dropping the column. - pub sql_catalog: Option>, } impl ConvertContext { @@ -142,22 +136,6 @@ impl ConvertContext { self.purpose == PlanningPurpose::Metadata } - /// The catalog a column DEFAULT is evaluated against. - /// - /// Raises when the converter holds none. A DEFAULT that reads the catalog - /// must fail loudly: omitting the column stores NULL where the - /// declaration promised a value. - pub fn sql_catalog( - &self, - ) -> crate::Result<&(dyn nodedb_sql::catalog::SqlCatalog + Send + Sync)> { - self.sql_catalog - .as_deref() - .ok_or_else(|| crate::Error::PlanError { - detail: "plan conversion holds no catalog, so a column DEFAULT that reads one cannot be evaluated" - .into(), - }) - } - /// Resolve an existing surrogate without creating a mapping while planning /// metadata. Execute planning retains the allocating assignment behavior. pub fn surrogate_for_pk( @@ -329,7 +307,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 0, shuffle_agg_threshold: 0, - sql_catalog: None, } } diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/insert/convert.rs b/nodedb/src/control/planner/sql_plan_convert/dml/insert/convert.rs index dc4a3b110..970986c4e 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/insert/convert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/insert/convert.rs @@ -8,7 +8,7 @@ use crate::types::{TenantId, VShardId}; use nodedb_physical::physical_plan::*; use super::super::super::convert::ConvertContext; -use super::super::super::value::{expand_row_defaults, row_to_msgpack, rows_to_msgpack_array}; +use super::super::super::value::{row_to_msgpack, rows_to_msgpack_array}; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use super::identity::{ @@ -22,8 +22,9 @@ pub(in super::super::super) struct ConvertInsertArgs<'a> { pub collection: &'a str, /// The lowering these rows take, decided by `nodedb-sql`. pub route: WriteRoute, + /// Rows with every declared DEFAULT already materialized and every + /// literal coerced to its declared column type by the planner. pub rows: &'a [Vec<(String, SqlValue)>], - pub column_defaults: &'a [(String, String)], pub column_schema: &'a [(String, String)], pub if_absent: bool, pub primary_key: &'a str, @@ -38,7 +39,6 @@ pub(in super::super::super) fn convert_insert( collection, route, rows, - column_defaults, column_schema, if_absent, primary_key, @@ -89,11 +89,6 @@ pub(in super::super::super) fn convert_insert( let mut balanced_documents: Vec<(String, Vec)> = Vec::new(); let mut balanced_surrogates: Vec = Vec::new(); - // Every engine's rows expand their DEFAULTs here, ahead of identity - // derivation. A DEFAULT materialized after the primary-key NOT NULL gate - // refuses a key the declaration supplies. - let expanded_rows = expand_row_defaults(rows, column_defaults, tenant_id, ctx)?; - // One catalog read for the whole statement. `_rowid` carries no // declaration, so it skips the read. let declared_pk = if is_auto_rowid_pk(primary_key) { @@ -102,7 +97,7 @@ pub(in super::super::super) fn convert_insert( declared_primary_key_name(ctx, collection)? }; - for row in &expanded_rows { + for row in rows { match route { WriteRoute::ColumnarFamily => { columnar_rows.push(row); @@ -283,7 +278,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; @@ -305,7 +299,6 @@ mod tests { collection: "crdt_coll", route: WriteRoute::Document, rows: &rows, - column_defaults: &[], column_schema: &[], if_absent: false, primary_key: "id", @@ -337,7 +330,6 @@ mod tests { collection: "plain", route: WriteRoute::Document, rows: &rows, - column_defaults: &[], column_schema: &[], if_absent: false, primary_key: "id", diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs b/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs index 5b3130117..b317b82f4 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs @@ -226,7 +226,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), } diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs index 6a92304b8..2161fb4d2 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs @@ -265,7 +265,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs index bfd2894fb..f9c75d93f 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs @@ -386,7 +386,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs b/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs index 70a5d679e..b93b0fed7 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs @@ -14,9 +14,7 @@ use nodedb_physical::physical_plan::ColumnarInsertIntent; use nodedb_physical::physical_plan::*; use super::super::convert::ConvertContext; -use super::super::value::{ - assignments_to_update_values, expand_row_defaults, row_to_msgpack, rows_to_msgpack_array, -}; +use super::super::value::{assignments_to_update_values, row_to_msgpack, rows_to_msgpack_array}; use super::insert::{ build_schema_bytes, columnar_row_surrogates, declared_primary_key_name, is_auto_rowid_pk, resolve_doc_identity_with_declared, @@ -28,8 +26,9 @@ pub(in super::super) struct ConvertUpsertArgs<'a> { pub collection: &'a str, /// The lowering these rows take, decided by `nodedb-sql`. pub route: WriteRoute, + /// Rows with every declared DEFAULT already materialized and every + /// literal coerced to its declared column type by the planner. pub rows: &'a [Vec<(String, SqlValue)>], - pub column_defaults: &'a [(String, String)], pub column_schema: &'a [(String, String)], pub on_conflict_updates: &'a [(String, SqlExpr)], pub primary_key: &'a str, @@ -44,7 +43,6 @@ pub(in super::super) fn convert_upsert( collection, route, rows, - column_defaults, column_schema, on_conflict_updates, primary_key, @@ -79,11 +77,6 @@ pub(in super::super) fn convert_upsert( let mut columnar_rows: Vec<&Vec<(String, SqlValue)>> = Vec::new(); - // Every engine's rows expand their DEFAULTs here, ahead of identity - // derivation, so the primary-key NOT NULL gate reads the row the - // declaration promises — see `expand_row_defaults`. - let expanded_rows = expand_row_defaults(rows, column_defaults, tenant_id, ctx)?; - // One catalog read for the whole statement, mirroring `convert_insert`. // `_rowid` carries no declaration, so it skips the read. let declared_pk = if is_auto_rowid_pk(primary_key) { @@ -92,7 +85,7 @@ pub(in super::super) fn convert_upsert( declared_primary_key_name(ctx, collection)? }; - for row in &expanded_rows { + for row in rows { match route { WriteRoute::Document => { let value_bytes = row_to_msgpack(row)?; diff --git a/nodedb/src/control/planner/sql_plan_convert/filter.rs b/nodedb/src/control/planner/sql_plan_convert/filter/expr_lower.rs similarity index 68% rename from nodedb/src/control/planner/sql_plan_convert/filter.rs rename to nodedb/src/control/planner/sql_plan_convert/filter/expr_lower.rs index 6cef153d8..cb325c4cf 100644 --- a/nodedb/src/control/planner/sql_plan_convert/filter.rs +++ b/nodedb/src/control/planner/sql_plan_convert/filter/expr_lower.rs @@ -1,168 +1,27 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Filter serialization: SqlPlan filters → ScanFilter msgpack bytes. +//! Raw WHERE `SqlExpr` → `ScanFilter` list. //! -//! This is the boundary between the Control Plane planner and the Data Plane -//! scan evaluator. Filter expressions the planner can reduce to simple -//! `(field, op, value)` triples travel as native `ScanFilter` records; any -//! expression the planner cannot reduce — scalar functions in WHERE, -//! non-literal BETWEEN bounds, column arithmetic, `NOT(...)`, IN with -//! computed elements — is shipped verbatim as a `FilterOp::Expr` carrying -//! a `nodedb_query::expr::SqlExpr`. The Data Plane evaluates that against -//! each candidate row via the shared evaluator. +//! Filter expressions the planner can reduce to simple `(field, op, value)` +//! triples travel as native `ScanFilter` records; any expression the planner +//! cannot reduce — scalar functions in WHERE, non-literal BETWEEN bounds, +//! column arithmetic, `NOT(...)`, IN with computed elements — is shipped +//! verbatim as a `FilterOp::Expr` carrying a `nodedb_query::expr::SqlExpr`. +//! The Data Plane evaluates that against each candidate row via the shared +//! evaluator. use nodedb_sql::planner::qualified_name; -use nodedb_sql::types::{Filter, FilterExpr, SqlExpr, SqlValue}; +use nodedb_sql::types::{SqlExpr, SqlValue}; -use super::expr::sql_expr_to_bridge_expr; -use super::value::sql_value_to_nodedb_value; - -/// Convert SqlPlan filters to ScanFilter msgpack bytes. -pub(super) fn serialize_filters(filters: &[Filter]) -> crate::Result> { - if filters.is_empty() { - return Ok(Vec::new()); - } - let scan_filters: Vec = filters - .iter() - .flat_map(|f| filter_to_scan_filters(&f.expr)) - .collect(); - if scan_filters.is_empty() { - return Ok(Vec::new()); - } - encode_scan_filters(&scan_filters) -} - -/// Serialize post-join WHERE filters, preserving table qualifiers inside full -/// expression predicates so they resolve against alias-prefixed merged rows. -pub(super) fn serialize_join_post_filters(filters: &[Filter]) -> crate::Result> { - if filters.is_empty() { - return Ok(Vec::new()); - } - let scan_filters = filters - .iter() - .flat_map(|filter| filter_to_join_scan_filters(&filter.expr)) - .collect::>(); - encode_scan_filters(&scan_filters) -} - -pub(super) fn encode_scan_filters( - filters: &Vec, -) -> crate::Result> { - if filters.is_empty() { - return Ok(Vec::new()); - } - zerompk::to_msgpack_vec(filters).map_err(|e| crate::Error::Serialization { - format: "msgpack".into(), - detail: format!("filter serialization: {e}"), - }) -} - -fn filter_to_join_scan_filters(expr: &FilterExpr) -> Vec { - use nodedb_query::scan_filter::{FilterOp, ScanFilter}; - - match expr { - FilterExpr::And(filters) => filters - .iter() - .flat_map(|filter| filter_to_join_scan_filters(&filter.expr)) - .collect(), - FilterExpr::Or(filters) => vec![ScanFilter { - field: String::new(), - op: FilterOp::Or, - value: nodedb_types::Value::Null, - clauses: filters - .iter() - .map(|filter| filter_to_join_scan_filters(&filter.expr)) - .collect(), - expr: None, - }], - FilterExpr::Expr(sql_expr) => sql_expr_to_join_scan_filters(sql_expr), - _ => filter_to_scan_filters(expr), - } -} - -pub(super) fn filter_to_scan_filters( - expr: &FilterExpr, -) -> Vec { - use nodedb_query::scan_filter::{FilterOp, ScanFilter}; - - match expr { - FilterExpr::Comparison { field, op, value } => { - let filter_op = match op { - nodedb_sql::types::CompareOp::Eq => FilterOp::Eq, - nodedb_sql::types::CompareOp::Ne => FilterOp::Ne, - nodedb_sql::types::CompareOp::Gt => FilterOp::Gt, - nodedb_sql::types::CompareOp::Ge => FilterOp::Gte, - nodedb_sql::types::CompareOp::Lt => FilterOp::Lt, - nodedb_sql::types::CompareOp::Le => FilterOp::Lte, - }; - vec![ScanFilter { - field: field.clone(), - op: filter_op, - value: sql_value_to_nodedb_value(value), - clauses: Vec::new(), - expr: None, - }] - } - FilterExpr::InList { field, values } => { - let arr = values.iter().map(sql_value_to_nodedb_value).collect(); - vec![ScanFilter { - field: field.clone(), - op: FilterOp::In, - value: nodedb_types::Value::Array(arr), - clauses: Vec::new(), - expr: None, - }] - } - FilterExpr::IsNull { field } => { - vec![ScanFilter { - field: field.clone(), - op: FilterOp::IsNull, - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }] - } - FilterExpr::IsNotNull { field } => { - vec![ScanFilter { - field: field.clone(), - op: FilterOp::IsNotNull, - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }] - } - FilterExpr::And(filters) => filters - .iter() - .flat_map(|f| filter_to_scan_filters(&f.expr)) - .collect(), - FilterExpr::Or(filters) => { - let clauses: Vec> = filters - .iter() - .map(|f| filter_to_scan_filters(&f.expr)) - .collect(); - vec![ScanFilter { - field: String::new(), - op: FilterOp::Or, - value: nodedb_types::Value::Null, - clauses, - expr: None, - }] - } - FilterExpr::Expr(sql_expr) => sql_expr_to_scan_filters(sql_expr), - _ => vec![ScanFilter { - field: String::new(), - op: FilterOp::MatchAll, - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }], - } -} +use crate::control::planner::sql_plan_convert::expr::{ + sql_expr_to_bridge_expr, sql_expr_to_bridge_expr_qualified, +}; +use crate::control::planner::sql_plan_convert::value::sql_value_to_nodedb_value; /// Build a `ScanFilter` carrying a full expression predicate. Used whenever /// the planner cannot reduce the WHERE expression to a simple /// `(field, op, value)` tuple. -pub(super) fn expr_filter(expr: &SqlExpr) -> nodedb_query::scan_filter::ScanFilter { +pub(crate) fn expr_filter(expr: &SqlExpr) -> nodedb_query::scan_filter::ScanFilter { nodedb_query::scan_filter::ScanFilter { field: String::new(), op: nodedb_query::scan_filter::FilterOp::Expr, @@ -174,13 +33,13 @@ pub(super) fn expr_filter(expr: &SqlExpr) -> nodedb_query::scan_filter::ScanFilt /// Like [`expr_filter`] but qualifies column references with table names /// for evaluation against join-merged documents. -pub(super) fn expr_filter_qualified(expr: &SqlExpr) -> nodedb_query::scan_filter::ScanFilter { +pub(crate) fn expr_filter_qualified(expr: &SqlExpr) -> nodedb_query::scan_filter::ScanFilter { nodedb_query::scan_filter::ScanFilter { field: String::new(), op: nodedb_query::scan_filter::FilterOp::Expr, value: nodedb_types::Value::Null, clauses: Vec::new(), - expr: Some(super::expr::sql_expr_to_bridge_expr_qualified(expr)), + expr: Some(sql_expr_to_bridge_expr_qualified(expr)), } } @@ -191,7 +50,9 @@ pub(super) fn expr_filter_qualified(expr: &SqlExpr) -> nodedb_query::scan_filter /// use its fast pre-filtered path. Anything that doesn't fit — scalar /// functions on the LHS, arithmetic, NOT, non-literal bounds — is shipped /// as a single `FilterOp::Expr` carrying the whole expression tree. -fn sql_expr_to_join_scan_filters(root: &SqlExpr) -> Vec { +pub(super) fn sql_expr_to_join_scan_filters( + root: &SqlExpr, +) -> Vec { use nodedb_query::scan_filter::{FilterOp, ScanFilter}; match root { @@ -229,7 +90,9 @@ fn sql_expr_to_join_scan_filters(root: &SqlExpr) -> Vec Vec { +pub(super) fn sql_expr_to_scan_filters( + root: &SqlExpr, +) -> Vec { use nodedb_query::scan_filter::{FilterOp, ScanFilter}; match root { diff --git a/nodedb/src/control/planner/sql_plan_convert/filter/mod.rs b/nodedb/src/control/planner/sql_plan_convert/filter/mod.rs new file mode 100644 index 000000000..903fd27a1 --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/filter/mod.rs @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Filter serialization: SqlPlan filters → ScanFilter msgpack bytes. +//! +//! The boundary between the Control Plane planner and the Data Plane scan +//! evaluator. `serialize` walks the planner's `Filter` tree and encodes the +//! `ScanFilter` list; `expr_lower` reduces a raw WHERE `SqlExpr` to native +//! `(field, op, value)` triples where it can and ships anything else verbatim +//! as a `FilterOp::Expr`. + +mod expr_lower; +mod serialize; + +pub(super) use expr_lower::{expr_filter, expr_filter_qualified}; +pub(super) use serialize::{ + encode_scan_filters, filter_to_scan_filters, serialize_filters, serialize_join_post_filters, +}; diff --git a/nodedb/src/control/planner/sql_plan_convert/filter/serialize.rs b/nodedb/src/control/planner/sql_plan_convert/filter/serialize.rs new file mode 100644 index 000000000..b9d974a0b --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/filter/serialize.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `Filter` tree → `ScanFilter` list → msgpack bytes. +//! +//! A `FilterExpr` the planner already reduced to a field/op/value triple maps +//! one-to-one onto a `ScanFilter`; a `FilterExpr::Expr` is handed to +//! [`super::expr_lower`] for reduction. + +use nodedb_sql::types::{Filter, FilterExpr}; + +use super::expr_lower::{sql_expr_to_join_scan_filters, sql_expr_to_scan_filters}; +use crate::control::planner::sql_plan_convert::value::sql_value_to_nodedb_value; + +/// Convert SqlPlan filters to ScanFilter msgpack bytes. +pub(crate) fn serialize_filters(filters: &[Filter]) -> crate::Result> { + if filters.is_empty() { + return Ok(Vec::new()); + } + let scan_filters: Vec = filters + .iter() + .flat_map(|f| filter_to_scan_filters(&f.expr)) + .collect(); + if scan_filters.is_empty() { + return Ok(Vec::new()); + } + encode_scan_filters(&scan_filters) +} + +/// Serialize post-join WHERE filters, preserving table qualifiers inside full +/// expression predicates so they resolve against alias-prefixed merged rows. +pub(crate) fn serialize_join_post_filters(filters: &[Filter]) -> crate::Result> { + if filters.is_empty() { + return Ok(Vec::new()); + } + let scan_filters = filters + .iter() + .flat_map(|filter| filter_to_join_scan_filters(&filter.expr)) + .collect::>(); + encode_scan_filters(&scan_filters) +} + +pub(crate) fn encode_scan_filters( + filters: &Vec, +) -> crate::Result> { + if filters.is_empty() { + return Ok(Vec::new()); + } + zerompk::to_msgpack_vec(filters).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("filter serialization: {e}"), + }) +} + +fn filter_to_join_scan_filters(expr: &FilterExpr) -> Vec { + use nodedb_query::scan_filter::{FilterOp, ScanFilter}; + + match expr { + FilterExpr::And(filters) => filters + .iter() + .flat_map(|filter| filter_to_join_scan_filters(&filter.expr)) + .collect(), + FilterExpr::Or(filters) => vec![ScanFilter { + field: String::new(), + op: FilterOp::Or, + value: nodedb_types::Value::Null, + clauses: filters + .iter() + .map(|filter| filter_to_join_scan_filters(&filter.expr)) + .collect(), + expr: None, + }], + FilterExpr::Expr(sql_expr) => sql_expr_to_join_scan_filters(sql_expr), + _ => filter_to_scan_filters(expr), + } +} + +pub(crate) fn filter_to_scan_filters( + expr: &FilterExpr, +) -> Vec { + use nodedb_query::scan_filter::{FilterOp, ScanFilter}; + + match expr { + FilterExpr::Comparison { field, op, value } => { + let filter_op = match op { + nodedb_sql::types::CompareOp::Eq => FilterOp::Eq, + nodedb_sql::types::CompareOp::Ne => FilterOp::Ne, + nodedb_sql::types::CompareOp::Gt => FilterOp::Gt, + nodedb_sql::types::CompareOp::Ge => FilterOp::Gte, + nodedb_sql::types::CompareOp::Lt => FilterOp::Lt, + nodedb_sql::types::CompareOp::Le => FilterOp::Lte, + }; + vec![ScanFilter { + field: field.clone(), + op: filter_op, + value: sql_value_to_nodedb_value(value), + clauses: Vec::new(), + expr: None, + }] + } + FilterExpr::InList { field, values } => { + let arr = values.iter().map(sql_value_to_nodedb_value).collect(); + vec![ScanFilter { + field: field.clone(), + op: FilterOp::In, + value: nodedb_types::Value::Array(arr), + clauses: Vec::new(), + expr: None, + }] + } + FilterExpr::IsNull { field } => { + vec![ScanFilter { + field: field.clone(), + op: FilterOp::IsNull, + value: nodedb_types::Value::Null, + clauses: Vec::new(), + expr: None, + }] + } + FilterExpr::IsNotNull { field } => { + vec![ScanFilter { + field: field.clone(), + op: FilterOp::IsNotNull, + value: nodedb_types::Value::Null, + clauses: Vec::new(), + expr: None, + }] + } + FilterExpr::And(filters) => filters + .iter() + .flat_map(|f| filter_to_scan_filters(&f.expr)) + .collect(), + FilterExpr::Or(filters) => { + let clauses: Vec> = filters + .iter() + .map(|f| filter_to_scan_filters(&f.expr)) + .collect(); + vec![ScanFilter { + field: String::new(), + op: FilterOp::Or, + value: nodedb_types::Value::Null, + clauses, + expr: None, + }] + } + FilterExpr::Expr(sql_expr) => sql_expr_to_scan_filters(sql_expr), + _ => vec![ScanFilter { + field: String::new(), + op: FilterOp::MatchAll, + value: nodedb_types::Value::Null, + clauses: Vec::new(), + expr: None, + }], + } +} diff --git a/nodedb/src/control/planner/sql_plan_convert/scan/timeseries.rs b/nodedb/src/control/planner/sql_plan_convert/scan/timeseries.rs index 93ae46a3b..8333cd819 100644 --- a/nodedb/src/control/planner/sql_plan_convert/scan/timeseries.rs +++ b/nodedb/src/control/planner/sql_plan_convert/scan/timeseries.rs @@ -14,7 +14,7 @@ use super::super::aggregate::{ use super::super::expr::convert_sort_keys; use super::super::filter::serialize_filters; use super::super::scan_params::TimeseriesScanParams; -use super::super::value::{row_to_msgpack, write_msgpack_array_header}; +use super::super::value::{InstantForm, row_to_msgpack_with, write_msgpack_array_header}; use super::helpers::valid_at_from_scope; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; @@ -105,7 +105,10 @@ pub(in crate::control::planner::sql_plan_convert) fn convert_timeseries_ingest( write_msgpack_array_header(&mut payload, rows.len()); let mut surrogates: Vec = Vec::with_capacity(rows.len()); for row in rows { - let row_bytes = row_to_msgpack(row)?; + // A declared `TIMESTAMP` / `TIMESTAMPTZ` value travels as the typed + // instant ext: the ingest decoder takes it on the same arm a restore + // reissue's typed cell takes, so both spellings converge there. + let row_bytes = row_to_msgpack_with(row, InstantForm::Ext)?; payload.extend_from_slice(&row_bytes); // A timeseries row's natural identity is its (timestamp, tag-set) // tuple, which is not a cross-engine surrogate and carries no PK diff --git a/nodedb/src/control/planner/sql_plan_convert/set_ops.rs b/nodedb/src/control/planner/sql_plan_convert/set_ops.rs index a9d435fe6..c24534444 100644 --- a/nodedb/src/control/planner/sql_plan_convert/set_ops.rs +++ b/nodedb/src/control/planner/sql_plan_convert/set_ops.rs @@ -412,7 +412,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }, @@ -472,7 +471,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }, diff --git a/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs b/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs deleted file mode 100644 index cb6e4aba2..000000000 --- a/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! The one DEFAULT materialization point for the row-shaped DML converters. - -use nodedb_sql::types::SqlValue; - -use super::super::convert::ConvertContext; -use crate::control::planner::plan_error_map::map_plan_error; -use crate::types::TenantId; - -/// Expand each row's omitted DEFAULT columns, before engine dispatch. -/// -/// INSERT and UPSERT call this once per statement, so identity derivation, -/// the primary-key NOT NULL gate, and the stored payload all read the same -/// row. A DEFAULT materialized per engine after that gate refuses a key the -/// declaration supplies. -/// -/// Each DEFAULT compiles once per statement and evaluates once per row, so a -/// `nextval` DEFAULT allocates exactly one value per row of a multi-row VALUES -/// clause and the expression is parsed once however many rows it fills. -/// -/// `column_defaults` empty means nothing to expand: the rows pass through and -/// the catalog is never read. -pub(in super::super) fn expand_row_defaults( - rows: &[Vec<(String, SqlValue)>], - column_defaults: &[(String, String)], - tenant_id: TenantId, - ctx: &ConvertContext, -) -> crate::Result>> { - if column_defaults.is_empty() { - return Ok(rows.to_vec()); - } - let catalog = ctx.sql_catalog()?; - let compiled = nodedb_sql::planner::defaults::ColumnDefaults::compile_pairs(column_defaults) - .map_err(|e| map_plan_error(e, tenant_id))?; - let mut expanded = Vec::with_capacity(rows.len()); - for row in rows { - let mut row = row.clone(); - compiled - .materialize_row(&mut row, catalog) - .map_err(|e| map_plan_error(e, tenant_id))?; - expanded.push(row); - } - Ok(expanded) -} diff --git a/nodedb/src/control/planner/sql_plan_convert/value/mod.rs b/nodedb/src/control/planner/sql_plan_convert/value/mod.rs index 851ae2e54..2cf9852fd 100644 --- a/nodedb/src/control/planner/sql_plan_convert/value/mod.rs +++ b/nodedb/src/control/planner/sql_plan_convert/value/mod.rs @@ -1,11 +1,10 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Value conversion utilities: SqlValue ↔ nodedb_types::Value, msgpack -//! encoding, and the one DEFAULT materialization point. +//! Value conversion utilities: SqlValue ↔ nodedb_types::Value and msgpack +//! encoding. pub(super) mod assignments; pub(super) mod convert; -pub(super) mod defaults; pub(super) mod msgpack_write; pub(super) mod rows; @@ -15,9 +14,8 @@ pub(super) use assignments::{ pub(super) use convert::{ sql_value_to_bytes, sql_value_to_msgpack, sql_value_to_nodedb_value, sql_value_to_string, }; -pub(super) use defaults::expand_row_defaults; pub(super) use msgpack_write::{ - row_to_msgpack, write_msgpack_array_header, write_msgpack_map_header, write_msgpack_str, - write_msgpack_value, + InstantForm, row_to_msgpack, row_to_msgpack_with, write_msgpack_array_header, + write_msgpack_map_header, write_msgpack_str, write_msgpack_value, }; pub(super) use rows::rows_to_msgpack_array; diff --git a/nodedb/src/control/planner/sql_plan_convert/value/msgpack_write.rs b/nodedb/src/control/planner/sql_plan_convert/value/msgpack_write.rs index 41dc91abb..4a0f2f81b 100644 --- a/nodedb/src/control/planner/sql_plan_convert/value/msgpack_write.rs +++ b/nodedb/src/control/planner/sql_plan_convert/value/msgpack_write.rs @@ -5,15 +5,39 @@ //! These are the *only* msgpack producers used by the DML path — no JSON or //! zerompk intermediary. Format matches the on-wire layout read by //! `json_from_msgpack` and the Data Plane row decoders. +//! +//! A typed instant (`SqlValue::Timestamp` / `Timestamptz`) has two +//! spellings, chosen per payload by [`InstantForm`]: ISO 8601 text for the +//! document, KV, and CRDT row decoders, and the `fixext8` instant ext for +//! the timeseries ingest payload, whose decoder takes the typed form. use nodedb_sql::types::SqlValue; +use nodedb_types::{InstantKind, write_instant}; + +/// How a typed instant is written into a row payload. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum InstantForm { + /// ISO 8601 text, decoded by the Data Plane's text-coercion path. + Iso8601, + /// The ten-byte `fixext8` instant ext, decoded as a typed instant. + Ext, +} +/// A row map with instants as ISO 8601 text. pub(crate) fn row_to_msgpack(row: &[(String, SqlValue)]) -> crate::Result> { + row_to_msgpack_with(row, InstantForm::Iso8601) +} + +/// A row map with instants in the given form. +pub(crate) fn row_to_msgpack_with( + row: &[(String, SqlValue)], + instants: InstantForm, +) -> crate::Result> { let mut buf = Vec::with_capacity(row.len() * 32); write_msgpack_map_header(&mut buf, row.len()); for (key, val) in row { write_msgpack_str(&mut buf, key); - write_msgpack_value(&mut buf, val); + write_msgpack_value_with(&mut buf, val, instants); } Ok(buf) } @@ -60,7 +84,13 @@ pub(crate) fn write_msgpack_str(buf: &mut Vec, s: &str) { buf.extend_from_slice(bytes); } +/// Write one value with instants as ISO 8601 text. pub(crate) fn write_msgpack_value(buf: &mut Vec, val: &SqlValue) { + write_msgpack_value_with(buf, val, InstantForm::Iso8601) +} + +/// Write one value with instants in the given form. +pub(crate) fn write_msgpack_value_with(buf: &mut Vec, val: &SqlValue, instants: InstantForm) { match val { SqlValue::Null => buf.push(0xC0), SqlValue::Bool(true) => buf.push(0xC3), @@ -82,15 +112,18 @@ pub(crate) fn write_msgpack_value(buf: &mut Vec, val: &SqlValue) { SqlValue::Array(arr) => { write_msgpack_array_header(buf, arr.len()); for item in arr { - write_msgpack_value(buf, item); + write_msgpack_value_with(buf, item, instants); } } SqlValue::Bytes(b) => write_msgpack_bin(buf, b), - // Timestamp/Timestamptz: write as ISO 8601 string for the Data Plane - // to decode via its standard text-coercion path. - SqlValue::Timestamp(dt) | SqlValue::Timestamptz(dt) => { - write_msgpack_str(buf, &dt.to_iso8601()) - } + SqlValue::Timestamp(dt) => match instants { + InstantForm::Iso8601 => write_msgpack_str(buf, &dt.to_iso8601()), + InstantForm::Ext => write_instant(buf, InstantKind::Naive, dt.micros), + }, + SqlValue::Timestamptz(dt) => match instants { + InstantForm::Iso8601 => write_msgpack_str(buf, &dt.to_iso8601()), + InstantForm::Ext => write_instant(buf, InstantKind::Utc, dt.micros), + }, } } @@ -164,4 +197,39 @@ mod tests { write_msgpack_value(&mut buf, &SqlValue::String("hi".into())); assert_eq!(buf, vec![0xA2, b'h', b'i']); } + + /// `2020-03-05T10:00:00Z` in epoch microseconds. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + /// The default form spells an instant as ISO 8601 text. + #[test] + fn an_instant_is_iso8601_text_by_default() { + let dt = nodedb_types::NdbDateTime::from_micros(EARLY_MICROS); + for val in [SqlValue::Timestamp(dt), SqlValue::Timestamptz(dt)] { + let mut buf = Vec::new(); + write_msgpack_value(&mut buf, &val); + let mut expected = Vec::new(); + write_msgpack_str(&mut expected, "2020-03-05T10:00:00.000000Z"); + assert_eq!(buf, expected); + } + } + + /// The ext form writes the `fixext8` instant, tagged by the SQL type. + #[test] + fn an_instant_in_ext_form_is_a_typed_fixext8() { + let dt = nodedb_types::NdbDateTime::from_micros(EARLY_MICROS); + let cases = [ + (SqlValue::Timestamp(dt), InstantKind::Naive), + (SqlValue::Timestamptz(dt), InstantKind::Utc), + ]; + for (val, kind) in cases { + let mut buf = Vec::new(); + write_msgpack_value_with(&mut buf, &val, InstantForm::Ext); + assert_eq!( + nodedb_types::read_instant(&buf, 0), + Some((kind, EARLY_MICROS)), + "{val:?}" + ); + } + } } diff --git a/nodedb/src/control/planner/sql_plan_convert/value/rows.rs b/nodedb/src/control/planner/sql_plan_convert/value/rows.rs index ab35416aa..b8f29818c 100644 --- a/nodedb/src/control/planner/sql_plan_convert/value/rows.rs +++ b/nodedb/src/control/planner/sql_plan_convert/value/rows.rs @@ -6,10 +6,11 @@ use nodedb_sql::types::SqlValue; use super::convert::sql_value_to_nodedb_value; -/// Encode already-expanded rows as one msgpack array of maps. +/// Encode planner-typed rows as one msgpack array of maps. /// -/// Callers materialize DEFAULTs through `expand_row_defaults` before routing, -/// so every column the declaration promises is already present in `rows`. +/// The planner materializes every declared DEFAULT before the plan reaches +/// conversion, so every column the declaration promises is already present +/// in `rows`. pub(crate) fn rows_to_msgpack_array(rows: &[&Vec<(String, SqlValue)>]) -> crate::Result> { let mut arr: Vec = Vec::with_capacity(rows.len()); for row in rows { diff --git a/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs b/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs index 5b16ac971..607fb34f6 100644 --- a/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs +++ b/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs @@ -13,7 +13,6 @@ macro_rules! impl_dml_arms_for_convert_visitor { engine: _engine, route, rows, - column_defaults, if_absent, column_schema, primary_key, @@ -28,7 +27,6 @@ macro_rules! impl_dml_arms_for_convert_visitor { collection, route, rows, - column_defaults, column_schema, if_absent, primary_key, @@ -46,7 +44,6 @@ macro_rules! impl_dml_arms_for_convert_visitor { engine: _engine, route, rows, - column_defaults, on_conflict_updates, column_schema, primary_key, @@ -61,7 +58,6 @@ macro_rules! impl_dml_arms_for_convert_visitor { collection, route, rows, - column_defaults, column_schema, on_conflict_updates, primary_key, diff --git a/nodedb/src/control/security/catalog/rls.rs b/nodedb/src/control/security/catalog/rls.rs index 169fc1aff..b252fc82d 100644 --- a/nodedb/src/control/security/catalog/rls.rs +++ b/nodedb/src/control/security/catalog/rls.rs @@ -2,20 +2,30 @@ //! RLS policy persistence in the system catalog. //! -//! `RlsPolicy` (the runtime shape) carries `Option` and -//! `DenyMode`, both of which are serde-only and don't fit zerompk's -//! `ToMessagePack` derive. `StoredRlsPolicy` flattens those parts into -//! JSON strings (via sonic_rs) so the whole record can be msgpack-encoded -//! by zerompk like every other catalog row. +//! `StoredRlsPolicy` is the catalog row: the policy's `USING` text, its +//! database, and the scalar settings. The compiled predicate is never +//! persisted. Every reader recompiles the text against the collection's +//! current declared columns through [`StoredRlsPolicy::to_runtime`], so a +//! literal compared against a `TIMESTAMP` column is typed the same way at +//! boot, on Raft apply, and after a schema change as it was at +//! `CREATE RLS POLICY`. `DenyMode` is serde-only and is carried as a +//! sonic_rs JSON string so the row can derive zerompk like every other +//! catalog row. //! //! Conversions: [`StoredRlsPolicy::from_runtime`] for serialization, -//! [`StoredRlsPolicy::to_runtime`] for replay on apply / boot. +//! [`StoredRlsPolicy::to_runtime`] for replay on apply / boot, and +//! [`StoredRlsPolicy::rehydrate`] for the load paths that must install a +//! policy whatever happens: a row that cannot be compiled installs as a +//! restrictive deny-all so it can never widen access. use redb::{ReadableDatabase, ReadableTable, TableDefinition}; +use tracing::error; use crate::control::security::deny::DenyMode; use crate::control::security::predicate::{PolicyMode, RlsPredicate}; +use crate::control::security::rls::compile::{compile_policy_predicate, declared_columns}; use crate::control::security::rls::{PolicyType, RlsPolicy}; +use crate::types::DatabaseId; use super::types::{SystemCatalog, catalog_err}; @@ -24,29 +34,29 @@ use super::types::{SystemCatalog, catalog_err}; pub(super) const RLS_POLICIES: TableDefinition<&str, &[u8]> = TableDefinition::new("_system.rls_policies"); -/// Catalog-shape RLS policy. JSON strings are sonic_rs-encoded -/// versions of the runtime types so zerompk can derive the encoder. +/// Catalog-shape RLS policy. /// -/// Map-encoded (`#[msgpack(map)]`) so `display_collection` could be added -/// with `#[msgpack(default)]`: records written before that field decode -/// with `display_collection = ""` instead of failing outright. +/// Map-encoded (`#[msgpack(map)]`) so a field can carry `#[msgpack(default)]`. #[derive(zerompk::ToMessagePack, zerompk::FromMessagePack, Debug, Clone)] #[msgpack(map)] pub struct StoredRlsPolicy { pub tenant_id: u64, + /// Database the policy's collection lives in. With `display_collection` + /// it names the catalog collection whose declared columns type the + /// predicate's literals. + pub database_id: u64, /// Qualified with the owning database ID — the lookup key. Never /// shown to a user; see `display_collection`. pub collection: String, /// The collection name as the user wrote it, unqualified. - /// Display-only: falls back to `collection` when empty (records - /// written before this field existed). + /// Falls back to `collection` when empty. #[msgpack(default)] pub display_collection: String, pub name: String, /// 0 = Read, 1 = Write, 2 = All. pub policy_type_tag: u8, - /// JSON-serialized `Option`. Empty string = None. - pub compiled_predicate_json: String, + /// The `USING (...)` text as written. Empty = no row filter (vacuous). + pub predicate_text: String, /// 0 = Permissive, 1 = Restrictive. pub mode_tag: u8, /// JSON-serialized `DenyMode`. @@ -57,15 +67,18 @@ pub struct StoredRlsPolicy { } impl StoredRlsPolicy { - pub fn from_runtime(p: &RlsPolicy) -> crate::Result { - let compiled_predicate_json = match &p.compiled_predicate { - Some(rp) => sonic_rs::to_string(rp).map_err(|e| catalog_err("ser compiled rls", e))?, - None => String::new(), - }; + /// The catalog row for `p`, whose predicate was compiled from + /// `predicate_text` against the collection in `database_id`. + pub fn from_runtime( + p: &RlsPolicy, + database_id: DatabaseId, + predicate_text: &str, + ) -> crate::Result { let on_deny_json = sonic_rs::to_string(&p.on_deny).map_err(|e| catalog_err("ser deny mode", e))?; Ok(Self { tenant_id: p.tenant_id, + database_id: database_id.as_u64(), collection: p.collection.clone(), display_collection: p.display_collection.clone(), name: p.name.clone(), @@ -74,7 +87,7 @@ impl StoredRlsPolicy { PolicyType::Write => 1, PolicyType::All => 2, }, - compiled_predicate_json, + predicate_text: predicate_text.to_string(), mode_tag: match p.mode { PolicyMode::Permissive => 0, PolicyMode::Restrictive => 1, @@ -86,18 +99,14 @@ impl StoredRlsPolicy { }) } - pub fn to_runtime(&self) -> crate::Result { - let policy_type = match self.policy_type_tag { - 0 => PolicyType::Read, - 1 => PolicyType::Write, - 2 => PolicyType::All, - other => { - return Err(catalog_err( - "deser rls", - format!("invalid policy_type_tag {other}"), - )); - } - }; + /// The runtime policy, with `predicate_text` compiled against the + /// collection's current declared columns. + /// + /// Errors when a tag is invalid, the deny mode does not decode, the + /// collection is absent from `catalog`, or the text does not compile + /// against its columns. + pub fn to_runtime(&self, catalog: &SystemCatalog) -> crate::Result { + let policy_type = self.policy_type()?; let mode = match self.mode_tag { 0 => PolicyMode::Permissive, 1 => PolicyMode::Restrictive, @@ -108,27 +117,23 @@ impl StoredRlsPolicy { )); } }; - let compiled_predicate: Option = if self.compiled_predicate_json.is_empty() { - None - } else { - Some( - sonic_rs::from_str(&self.compiled_predicate_json) - .map_err(|e| catalog_err("deser compiled rls", e))?, - ) - }; let on_deny: DenyMode = sonic_rs::from_str(&self.on_deny_json) .map_err(|e| catalog_err("deser deny mode", e))?; - // Records written before `display_collection` existed decode it - // empty; fall back to `collection`, which was unqualified then. - let display_collection = if self.display_collection.is_empty() { - self.collection.clone() + let compiled_predicate = if self.predicate_text.is_empty() { + None } else { - self.display_collection.clone() + let columns = declared_columns( + catalog, + DatabaseId::new(self.database_id), + self.tenant_id, + self.display_name(), + )?; + Some(compile_policy_predicate(&self.predicate_text, &columns)?) }; Ok(RlsPolicy { name: self.name.clone(), collection: self.collection.clone(), - display_collection, + display_collection: self.display_name().to_string(), tenant_id: self.tenant_id, policy_type, compiled_predicate, @@ -140,6 +145,70 @@ impl StoredRlsPolicy { }) } + /// [`Self::to_runtime`], or a restrictive deny-all policy under the same + /// key when the row cannot be compiled. + /// + /// For the load paths that install every stored row: a policy that + /// cannot be typed against its collection must still govern the + /// collection, and the only safe form is one that admits no row. The + /// error is logged at `error` level with the policy's key. + pub fn rehydrate(&self, catalog: &SystemCatalog) -> RlsPolicy { + match self.to_runtime(catalog) { + Ok(policy) => policy, + Err(e) => { + error!( + policy = %self.name, + collection = %self.collection, + tenant = self.tenant_id, + error = %e, + "RLS policy cannot be compiled; installed as restrictive deny-all" + ); + self.deny_all() + } + } + } + + /// The restrictive deny-all form of this row: same key, same enabled + /// flag, a predicate no row passes, AND-combined with every other policy. + fn deny_all(&self) -> RlsPolicy { + RlsPolicy { + name: self.name.clone(), + collection: self.collection.clone(), + display_collection: self.display_name().to_string(), + tenant_id: self.tenant_id, + // An invalid tag cannot say which path the policy governs, so + // the deny-all governs both. + policy_type: self.policy_type().unwrap_or(PolicyType::All), + compiled_predicate: Some(RlsPredicate::AlwaysFalse), + mode: PolicyMode::Restrictive, + on_deny: DenyMode::default(), + enabled: self.enabled, + created_by: self.created_by.clone(), + created_at: self.created_at, + } + } + + fn policy_type(&self) -> crate::Result { + match self.policy_type_tag { + 0 => Ok(PolicyType::Read), + 1 => Ok(PolicyType::Write), + 2 => Ok(PolicyType::All), + other => Err(catalog_err( + "deser rls", + format!("invalid policy_type_tag {other}"), + )), + } + } + + /// The unqualified collection name, falling back to `collection`. + fn display_name(&self) -> &str { + if self.display_collection.is_empty() { + &self.collection + } else { + &self.display_collection + } + } + fn redb_key(&self) -> String { rls_key(self.tenant_id, &self.collection, &self.name) } @@ -308,11 +377,11 @@ mod tests { fn put_get_delete_roundtrip() { let catalog = make_catalog(); let runtime = sample_policy(1, "users", "p1"); - let stored = StoredRlsPolicy::from_runtime(&runtime).unwrap(); + let stored = StoredRlsPolicy::from_runtime(&runtime, DatabaseId::DEFAULT, "").unwrap(); catalog.put_rls_policy(&stored).unwrap(); let loaded = catalog.get_rls_policy(1, "users", "p1").unwrap().unwrap(); - let runtime2 = loaded.to_runtime().unwrap(); + let runtime2 = loaded.to_runtime(&catalog).unwrap(); assert_eq!(runtime2.name, "p1"); assert_eq!(runtime2.collection, "users"); assert!(matches!(runtime2.policy_type, PolicyType::Read)); @@ -325,10 +394,38 @@ mod tests { fn load_all_returns_every_tenant() { let catalog = make_catalog(); for (tenant, name) in [(1, "a"), (1, "b"), (2, "c")] { - let stored = StoredRlsPolicy::from_runtime(&sample_policy(tenant, "x", name)).unwrap(); + let stored = StoredRlsPolicy::from_runtime( + &sample_policy(tenant, "x", name), + DatabaseId::DEFAULT, + "", + ) + .unwrap(); catalog.put_rls_policy(&stored).unwrap(); } let all = catalog.load_all_rls_policies().unwrap(); assert_eq!(all.len(), 3); } + + /// A row whose collection is absent from the catalog cannot be typed; + /// `rehydrate` installs it as a restrictive deny-all under the same key + /// instead of dropping it or admitting every row. + #[test] + fn rehydrate_without_the_collection_installs_a_restrictive_deny_all() { + let catalog = make_catalog(); + let mut runtime = sample_policy(1, "ghost", "p1"); + runtime.mode = PolicyMode::Permissive; + let stored = + StoredRlsPolicy::from_runtime(&runtime, DatabaseId::DEFAULT, "owner = $auth.id") + .unwrap(); + + assert!(stored.to_runtime(&catalog).is_err()); + let installed = stored.rehydrate(&catalog); + assert_eq!(installed.name, "p1"); + assert_eq!(installed.collection, "ghost"); + assert_eq!(installed.mode, PolicyMode::Restrictive); + assert!(matches!( + installed.compiled_predicate, + Some(RlsPredicate::AlwaysFalse) + )); + } } diff --git a/nodedb/src/control/security/explain.rs b/nodedb/src/control/security/explain.rs index c5b628c04..4bc204bf4 100644 --- a/nodedb/src/control/security/explain.rs +++ b/nodedb/src/control/security/explain.rs @@ -169,10 +169,12 @@ pub fn lint_predicate(predicate: &super::predicate::RlsPredicate) -> Vec super::predicate::RlsPredicate::AlwaysFalse => { warnings.push("contradiction: predicate is always false (blocks everything)".into()); } - super::predicate::RlsPredicate::Compare { value, .. } - if !value.is_auth_ref() - && matches!(value, super::predicate::PredicateValue::Literal(_)) => - { + super::predicate::RlsPredicate::Compare { + value: + super::predicate::PredicateValue::Literal(_) + | super::predicate::PredicateValue::Instant { .. }, + .. + } => { warnings .push("static predicate: no $auth reference — same result for all users".into()); } diff --git a/nodedb/src/control/security/mod.rs b/nodedb/src/control/security/mod.rs index 78275f8eb..8f73759b8 100644 --- a/nodedb/src/control/security/mod.rs +++ b/nodedb/src/control/security/mod.rs @@ -33,6 +33,7 @@ pub mod permission_tree; pub mod predicate; pub mod predicate_eval; pub mod predicate_parser; +pub mod predicate_typing; pub mod random; pub mod ratelimit; pub mod redaction; diff --git a/nodedb/src/control/security/predicate.rs b/nodedb/src/control/security/predicate.rs index 84834bc03..65395d2d8 100644 --- a/nodedb/src/control/security/predicate.rs +++ b/nodedb/src/control/security/predicate.rs @@ -31,9 +31,12 @@ //! - **Restrictive**: AND-combined. ALL restrictive policies must pass. //! - Final: `(any permissive passes) AND (all restrictive pass)` +use nodedb_types::datetime::NdbDateTime; +use nodedb_types::json_msgpack::InstantKind; use serde::{Deserialize, Serialize}; use super::auth_context::AuthContext; +use crate::bridge::scan_filter::FilterOp; /// A compiled RLS predicate expression. /// @@ -101,21 +104,21 @@ pub enum CompareOp { } impl CompareOp { - /// Convert to the ScanFilter operator string. - pub fn as_filter_op(&self) -> &'static str { + /// The `ScanFilter` operator this comparison lowers to. + pub fn as_filter_op(&self) -> FilterOp { match self { - Self::Eq => "eq", - Self::Ne => "ne", - Self::Gt => "gt", - Self::Gte => "gte", - Self::Lt => "lt", - Self::Lte => "lte", - Self::In => "in", - Self::NotIn => "not_in", - Self::Like => "like", - Self::ILike => "ilike", - Self::IsNull => "is_null", - Self::IsNotNull => "is_not_null", + Self::Eq => FilterOp::Eq, + Self::Ne => FilterOp::Ne, + Self::Gt => FilterOp::Gt, + Self::Gte => FilterOp::Gte, + Self::Lt => FilterOp::Lt, + Self::Lte => FilterOp::Lte, + Self::In => FilterOp::In, + Self::NotIn => FilterOp::NotIn, + Self::Like => FilterOp::Like, + Self::ILike => FilterOp::Ilike, + Self::IsNull => FilterOp::IsNull, + Self::IsNotNull => FilterOp::IsNotNull, } } @@ -149,6 +152,13 @@ impl CompareOp { pub enum PredicateValue { /// A literal JSON value (string, number, bool, array, null). Literal(serde_json::Value), + /// A literal typed against the declared `TIMESTAMP` / `TIMESTAMPTZ` + /// column it is compared with. Policy compilation resolves a numeric + /// (epoch milliseconds) or text (ISO-8601) literal to this at + /// `CREATE RLS POLICY` and again at every load, by the same rule the + /// planner applies to a query predicate, so the Data Plane compares an + /// instant with an instant. + Instant { at: NdbDateTime, kind: InstantKind }, /// A document field reference (resolved at Data Plane scan time). Field(String), /// A session variable reference: `$auth.id`, `$auth.roles`, etc. @@ -168,7 +178,8 @@ impl PredicateValue { /// Resolve this value using the given `AuthContext`. /// /// - `Literal`: returned as-is. - /// - `Field`: returned as-is (resolved at scan time by Data Plane). + /// - `Instant`: its ISO-8601 rendering. + /// - `Field`: `None` (resolved at scan time by the Data Plane). /// - `AuthRef`: resolved via `AuthContext::resolve_variable()`. /// - `AuthFunc`: resolved via `AuthContext` metadata (pre-computed). /// @@ -176,6 +187,7 @@ impl PredicateValue { pub fn resolve(&self, auth: &AuthContext) -> Option { match self { Self::Literal(v) => Some(v.clone()), + Self::Instant { at, .. } => Some(serde_json::Value::String(at.to_iso8601())), Self::Field(_) => None, Self::AuthRef(field) => auth.resolve_variable(field), Self::AuthFunc { func, args } => { @@ -187,6 +199,21 @@ impl PredicateValue { } } } + + /// Resolve this value to the `nodedb_types::Value` a `ScanFilter` carries. + /// + /// An `Instant` keeps its typed kind (`Value::NaiveDateTime` for a + /// `TIMESTAMP` column, `Value::DateTime` for `TIMESTAMPTZ`); every other + /// variant goes through [`Self::resolve`]. Returns `None` when a `Field` + /// is asked for a value, or an `$auth.*` reference cannot be resolved. + pub fn resolve_scan_value(&self, auth: &AuthContext) -> Option { + match self { + Self::Instant { at, kind } => Some(kind.value(*at)), + Self::Literal(_) | Self::Field(_) | Self::AuthRef(_) | Self::AuthFunc { .. } => { + self.resolve(auth).map(nodedb_types::Value::from) + } + } + } } /// Whether a policy is permissive (OR-combined) or restrictive (AND-combined). @@ -327,7 +354,7 @@ mod tests { assert_eq!(filters[0].field, "allowed_users"); assert_eq!( filters[0].op, - crate::bridge::scan_filter::FilterOp::Contains + crate::bridge::scan_filter::FilterOp::ArrayContains ); assert_eq!(filters[0].value, nodedb_types::Value::String("123".into())); } diff --git a/nodedb/src/control/security/predicate_eval.rs b/nodedb/src/control/security/predicate_eval.rs deleted file mode 100644 index 5922d66c1..000000000 --- a/nodedb/src/control/security/predicate_eval.rs +++ /dev/null @@ -1,500 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Plan-time predicate evaluation: substitute `$auth.*` references and -//! combine policies into concrete `ScanFilter` values. -//! -//! This module converts compiled [`RlsPredicate`] trees into static -//! `ScanFilter` lists that the Data Plane can evaluate without session -//! awareness. - -use super::auth_context::AuthContext; -use super::predicate::{CompareOp, PolicyMode, PredicateValue, RlsPredicate}; -use crate::bridge::scan_filter::{FilterOp, ScanFilter}; - -/// Substitute `$auth.*` references in a predicate tree and produce -/// concrete `ScanFilter` values for the Data Plane. -/// -/// This is the core plan-time substitution. After this, the resulting -/// `ScanFilter` contains only literal values and field references — no -/// session variables. The Data Plane evaluates these without any auth -/// awareness. -/// -/// Returns `None` if any required `$auth` reference cannot be resolved -/// (e.g., `$auth.org_id` when no org context). This causes the predicate -/// to evaluate as **deny** (fail-closed). -pub fn substitute_to_scan_filters( - predicate: &RlsPredicate, - auth: &AuthContext, -) -> Option> { - match predicate { - RlsPredicate::AlwaysTrue => Some(vec![ScanFilter { - field: String::new(), - op: "match_all".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]), - - RlsPredicate::AlwaysFalse => { - // Emit a filter that never matches: non-existent field must be non-null. - Some(vec![ScanFilter { - field: "__rls_deny__".into(), - op: "is_not_null".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } - - RlsPredicate::Compare { field, op, value } => { - let resolved = match value { - PredicateValue::Literal(v) => v.clone(), - PredicateValue::AuthRef(auth_field) => auth.resolve_variable(auth_field)?, - PredicateValue::AuthFunc { .. } => value.resolve(auth)?, - PredicateValue::Field(_) => { - // Field-to-field comparison not supported in ScanFilter. - return None; - } - }; - - Some(vec![ScanFilter { - field: field.clone(), - op: op.as_filter_op().into(), - value: nodedb_types::Value::from(resolved), - clauses: Vec::new(), - expr: None, - }]) - } - - RlsPredicate::Contains { set, element } => substitute_contains(set, element, auth), - - RlsPredicate::Intersects { left, right } => substitute_intersects(left, right, auth), - - RlsPredicate::And(children) => { - let mut combined = Vec::new(); - for child in children { - combined.extend(substitute_to_scan_filters(child, auth)?); - } - Some(combined) - } - - RlsPredicate::Or(children) => { - let mut clause_groups: Vec> = Vec::new(); - for child in children { - if let Some(filters) = substitute_to_scan_filters(child, auth) { - // Check for match_all (always-true) — short-circuit. - if filters.len() == 1 && filters[0].op == FilterOp::MatchAll { - return Some(filters); - } - clause_groups.push(filters); - } - } - - if clause_groups.is_empty() { - return Some(vec![ScanFilter { - field: "__rls_deny__".into(), - op: "is_not_null".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]); - } - - if clause_groups.len() == 1 { - return Some(clause_groups.into_iter().next().unwrap_or_default()); - } - - Some(vec![ScanFilter { - field: String::new(), - op: "or".into(), - value: nodedb_types::Value::Null, - clauses: clause_groups, - expr: None, - }]) - } - - RlsPredicate::Not(inner) => substitute_not(inner, auth), - } -} - -/// Combine multiple policies according to their modes. -/// -/// Final result: `(any permissive passes) AND (all restrictive pass)`. -/// -/// Returns the combined `ScanFilter` list to inject into the query. -/// Empty return = no RLS policies (allow all). -pub fn combine_policies( - policies: &[(RlsPredicate, PolicyMode)], - auth: &AuthContext, -) -> Option> { - if policies.is_empty() { - return Some(Vec::new()); // No policies → allow all - } - - let mut permissive: Vec<&RlsPredicate> = Vec::new(); - let mut restrictive: Vec<&RlsPredicate> = Vec::new(); - - for (pred, mode) in policies { - match mode { - PolicyMode::Permissive => permissive.push(pred), - PolicyMode::Restrictive => restrictive.push(pred), - } - } - - let mut combined = Vec::new(); - - // Permissive: OR-combine. If no permissive policies exist, default allow. - if permissive.len() == 1 { - combined.extend(substitute_to_scan_filters(permissive[0], auth)?); - } else if permissive.len() > 1 { - let or_children: Vec = permissive.iter().map(|p| (*p).clone()).collect(); - let or_pred = RlsPredicate::Or(or_children); - combined.extend(substitute_to_scan_filters(&or_pred, auth)?); - } - - // Restrictive: AND-combine (each becomes additional filters). - for pred in &restrictive { - combined.extend(substitute_to_scan_filters(pred, auth)?); - } - - Some(combined) -} - -// --------------------------------------------------------------------------- -// Internal helpers -// --------------------------------------------------------------------------- - -fn substitute_contains( - set: &PredicateValue, - element: &PredicateValue, - auth: &AuthContext, -) -> Option> { - match (set, element) { - // $auth.roles CONTAINS 'admin' → resolved at plan time. - (PredicateValue::AuthRef(auth_field), PredicateValue::Literal(lit)) => { - let auth_val = auth.resolve_variable(auth_field)?; - if let Some(arr) = auth_val.as_array() { - if arr.contains(lit) { - Some(vec![ScanFilter { - field: String::new(), - op: "match_all".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } else { - Some(vec![ScanFilter { - field: "__rls_deny__".into(), - op: "is_not_null".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } - } else { - None // Expected array, got scalar → deny - } - } - - // $auth.scope_status('pro:all') CONTAINS 'active' → resolved at plan time. - (PredicateValue::AuthFunc { .. }, PredicateValue::Literal(lit)) => { - let auth_val = set.resolve(auth)?; - if let Some(arr) = auth_val.as_array() { - if arr.contains(lit) { - Some(vec![ScanFilter { - field: String::new(), - op: "match_all".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } else { - Some(vec![ScanFilter { - field: "__rls_deny__".into(), - op: "is_not_null".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } - } else { - None // Expected array, got scalar → deny - } - } - - // doc_field CONTAINS $auth.id → field "contains" resolved_value. - (PredicateValue::Field(doc_field), PredicateValue::AuthRef(auth_field)) => { - let auth_val = auth.resolve_variable(auth_field)?; - Some(vec![ScanFilter { - field: doc_field.clone(), - op: "contains".into(), - value: nodedb_types::Value::from(auth_val), - clauses: Vec::new(), - expr: None, - }]) - } - - // doc_field CONTAINS $auth.scope_status('pro:all') → field "contains" resolved_value. - (PredicateValue::Field(doc_field), PredicateValue::AuthFunc { .. }) => { - let auth_val = element.resolve(auth)?; - Some(vec![ScanFilter { - field: doc_field.clone(), - op: "contains".into(), - value: nodedb_types::Value::from(auth_val), - clauses: Vec::new(), - expr: None, - }]) - } - - // doc_field CONTAINS 'literal' → field "contains" literal. - (PredicateValue::Field(doc_field), PredicateValue::Literal(lit)) => { - Some(vec![ScanFilter { - field: doc_field.clone(), - op: "contains".into(), - value: nodedb_types::Value::from(lit.clone()), - clauses: Vec::new(), - expr: None, - }]) - } - - _ => None, // Unsupported combination → deny - } -} - -fn substitute_intersects( - left: &PredicateValue, - right: &PredicateValue, - auth: &AuthContext, -) -> Option> { - match (left, right) { - // doc_field INTERSECTS $auth.groups → "any_in" operator. - (PredicateValue::Field(doc_field), PredicateValue::AuthRef(auth_field)) - | (PredicateValue::AuthRef(auth_field), PredicateValue::Field(doc_field)) => { - let auth_val = auth.resolve_variable(auth_field)?; - Some(vec![ScanFilter { - field: doc_field.clone(), - op: "any_in".into(), - value: nodedb_types::Value::from(auth_val), - clauses: Vec::new(), - expr: None, - }]) - } - - // doc_field INTERSECTS $auth.scope_status('pro:all') → "any_in" operator. - (PredicateValue::Field(doc_field), PredicateValue::AuthFunc { .. }) => { - let auth_val = right.resolve(auth)?; - Some(vec![ScanFilter { - field: doc_field.clone(), - op: "any_in".into(), - value: nodedb_types::Value::from(auth_val), - clauses: Vec::new(), - expr: None, - }]) - } - (PredicateValue::AuthFunc { .. }, PredicateValue::Field(doc_field)) => { - let auth_val = left.resolve(auth)?; - Some(vec![ScanFilter { - field: doc_field.clone(), - op: "any_in".into(), - value: nodedb_types::Value::from(auth_val), - clauses: Vec::new(), - expr: None, - }]) - } - - // $auth.groups INTERSECTS $auth.allowed → plan-time evaluation. - (PredicateValue::AuthRef(left_field), PredicateValue::AuthRef(right_field)) => { - let left_val = auth.resolve_variable(left_field)?; - let right_val = auth.resolve_variable(right_field)?; - let intersects = if let (Some(l), Some(r)) = (left_val.as_array(), right_val.as_array()) - { - l.iter().any(|v| r.contains(v)) - } else { - false - }; - - if intersects { - Some(vec![ScanFilter { - field: String::new(), - op: "match_all".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } else { - Some(vec![ScanFilter { - field: "__rls_deny__".into(), - op: "is_not_null".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } - } - - // AuthFunc on either or both sides → plan-time evaluation via resolve(). - (PredicateValue::AuthFunc { .. }, PredicateValue::AuthFunc { .. }) - | (PredicateValue::AuthRef(_), PredicateValue::AuthFunc { .. }) - | (PredicateValue::AuthFunc { .. }, PredicateValue::AuthRef(_)) => { - let left_val = left.resolve(auth)?; - let right_val = right.resolve(auth)?; - let intersects = if let (Some(l), Some(r)) = (left_val.as_array(), right_val.as_array()) - { - l.iter().any(|v| r.contains(v)) - } else { - false - }; - - if intersects { - Some(vec![ScanFilter { - field: String::new(), - op: "match_all".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } else { - Some(vec![ScanFilter { - field: "__rls_deny__".into(), - op: "is_not_null".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } - } - - _ => None, - } -} - -fn substitute_not(inner: &RlsPredicate, auth: &AuthContext) -> Option> { - match inner { - RlsPredicate::AlwaysTrue => substitute_to_scan_filters(&RlsPredicate::AlwaysFalse, auth), - RlsPredicate::AlwaysFalse => substitute_to_scan_filters(&RlsPredicate::AlwaysTrue, auth), - RlsPredicate::Compare { field, op, value } => { - let negated_op = match op { - CompareOp::Eq => CompareOp::Ne, - CompareOp::Ne => CompareOp::Eq, - CompareOp::Gt => CompareOp::Lte, - CompareOp::Gte => CompareOp::Lt, - CompareOp::Lt => CompareOp::Gte, - CompareOp::Lte => CompareOp::Gt, - CompareOp::In => CompareOp::NotIn, - CompareOp::NotIn => CompareOp::In, - CompareOp::IsNull => CompareOp::IsNotNull, - CompareOp::IsNotNull => CompareOp::IsNull, - _ => return None, // Can't negate LIKE/ILIKE simply - }; - substitute_to_scan_filters( - &RlsPredicate::Compare { - field: field.clone(), - op: negated_op, - value: value.clone(), - }, - auth, - ) - } - _ => None, // Complex NOT not supported → deny - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::control::security::auth_context::AuthContext; - use crate::control::security::identity::{ - AuthMethod, AuthenticatedIdentity, DatabaseSet, Role, - }; - use crate::control::security::predicate::{ - CompareOp, PolicyMode, PredicateValue, RlsPredicate, - }; - use crate::types::TenantId; - use nodedb_types::id::DatabaseId; - - fn test_identity() -> AuthenticatedIdentity { - AuthenticatedIdentity::new_regular( - 42, - "alice", - TenantId::new(1), - AuthMethod::ScramSha256, - vec![Role::ReadWrite], - None, - DatabaseSet::Some(smallvec::smallvec![DatabaseId::DEFAULT]), - ) - } - - fn auth_with_database(db_id: DatabaseId) -> AuthContext { - let mut ctx = AuthContext::from_identity(&test_identity(), "s_test".into()); - ctx.database_id = Some(db_id); - ctx - } - - fn auth_without_database() -> AuthContext { - AuthContext::from_identity(&test_identity(), "s_test".into()) - } - - /// `$auth.database_id` resolves to the session's database id and the - /// predicate produces the correct ScanFilter value. - #[test] - fn database_id_auth_ref_substitutes_correctly() { - let db_id = DatabaseId::new(99); - let auth = auth_with_database(db_id); - - let predicate = RlsPredicate::Compare { - field: "owning_db".into(), - op: CompareOp::Eq, - value: PredicateValue::AuthRef("database_id".into()), - }; - - let filters = substitute_to_scan_filters(&predicate, &auth) - .expect("should resolve when database_id is set"); - assert_eq!(filters.len(), 1); - assert_eq!(filters[0].field, "owning_db"); - // The value should be the numeric database id. - match &filters[0].value { - nodedb_types::Value::Integer(n) => assert_eq!(*n as u64, db_id.as_u64()), - other => panic!("expected numeric value, got {:?}", other), - } - } - - /// When `database_id` is `None` the predicate fails closed (returns None). - #[test] - fn database_id_auth_ref_fails_closed_when_none() { - let auth = auth_without_database(); - - let predicate = RlsPredicate::Compare { - field: "owning_db".into(), - op: CompareOp::Eq, - value: PredicateValue::AuthRef("database_id".into()), - }; - - // The substitution must return None so that RLS defaults to deny. - let result = substitute_to_scan_filters(&predicate, &auth); - assert!( - result.is_none(), - "predicate must fail closed when database_id is None" - ); - } - - /// `combine_policies` with a single permissive database_id policy produces - /// the correct ScanFilter when the session has a bound database. - #[test] - fn combine_database_id_policy_passes_when_set() { - let db_id = DatabaseId::new(77); - let auth = auth_with_database(db_id); - - let predicate = RlsPredicate::Compare { - field: "db".into(), - op: CompareOp::Eq, - value: PredicateValue::AuthRef("database_id".into()), - }; - - let policies = [(predicate, PolicyMode::Permissive)]; - let filters = combine_policies(&policies, &auth); - assert!( - filters.as_ref().is_some_and(|v| !v.is_empty()), - "should produce scan filters when database_id is bound" - ); - } -} diff --git a/nodedb/src/control/security/predicate_eval/filters.rs b/nodedb/src/control/security/predicate_eval/filters.rs new file mode 100644 index 000000000..e228ba4cd --- /dev/null +++ b/nodedb/src/control/security/predicate_eval/filters.rs @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The two constant `ScanFilter` shapes policy lowering emits. + +use crate::bridge::scan_filter::{FilterOp, ScanFilter}; + +/// A filter every row passes. +pub fn match_all_filter() -> ScanFilter { + ScanFilter { + field: String::new(), + op: FilterOp::MatchAll, + value: nodedb_types::Value::Null, + clauses: Vec::new(), + expr: None, + } +} + +/// A filter no row passes: a field no document carries must be non-null. +pub fn deny_filter() -> ScanFilter { + ScanFilter { + field: "__rls_deny__".into(), + op: FilterOp::IsNotNull, + value: nodedb_types::Value::Null, + clauses: Vec::new(), + expr: None, + } +} + +/// `field value` as one filter. +pub(super) fn compare_filter(field: &str, op: FilterOp, value: nodedb_types::Value) -> ScanFilter { + ScanFilter { + field: field.to_string(), + op, + value, + clauses: Vec::new(), + expr: None, + } +} + +/// `match_all` when `passes`, the deny filter otherwise. +pub(super) fn verdict_filter(passes: bool) -> ScanFilter { + if passes { + match_all_filter() + } else { + deny_filter() + } +} diff --git a/nodedb/src/control/security/predicate_eval/mod.rs b/nodedb/src/control/security/predicate_eval/mod.rs new file mode 100644 index 000000000..242128b77 --- /dev/null +++ b/nodedb/src/control/security/predicate_eval/mod.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Plan-time predicate evaluation: substitute `$auth.*` references and +//! combine policies into concrete `ScanFilter` values. +//! +//! Converts compiled [`super::predicate::RlsPredicate`] trees into static +//! `ScanFilter` lists that the Data Plane evaluates without session +//! awareness. +//! +//! - [`substitute`] — `substitute_to_scan_filters` and `combine_policies`. +//! - [`sets`] — `CONTAINS` / `INTERSECTS` lowering. +//! - [`filters`] — the `match_all` and deny filter constructors. + +pub mod filters; +pub mod sets; +pub mod substitute; + +pub use filters::{deny_filter, match_all_filter}; +pub use substitute::{combine_policies, substitute_to_scan_filters}; diff --git a/nodedb/src/control/security/predicate_eval/sets.rs b/nodedb/src/control/security/predicate_eval/sets.rs new file mode 100644 index 000000000..dc3a99e99 --- /dev/null +++ b/nodedb/src/control/security/predicate_eval/sets.rs @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `CONTAINS` / `INTERSECTS` lowering. +//! +//! A set that lives on the session (`$auth.roles`, `$auth.scope_status(..)`) +//! is decided at plan time and lowers to `match_all` or the deny filter. A +//! set that lives on the document lowers to an `array_contains` / +//! `array_overlap` filter the Data Plane evaluates per row. A combination with no set on either +//! side, or a document set on both sides, has no `ScanFilter` form and +//! denies. + +use super::filters::{compare_filter, verdict_filter}; +use crate::bridge::scan_filter::{FilterOp, ScanFilter}; +use crate::control::security::auth_context::AuthContext; +use crate::control::security::predicate::PredicateValue; + +/// Where one operand of a set predicate resolves. +enum Operand<'a> { + /// A document field, evaluated per row by the Data Plane. + Doc(&'a str), + /// A session value, resolved at plan time. + Session(&'a PredicateValue), + /// A constant. + Constant(&'a PredicateValue), +} + +fn classify(value: &PredicateValue) -> Operand<'_> { + match value { + PredicateValue::Field(name) => Operand::Doc(name), + PredicateValue::AuthRef(_) | PredicateValue::AuthFunc { .. } => Operand::Session(value), + PredicateValue::Literal(_) | PredicateValue::Instant { .. } => Operand::Constant(value), + } +} + +/// `set CONTAINS element`. +pub(super) fn substitute_contains( + set: &PredicateValue, + element: &PredicateValue, + auth: &AuthContext, +) -> Option> { + match (classify(set), classify(element)) { + // `$auth.roles CONTAINS 'admin'`: decided at plan time. + (Operand::Session(set), Operand::Constant(element)) => { + let members = set.resolve(auth)?; + let needle = element.resolve(auth)?; + let passes = members.as_array()?.contains(&needle); + Some(vec![verdict_filter(passes)]) + } + // `doc_field CONTAINS `: per row. The + // document side is an array, so this is `array_contains` + // (membership), not `contains` (substring). + (Operand::Doc(field), Operand::Session(element) | Operand::Constant(element)) => { + let needle = element.resolve_scan_value(auth)?; + Some(vec![compare_filter(field, FilterOp::ArrayContains, needle)]) + } + (Operand::Session(_), Operand::Session(_) | Operand::Doc(_)) + | (Operand::Doc(_), Operand::Doc(_)) + | (Operand::Constant(_), _) => None, + } +} + +/// `left INTERSECTS right`. +pub(super) fn substitute_intersects( + left: &PredicateValue, + right: &PredicateValue, + auth: &AuthContext, +) -> Option> { + match (classify(left), classify(right)) { + // `doc_field INTERSECTS $auth.groups`, either orientation: per row. + (Operand::Doc(field), Operand::Session(other)) + | (Operand::Session(other), Operand::Doc(field)) => { + let members = other.resolve_scan_value(auth)?; + Some(vec![compare_filter(field, FilterOp::ArrayOverlap, members)]) + } + // `$auth.groups INTERSECTS $auth.allowed`: decided at plan time. + (Operand::Session(left), Operand::Session(right)) => { + let left = left.resolve(auth)?; + let right = right.resolve(auth)?; + let passes = match (left.as_array(), right.as_array()) { + (Some(l), Some(r)) => l.iter().any(|v| r.contains(v)), + (None, _) | (_, None) => false, + }; + Some(vec![verdict_filter(passes)]) + } + (Operand::Doc(_), Operand::Doc(_) | Operand::Constant(_)) + | (Operand::Session(_), Operand::Constant(_)) + | (Operand::Constant(_), _) => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::security::identity::{ + AuthMethod, AuthenticatedIdentity, DatabaseSet, Role, + }; + use crate::types::TenantId; + use nodedb_types::id::DatabaseId; + + fn auth() -> AuthContext { + let identity = AuthenticatedIdentity::new_regular( + 42, + "alice", + TenantId::new(1), + AuthMethod::ScramSha256, + vec![Role::ReadWrite], + None, + DatabaseSet::Some(smallvec::smallvec![DatabaseId::DEFAULT]), + ); + AuthContext::from_identity(&identity, "s_test".into()) + } + + /// A session set that holds the constant admits every row; one that + /// does not denies every row. + #[test] + fn session_set_contains_constant_is_decided_at_plan_time() { + let auth = auth(); + let held = substitute_contains( + &PredicateValue::AuthRef("roles".into()), + &PredicateValue::Literal(serde_json::json!("readwrite")), + &auth, + ) + .expect("resolves"); + assert_eq!(held[0].op, FilterOp::MatchAll); + + let missing = substitute_contains( + &PredicateValue::AuthRef("roles".into()), + &PredicateValue::Literal(serde_json::json!("admin")), + &auth, + ) + .expect("resolves"); + assert_eq!(missing[0].op, FilterOp::IsNotNull); + assert_eq!(missing[0].field, "__rls_deny__"); + } + + /// A session scalar where a set is required denies. + #[test] + fn session_scalar_as_set_denies() { + let auth = auth(); + assert!( + substitute_contains( + &PredicateValue::AuthRef("username".into()), + &PredicateValue::Literal(serde_json::json!("alice")), + &auth, + ) + .is_none() + ); + } + + /// A constant on the set side has no filter form and denies. + #[test] + fn constant_set_denies() { + let auth = auth(); + assert!( + substitute_contains( + &PredicateValue::Literal(serde_json::json!(["a"])), + &PredicateValue::Field("tags".into()), + &auth, + ) + .is_none() + ); + assert!( + substitute_intersects( + &PredicateValue::Literal(serde_json::json!(["a"])), + &PredicateValue::Field("tags".into()), + &auth, + ) + .is_none() + ); + } + + /// `doc_field INTERSECTS $auth.groups` lowers to `array_overlap` in + /// either orientation. + #[test] + fn doc_field_intersects_session_set_in_either_orientation() { + let auth = auth(); + let field = PredicateValue::Field("allowed".into()); + let session = PredicateValue::AuthRef("roles".into()); + for (l, r) in [(&field, &session), (&session, &field)] { + let filters = substitute_intersects(l, r, &auth).expect("resolves"); + assert_eq!(filters[0].field, "allowed"); + assert_eq!(filters[0].op, FilterOp::ArrayOverlap); + } + } +} diff --git a/nodedb/src/control/security/predicate_eval/substitute.rs b/nodedb/src/control/security/predicate_eval/substitute.rs new file mode 100644 index 000000000..d11f1baae --- /dev/null +++ b/nodedb/src/control/security/predicate_eval/substitute.rs @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `$auth.*` substitution over a predicate tree and policy combination. + +use super::filters::{compare_filter, deny_filter, match_all_filter}; +use super::sets::{substitute_contains, substitute_intersects}; +use crate::bridge::scan_filter::{FilterOp, ScanFilter}; +use crate::control::security::auth_context::AuthContext; +use crate::control::security::predicate::{CompareOp, PolicyMode, PredicateValue, RlsPredicate}; + +/// Substitute `$auth.*` references in a predicate tree and produce +/// concrete `ScanFilter` values for the Data Plane. +/// +/// This is the core plan-time substitution. After this, the resulting +/// `ScanFilter` contains only literal values and field references — no +/// session variables. The Data Plane evaluates these without any auth +/// awareness. +/// +/// Returns `None` if any required `$auth` reference cannot be resolved +/// (e.g., `$auth.org_id` when no org context). This causes the predicate +/// to evaluate as **deny** (fail-closed). +pub fn substitute_to_scan_filters( + predicate: &RlsPredicate, + auth: &AuthContext, +) -> Option> { + match predicate { + RlsPredicate::AlwaysTrue => Some(vec![match_all_filter()]), + RlsPredicate::AlwaysFalse => Some(vec![deny_filter()]), + + RlsPredicate::Compare { field, op, value } => { + // Field-to-field comparison is not expressible as a `ScanFilter`. + if matches!(value, PredicateValue::Field(_)) { + return None; + } + let resolved = value.resolve_scan_value(auth)?; + Some(vec![compare_filter(field, op.as_filter_op(), resolved)]) + } + + RlsPredicate::Contains { set, element } => substitute_contains(set, element, auth), + + RlsPredicate::Intersects { left, right } => substitute_intersects(left, right, auth), + + RlsPredicate::And(children) => { + let mut combined = Vec::new(); + for child in children { + combined.extend(substitute_to_scan_filters(child, auth)?); + } + Some(combined) + } + + RlsPredicate::Or(children) => substitute_or(children, auth), + + RlsPredicate::Not(inner) => substitute_not(inner, auth), + } +} + +/// Lower an `OR` node. A child that cannot be resolved contributes nothing; +/// a child that lowers to `match_all` short-circuits the whole disjunction. +fn substitute_or(children: &[RlsPredicate], auth: &AuthContext) -> Option> { + let mut clause_groups: Vec> = Vec::new(); + for child in children { + if let Some(filters) = substitute_to_scan_filters(child, auth) { + if filters.len() == 1 && filters[0].op == FilterOp::MatchAll { + return Some(filters); + } + clause_groups.push(filters); + } + } + + if clause_groups.len() == 1 + && let Some(single) = clause_groups.pop() + { + return Some(single); + } + if clause_groups.is_empty() { + return Some(vec![deny_filter()]); + } + + Some(vec![ScanFilter { + field: String::new(), + op: FilterOp::Or, + value: nodedb_types::Value::Null, + clauses: clause_groups, + expr: None, + }]) +} + +/// Combine multiple policies according to their modes. +/// +/// Final result: `(any permissive passes) AND (all restrictive pass)`. +/// +/// Returns the combined `ScanFilter` list to inject into the query. +/// Empty return = no RLS policies (allow all). +pub fn combine_policies( + policies: &[(RlsPredicate, PolicyMode)], + auth: &AuthContext, +) -> Option> { + if policies.is_empty() { + return Some(Vec::new()); // No policies → allow all + } + + let mut permissive: Vec<&RlsPredicate> = Vec::new(); + let mut restrictive: Vec<&RlsPredicate> = Vec::new(); + + for (pred, mode) in policies { + match mode { + PolicyMode::Permissive => permissive.push(pred), + PolicyMode::Restrictive => restrictive.push(pred), + } + } + + let mut combined = Vec::new(); + + // Permissive: OR-combine. If no permissive policies exist, default allow. + if permissive.len() == 1 { + combined.extend(substitute_to_scan_filters(permissive[0], auth)?); + } else if permissive.len() > 1 { + let or_children: Vec = permissive.iter().map(|p| (*p).clone()).collect(); + let or_pred = RlsPredicate::Or(or_children); + combined.extend(substitute_to_scan_filters(&or_pred, auth)?); + } + + // Restrictive: AND-combine (each becomes additional filters). + for pred in &restrictive { + combined.extend(substitute_to_scan_filters(pred, auth)?); + } + + Some(combined) +} + +/// Lower a `NOT` node by negating the operator of the one comparison it +/// wraps. `LIKE` / `ILIKE` and composite children have no single negated +/// filter, so they deny. +fn substitute_not(inner: &RlsPredicate, auth: &AuthContext) -> Option> { + match inner { + RlsPredicate::AlwaysTrue => substitute_to_scan_filters(&RlsPredicate::AlwaysFalse, auth), + RlsPredicate::AlwaysFalse => substitute_to_scan_filters(&RlsPredicate::AlwaysTrue, auth), + RlsPredicate::Compare { field, op, value } => { + let negated_op = match op { + CompareOp::Eq => CompareOp::Ne, + CompareOp::Ne => CompareOp::Eq, + CompareOp::Gt => CompareOp::Lte, + CompareOp::Gte => CompareOp::Lt, + CompareOp::Lt => CompareOp::Gte, + CompareOp::Lte => CompareOp::Gt, + CompareOp::In => CompareOp::NotIn, + CompareOp::NotIn => CompareOp::In, + CompareOp::IsNull => CompareOp::IsNotNull, + CompareOp::IsNotNull => CompareOp::IsNull, + CompareOp::Like | CompareOp::ILike => return None, + }; + substitute_to_scan_filters( + &RlsPredicate::Compare { + field: field.clone(), + op: negated_op, + value: value.clone(), + }, + auth, + ) + } + RlsPredicate::Contains { .. } + | RlsPredicate::Intersects { .. } + | RlsPredicate::And(_) + | RlsPredicate::Or(_) + | RlsPredicate::Not(_) => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::security::identity::{ + AuthMethod, AuthenticatedIdentity, DatabaseSet, Role, + }; + use crate::types::TenantId; + use nodedb_types::datetime::NdbDateTime; + use nodedb_types::id::DatabaseId; + use nodedb_types::json_msgpack::InstantKind; + + fn test_identity() -> AuthenticatedIdentity { + AuthenticatedIdentity::new_regular( + 42, + "alice", + TenantId::new(1), + AuthMethod::ScramSha256, + vec![Role::ReadWrite], + None, + DatabaseSet::Some(smallvec::smallvec![DatabaseId::DEFAULT]), + ) + } + + fn auth_with_database(db_id: DatabaseId) -> AuthContext { + let mut ctx = AuthContext::from_identity(&test_identity(), "s_test".into()); + ctx.database_id = Some(db_id); + ctx + } + + fn auth_without_database() -> AuthContext { + AuthContext::from_identity(&test_identity(), "s_test".into()) + } + + /// `$auth.database_id` resolves to the session's database id and the + /// predicate produces the correct ScanFilter value. + #[test] + fn database_id_auth_ref_substitutes_correctly() { + let db_id = DatabaseId::new(99); + let auth = auth_with_database(db_id); + + let predicate = RlsPredicate::Compare { + field: "owning_db".into(), + op: CompareOp::Eq, + value: PredicateValue::AuthRef("database_id".into()), + }; + + let filters = substitute_to_scan_filters(&predicate, &auth) + .expect("should resolve when database_id is set"); + assert_eq!(filters.len(), 1); + assert_eq!(filters[0].field, "owning_db"); + match &filters[0].value { + nodedb_types::Value::Integer(n) => assert_eq!(*n as u64, db_id.as_u64()), + other => panic!("expected numeric value, got {:?}", other), + } + } + + /// When `database_id` is `None` the predicate fails closed (returns None). + #[test] + fn database_id_auth_ref_fails_closed_when_none() { + let auth = auth_without_database(); + + let predicate = RlsPredicate::Compare { + field: "owning_db".into(), + op: CompareOp::Eq, + value: PredicateValue::AuthRef("database_id".into()), + }; + + let result = substitute_to_scan_filters(&predicate, &auth); + assert!( + result.is_none(), + "predicate must fail closed when database_id is None" + ); + } + + /// `combine_policies` with a single permissive database_id policy produces + /// the correct ScanFilter when the session has a bound database. + #[test] + fn combine_database_id_policy_passes_when_set() { + let db_id = DatabaseId::new(77); + let auth = auth_with_database(db_id); + + let predicate = RlsPredicate::Compare { + field: "db".into(), + op: CompareOp::Eq, + value: PredicateValue::AuthRef("database_id".into()), + }; + + let policies = [(predicate, PolicyMode::Permissive)]; + let filters = combine_policies(&policies, &auth); + assert!( + filters.as_ref().is_some_and(|v| !v.is_empty()), + "should produce scan filters when database_id is bound" + ); + } + + /// A typed instant literal lowers to the typed `ScanFilter.value` of its + /// declared kind, so the Data Plane compares an instant with an instant. + #[test] + fn instant_literal_lowers_to_a_typed_scan_value() { + let auth = auth_without_database(); + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + + let naive = RlsPredicate::Compare { + field: "captured_at".into(), + op: CompareOp::Gte, + value: PredicateValue::Instant { + at, + kind: InstantKind::Naive, + }, + }; + let filters = substitute_to_scan_filters(&naive, &auth).expect("an instant resolves"); + assert_eq!(filters.len(), 1); + assert_eq!(filters[0].field, "captured_at"); + assert_eq!(filters[0].op, FilterOp::Gte); + assert_eq!(filters[0].value, nodedb_types::Value::NaiveDateTime(at)); + + let zoned = RlsPredicate::Compare { + field: "captured_at".into(), + op: CompareOp::Lt, + value: PredicateValue::Instant { + at, + kind: InstantKind::Utc, + }, + }; + let filters = substitute_to_scan_filters(&zoned, &auth).expect("an instant resolves"); + assert_eq!(filters[0].value, nodedb_types::Value::DateTime(at)); + } + + /// `NOT` over an instant comparison negates the operator and keeps the + /// typed value. + #[test] + fn negated_instant_comparison_keeps_the_typed_value() { + let auth = auth_without_database(); + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + let predicate = RlsPredicate::Not(Box::new(RlsPredicate::Compare { + field: "captured_at".into(), + op: CompareOp::Gte, + value: PredicateValue::Instant { + at, + kind: InstantKind::Naive, + }, + })); + let filters = substitute_to_scan_filters(&predicate, &auth).expect("resolves"); + assert_eq!(filters[0].op, FilterOp::Lt); + assert_eq!(filters[0].value, nodedb_types::Value::NaiveDateTime(at)); + } +} diff --git a/nodedb/src/control/security/predicate_parser.rs b/nodedb/src/control/security/predicate_parser.rs index f66421462..5587ec2f1 100644 --- a/nodedb/src/control/security/predicate_parser.rs +++ b/nodedb/src/control/security/predicate_parser.rs @@ -198,19 +198,18 @@ fn parse_atom(tokens: &[String], pos: &mut usize) -> Result= tokens.len() { // Standalone value — treat as boolean (truthy). - return match left_value { - PredicateValue::AuthRef(_) - | PredicateValue::Field(_) - | PredicateValue::AuthFunc { .. } => Ok(RlsPredicate::Compare { - field: match &left_value { - PredicateValue::Field(f) => f.clone(), - _ => String::new(), - }, - op: CompareOp::IsNotNull, - value: left_value, - }), - PredicateValue::Literal(_) => Ok(RlsPredicate::AlwaysTrue), + let field = match &left_value { + PredicateValue::Field(f) => f.clone(), + PredicateValue::AuthRef(_) | PredicateValue::AuthFunc { .. } => String::new(), + PredicateValue::Literal(_) | PredicateValue::Instant { .. } => { + return Ok(RlsPredicate::AlwaysTrue); + } }; + return Ok(RlsPredicate::Compare { + field, + op: CompareOp::IsNotNull, + value: left_value, + }); } let op_token = tokens[*pos].to_uppercase(); @@ -430,7 +429,9 @@ pub fn validate_auth_refs(predicate: &RlsPredicate) -> crate::Result<()> { }); } } - _ => {} + PredicateValue::Literal(_) + | PredicateValue::Instant { .. } + | PredicateValue::Field(_) => {} } Ok(()) } diff --git a/nodedb/src/control/security/predicate_typing.rs b/nodedb/src/control/security/predicate_typing.rs new file mode 100644 index 000000000..165e36ae1 --- /dev/null +++ b/nodedb/src/control/security/predicate_typing.rs @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Type the literals of a compiled RLS predicate against the collection's +//! declared columns. +//! +//! A policy literal reaches the parser as whatever the author typed: +//! `captured_at > 1583402400000` or `captured_at > '2020-03-05'`. The Data +//! Plane compares an instant column against a typed instant, and an untyped +//! integer has no order against one, so a policy left untyped matches +//! nothing on such a column. This pass resolves every literal compared +//! against a declared `TIMESTAMP` / `TIMESTAMPTZ` column through +//! [`nodedb_sql::planner::predicate_coerce::coerce_read_literal`], the same +//! rule the planner applies to a query's `WHERE` literal, so a policy and a +//! query on the same column agree on the instant. +//! +//! A literal the declared type cannot read (`captured_at >= true`) is an +//! error naming the column and the literal. The caller refuses the policy: +//! a policy that cannot be enforced as written is never stored. + +use nodedb_sql::planner::predicate_coerce::coerce_read_literal; +use nodedb_sql::types::{ColumnInfo, SqlValue}; +use nodedb_types::json_msgpack::InstantKind; + +use super::predicate::{PredicateValue, RlsPredicate}; + +/// Type every literal in `predicate` that a `Compare` node pairs with a +/// declared column of `columns`. +/// +/// Literals on `CONTAINS` / `INTERSECTS` nodes and on plan-time-only +/// comparisons (`$auth.x = 'lit'`, which name no document field) are left +/// as written: neither compares a literal against a declared scalar column. +pub fn type_predicate_literals( + predicate: RlsPredicate, + columns: &[ColumnInfo], +) -> crate::Result { + Ok(match predicate { + RlsPredicate::Compare { field, op, value } => { + let value = type_compare_literal(&field, value, columns)?; + RlsPredicate::Compare { field, op, value } + } + RlsPredicate::And(children) => RlsPredicate::And(type_children(children, columns)?), + RlsPredicate::Or(children) => RlsPredicate::Or(type_children(children, columns)?), + RlsPredicate::Not(inner) => { + RlsPredicate::Not(Box::new(type_predicate_literals(*inner, columns)?)) + } + RlsPredicate::Contains { .. } + | RlsPredicate::Intersects { .. } + | RlsPredicate::AlwaysTrue + | RlsPredicate::AlwaysFalse => predicate, + }) +} + +fn type_children( + children: Vec, + columns: &[ColumnInfo], +) -> crate::Result> { + children + .into_iter() + .map(|child| type_predicate_literals(child, columns)) + .collect() +} + +/// Type the right-hand side of `field value` when it is a literal and +/// `field` names a declared column. +fn type_compare_literal( + field: &str, + value: PredicateValue, + columns: &[ColumnInfo], +) -> crate::Result { + let PredicateValue::Literal(literal) = value else { + return Ok(value); + }; + let Some(column) = columns + .iter() + .find(|column| column.name.eq_ignore_ascii_case(field)) + else { + return Ok(PredicateValue::Literal(literal)); + }; + let Some(sql_literal) = json_to_sql_literal(&literal) else { + return Ok(PredicateValue::Literal(literal)); + }; + let coerced = + coerce_read_literal(column, sql_literal).map_err(|error| crate::Error::BadRequest { + detail: format!("RLS predicate: {error}"), + })?; + Ok(sql_literal_to_predicate_value(coerced, literal)) +} + +/// The `SqlValue` a policy literal denotes, for the literal kinds the policy +/// parser produces. An array or object literal has no scalar form and stays +/// as written. +fn json_to_sql_literal(literal: &serde_json::Value) -> Option { + match literal { + serde_json::Value::Null => Some(SqlValue::Null), + serde_json::Value::Bool(b) => Some(SqlValue::Bool(*b)), + serde_json::Value::Number(n) => n + .as_i64() + .map(SqlValue::Int) + .or_else(|| n.as_f64().map(SqlValue::Float)), + serde_json::Value::String(s) => Some(SqlValue::String(s.clone())), + serde_json::Value::Array(_) | serde_json::Value::Object(_) => None, + } +} + +/// The predicate value a coerced literal becomes: a typed instant, or the +/// original literal when the declared type imposed no representation. +fn sql_literal_to_predicate_value( + coerced: SqlValue, + original: serde_json::Value, +) -> PredicateValue { + match coerced { + SqlValue::Timestamp(at) => PredicateValue::Instant { + at, + kind: InstantKind::Naive, + }, + SqlValue::Timestamptz(at) => PredicateValue::Instant { + at, + kind: InstantKind::Utc, + }, + SqlValue::Null + | SqlValue::Bool(_) + | SqlValue::Int(_) + | SqlValue::Float(_) + | SqlValue::Decimal(_) + | SqlValue::String(_) + | SqlValue::Bytes(_) + | SqlValue::Array(_) => PredicateValue::Literal(original), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::security::predicate::CompareOp; + use crate::control::security::predicate_parser::parse_predicate; + use nodedb_sql::types::SqlDataType; + use nodedb_types::datetime::NdbDateTime; + + /// `2020-03-05T10:00:00Z` as epoch milliseconds. + const EARLY_MS: i64 = 1_583_402_400_000; + + fn early() -> NdbDateTime { + NdbDateTime::from_micros(EARLY_MS * 1_000) + } + + fn column(name: &str, data_type: SqlDataType, is_primary_key: bool) -> ColumnInfo { + ColumnInfo { + name: name.to_string(), + data_type, + nullable: true, + is_primary_key, + default: None, + raw_type: None, + int_width: None, + float_width: None, + } + } + + fn columns() -> Vec { + vec![ + column("id", SqlDataType::String, true), + column("captured_at", SqlDataType::Timestamp, false), + column("seen_at", SqlDataType::Timestamptz, false), + column("n", SqlDataType::Int64, false), + ] + } + + fn typed(text: &str) -> crate::Result { + let parsed = parse_predicate(text).expect("policy text parses"); + type_predicate_literals(parsed, &columns()) + } + + fn compare_value(predicate: &RlsPredicate) -> &PredicateValue { + match predicate { + RlsPredicate::Compare { value, .. } => value, + other => panic!("expected a comparison, got {other:?}"), + } + } + + /// A numeric literal against a `TIMESTAMP` column is epoch milliseconds + /// and becomes a naive instant. + #[test] + fn numeric_literal_on_an_instant_column_becomes_a_naive_instant() { + let predicate = typed(&format!("captured_at >= {EARLY_MS}")).expect("types"); + assert!(matches!( + compare_value(&predicate), + PredicateValue::Instant { at, kind: InstantKind::Naive } if *at == early() + )); + } + + /// A text literal against a `TIMESTAMPTZ` column is parsed as ISO-8601 + /// and becomes a zoned instant. + #[test] + fn text_literal_on_a_zoned_column_becomes_a_utc_instant() { + let predicate = typed("seen_at > '2020-03-05 10:00:00'").expect("types"); + assert!(matches!( + compare_value(&predicate), + PredicateValue::Instant { at, kind: InstantKind::Utc } if *at == early() + )); + } + + /// Typing reaches every comparison under `AND` / `OR` / `NOT`. + #[test] + fn typing_descends_through_composite_nodes() { + let predicate = typed(&format!( + "(captured_at >= {EARLY_MS} OR owner = $auth.username) AND NOT seen_at < '2020-03-05'" + )) + .expect("types"); + let RlsPredicate::And(children) = &predicate else { + panic!("expected AND, got {predicate:?}"); + }; + let RlsPredicate::Or(or_children) = &children[0] else { + panic!("expected OR, got {:?}", children[0]); + }; + assert!(matches!( + compare_value(&or_children[0]), + PredicateValue::Instant { .. } + )); + let RlsPredicate::Not(inner) = &children[1] else { + panic!("expected NOT, got {:?}", children[1]); + }; + assert!(matches!( + compare_value(inner), + PredicateValue::Instant { .. } + )); + } + + /// A literal no instant can be read from is refused, naming the column. + #[test] + fn non_instant_literal_on_an_instant_column_is_refused_naming_the_column() { + let error = typed("captured_at >= true").expect_err("a boolean is refused"); + let detail = error.to_string(); + assert!( + detail.contains("captured_at") && detail.contains("boolean"), + "error must name the column and the literal kind: {detail}" + ); + let error = typed("captured_at >= 'not a date'").expect_err("non-datetime text is refused"); + let detail = error.to_string(); + assert!( + detail.contains("captured_at") && detail.contains("not a date"), + "error must name the column and the literal: {detail}" + ); + } + + /// A column that is not an instant, an undeclared field, the primary + /// key, and an `$auth.*` reference are all left as written. + #[test] + fn non_instant_targets_are_left_as_written() { + for text in [ + "n > 5", + "undeclared > 5", + "id = '2020-03-05'", + "captured_at = $auth.id", + ] { + let predicate = typed(text).unwrap_or_else(|e| panic!("{text}: {e}")); + assert!( + !matches!(compare_value(&predicate), PredicateValue::Instant { .. }), + "{text} must not be typed as an instant: {predicate:?}" + ); + } + let predicate = typed("n > 5").expect("types"); + assert!(matches!( + predicate, + RlsPredicate::Compare { + op: CompareOp::Gt, + value: PredicateValue::Literal(_), + .. + } + )); + } +} diff --git a/nodedb/src/control/security/redaction/apply.rs b/nodedb/src/control/security/redaction/apply.rs index 685a732b3..4f488f54d 100644 --- a/nodedb/src/control/security/redaction/apply.rs +++ b/nodedb/src/control/security/redaction/apply.rs @@ -7,13 +7,21 @@ //! per-row [`RedactionStore::apply_flat_row`] go through it, so the mask / hash //! / null semantics exist exactly once. //! -//! `apply_flat_row` is the SELECT-path entry point: it rewrites one already -//! flattened result row, whose columns may come from several source -//! collections at once (a join), rather than one document belonging to a -//! single collection. +//! `apply_flat_row` (JSON cells) and `apply_flat_row_typed` (typed +//! `nodedb_types::Value` cells) are the SELECT-path entry points: each +//! rewrites one already flattened result row, whose columns may come from +//! several source collections at once (a join), rather than one document +//! belonging to a single collection. Both share one rule-resolution body +//! through [`FlatRow`], and a typed cell hashes through the same JSON text +//! its wire rendering produces, so the two cannot drift. +use std::collections::BTreeMap; + +use nodedb_types::Value; use serde_json::{Map, Value as JsonValue}; +use crate::util::wire_json::value_to_wire_json; + use super::store::RedactionStore; use super::types::{RedactionMode, RedactionRule, policy_key}; @@ -45,13 +53,66 @@ fn hash_value(value: &JsonValue) -> String { format!("hash:{digest:x}") } -/// Rewrite `row[key]` per `mode`, if the row actually carries that key. -fn redact_key(row: &mut Map, key: &str, mode: &RedactionMode) { - if !row.contains_key(key) { - return; +/// The value `mode` produces for a typed field whose present value is +/// `current`. A mask or hash renders through the JSON path: the hash input +/// is the cell's wire JSON, exactly what the JSON row would have held. +fn redacted_typed_value(mode: &RedactionMode, current: Option<&Value>) -> Value { + match mode { + RedactionMode::Mask(mask) => Value::String(mask.clone()), + RedactionMode::Hash => { + let wire = current.map(value_to_wire_json); + Value::String(hash_value(wire.as_ref().unwrap_or(&JsonValue::Null))) + } + RedactionMode::Null => Value::Null, + } +} + +/// One flattened result row, whatever its cell type. +/// +/// The rule-resolution body in `RedactionStore::apply_flat_row_impl` is +/// written once against this trait; the JSON and typed row maps each +/// implement it. +trait FlatRow { + fn is_empty(&self) -> bool; + fn keys(&self) -> impl Iterator; + /// Rewrite `self[key]` per `mode`, if the row actually carries that key. + fn redact_key(&mut self, key: &str, mode: &RedactionMode); +} + +impl FlatRow for Map { + fn is_empty(&self) -> bool { + Map::is_empty(self) + } + + fn keys(&self) -> impl Iterator { + Map::keys(self).map(String::as_str) + } + + fn redact_key(&mut self, key: &str, mode: &RedactionMode) { + if !self.contains_key(key) { + return; + } + let value = redacted_value(mode, self.get(key)); + self.insert(key.to_string(), value); + } +} + +impl FlatRow for BTreeMap { + fn is_empty(&self) -> bool { + BTreeMap::is_empty(self) + } + + fn keys(&self) -> impl Iterator { + BTreeMap::keys(self).map(String::as_str) + } + + fn redact_key(&mut self, key: &str, mode: &RedactionMode) { + if !self.contains_key(key) { + return; + } + let value = redacted_typed_value(mode, self.get(key)); + self.insert(key.to_string(), value); } - let value = redacted_value(mode, row.get(key)); - row.insert(key.to_string(), value); } /// How many of the plan's sources a row-map key can be attributed to. @@ -73,7 +134,7 @@ fn attribution_count(key: &str, sources: &[(&str, Vec<&RedactionRule>)]) -> usiz } impl RedactionStore { - /// Redact one already-flattened SELECT result row in place. + /// Redact one already-flattened SELECT result row of JSON cells in place. /// /// `collections` lists the plan's source collections as /// `(qualifier, collection)`, where `qualifier` is the prefix that appears @@ -100,6 +161,30 @@ impl RedactionStore { roles: &[String], collections: &[(String, String)], row: &mut Map, + ) { + self.apply_flat_row_impl(tenant_id, roles, collections, row); + } + + /// Redact one already-flattened SELECT result row of typed cells in + /// place. Same matching rules as [`RedactionStore::apply_flat_row`]; a + /// `Null` mode writes `Value::Null`, a mask or hash writes the same text + /// the JSON path writes. + pub fn apply_flat_row_typed( + &self, + tenant_id: u64, + roles: &[String], + collections: &[(String, String)], + row: &mut BTreeMap, + ) { + self.apply_flat_row_impl(tenant_id, roles, collections, row); + } + + fn apply_flat_row_impl( + &self, + tenant_id: u64, + roles: &[String], + collections: &[(String, String)], + row: &mut R, ) { if roles.is_empty() || collections.is_empty() || row.is_empty() { return; @@ -122,14 +207,14 @@ impl RedactionStore { if let [(_, rules)] = sources.as_slice() { for rule in rules { - redact_key(row, &rule.field, &rule.mode); + row.redact_key(&rule.field, &rule.mode); } return; } for (qualifier, rules) in &sources { for rule in rules { - redact_key(row, &format!("{qualifier}.{}", rule.field), &rule.mode); + row.redact_key(&format!("{qualifier}.{}", rule.field), &rule.mode); } } @@ -137,7 +222,7 @@ impl RedactionStore { let unattributed: Vec = row .keys() .filter(|key| attribution_count(key, &sources) != 1) - .cloned() + .map(str::to_owned) .collect(); for key in unattributed { let bare = key.rfind('.').map_or(key.as_str(), |dot| &key[dot + 1..]); @@ -147,7 +232,7 @@ impl RedactionStore { .find(|rule| rule.field == bare || rule.field == key) .map(|rule| rule.mode.clone()); if let Some(mode) = mode { - redact_key(row, &key, &mode); + row.redact_key(&key, &mode); } } } @@ -273,6 +358,93 @@ mod tests { assert_eq!(r["email"], JsonValue::Null); } + fn typed_row(pairs: &[(&str, Value)]) -> BTreeMap { + pairs + .iter() + .map(|(k, v)| ((*k).to_string(), v.clone())) + .collect() + } + + /// The typed entry point resolves rules exactly as the JSON one: same + /// mask on the ruled side of a join, clear value on the other, and a + /// `Null` mode keeps the key present. + #[test] + fn typed_row_matches_the_json_rules() { + let store = store_with( + "workspaces", + "support", + vec![ + mask("id", "***"), + RedactionRule { + field: "note".into(), + mode: RedactionMode::Null, + }, + ], + ); + let mut r = typed_row(&[ + ("w.id", Value::String("w1".into())), + ("b.id", Value::String("b1".into())), + ("w.note", Value::Integer(7)), + ]); + store.apply_flat_row_typed( + 1, + &["support".into()], + &[ + ("w".into(), "workspaces".into()), + ("b".into(), "boards".into()), + ], + &mut r, + ); + assert_eq!(r["w.id"], Value::String("***".into())); + assert_eq!(r["b.id"], Value::String("b1".into())); + assert_eq!(r["w.note"], Value::Null); + } + + /// A hashed typed cell yields the digest the JSON path yields for the + /// same cell: a string hashes its bytes, a number hashes its JSON text, + /// and bytes hash their base64 wire text. + #[test] + fn typed_hash_matches_the_json_hash() { + let store = store_with( + "users", + "support", + vec![ + RedactionRule { + field: "email".into(), + mode: RedactionMode::Hash, + }, + RedactionRule { + field: "n".into(), + mode: RedactionMode::Hash, + }, + RedactionRule { + field: "blob".into(), + mode: RedactionMode::Hash, + }, + ], + ); + let roles = ["support".to_string()]; + let sources = [(String::new(), "users".to_string())]; + + let mut typed = typed_row(&[ + ("email", Value::String("a@b.c".into())), + ("n", Value::Integer(42)), + ("blob", Value::Bytes(vec![0, 255, 7])), + ]); + store.apply_flat_row_typed(1, &roles, &sources, &mut typed); + + let mut json = row(json!({"email": "a@b.c", "n": 42, "blob": "AP8H"})); + store.apply_flat_row(1, &roles, &sources, &mut json); + + for key in ["email", "n", "blob"] { + assert_eq!( + typed[key], + Value::String(json[key].as_str().expect("hashed text").to_string()), + "{key}" + ); + } + } + #[test] fn no_policy_leaves_the_row_untouched() { let store = RedactionStore::new(); diff --git a/nodedb/src/control/security/rls/compile.rs b/nodedb/src/control/security/rls/compile.rs new file mode 100644 index 000000000..dd0e89d9a --- /dev/null +++ b/nodedb/src/control/security/rls/compile.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Compile a policy's `USING` text against its collection's declared columns. +//! +//! One entry point, [`compile_policy_predicate`], serves `CREATE RLS POLICY` +//! and every load of a stored policy (boot replay, Raft apply, recovery +//! reload, schema change), so a policy is typed against the collection's +//! current declared columns wherever it is materialized. The compiled +//! predicate is never persisted: the catalog keeps the text, and every +//! reader recompiles. + +use nodedb_sql::types::ColumnInfo; +use nodedb_types::DatabaseId; + +use crate::control::planner::catalog_adapter::convert_collection_type; +use crate::control::security::catalog::SystemCatalog; +use crate::control::security::predicate::RlsPredicate; +use crate::control::security::predicate_parser::{parse_predicate, validate_auth_refs}; +use crate::control::security::predicate_typing::type_predicate_literals; + +/// Parse, validate, and type `text` against `columns`. +/// +/// Errors: a parse error, an unknown `$auth.*` reference, or a literal the +/// declared column type cannot read. Each names what was refused. +pub fn compile_policy_predicate(text: &str, columns: &[ColumnInfo]) -> crate::Result { + let parsed = parse_predicate(text).map_err(|error| crate::Error::BadRequest { + detail: format!("predicate parse error: {error}"), + })?; + validate_auth_refs(&parsed)?; + type_predicate_literals(parsed, columns) +} + +/// The declared columns of the active collection `collection` in +/// `database_id` / `tenant_id`, as the planner sees them. +/// +/// Errors with `CollectionNotFound` when the collection is absent or +/// dropped: a policy cannot be typed without its schema. +pub fn declared_columns( + catalog: &SystemCatalog, + database_id: DatabaseId, + tenant_id: u64, + collection: &str, +) -> crate::Result> { + let stored = catalog + .get_collection(database_id, tenant_id, collection)? + .filter(|stored| stored.is_active) + .ok_or_else(|| crate::Error::CollectionNotFound { + tenant_id: crate::types::TenantId::new(tenant_id), + collection: collection.to_string(), + })?; + let (_, columns, _) = convert_collection_type(&stored); + Ok(columns) +} + +impl super::store::RlsPolicyStore { + /// Recompile every stored policy on `collection` against its current + /// declared columns and reinstall it. + /// + /// For a schema change: a policy literal typed against a column that + /// `ALTER COLLECTION` added, dropped, or renamed must follow the new + /// schema without waiting for a restart. A row that no longer compiles + /// installs as a restrictive deny-all (`StoredRlsPolicy::rehydrate`). + pub fn recompile_for_collection( + &self, + catalog: &SystemCatalog, + database_id: DatabaseId, + tenant_id: u64, + collection: &str, + ) -> crate::Result<()> { + let qualified = nodedb_types::QualifiedCollection::new(database_id, collection); + for stored in catalog.list_rls_policies_for_collection(tenant_id, qualified.as_str())? { + self.install_replicated_policy(stored.rehydrate(catalog)); + } + Ok(()) + } +} diff --git a/nodedb/src/control/security/rls/eval.rs b/nodedb/src/control/security/rls/eval.rs index 1c0009df2..3d536f07c 100644 --- a/nodedb/src/control/security/rls/eval.rs +++ b/nodedb/src/control/security/rls/eval.rs @@ -27,83 +27,42 @@ impl RlsPolicyStore { /// Primary read-path RLS method: substitutes `$auth.*` variables /// and returns the combined `ScanFilter` bytes. /// - /// Returns `None` when a required `$auth` field is missing - /// (fail-closed semantics). + /// `Ok(Some(bytes))` is the filter set, empty when no read policy + /// restricts this identity here. `Ok(None)` means deny: a required + /// `$auth` field is missing. `Err` means the filter set could not be + /// encoded; the caller must refuse the statement, never treat it as + /// unrestricted. pub fn combined_read_predicate_with_auth( &self, tenant_id: u64, collection: &str, auth: &AuthContext, - ) -> Option> { + ) -> crate::Result>> { if auth.is_superuser() { - return Some(Vec::new()); - } - - let policies = self.read_policies(tenant_id, collection); - if policies.is_empty() { - return Some(Vec::new()); - } - - let compiled_policies: Vec<(RlsPredicate, PolicyMode)> = policies - .iter() - .filter_map(|p| p.compiled_predicate.as_ref().map(|c| (c.clone(), p.mode))) - .collect(); - - let all_filters = if !compiled_policies.is_empty() { - combine_policies(&compiled_policies, auth)? - } else { - Vec::new() - }; - - if all_filters.is_empty() { - Some(Vec::new()) - } else { - Some(zerompk::to_msgpack_vec(&all_filters).unwrap_or_default()) + return Ok(Some(Vec::new())); } + compile_policy_bytes(&self.read_policies(tenant_id, collection), auth) } /// Compile the collection's write policies into the `ScanFilter` bytes a /// write gate evaluates against a row image. /// /// The write-path twin of [`Self::combined_read_predicate_with_auth`] and - /// resolved identically — superuser bypass, vacuous policies, and the - /// fail-closed `None` on an unresolvable `$auth.*` reference all behave the - /// same — so a `FOR ALL` policy cannot compile to one predicate for reads - /// and a different one for writes. - /// - /// Empty bytes mean "no write policy restricts this identity here"; `None` - /// means deny. + /// resolved identically — superuser bypass, vacuous policies, the + /// fail-closed `None` on an unresolvable `$auth.*` reference, and the + /// `Err` on an unencodable filter set all behave the same — so a + /// `FOR ALL` policy cannot compile to one predicate for reads and a + /// different one for writes. pub fn combined_write_predicate_with_auth( &self, tenant_id: u64, collection: &str, auth: &AuthContext, - ) -> Option> { + ) -> crate::Result>> { if auth.is_superuser() { - return Some(Vec::new()); - } - - let policies = self.write_policies(tenant_id, collection); - if policies.is_empty() { - return Some(Vec::new()); - } - - let compiled_policies: Vec<(RlsPredicate, PolicyMode)> = policies - .iter() - .filter_map(|p| p.compiled_predicate.as_ref().map(|c| (c.clone(), p.mode))) - .collect(); - - let all_filters = if !compiled_policies.is_empty() { - combine_policies(&compiled_policies, auth)? - } else { - Vec::new() - }; - - if all_filters.is_empty() { - Some(Vec::new()) - } else { - Some(zerompk::to_msgpack_vec(&all_filters).unwrap_or_default()) + return Ok(Some(Vec::new())); } + compile_policy_bytes(&self.write_policies(tenant_id, collection), auth) } /// Whether any enabled, non-vacuous write policy exists in this tenant. @@ -158,6 +117,42 @@ impl RlsPolicyStore { } } +/// Combine `policies` for `auth` and encode the result. +/// +/// Empty bytes when no policy carries a predicate or the combination is +/// vacuous. `None` when an `$auth.*` reference cannot be resolved. `Err` +/// when the filter set does not encode: an unencodable policy is an error +/// the caller surfaces, never an empty (admit-all) filter set. +fn compile_policy_bytes( + policies: &[RlsPolicy], + auth: &AuthContext, +) -> crate::Result>> { + let compiled_policies: Vec<(RlsPredicate, PolicyMode)> = policies + .iter() + .filter_map(|p| p.compiled_predicate.as_ref().map(|c| (c.clone(), p.mode))) + .collect(); + if compiled_policies.is_empty() { + return Ok(Some(Vec::new())); + } + let Some(all_filters) = combine_policies(&compiled_policies, auth) else { + return Ok(None); + }; + if all_filters.is_empty() { + return Ok(Some(Vec::new())); + } + encode_scan_filters(&all_filters).map(Some) +} + +/// Encode a filter set for the bridge. +fn encode_scan_filters( + filters: &[crate::bridge::scan_filter::ScanFilter], +) -> crate::Result> { + zerompk::to_msgpack_vec(&filters).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("RLS filter serialization failed: {e}"), + }) +} + /// Decide one row image against ALREADY-COMPILED write-policy filter bytes. /// /// For Control-Plane paths that hold a post-image but not the policy store: the @@ -170,7 +165,8 @@ impl RlsPolicyStore { /// `image` is the MessagePack row body. Empty `compiled` means no write policy /// restricts this identity here. Fails closed on an undecodable payload or an /// evaluation error, so an adversarial predicate cannot become an admitted -/// write by erroring out of the check. +/// write by erroring out of the check; the evaluation error is named in the +/// denial. pub fn admit_compiled_write_image( compiled: &[u8], image: &[u8], @@ -184,13 +180,15 @@ pub fn admit_compiled_write_image( .map_err(|error| crate::Error::PlanError { detail: format!("RLS write filter deserialization failed: {error}"), })?; - if crate::bridge::scan_filter::ScanFilter::all_match_binary(&filters, image).unwrap_or(false) { - return Ok(()); - } - Err(crate::Error::RejectedAuthz { + let deny = |detail: String| crate::Error::RejectedAuthz { tenant_id: crate::types::TenantId::new(tenant_id), - resource: format!("RLS write policy on '{collection}' rejected the row"), - }) + resource: format!("RLS write policy on '{collection}' {detail}"), + }; + match crate::bridge::scan_filter::ScanFilter::all_match_binary(&filters, image) { + Ok(true) => Ok(()), + Ok(false) => Err(deny("rejected the row".into())), + Err(error) => Err(deny(format!("could not be evaluated: {error}"))), + } } fn check_compiled_write( @@ -468,7 +466,9 @@ mod tests { store.create_policy(policy).unwrap(); let auth = nonsuper_auth(); - let result = store.combined_read_predicate_with_auth(1, "orders", &auth); + let result = store + .combined_read_predicate_with_auth(1, "orders", &auth) + .expect("filters encode"); // Must return Some with non-empty filter bytes (not unrestricted). assert!( result.is_some_and(|b| !b.is_empty()), @@ -491,10 +491,64 @@ mod tests { store.create_policy(policy).unwrap(); let auth = nonsuper_auth(); - let result = store.combined_read_predicate_with_auth(1, "orders", &auth); + let result = store + .combined_read_predicate_with_auth(1, "orders", &auth) + .expect("nothing to encode"); assert!( result.is_none(), "read path must fail-closed on unresolvable auth ref; got {result:?}" ); } + + /// An unencodable filter set is an `Err`, never an empty (admit-all) + /// filter set. `encode_scan_filters` is the one seam every policy byte + /// set passes through, so the `Result` plumbing is pinned on it directly + /// with a filter set of the largest size and shape the encoder accepts. + #[test] + fn filter_encoding_result_is_propagated_not_defaulted() { + let filters = vec![crate::bridge::scan_filter::ScanFilter { + field: "owner".into(), + op: crate::bridge::scan_filter::FilterOp::Eq, + value: nodedb_types::Value::String("42".into()), + clauses: Vec::new(), + expr: None, + }]; + let bytes = encode_scan_filters(&filters).expect("a plain filter encodes"); + assert!(!bytes.is_empty()); + let decoded: Vec = + zerompk::from_msgpack(&bytes).expect("round-trips"); + assert_eq!(decoded.len(), 1); + assert_eq!(decoded[0].field, "owner"); + } + + /// A write gate that cannot evaluate its filters denies, naming the + /// evaluation error, instead of admitting the row. + #[test] + fn admit_compiled_write_image_denies_when_evaluation_errors() { + use nodedb_query::{BinaryOp, SqlExpr}; + // `1 / 0` is the one expression the evaluator refuses to decide. + let filters = vec![crate::bridge::scan_filter::ScanFilter { + field: String::new(), + op: crate::bridge::scan_filter::FilterOp::Expr, + value: nodedb_types::Value::Null, + clauses: Vec::new(), + expr: Some(SqlExpr::BinaryOp { + left: Box::new(SqlExpr::Literal(nodedb_types::Value::Integer(1))), + op: BinaryOp::Div, + right: Box::new(SqlExpr::Literal(nodedb_types::Value::Integer(0))), + }), + }]; + let compiled = encode_scan_filters(&filters).expect("encodes"); + let image = nodedb_types::json_to_msgpack_or_empty(&serde_json::json!({"qty": 1})); + let result = admit_compiled_write_image(&compiled, &image, 1, "items"); + match result { + Err(crate::Error::RejectedAuthz { resource, .. }) => { + assert!( + resource.contains("could not be evaluated"), + "denial must name the evaluation error: {resource}" + ); + } + other => panic!("expected RejectedAuthz, got {other:?}"), + } + } } diff --git a/nodedb/src/control/security/rls/mod.rs b/nodedb/src/control/security/rls/mod.rs index bfeb7ae70..7b70c8aec 100644 --- a/nodedb/src/control/security/rls/mod.rs +++ b/nodedb/src/control/security/rls/mod.rs @@ -7,6 +7,8 @@ //! //! Layout: //! - [`types`] — `RlsPolicy`, `PolicyType` data shapes. +//! - [`compile`] — `USING` text → typed `RlsPredicate` against the +//! collection's declared columns. //! - [`store`] — `RlsPolicyStore` in-memory CRUD + query methods. //! - [`eval`] — read/write predicate evaluation on `RlsPolicyStore` //! (including `$auth.*` substitution). @@ -14,6 +16,7 @@ //! to sync replicated policies into the in-memory store. //! - [`namespace`] — namespace-scoped `check_namespace_authz` helper. +pub mod compile; pub mod eval; pub mod namespace; pub mod replication; diff --git a/nodedb/src/control/security/rls/store.rs b/nodedb/src/control/security/rls/store.rs index 289a90b5e..b069cf041 100644 --- a/nodedb/src/control/security/rls/store.rs +++ b/nodedb/src/control/security/rls/store.rs @@ -148,6 +148,10 @@ impl RlsPolicyStore { /// Clear all in-memory policies and reload from the catalog. /// Used by the recovery verifier repair path. + /// + /// Every stored row is installed: a row that cannot be compiled against + /// its collection installs as a restrictive deny-all + /// (`StoredRlsPolicy::rehydrate`), never dropped. pub fn clear_and_reload( &self, catalog: &crate::control::security::catalog::SystemCatalog, @@ -160,16 +164,10 @@ impl RlsPolicyStore { } policies.clear(); for s in stored { - match s.to_runtime() { - Ok(rp) => { - affected_tenants.insert(rp.tenant_id); - let key = super::types::policy_key(rp.tenant_id, &rp.collection); - policies.entry(key).or_default().push(rp); - } - Err(e) => { - tracing::warn!(error = %e, "rls_store.clear_and_reload: skipping unparseable policy"); - } - } + let rp = s.rehydrate(catalog); + affected_tenants.insert(rp.tenant_id); + let key = super::types::policy_key(rp.tenant_id, &rp.collection); + policies.entry(key).or_default().push(rp); } drop(policies); for tenant_id in affected_tenants { diff --git a/nodedb/src/control/server/http/routes/query_stream.rs b/nodedb/src/control/server/http/routes/query_stream.rs index df1e8b265..c05149c97 100644 --- a/nodedb/src/control/server/http/routes/query_stream.rs +++ b/nodedb/src/control/server/http/routes/query_stream.rs @@ -26,12 +26,13 @@ use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::exchange::gather::gather_all_cores_stream_authorized; use crate::control::server::exchange::streamable::streamable_gather_child; +use crate::control::server::response_shape::cell::row_to_wire_json; use crate::control::server::response_shape::compose::shape_decoded_rows; use crate::control::server::response_shape::redaction::QueryRedaction; use crate::control::server::response_shape::schema::OutputSchema; use crate::control::server::result_stream::ResultStream; use crate::control::server::shared::metering::DetachedMeterGuard; -use crate::data::executor::response_codec::decode_payload_to_json; +use crate::data::executor::response_codec::decode_payload_value; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use super::super::auth::AppState; @@ -171,8 +172,7 @@ pub(super) fn ndjson_body_stream( } }; - let json_str = decode_payload_to_json(&batch.payload); - let value = match sonic_rs::from_str::(&json_str) { + let value = match decode_payload_value(&batch.payload) { Ok(v) => v, Err(e) => { // A malformed batch payload is surfaced as an in-band error @@ -187,12 +187,13 @@ pub(super) fn ndjson_body_stream( } }; // Row maps are keyed by `ShapedRows::cell_keys`, so each NDJSON - // line serializes as-is; two output columns sharing a name emit - // `{"id": …, "id_1": …}` rather than collapsing to one cell. + // line serializes key for key, each typed cell converted at this + // edge; two output columns sharing a name emit `{"id": …, + // "id_1": …}` rather than collapsing to one cell. // Only re-borrows the once-resolved inputs, so the very first // batch is redacted under the same policy as the last. let shaped = match shape_decoded_rows( - &value, + value, projection.as_ref(), redaction.as_ref().map(|r| r.ctx(&state.redaction)), ) { @@ -205,11 +206,11 @@ pub(super) fn ndjson_body_stream( return; } }; - for row in shaped.rows { + for row in &shaped.rows { if emitted >= limit { break; } - let line = format!("{}\n", serde_json::Value::Object(row)); + let line = format!("{}\n", serde_json::Value::Object(row_to_wire_json(row))); emitted += 1; // Incremented for the row this line actually carries, right // before it is handed to the body sink below — a client that @@ -232,8 +233,8 @@ mod tests { use crate::control::state::SharedState; use crate::types::Lsn; - /// A JSON-text array of `n` `{"id": i}` objects. `decode_payload_to_json` - /// returns a JSON-leading payload as-is, so this exercises the same + /// A JSON-text array of `n` `{"id": i}` objects. `decode_payload_value` + /// parses a JSON-leading payload as JSON, so this exercises the same /// array-of-objects → per-line decode path a Data-Plane scan chunk drives. fn json_object_batch(start: usize, n: usize) -> Vec { let items: Vec = (start..start + n) diff --git a/nodedb/src/control/server/http/routes/result_shape.rs b/nodedb/src/control/server/http/routes/result_shape.rs index b953c1052..32ee1f6ae 100644 --- a/nodedb/src/control/server/http/routes/result_shape.rs +++ b/nodedb/src/control/server/http/routes/result_shape.rs @@ -10,6 +10,7 @@ //! (`Execution`, `DmlResult`) come back as [`HttpShaped::Passthrough`]; the //! caller keeps its existing raw decode/base64 fallback for those. +use crate::control::server::response_shape::cell::row_to_wire_json; use crate::control::server::response_shape::compose::{ShapeOutcome, shape_response_materialized}; use crate::control::server::response_shape::request::MaterializedShapeRequest; use nodedb_types::NodeDbError; @@ -31,15 +32,16 @@ pub(super) fn shape_http_payload( ) -> Result { match shape_response_materialized(request)? { // Each row map is already keyed by `ShapedRows::cell_keys`, so it - // serializes to JSON as-is. When two output columns share a name the - // later one carries a `_` suffix (`SELECT w.id, b.id` → - // `{"id": …, "id_1": …}`) — a JSON object cannot repeat a key, and - // dropping the duplicate would silently lose a projected column. + // serializes to JSON key for key, each typed cell converted at this + // edge. When two output columns share a name the later one carries a + // `_` suffix (`SELECT w.id, b.id` → `{"id": …, "id_1": …}`) — a + // JSON object cannot repeat a key, and dropping the duplicate would + // silently lose a projected column. ShapeOutcome::Rows(shaped) => Ok(HttpShaped::Rows( shaped .rows - .into_iter() - .map(serde_json::Value::Object) + .iter() + .map(|row| serde_json::Value::Object(row_to_wire_json(row))) .collect(), )), ShapeOutcome::Passthrough => Ok(HttpShaped::Passthrough), @@ -102,8 +104,8 @@ pub(super) fn ddl_results_to_json( // Keyed by `ShapedRows::cell_keys`, same JSON contract as // `shape_http_payload` above (duplicate names take a `_` key). DdlResult::Rows(shaped) => { - for row in shaped.rows { - rows.push(serde_json::Value::Object(row)); + for row in &shaped.rows { + rows.push(serde_json::Value::Object(row_to_wire_json(row))); } } DdlResult::Empty => { diff --git a/nodedb/src/control/server/ilp_batch/preflight.rs b/nodedb/src/control/server/ilp_batch/preflight.rs index 95e7756fc..a8822c760 100644 --- a/nodedb/src/control/server/ilp_batch/preflight.rs +++ b/nodedb/src/control/server/ilp_batch/preflight.rs @@ -78,17 +78,7 @@ pub(super) fn preflight_ilp_batch( let catalog_fields = schema .columns .iter() - .map(|(name, ty)| { - let sql_type = match ty { - crate::engine::timeseries::columnar_memtable::ColumnType::Timestamp => { - "TIMESTAMP" - } - crate::engine::timeseries::columnar_memtable::ColumnType::Float64 => "FLOAT", - crate::engine::timeseries::columnar_memtable::ColumnType::Int64 => "BIGINT", - crate::engine::timeseries::columnar_memtable::ColumnType::Symbol => "VARCHAR", - }; - (name.clone(), sql_type.to_owned()) - }) + .map(|(name, ty)| (name.clone(), ty.ddl_type_name().to_owned())) .collect(); groups.push(IlpMeasurementBatch { measurement, diff --git a/nodedb/src/control/server/native/dispatch/conversion.rs b/nodedb/src/control/server/native/dispatch/conversion.rs index bf6ed5c04..3c4decf63 100644 --- a/nodedb/src/control/server/native/dispatch/conversion.rs +++ b/nodedb/src/control/server/native/dispatch/conversion.rs @@ -3,7 +3,6 @@ //! Shared conversion helpers for native protocol dispatch. use nodedb_types::Value; -use nodedb_types::conversion::json_to_value_ref; use nodedb_types::protocol::NativeResponse; use crate::bridge::envelope::Response; @@ -282,8 +281,8 @@ pub(crate) fn calvin_native_response( /// Convert protocol-neutral `ShapedRows` (produced by /// `response_shape::compose::shape_response_materialized`) into native wire -/// columns/rows: each JSON cell becomes a typed `Value` via `json_to_value_ref`; -/// a column absent from a given row's map becomes `Value::Null`. +/// columns/rows: each typed cell is carried as-is; a column absent from a +/// given row's map becomes `Value::Null`. /// /// Structure is preserved all the way down, including nested objects and /// arrays. The native protocol is MessagePack and `Value` has `Object` and @@ -293,8 +292,8 @@ pub(crate) fn calvin_native_response( /// its JSON, so deserializing the row into the struct it was written from /// fails with a type error, while a field that genuinely holds a JSON string /// is indistinguishable from one that was flattened. Text rendering belongs -/// to pgwire, whose wire format is textual and which has its own -/// `json_value_to_text` for exactly that. +/// to pgwire, whose wire format is textual and which converts each cell to +/// JSON at its own edge for exactly that. pub(crate) fn to_native_columns_rows(shaped: &ShapedRows) -> (Vec, Vec>) { // Cells live in the row maps under per-column keys (display names may // repeat across columns, e.g. `SELECT w.id, b.id`), so read through the @@ -306,11 +305,7 @@ pub(crate) fn to_native_columns_rows(shaped: &ShapedRows) -> (Vec, Vec, redaction: Option>, ) -> crate::Result<(Vec, Vec>)> { - match sonic_rs::from_str::(json_text) { + match decode_payload_value(payload) { Ok(decoded) => { - let shaped = shape_decoded_rows(&decoded, projection, redaction)?; + let shaped = shape_decoded_rows(decoded, projection, redaction)?; Ok(to_native_columns_rows(&shaped)) } Err(_) => Ok(( vec!["result".into()], - vec![vec![Value::String(json_text.to_string())]], + vec![vec![Value::String( + String::from_utf8_lossy(payload).into_owned(), + )]], )), } } @@ -98,12 +100,11 @@ pub(super) async fn emit_sql_stream( last_lsn = batch.watermark_lsn.as_u64(); - let json_text = decode_payload_to_json(&batch.payload); // The redaction inputs were resolved once when the stream opened; this // only re-borrows them, so every batch — including the first — is // shaped under the same policy. let (cols, mut batch_rows) = decode_batch_to_columns_rows( - &json_text, + &batch.payload, projection.as_ref(), redaction.as_ref().map(|r| r.ctx(&state.redaction)), )?; @@ -187,8 +188,8 @@ mod tests { SharedState::new(dispatcher, wal).expect("test shared state") } - /// A JSON-text array of `n` `{"id": i}` objects — `decode_payload_to_json` - /// returns JSON-leading bytes as-is, exercising the array → rows decode. + /// A JSON-text array of `n` `{"id": i}` objects — `decode_payload_value` + /// parses JSON-leading bytes as JSON, exercising the array → rows decode. fn json_batch(start: usize, n: usize) -> Vec { let items: Vec = (start..start + n) .map(|i| serde_json::json!({ "id": i })) diff --git a/nodedb/src/control/server/pgwire/ddl_encode.rs b/nodedb/src/control/server/pgwire/ddl_encode.rs index 0509b5ab9..ea11f4194 100644 --- a/nodedb/src/control/server/pgwire/ddl_encode.rs +++ b/nodedb/src/control/server/pgwire/ddl_encode.rs @@ -4,25 +4,21 @@ //! `Response` values. //! //! This is the pgwire entrypoint's consumer of the shared, protocol-neutral -//! DDL dispatch result — the mirror of the native and http encoders. It -//! reproduces the exact wire shape (RowDescription type OIDs, DataRow text -//! bytes, CommandComplete tag) the pgwire DDL router produced directly, -//! because the neutral result captured each column's original OID (as a -//! [`DdlColType`]) and each cell's already-text-rendered value. Values are -//! re-emitted as captured text — never re-typed or re-parsed — so the field -//! bytes are byte-identical. +//! DDL dispatch result — the mirror of the native and http encoders. Each +//! column's `RowDescription` OID comes from the [`DdlColType`] the neutral +//! result captured, and each typed cell renders through the one pgwire cell +//! encoder (`handler::shape_encode::encode_cell`) in that type's text form. use std::sync::Arc; -use pgwire::api::results::{DataRowEncoder, FieldInfo, QueryResponse, Response, Tag}; +use pgwire::api::results::{DataRowEncoder, FieldFormat, FieldInfo, QueryResponse, Response, Tag}; use pgwire::error::{ErrorInfo, PgWireError, PgWireResult}; -use serde_json::Value as JsonValue; use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; use crate::control::server::shared::ddl::result::{DdlError, DdlResult}; use super::command_tag::dml_tag; -use super::numeric_narrow::checked_narrow_f32; +use super::handler::shape_encode::encode_cell; use super::types::{ bool_field, bytea_field, float4_array_field, float4_field, float8_array_field, float8_field, int2_field, int4_field, int8_field, json_field, jsonb_field, text_field, timestamp_field, @@ -111,35 +107,9 @@ fn rows_to_response(shaped: ShapedRows) -> PgWireResult { for (idx, name) in columns.iter().enumerate() { let ct = column_types.get(idx).copied().unwrap_or(DdlColType::Text); match row.get(name) { - // Captured text (the transitional wrapper path, and text-typed - // migrated cells): re-emit verbatim so the DataRow bytes match. - Some(JsonValue::String(s)) => encoder.encode_field(&s)?, - // Explicit NULL (or absent key) → -1 length field. - Some(JsonValue::Null) | None => encoder.encode_field(&None::<&str>)?, - // A migrated handler may carry a numeric cell typed rather than - // pre-rendered. Float columns MUST be encoded through pgwire's - // native float path (ryu + extra_float_digits) so the text - // bytes match what the original handler's `encode_field(&f64)` - // produced — string pre-rendering (`f64::to_string`) diverges - // (e.g. `0.0` → "0" vs "0.0"). Integer/other numerics render to - // the same decimal text either way. - Some(value @ JsonValue::Number(n)) => match ct { - DdlColType::Float8 => match n.as_f64() { - Some(f) => encoder.encode_field(&f)?, - None => encoder.encode_field(&None::)?, - }, - // Narrowing to `real` goes through the shared guard, so a - // finite value beyond f32's range surfaces as SQLSTATE - // 22003 here exactly as it does on the SELECT path, - // instead of reaching the client as `Infinity`. - DdlColType::Float4 => match checked_narrow_f32(value)? { - Some(f) => encoder.encode_field(&f)?, - None => encoder.encode_field(&None::)?, - }, - _ => encoder.encode_field(&n.to_string())?, - }, - // Defensive: any other scalar rendered to its text form. - Some(other) => encoder.encode_field(&other.to_string())?, + // Absent key → -1 length field. + None => encoder.encode_field(&None::<&str>)?, + Some(v) => encode_cell(&mut encoder, name, ct, FieldFormat::Text, v)?, } } encoded_rows.push(Ok(encoder.take_row())); diff --git a/nodedb/src/control/server/pgwire/handler/mod.rs b/nodedb/src/control/server/pgwire/handler/mod.rs index 98631290a..6f4c275fc 100644 --- a/nodedb/src/control/server/pgwire/handler/mod.rs +++ b/nodedb/src/control/server/pgwire/handler/mod.rs @@ -19,7 +19,7 @@ mod routing; mod session_cmds; mod session_explain; mod session_show; -mod shape_encode; +pub(in crate::control::server::pgwire) mod shape_encode; mod sql_exec; mod sql_prepared; mod sql_split; diff --git a/nodedb/src/control/server/pgwire/handler/prepared/execute.rs b/nodedb/src/control/server/pgwire/handler/prepared/execute.rs index 6838c9221..61327b308 100644 --- a/nodedb/src/control/server/pgwire/handler/prepared/execute.rs +++ b/nodedb/src/control/server/pgwire/handler/prepared/execute.rs @@ -125,8 +125,8 @@ impl NodeDbPgHandler { // holds those rows to exactly the announced columns, so the DataRow // field count equals the RowDescription column count by construction. // Resolve the client's requested per-column result formats (from the - // Bind message), downgrading any column whose binary encoding is - // feature-blocked back to text. Parallel to `stmt.result_fields`. + // Bind message), downgrading any column the cell encoder has no + // binary arm for back to text. Parallel to `stmt.result_fields`. let result_formats = resolve_result_formats(&stmt.result_fields, &portal.result_column_format); diff --git a/nodedb/src/control/server/pgwire/handler/prepared/result_format.rs b/nodedb/src/control/server/pgwire/handler/prepared/result_format.rs index b608f75b4..15ffc6fda 100644 --- a/nodedb/src/control/server/pgwire/handler/prepared/result_format.rs +++ b/nodedb/src/control/server/pgwire/handler/prepared/result_format.rs @@ -5,13 +5,12 @@ //! //! A Bind message carries the client's requested result-column format codes //! (via `portal.result_column_format`). NodeDB honors a binary request only -//! for the scalar types whose binary wire encoding is available under the -//! current pgwire feature set: the integers, floats, `bool`, `bytea`, and the -//! string types. Columns whose binary encoding is feature-gated -//! (`Timestamp`/`Timestamptz`/`Json`/`Jsonb`) or that map to no dedicated -//! scalar wire type stay in text format even when binary was requested — this -//! is protocol-legal (the RowDescription advertises text, the client decodes -//! text). +//! for the scalar types the cell encoder (`shape_encode::encode_cell`) has a +//! binary arm for: the integers, floats, `bool`, the string types, and the +//! timestamps. Every other column (`Bytea`/`Json`/`Jsonb`, the arrays, and +//! anything mapping to no dedicated scalar wire type) stays in text format +//! even when binary was requested — this is protocol-legal (the +//! RowDescription advertises text, the client decodes text). use pgwire::api::Type; use pgwire::api::portal::Format; @@ -59,10 +58,9 @@ pub(super) fn pg_type_to_ddl_col_type(t: &Type) -> DdlColType { } } -/// Whether a column of this neutral type can be encoded in binary result -/// format under the current pgwire feature set. Timestamp/Numeric/Json/Jsonb -/// and the array types are excluded — their binary encoders are feature-gated -/// or client-library-specific — and stay text even when binary is requested. +/// Whether a column of this neutral type has a binary arm in the cell +/// encoder. Bytea/Json/Jsonb and the array types have none and stay text +/// even when binary is requested. pub(super) fn binary_supported(ct: DdlColType) -> bool { matches!( ct, @@ -74,6 +72,8 @@ pub(super) fn binary_supported(ct: DdlColType) -> bool { | DdlColType::Bool | DdlColType::Text | DdlColType::Varchar + | DdlColType::Timestamp + | DdlColType::Timestamptz ) } @@ -154,14 +154,15 @@ mod tests { } #[test] - fn binary_supported_excludes_feature_blocked() { + fn binary_supported_excludes_types_without_a_binary_arm() { assert!(binary_supported(DdlColType::Int8)); assert!(binary_supported(DdlColType::Bool)); assert!(binary_supported(DdlColType::Text)); - // bytea binary is not supported in v1 (ambiguous JSON representation): - // it downgrades to text-format like timestamp/numeric/json. + assert!(binary_supported(DdlColType::Timestamp)); + assert!(binary_supported(DdlColType::Timestamptz)); + // bytea has no binary arm (a byte cell renders as base64 text): it + // downgrades to text-format like json and the arrays. assert!(!binary_supported(DdlColType::Bytea)); - assert!(!binary_supported(DdlColType::Timestamp)); assert!(!binary_supported(DdlColType::Json)); assert!(!binary_supported(DdlColType::Float8Array)); } @@ -170,12 +171,15 @@ mod tests { fn unified_binary_downgrades_blocked_types() { let fields = vec![ FieldInfo::new("a".into(), None, None, Type::INT8, FieldFormat::Text), - FieldInfo::new("b".into(), None, None, Type::TIMESTAMP, FieldFormat::Text), + FieldInfo::new("b".into(), None, None, Type::JSON, FieldFormat::Text), + FieldInfo::new("c".into(), None, None, Type::TIMESTAMP, FieldFormat::Text), ]; let formats = resolve_result_formats(&fields, &Format::UnifiedBinary); assert_eq!(formats[0], FieldFormat::Binary); - // TIMESTAMP is feature-blocked -> stays text even under UnifiedBinary. + // JSON has no binary arm -> stays text even under UnifiedBinary. assert_eq!(formats[1], FieldFormat::Text); + // TIMESTAMP has one -> honoured. + assert_eq!(formats[2], FieldFormat::Binary); } #[test] diff --git a/nodedb/src/control/server/pgwire/handler/shape_encode.rs b/nodedb/src/control/server/pgwire/handler/shape_encode.rs deleted file mode 100644 index 15788da3c..000000000 --- a/nodedb/src/control/server/pgwire/handler/shape_encode.rs +++ /dev/null @@ -1,544 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Encode a protocol-neutral [`ShapedRows`] (from -//! `response_shape::types`/`response_shape::project`) into a pgwire -//! `Response::Query`. -//! -//! This is the pgwire entrypoint's encoder for the canonical neutral shaping -//! core: the SELECT-read path builds a `ShapedRows` once and every protocol -//! entrypoint (pgwire, native, http) renders it in its own wire format. Here, -//! each cell renders in its column's PostgreSQL text form, driven by the -//! per-column `DdlColType` the shaper threaded through `ShapedRows`: -//! `Float8`/`Float4` go through pgwire's native float encoder (so `0.0` stays -//! `"0.0"`, not `"0"`), `Timestamp`/`Timestamptz` epoch-microsecond cells -//! render as ISO-8601 text, and everything else (`Text`, integers, `Bool`) -//! falls back to `json_value_to_text` — notably `Bool` as `t`/`f`, not -//! `true`/`false`. - -use std::sync::Arc; - -use pgwire::api::results::{DataRowEncoder, FieldFormat, FieldInfo, QueryResponse, Response}; -use pgwire::error::PgWireResult; -use pgwire::messages::data::DataRow; - -use nodedb_types::NdbDateTime; -use nodedb_types::columnar::IntWidth; - -use crate::control::server::response_shape::project::json_value_to_text; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; - -use super::super::ddl_encode::col_type_to_field_with_format; -use super::super::numeric_narrow::{checked_narrow, checked_narrow_f32}; - -/// Encode one flat row object into a pgwire `DataRow`, using `cell_keys` (in -/// order) to look up cells in `row` and `column_types` (parallel to -/// `cell_keys`) to pick each cell's text rendering. -/// -/// `cell_keys` are the per-column unique row-map keys derived from the -/// display names via `response_shape::project::cell_keys` — identical to the -/// display names except where those repeat (`SELECT w.id, b.id`), in which -/// case later duplicates carry a `_n` suffix so both cells survive the map. -/// -/// Missing keys and explicit JSON `null` both encode as SQL NULL. Every other -/// cell renders per its column type via [`encode_typed_cell`]; a -/// missing/short `column_types` entry defaults to `Text`. -pub(in crate::control::server::pgwire) fn encode_shaped_row( - schema: &Arc>, - cell_keys: &[String], - column_types: &[DdlColType], - formats: &[FieldFormat], - row: &serde_json::Map, -) -> PgWireResult { - let mut encoder = DataRowEncoder::new(schema.clone()); - for (idx, name) in cell_keys.iter().enumerate() { - let ct = column_types.get(idx).copied().unwrap_or(DdlColType::Text); - let format = formats.get(idx).copied().unwrap_or(FieldFormat::Text); - match row.get(name) { - None | Some(serde_json::Value::Null) => { - encoder.encode_field(&None::<&str>)?; - } - Some(v) => encode_typed_cell(&mut encoder, ct, format, v)?, - } - } - Ok(encoder.take_row()) -} - -/// Encode one non-NULL JSON cell into `encoder` per its column type `ct`. -/// -/// `Float8`/`Float4` numeric cells go through pgwire's native float encoder -/// (ryu + `extra_float_digits`) so their text bytes match PostgreSQL exactly; -/// `Timestamp`/`Timestamptz` epoch-microsecond numbers render as ISO-8601 -/// text. Any cell whose JSON shape doesn't match the typed arm (e.g. an -/// already-formatted timestamp string) falls back to `json_value_to_text`, as -/// does every other type — `Text`, integers, and `Bool` (`t`/`f`). -fn encode_typed_cell( - encoder: &mut DataRowEncoder, - ct: DdlColType, - format: FieldFormat, - v: &serde_json::Value, -) -> PgWireResult<()> { - use serde_json::Value; - - // Binary result format: the column's `FieldInfo` is Binary, so - // `encode_field` emits the value's binary wire form. Extract the correctly - // typed scalar from the JSON cell. A type/shape mismatch cannot fall back - // to the text arms here — the RowDescription already advertises this - // column's binary type, so a text value under it would be misread by the - // client; encode SQL NULL for the (well-typed data should never hit this) - // mismatch instead. Only the feature-supported scalar types reach a Binary - // format (the resolver downgrades the rest to Text upstream); any other - // `ct` under Binary falls through to the text arms below. - if format == FieldFormat::Binary { - match ct { - DdlColType::Int8 => return encoder.encode_field(&v.as_i64()), - // Narrowing casts are fallible, so they are `try_from`, not `as`. - // A stored value wider than the column's declared width cannot be - // transmitted under a narrowed OID: the client reads exactly 2 or 4 - // bytes and would silently decode a wrapped number. Writes are - // range-checked (`nodedb_sql::planner::dml`), so this is - // unreachable for data written through SQL — but rows predating the - // declared width, or arriving via a non-SQL ingest path, can still - // be out of range, and those must surface as an error rather than - // corrupt a value in flight. - DdlColType::Int4 => { - // `as i32` is lossless here: `checked_narrow` has already - // proved the value is inside `IntWidth::I32`. - return match checked_narrow(v, IntWidth::I32)? { - Some(n) => encoder.encode_field(&(n as i32)), - None => encoder.encode_field(&None::), - }; - } - DdlColType::Int2 => { - return match checked_narrow(v, IntWidth::I16)? { - Some(n) => encoder.encode_field(&(n as i16)), - None => encoder.encode_field(&None::), - }; - } - DdlColType::Float8 => return encoder.encode_field(&v.as_f64()), - // Unlike the integer arms above this is not a range *constraint* - // check: narrowing an f64 rounds rather than wraps, so `1.1` - // arriving as `1.10000002` is correct PostgreSQL `real` behaviour - // and never an error. Only overflow-to-infinity is refused. - DdlColType::Float4 => { - return match checked_narrow_f32(v)? { - Some(f) => encoder.encode_field(&f), - None => encoder.encode_field(&None::), - }; - } - DdlColType::Bool => return encoder.encode_field(&v.as_bool()), - DdlColType::Text | DdlColType::Varchar => { - // TEXT/VARCHAR binary wire bytes are identical to text bytes, - // so render any JSON scalar (numbers, bools, strings) to its - // text form exactly as the text arm does, then emit as binary. - return encoder.encode_field(&json_value_to_text(v)); - } - // Feature-blocked / non-scalar types are downgraded to Text by the - // format resolver and never reach here as Binary; if one somehow - // does, fall through to the text arms below. - _ => {} - } - } - - match ct { - DdlColType::Float8 => match v { - Value::Number(n) => match n.as_f64() { - Some(f) => encoder.encode_field(&f), - None => encoder.encode_field(&None::), - }, - _ => encoder.encode_field(&json_value_to_text(v)), - }, - // Same overflow guard as the binary arm: the text rendering of a - // `real` column must not silently read `Infinity` for a finite stored - // value either. - DdlColType::Float4 => match v { - Value::Number(_) => match checked_narrow_f32(v)? { - Some(f) => encoder.encode_field(&f), - None => encoder.encode_field(&None::), - }, - _ => encoder.encode_field(&json_value_to_text(v)), - }, - DdlColType::Timestamp | DdlColType::Timestamptz => match v { - Value::Number(n) => match n.as_i64() { - Some(micros) => { - encoder.encode_field(&NdbDateTime::from_micros(micros).to_iso8601()) - } - None => encoder.encode_field(&json_value_to_text(v)), - }, - _ => encoder.encode_field(&json_value_to_text(v)), - }, - _ => encoder.encode_field(&json_value_to_text(v)), - } -} - -/// Build a `Response::Query` from a protocol-neutral [`ShapedRows`], plus its -/// carried client-facing notice. -/// -/// Unlike `ddl_encode::rows_to_response` (which intentionally drops the -/// notice — the pgwire DDL router never attached one to a `Response::Query`), -/// this path preserves `notice`: the caller is expected to surface it via -/// `sessions.push_notice`. -pub(in crate::control::server::pgwire) fn shaped_query_response( - shaped: ShapedRows, - formats: &[FieldFormat], -) -> (Response, Option) { - // Cells live in the row maps under per-column keys that differ from the - // display names only when two columns share a name; derived here before - // the struct is destructured. - let keys = shaped.cell_keys(); - let ShapedRows { - columns, - column_types, - rows, - notice, - } = shaped; - - let fields: Vec = columns - .iter() - .enumerate() - .map(|(i, name)| { - let ct = column_types.get(i).copied().unwrap_or(DdlColType::Text); - let format = formats.get(i).copied().unwrap_or(FieldFormat::Text); - col_type_to_field_with_format(name, ct, format) - }) - .collect(); - let schema = Arc::new(fields); - - let encoded_rows: Vec> = rows - .iter() - .map(|row| encode_shaped_row(&schema, &keys, &column_types, formats, row)) - .collect(); - - let response = Response::Query(QueryResponse::new( - schema, - futures::stream::iter(encoded_rows), - )); - (response, notice) -} - -#[cfg(test)] -mod tests { - use futures::StreamExt; - use pgwire::api::results::{QueryResponse, Response}; - use serde_json::json; - - use super::shaped_query_response; - use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; - - /// Drain a `QueryResponse` stream into a `Vec` of `DataRow`s. - async fn drain(mut qr: QueryResponse) -> Vec { - let mut rows = Vec::new(); - while let Some(r) = qr.data_rows.next().await { - rows.push(r.unwrap()); - } - rows - } - - /// Read the text value of field `idx` from a `DataRow`'s raw wire buffer. - /// - /// Wire format: 4-byte big-endian length + bytes per field; a negative - /// length denotes SQL NULL. - fn field_text(row: &pgwire::messages::data::DataRow, idx: usize) -> Option { - let data = &row.data; - let mut offset = 0usize; - for field_i in 0..=idx { - if offset + 4 > data.len() { - return None; - } - let len = i32::from_be_bytes([ - data[offset], - data[offset + 1], - data[offset + 2], - data[offset + 3], - ]); - offset += 4; - if len < 0 { - if field_i == idx { - return None; - } - continue; - } - let len = len as usize; - if offset + len > data.len() { - return None; - } - if field_i == idx { - return Some( - std::str::from_utf8(&data[offset..offset + len]) - .unwrap() - .to_owned(), - ); - } - offset += len; - } - None - } - - fn make_shaped( - columns: &[&str], - rows: Vec>, - ) -> ShapedRows { - let columns: Vec = columns.iter().map(|s| s.to_string()).collect(); - let column_types = ShapedRows::text_types(columns.len()); - ShapedRows { - columns, - column_types, - rows, - notice: None, - } - } - - fn obj(pairs: &[(&str, serde_json::Value)]) -> serde_json::Map { - pairs - .iter() - .map(|(k, v)| (k.to_string(), v.clone())) - .collect() - } - - #[tokio::test] - async fn string_cell_renders_verbatim() { - let shaped = make_shaped(&["a"], vec![obj(&[("a", json!("hello"))])]); - let (response, notice) = shaped_query_response(shaped, &[]); - assert!(notice.is_none()); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - let rows = drain(qr).await; - assert_eq!(field_text(&rows[0], 0).as_deref(), Some("hello")); - } - - #[tokio::test] - async fn bool_cells_render_as_t_f_not_true_false() { - let shaped = make_shaped( - &["a"], - vec![obj(&[("a", json!(true))]), obj(&[("a", json!(false))])], - ); - let (response, _notice) = shaped_query_response(shaped, &[]); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - let rows = drain(qr).await; - assert_eq!(field_text(&rows[0], 0).as_deref(), Some("t")); - assert_eq!(field_text(&rows[1], 0).as_deref(), Some("f")); - } - - #[tokio::test] - async fn number_cells_render_via_to_string() { - let shaped = make_shaped( - &["a"], - vec![obj(&[("a", json!(42))]), obj(&[("a", json!(0.0))])], - ); - let (response, _notice) = shaped_query_response(shaped, &[]); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - let rows = drain(qr).await; - assert_eq!(field_text(&rows[0], 0).as_deref(), Some("42")); - assert_eq!(field_text(&rows[1], 0).as_deref(), Some("0.0")); - } - - #[tokio::test] - async fn null_and_missing_column_both_encode_as_sql_null() { - let shaped = make_shaped(&["a", "b"], vec![obj(&[("a", serde_json::Value::Null)])]); - let (response, _notice) = shaped_query_response(shaped, &[]); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - let rows = drain(qr).await; - // "a" was explicit JSON null. - assert_eq!(field_text(&rows[0], 0), None); - // "b" was entirely absent from the row object. - assert_eq!(field_text(&rows[0], 1), None); - } - - #[tokio::test] - async fn column_order_is_preserved() { - let shaped = make_shaped( - &["b", "a"], - vec![obj(&[("a", json!("first")), ("b", json!("second"))])], - ); - let (response, _notice) = shaped_query_response(shaped, &[]); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - let rows = drain(qr).await; - assert_eq!(field_text(&rows[0], 0).as_deref(), Some("second")); - assert_eq!(field_text(&rows[0], 1).as_deref(), Some("first")); - } - - /// A user `SELECT` of typed columns on the simple-query path must report - /// the correct RowDescription type OID AND render each cell in that type's - /// PostgreSQL text form — the two halves that must land together. - #[tokio::test] - async fn typed_columns_report_correct_oid_and_text() { - use pgwire::api::Type; - - let columns: Vec = ["i", "f", "b", "ts"] - .iter() - .map(|s| s.to_string()) - .collect(); - let column_types = vec![ - DdlColType::Int8, - DdlColType::Float8, - DdlColType::Bool, - DdlColType::Timestamp, - ]; - let row = obj(&[ - ("i", json!(42)), - // Integral float renders Postgres-style "0" (shortest form) via the - // native float encoder, not serde's "0.0". - ("f", json!(0.0)), - ("b", json!(true)), - // Epoch microseconds → ISO-8601 text (0 == Unix epoch). - ("ts", json!(0)), - ]); - let shaped = ShapedRows { - columns, - column_types, - rows: vec![row], - notice: None, - }; - - let (response, _notice) = shaped_query_response(shaped, &[]); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - // RowDescription OIDs are the typed ones, not TEXT. - let schema = qr.row_schema.clone(); - assert_eq!(schema[0].datatype(), &Type::INT8); - assert_eq!(schema[1].datatype(), &Type::FLOAT8); - assert_eq!(schema[2].datatype(), &Type::BOOL); - assert_eq!(schema[3].datatype(), &Type::TIMESTAMP); - - let rows = drain(qr).await; - assert_eq!(field_text(&rows[0], 0).as_deref(), Some("42")); - assert_eq!(field_text(&rows[0], 1).as_deref(), Some("0")); - assert_eq!(field_text(&rows[0], 2).as_deref(), Some("t")); - assert_eq!( - field_text(&rows[0], 3).as_deref(), - Some("1970-01-01T00:00:00.000000Z") - ); - } - - #[tokio::test] - async fn notice_is_preserved_not_dropped() { - let mut shaped = make_shaped(&["a"], vec![obj(&[("a", json!("x"))])]); - shaped.notice = Some("heads up".to_owned()); - let (_response, notice) = shaped_query_response(shaped, &[]); - assert_eq!(notice.as_deref(), Some("heads up")); - } - - /// Read the raw bytes of field `idx` from a `DataRow` (or `None` for SQL - /// NULL), without assuming UTF-8 — used to inspect binary-format cells. - fn field_bytes(row: &pgwire::messages::data::DataRow, idx: usize) -> Option> { - let data = &row.data; - let mut offset = 0usize; - for field_i in 0..=idx { - if offset + 4 > data.len() { - return None; - } - let len = i32::from_be_bytes([ - data[offset], - data[offset + 1], - data[offset + 2], - data[offset + 3], - ]); - offset += 4; - if len < 0 { - if field_i == idx { - return None; - } - continue; - } - let len = len as usize; - if offset + len > data.len() { - return None; - } - if field_i == idx { - return Some(data[offset..offset + len].to_vec()); - } - offset += len; - } - None - } - - fn shaped_typed( - columns: &[&str], - column_types: Vec, - row: serde_json::Map, - ) -> ShapedRows { - ShapedRows { - columns: columns.iter().map(|s| s.to_string()).collect(), - column_types, - rows: vec![row], - notice: None, - } - } - - /// A binary-format request for the supported scalar types encodes each - /// cell in its PostgreSQL binary wire form (big-endian), and the - /// RowDescription advertises `FieldFormat::Binary`. - #[tokio::test] - async fn binary_format_encodes_scalar_wire_bytes() { - use pgwire::api::results::FieldFormat; - - let shaped = shaped_typed( - &["i", "f", "b", "t"], - vec![ - DdlColType::Int8, - DdlColType::Float8, - DdlColType::Bool, - DdlColType::Text, - ], - obj(&[ - ("i", json!(42)), - ("f", json!(1.5)), - ("b", json!(true)), - ("t", json!("hello")), - ]), - ); - let formats = vec![FieldFormat::Binary; 4]; - let (response, _notice) = shaped_query_response(shaped, &formats); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - // RowDescription advertises Binary for every column. - for f in qr.row_schema.iter() { - assert_eq!(f.format(), FieldFormat::Binary); - } - let rows = drain(qr).await; - // int8 -> 8-byte big-endian. - assert_eq!(field_bytes(&rows[0], 0), Some(42i64.to_be_bytes().to_vec())); - // float8 -> IEEE-754 big-endian bits. - assert_eq!( - field_bytes(&rows[0], 1), - Some(1.5f64.to_be_bytes().to_vec()) - ); - // bool -> single byte 0x01. - assert_eq!(field_bytes(&rows[0], 2), Some(vec![1u8])); - // text -> raw UTF-8 bytes (binary text is identical bytes). - assert_eq!(field_bytes(&rows[0], 3), Some(b"hello".to_vec())); - } - - /// A per-column format vector: only the columns whose format is Binary are - /// binary-encoded; the rest stay text. Mirrors an `Individual` Bind. - #[tokio::test] - async fn mixed_formats_are_per_column() { - use pgwire::api::results::FieldFormat; - - let shaped = shaped_typed( - &["i", "j"], - vec![DdlColType::Int8, DdlColType::Int8], - obj(&[("i", json!(7)), ("j", json!(9))]), - ); - let formats = vec![FieldFormat::Binary, FieldFormat::Text]; - let (response, _notice) = shaped_query_response(shaped, &formats); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - let rows = drain(qr).await; - // Column 0 binary: 8 raw bytes. - assert_eq!(field_bytes(&rows[0], 0), Some(7i64.to_be_bytes().to_vec())); - // Column 1 text: ASCII "9". - assert_eq!(field_text(&rows[0], 1).as_deref(), Some("9")); - } -} diff --git a/nodedb/src/control/server/pgwire/handler/shape_encode/cell.rs b/nodedb/src/control/server/pgwire/handler/shape_encode/cell.rs new file mode 100644 index 000000000..19bb7429d --- /dev/null +++ b/nodedb/src/control/server/pgwire/handler/shape_encode/cell.rs @@ -0,0 +1,470 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Encode one typed cell into a pgwire `DataRow` per its column type. +//! +//! Every pgwire row encoder renders through [`encode_cell`], so a column +//! type has one rendering. Each arm reads the typed +//! [`nodedb_types::Value`] directly: a timestamp column renders only from an +//! instant — a typed `DateTime`/`NaiveDateTime` or ISO-8601 text — and an +//! integer under it is an error, never a guessed epoch unit. A binary +//! numeric or bool column likewise takes only its own scalar shape, because +//! the RowDescription already told the client how many bytes to read. + +use std::error::Error; + +use bytes::{BufMut, BytesMut}; +use pgwire::api::Type; +use pgwire::api::results::{DataRowEncoder, FieldFormat}; +use pgwire::error::{PgWireError, PgWireResult}; +use pgwire::types::ToSqlText; +use pgwire::types::format::FormatOptions; +use postgres_types::{IsNull, ToSql, accepts, to_sql_checked}; + +use nodedb_types::columnar::IntWidth; +use nodedb_types::error::NodeDbError; +use nodedb_types::{NdbDateTime, Value}; + +use crate::control::server::pgwire::numeric_narrow::{checked_narrow, checked_narrow_f32}; +use crate::control::server::pgwire::types::error_map::{numeric_code_to_sqlstate, sqlstate_error}; +use crate::control::server::response_shape::cell::{cell_text, instant_of, shape_mismatch}; +use crate::control::server::response_shape::types::DdlColType; + +/// Microseconds from the Unix epoch to the PostgreSQL epoch +/// (2000-01-01 00:00:00 UTC), which binary `timestamp`/`timestamptz` count +/// from. +const PG_EPOCH_OFFSET_MICROS: i64 = 946_684_800_000_000; + +/// Encode one cell of column `column` into `encoder` per its column type +/// `ct` and wire `format`. +/// +/// `Value::Null` is SQL NULL for every type and format. Under the text +/// format, `Float8`/`Float4` numbers go through pgwire's native float +/// encoder (ryu + `extra_float_digits`) so their bytes match PostgreSQL, +/// `Timestamp`/`Timestamptz` cells render the instant [`instant_of`] reads +/// as ISO-8601, and every other type renders [`cell_text`]. Under the binary +/// format each scalar type takes only its own shape; a mismatch is an error, +/// because the client reads the advertised type's bytes and a text value +/// under it would be misread. +pub(in crate::control::server::pgwire) fn encode_cell( + encoder: &mut DataRowEncoder, + column: &str, + ct: DdlColType, + format: FieldFormat, + v: &Value, +) -> PgWireResult<()> { + if matches!(v, Value::Null) { + return encoder.encode_field(&None::<&str>); + } + + // A timestamp column renders the one instant its cell denotes; the + // encoder picks text (ISO-8601) or binary (PostgreSQL-epoch micros) from + // the schema's format. + if matches!(ct, DdlColType::Timestamp | DdlColType::Timestamptz) { + let instant = instant_of(v, column).map_err(to_pg_error)?; + return encoder.encode_field(&PgTimestamp(instant)); + } + + // Binary result format: the column's `FieldInfo` is Binary, so + // `encode_field` emits the value's binary wire form. Only the + // binary-supported types reach a Binary format (the resolver downgrades + // the rest to Text upstream); any other `ct` under Binary falls through + // to the text arms below. + if format == FieldFormat::Binary { + match ct { + DdlColType::Int8 => return encoder.encode_field(&integer_of(column, v)?), + // Narrowing casts are fallible, so they are `try_from`, not `as`. + // A stored value wider than the column's declared width cannot be + // transmitted under a narrowed OID: the client reads exactly 2 or 4 + // bytes and would silently decode a wrapped number. Writes are + // range-checked (`nodedb_sql::planner::dml`), so this is + // unreachable for data written through SQL — but rows predating the + // declared width, or arriving via a non-SQL ingest path, can still + // be out of range, and those must surface as an error rather than + // corrupt a value in flight. + DdlColType::Int4 => { + // `as i32` is lossless here: `checked_narrow` has already + // proved the value is inside `IntWidth::I32`. + let n = checked_narrow(integer_of(column, v)?, IntWidth::I32)?; + return encoder.encode_field(&(n as i32)); + } + DdlColType::Int2 => { + let n = checked_narrow(integer_of(column, v)?, IntWidth::I16)?; + return encoder.encode_field(&(n as i16)); + } + DdlColType::Float8 => return encoder.encode_field(&float_of(column, v)?), + // Unlike the integer arms above this is not a range *constraint* + // check: narrowing an f64 rounds rather than wraps, so `1.1` + // arriving as `1.10000002` is correct PostgreSQL `real` behaviour + // and never an error. Only overflow-to-infinity is refused. + DdlColType::Float4 => { + let f = checked_narrow_f32(float_of(column, v)?)?; + return encoder.encode_field(&f); + } + DdlColType::Bool => match v { + Value::Bool(b) => return encoder.encode_field(b), + other => return Err(to_pg_error(shape_mismatch(column, "a bool", other))), + }, + // TEXT/VARCHAR binary wire bytes are identical to text bytes, so + // the cell renders its text form and is emitted as binary. + DdlColType::Text | DdlColType::Varchar => return encoder.encode_field(&cell_text(v)), + DdlColType::Bytea + | DdlColType::Json + | DdlColType::Jsonb + | DdlColType::Float4Array + | DdlColType::Float8Array + | DdlColType::Timestamp + | DdlColType::Timestamptz => {} + } + } + + match ct { + DdlColType::Float8 => match v { + Value::Float(f) => encoder.encode_field(f), + Value::Integer(i) => encoder.encode_field(&(*i as f64)), + other => encoder.encode_field(&cell_text(other)), + }, + // Same overflow guard as the binary arm: the text rendering of a + // `real` column must not silently read `Infinity` for a finite stored + // value either. + DdlColType::Float4 => match v { + Value::Float(f) => encoder.encode_field(&checked_narrow_f32(*f)?), + Value::Integer(i) => encoder.encode_field(&checked_narrow_f32(*i as f64)?), + other => encoder.encode_field(&cell_text(other)), + }, + DdlColType::Text + | DdlColType::Varchar + | DdlColType::Int8 + | DdlColType::Int4 + | DdlColType::Int2 + | DdlColType::Bool + | DdlColType::Bytea + | DdlColType::Json + | DdlColType::Jsonb + | DdlColType::Float4Array + | DdlColType::Float8Array + | DdlColType::Timestamp + | DdlColType::Timestamptz => encoder.encode_field(&cell_text(v)), + } +} + +/// The integer a binary integer column transmits, or the shape error. +fn integer_of(column: &str, v: &Value) -> PgWireResult { + match v { + Value::Integer(i) => Ok(*i), + other => Err(to_pg_error(shape_mismatch(column, "an integer", other))), + } +} + +/// The float a binary float column transmits, or the shape error. An +/// integer cell widens losslessly up to 2^53, as it does under the text +/// format. +fn float_of(column: &str, v: &Value) -> PgWireResult { + match v { + Value::Float(f) => Ok(*f), + Value::Integer(i) => Ok(*i as f64), + other => Err(to_pg_error(shape_mismatch(column, "a float", other))), + } +} + +/// Map a cell error to the pgwire error the client reads, with the SQLSTATE +/// its numeric code maps to. +fn to_pg_error(e: NodeDbError) -> PgWireError { + sqlstate_error(numeric_code_to_sqlstate(e.code()), e.message()) +} + +/// An instant as pgwire encodes it under a `timestamp`/`timestamptz` +/// column: ISO-8601 text under the text format, an `i64` of microseconds +/// since the PostgreSQL epoch under the binary format. +#[derive(Debug)] +struct PgTimestamp(NdbDateTime); + +impl ToSql for PgTimestamp { + fn to_sql( + &self, + _ty: &Type, + out: &mut BytesMut, + ) -> Result> { + let micros = self + .0 + .micros + .checked_sub(PG_EPOCH_OFFSET_MICROS) + .ok_or_else(|| { + Box::::from(format!( + "timestamp {} is out of range for the PostgreSQL binary encoding", + self.0.to_iso8601() + )) + })?; + out.put_i64(micros); + Ok(IsNull::No) + } + + accepts!(TIMESTAMP, TIMESTAMPTZ); + + to_sql_checked!(); +} + +impl ToSqlText for PgTimestamp { + fn to_sql_text( + &self, + _ty: &Type, + out: &mut BytesMut, + _format_options: &FormatOptions, + ) -> Result> { + out.put_slice(self.0.to_iso8601().as_bytes()); + Ok(IsNull::No) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use pgwire::api::results::FieldInfo; + + use super::*; + use crate::control::server::pgwire::ddl_encode::col_type_to_field_with_format; + + /// The one instant these tests use: 2020-03-05T10:00:00Z. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + /// Encode one cell under a one-column schema and return its raw field + /// bytes, or `None` for SQL NULL. + fn encode_one(ct: DdlColType, format: FieldFormat, v: &Value) -> PgWireResult>> { + let schema: Arc> = + Arc::new(vec![col_type_to_field_with_format("c", ct, format)]); + let mut encoder = DataRowEncoder::new(schema); + encode_cell(&mut encoder, "c", ct, format, v)?; + let row = encoder.take_row(); + let data = &row.data; + let len = i32::from_be_bytes([data[0], data[1], data[2], data[3]]); + if len < 0 { + return Ok(None); + } + Ok(Some(data[4..4 + len as usize].to_vec())) + } + + fn encode_text(ct: DdlColType, v: &Value) -> PgWireResult> { + Ok(encode_one(ct, FieldFormat::Text, v)? + .map(|b| String::from_utf8(b).expect("text cell is UTF-8"))) + } + + /// The SQLSTATE and message an encode error carries. + fn error_of(err: PgWireError) -> (String, String) { + let PgWireError::UserError(info) = err else { + panic!("expected a UserError, got {err:?}"); + }; + (info.code.clone(), info.message.clone()) + } + + #[test] + fn null_is_sql_null_under_every_type_and_format() { + for ct in [ + DdlColType::Text, + DdlColType::Int8, + DdlColType::Float8, + DdlColType::Bool, + DdlColType::Timestamp, + ] { + for format in [FieldFormat::Text, FieldFormat::Binary] { + assert_eq!( + encode_one(ct, format, &Value::Null).expect("null encodes"), + None, + "{ct:?}/{format:?} must encode NULL" + ); + } + } + } + + /// An integer under a timestamp column is an error naming the column, + /// under both formats. + #[test] + fn integer_under_timestamp_is_an_error_naming_the_column() { + for ct in [DdlColType::Timestamp, DdlColType::Timestamptz] { + for format in [FieldFormat::Text, FieldFormat::Binary] { + let err = encode_one(ct, format, &Value::Integer(EARLY_MICROS)) + .expect_err("an integer carries no unit"); + let (_, message) = error_of(err); + assert!( + message.contains("column \"c\" holds an integer where a timestamp is required"), + "{ct:?}/{format:?}: {message}" + ); + } + } + } + + #[test] + fn typed_instant_under_timestamp_renders_iso8601() { + let at = NdbDateTime::from_micros(EARLY_MICROS); + assert_eq!( + encode_text(DdlColType::Timestamp, &Value::NaiveDateTime(at)) + .expect("encodes") + .as_deref(), + Some("2020-03-05T10:00:00.000000Z") + ); + assert_eq!( + encode_text(DdlColType::Timestamptz, &Value::DateTime(at)) + .expect("encodes") + .as_deref(), + Some("2020-03-05T10:00:00.000000Z") + ); + } + + /// An ISO string re-parses, so its rendering is canonical, not verbatim. + #[test] + fn iso_text_under_timestamp_renders_canonical_iso8601() { + assert_eq!( + encode_text( + DdlColType::Timestamp, + &Value::String("2020-03-05 10:00:00".into()) + ) + .expect("encodes") + .as_deref(), + Some("2020-03-05T10:00:00.000000Z") + ); + } + + /// Binary `timestamp` is a big-endian `i64` of microseconds since + /// 2000-01-01, under both timestamp types. + #[test] + fn binary_timestamp_is_pg_epoch_micros_big_endian() { + let at = NdbDateTime::from_micros(EARLY_MICROS); + let expected = (EARLY_MICROS - PG_EPOCH_OFFSET_MICROS) + .to_be_bytes() + .to_vec(); + assert_eq!( + encode_one( + DdlColType::Timestamp, + FieldFormat::Binary, + &Value::NaiveDateTime(at) + ) + .expect("encodes"), + Some(expected.clone()) + ); + assert_eq!( + encode_one( + DdlColType::Timestamptz, + FieldFormat::Binary, + &Value::DateTime(at) + ) + .expect("encodes"), + Some(expected) + ); + } + + /// Binary `timestamp` reads back as the same instant through + /// postgres-types' own `SystemTime` decoder. + #[test] + fn binary_timestamp_round_trips_through_postgres_types() { + use postgres_types::FromSql; + + let at = NdbDateTime::from_micros(EARLY_MICROS); + let bytes = encode_one( + DdlColType::Timestamp, + FieldFormat::Binary, + &Value::NaiveDateTime(at), + ) + .expect("encodes") + .expect("not null"); + let decoded = std::time::SystemTime::from_sql(&Type::TIMESTAMP, &bytes).expect("decodes"); + assert_eq!( + decoded, + std::time::UNIX_EPOCH + std::time::Duration::from_micros(EARLY_MICROS as u64) + ); + } + + /// The text arms render the pinned PostgreSQL text forms. + #[test] + fn text_arms_render_postgres_text() { + assert_eq!( + encode_text(DdlColType::Bool, &Value::Bool(true)) + .expect("encodes") + .as_deref(), + Some("t") + ); + assert_eq!( + encode_text(DdlColType::Int8, &Value::Integer(42)) + .expect("encodes") + .as_deref(), + Some("42") + ); + // Float columns go through pgwire's float encoder: shortest form. + assert_eq!( + encode_text(DdlColType::Float8, &Value::Float(0.0)) + .expect("encodes") + .as_deref(), + Some("0") + ); + // A text column keeps the JSON text of a float. + assert_eq!( + encode_text(DdlColType::Text, &Value::Float(0.0)) + .expect("encodes") + .as_deref(), + Some("0.0") + ); + assert_eq!( + encode_text(DdlColType::Text, &Value::Bytes(vec![0, 255, 7])) + .expect("encodes") + .as_deref(), + Some("AP8H") + ); + } + + /// A finite `f64` beyond `f32` range is refused under `real`, in text + /// and binary alike. + #[test] + fn float4_overflow_is_refused_in_both_formats() { + for format in [FieldFormat::Text, FieldFormat::Binary] { + let err = encode_one(DdlColType::Float4, format, &Value::Float(1e39)) + .expect_err("overflow must be refused"); + assert_eq!(error_of(err).0, "22003"); + } + } + + /// A binary scalar column takes only its own shape. + #[test] + fn binary_scalar_shape_mismatch_is_an_error() { + let text = Value::String("42".into()); + for (ct, expected) in [ + (DdlColType::Int8, "an integer"), + (DdlColType::Int4, "an integer"), + (DdlColType::Float8, "a float"), + (DdlColType::Bool, "a bool"), + ] { + let err = encode_one(ct, FieldFormat::Binary, &text) + .expect_err("text under a binary scalar column must be refused"); + let (_, message) = error_of(err); + assert!( + message.contains(&format!("holds text where {expected} is required")), + "{ct:?}: {message}" + ); + } + } + + /// Text under a text-format integer column renders verbatim: a + /// schemaless row can hold `"42"` under a declared `INT`. + #[test] + fn text_under_text_format_integer_renders_verbatim() { + assert_eq!( + encode_text(DdlColType::Int8, &Value::String("42".into())) + .expect("encodes") + .as_deref(), + Some("42") + ); + } + + /// Binary integer narrowing is range-checked. + #[test] + fn binary_narrowing_rejects_out_of_range() { + let err = encode_one( + DdlColType::Int2, + FieldFormat::Binary, + &Value::Integer(i16::MAX as i64 + 1), + ) + .expect_err("out of range must be refused"); + assert_eq!(error_of(err).0, "22003"); + assert_eq!( + encode_one(DdlColType::Int2, FieldFormat::Binary, &Value::Integer(7)).expect("encodes"), + Some(7i16.to_be_bytes().to_vec()) + ); + } +} diff --git a/nodedb/src/control/server/pgwire/handler/shape_encode/mod.rs b/nodedb/src/control/server/pgwire/handler/shape_encode/mod.rs new file mode 100644 index 000000000..2c53367a2 --- /dev/null +++ b/nodedb/src/control/server/pgwire/handler/shape_encode/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Encode a protocol-neutral [`ShapedRows`](crate::control::server::response_shape::types::ShapedRows) +//! into pgwire `DataRow`s and a `Response::Query`. + +pub mod cell; +pub mod response; + +pub(in crate::control::server::pgwire) use cell::encode_cell; +pub(in crate::control::server::pgwire) use response::{encode_shaped_row, shaped_query_response}; diff --git a/nodedb/src/control/server/pgwire/handler/shape_encode/response.rs b/nodedb/src/control/server/pgwire/handler/shape_encode/response.rs new file mode 100644 index 000000000..0e1249e05 --- /dev/null +++ b/nodedb/src/control/server/pgwire/handler/shape_encode/response.rs @@ -0,0 +1,482 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Encode a protocol-neutral [`ShapedRows`] (from +//! `response_shape::types`/`response_shape::project`) into a pgwire +//! `Response::Query`. +//! +//! This is the pgwire entrypoint's encoder for the canonical neutral shaping +//! core: the SELECT-read path builds a `ShapedRows` once and every protocol +//! entrypoint (pgwire, native, http) renders it in its own wire format. Here, +//! each typed cell renders per the `DdlColType` the shaper threaded through +//! `ShapedRows`, through the one cell encoder [`encode_cell`]. + +use std::sync::Arc; + +use pgwire::api::results::{DataRowEncoder, FieldFormat, FieldInfo, QueryResponse, Response}; +use pgwire::error::PgWireResult; +use pgwire::messages::data::DataRow; + +use crate::control::server::pgwire::ddl_encode::col_type_to_field_with_format; +use crate::control::server::response_shape::types::{DdlColType, ShapedRow, ShapedRows}; + +use super::cell::encode_cell; + +/// Encode one flat row object into a pgwire `DataRow`, using `cell_keys` (in +/// order) to look up cells in `row` and `column_types` (parallel to +/// `cell_keys`) to pick each cell's rendering. +/// +/// `cell_keys` are the per-column unique row-map keys derived from the +/// display names via `response_shape::project::cell_keys` — identical to the +/// display names except where those repeat (`SELECT w.id, b.id`), in which +/// case later duplicates carry a `_n` suffix so both cells survive the map. +/// +/// A missing key encodes as SQL NULL. Every present cell renders per its +/// column type via [`encode_cell`]; a missing/short `column_types` entry +/// defaults to `Text`. +pub(in crate::control::server::pgwire) fn encode_shaped_row( + schema: &Arc>, + cell_keys: &[String], + column_types: &[DdlColType], + formats: &[FieldFormat], + row: &ShapedRow, +) -> PgWireResult { + let mut encoder = DataRowEncoder::new(schema.clone()); + for (idx, name) in cell_keys.iter().enumerate() { + let ct = column_types.get(idx).copied().unwrap_or(DdlColType::Text); + let format = formats.get(idx).copied().unwrap_or(FieldFormat::Text); + match row.get(name) { + None => encoder.encode_field(&None::<&str>)?, + Some(v) => encode_cell(&mut encoder, name, ct, format, v)?, + } + } + Ok(encoder.take_row()) +} + +/// Build a `Response::Query` from a protocol-neutral [`ShapedRows`], plus its +/// carried client-facing notice. +/// +/// Unlike `ddl_encode::rows_to_response` (which intentionally drops the +/// notice — the pgwire DDL router never attached one to a `Response::Query`), +/// this path preserves `notice`: the caller is expected to surface it via +/// `sessions.push_notice`. +pub(in crate::control::server::pgwire) fn shaped_query_response( + shaped: ShapedRows, + formats: &[FieldFormat], +) -> (Response, Option) { + // Cells live in the row maps under per-column keys that differ from the + // display names only when two columns share a name; derived here before + // the struct is destructured. + let keys = shaped.cell_keys(); + let ShapedRows { + columns, + column_types, + rows, + notice, + } = shaped; + + let fields: Vec = columns + .iter() + .enumerate() + .map(|(i, name)| { + let ct = column_types.get(i).copied().unwrap_or(DdlColType::Text); + let format = formats.get(i).copied().unwrap_or(FieldFormat::Text); + col_type_to_field_with_format(name, ct, format) + }) + .collect(); + let schema = Arc::new(fields); + + let encoded_rows: Vec> = rows + .iter() + .map(|row| encode_shaped_row(&schema, &keys, &column_types, formats, row)) + .collect(); + + let response = Response::Query(QueryResponse::new( + schema, + futures::stream::iter(encoded_rows), + )); + (response, notice) +} + +#[cfg(test)] +mod tests { + use futures::StreamExt; + use nodedb_types::Value; + use pgwire::api::results::{FieldFormat, QueryResponse, Response}; + use pgwire::error::PgWireError; + + use super::shaped_query_response; + use crate::control::server::response_shape::types::{DdlColType, ShapedRow, ShapedRows}; + + type DataRow = pgwire::messages::data::DataRow; + + /// Drain a `QueryResponse` stream into a `Vec` of per-row results. + async fn drain_results(mut qr: QueryResponse) -> Vec> { + let mut rows = Vec::new(); + while let Some(r) = qr.data_rows.next().await { + rows.push(r); + } + rows + } + + /// Drain a `QueryResponse` stream into a `Vec` of `DataRow`s. + async fn drain(qr: QueryResponse) -> Vec { + drain_results(qr) + .await + .into_iter() + .map(|r| r.unwrap()) + .collect() + } + + /// Read the raw bytes of field `idx` from a `DataRow` (or `None` for SQL + /// NULL), without assuming UTF-8 — used to inspect binary-format cells. + /// + /// Wire format: 4-byte big-endian length + bytes per field; a negative + /// length denotes SQL NULL. + fn field_bytes(row: &DataRow, idx: usize) -> Option> { + let data = &row.data; + let mut offset = 0usize; + for field_i in 0..=idx { + if offset + 4 > data.len() { + return None; + } + let len = i32::from_be_bytes([ + data[offset], + data[offset + 1], + data[offset + 2], + data[offset + 3], + ]); + offset += 4; + if len < 0 { + if field_i == idx { + return None; + } + continue; + } + let len = len as usize; + if offset + len > data.len() { + return None; + } + if field_i == idx { + return Some(data[offset..offset + len].to_vec()); + } + offset += len; + } + None + } + + /// Read the text value of field `idx` from a `DataRow`'s raw wire buffer. + fn field_text(row: &DataRow, idx: usize) -> Option { + field_bytes(row, idx).map(|b| String::from_utf8(b).unwrap()) + } + + fn make_shaped(columns: &[&str], rows: Vec) -> ShapedRows { + let columns: Vec = columns.iter().map(|s| s.to_string()).collect(); + let column_types = ShapedRows::text_types(columns.len()); + ShapedRows { + columns, + column_types, + rows, + notice: None, + } + } + + fn shaped_typed(columns: &[&str], column_types: Vec, row: ShapedRow) -> ShapedRows { + ShapedRows { + columns: columns.iter().map(|s| s.to_string()).collect(), + column_types, + rows: vec![row], + notice: None, + } + } + + fn obj(pairs: &[(&str, Value)]) -> ShapedRow { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.clone())) + .collect() + } + + fn text(s: &str) -> Value { + Value::String(s.to_string()) + } + + fn query_of(response: Response) -> QueryResponse { + let Response::Query(qr) = response else { + panic!("expected Query response"); + }; + qr + } + + #[tokio::test] + async fn string_cell_renders_verbatim() { + let shaped = make_shaped(&["a"], vec![obj(&[("a", text("hello"))])]); + let (response, notice) = shaped_query_response(shaped, &[]); + assert!(notice.is_none()); + let rows = drain(query_of(response)).await; + assert_eq!(field_text(&rows[0], 0).as_deref(), Some("hello")); + } + + #[tokio::test] + async fn bool_cells_render_as_t_f_not_true_false() { + let shaped = make_shaped( + &["a"], + vec![ + obj(&[("a", Value::Bool(true))]), + obj(&[("a", Value::Bool(false))]), + ], + ); + let (response, _notice) = shaped_query_response(shaped, &[]); + let rows = drain(query_of(response)).await; + assert_eq!(field_text(&rows[0], 0).as_deref(), Some("t")); + assert_eq!(field_text(&rows[1], 0).as_deref(), Some("f")); + } + + #[tokio::test] + async fn number_cells_render_via_to_string() { + let shaped = make_shaped( + &["a"], + vec![ + obj(&[("a", Value::Integer(42))]), + obj(&[("a", Value::Float(0.0))]), + ], + ); + let (response, _notice) = shaped_query_response(shaped, &[]); + let rows = drain(query_of(response)).await; + assert_eq!(field_text(&rows[0], 0).as_deref(), Some("42")); + assert_eq!(field_text(&rows[1], 0).as_deref(), Some("0.0")); + } + + #[tokio::test] + async fn null_and_missing_column_both_encode_as_sql_null() { + let shaped = make_shaped(&["a", "b"], vec![obj(&[("a", Value::Null)])]); + let (response, _notice) = shaped_query_response(shaped, &[]); + let rows = drain(query_of(response)).await; + // "a" was an explicit NULL cell. + assert_eq!(field_text(&rows[0], 0), None); + // "b" was entirely absent from the row object. + assert_eq!(field_text(&rows[0], 1), None); + } + + #[tokio::test] + async fn column_order_is_preserved() { + let shaped = make_shaped( + &["b", "a"], + vec![obj(&[("a", text("first")), ("b", text("second"))])], + ); + let (response, _notice) = shaped_query_response(shaped, &[]); + let rows = drain(query_of(response)).await; + assert_eq!(field_text(&rows[0], 0).as_deref(), Some("second")); + assert_eq!(field_text(&rows[0], 1).as_deref(), Some("first")); + } + + /// A user `SELECT` of typed columns on the simple-query path must report + /// the correct RowDescription type OID AND render each cell in that type's + /// PostgreSQL text form — the two halves that must land together. + #[tokio::test] + async fn typed_columns_report_correct_oid_and_text() { + use pgwire::api::Type; + + let at = nodedb_types::NdbDateTime::from_micros(0); + let shaped = shaped_typed( + &["i", "f", "b", "ts"], + vec![ + DdlColType::Int8, + DdlColType::Float8, + DdlColType::Bool, + DdlColType::Timestamp, + ], + obj(&[ + ("i", Value::Integer(42)), + // Integral float renders Postgres-style "0" (shortest form) via + // the native float encoder, not serde's "0.0". + ("f", Value::Float(0.0)), + ("b", Value::Bool(true)), + // The Unix epoch as a typed instant → ISO-8601 text. + ("ts", Value::NaiveDateTime(at)), + ]), + ); + + let (response, _notice) = shaped_query_response(shaped, &[]); + let qr = query_of(response); + // RowDescription OIDs are the typed ones, not TEXT. + let schema = qr.row_schema.clone(); + assert_eq!(schema[0].datatype(), &Type::INT8); + assert_eq!(schema[1].datatype(), &Type::FLOAT8); + assert_eq!(schema[2].datatype(), &Type::BOOL); + assert_eq!(schema[3].datatype(), &Type::TIMESTAMP); + + let rows = drain(qr).await; + assert_eq!(field_text(&rows[0], 0).as_deref(), Some("42")); + assert_eq!(field_text(&rows[0], 1).as_deref(), Some("0")); + assert_eq!(field_text(&rows[0], 2).as_deref(), Some("t")); + assert_eq!( + field_text(&rows[0], 3).as_deref(), + Some("1970-01-01T00:00:00.000000Z") + ); + } + + /// An instant cell renders as its ISO-8601 text under a TEXT column and + /// under a TIMESTAMP column alike, and a byte cell as unpadded base64. + #[tokio::test] + async fn instant_and_byte_cells_render_through_the_edge_conversion() { + let at = nodedb_types::NdbDateTime::from_micros(1_583_402_400_000_000); + let shaped = shaped_typed( + &["t", "ts", "blob"], + vec![DdlColType::Text, DdlColType::Timestamp, DdlColType::Text], + obj(&[ + ("t", Value::NaiveDateTime(at)), + ("ts", Value::NaiveDateTime(at)), + ("blob", Value::Bytes(vec![0, 255, 7])), + ]), + ); + let (response, _notice) = shaped_query_response(shaped, &[]); + let rows = drain(query_of(response)).await; + assert_eq!( + field_text(&rows[0], 0).as_deref(), + Some("2020-03-05T10:00:00.000000Z") + ); + assert_eq!( + field_text(&rows[0], 1).as_deref(), + Some("2020-03-05T10:00:00.000000Z") + ); + assert_eq!(field_text(&rows[0], 2).as_deref(), Some("AP8H")); + } + + /// An ISO string under a TIMESTAMP column re-parses, so the row renders + /// the canonical ISO-8601 form, not the stored spelling. + #[tokio::test] + async fn iso_text_under_timestamp_renders_canonical_iso8601() { + let shaped = shaped_typed( + &["ts"], + vec![DdlColType::Timestamp], + obj(&[("ts", text("2020-03-05 10:00:00"))]), + ); + let (response, _notice) = shaped_query_response(shaped, &[]); + let rows = drain(query_of(response)).await; + assert_eq!( + field_text(&rows[0], 0).as_deref(), + Some("2020-03-05T10:00:00.000000Z") + ); + } + + /// An integer under a TIMESTAMP column is a row-encode error naming the + /// column — never a guessed epoch unit. + #[tokio::test] + async fn integer_under_timestamp_column_is_an_error_naming_the_column() { + let shaped = shaped_typed( + &["created_at"], + vec![DdlColType::Timestamp], + obj(&[("created_at", Value::Integer(1_583_402_400_000_000))]), + ); + let (response, _notice) = shaped_query_response(shaped, &[]); + let results = drain_results(query_of(response)).await; + let err = results + .into_iter() + .next() + .expect("one row") + .expect_err("an integer under a timestamp column must not encode"); + let PgWireError::UserError(info) = err else { + panic!("expected a UserError, got {err:?}"); + }; + assert!( + info.message + .contains("column \"created_at\" holds an integer where a timestamp is required"), + "message must name the column, got: {}", + info.message + ); + } + + /// Binary `timestamp` cells are the big-endian `i64` of microseconds + /// since the PostgreSQL epoch (2000-01-01), and the RowDescription + /// advertises `FieldFormat::Binary`. + #[tokio::test] + async fn binary_timestamp_encodes_pg_epoch_micros() { + let micros = 1_583_402_400_000_000i64; + let at = nodedb_types::NdbDateTime::from_micros(micros); + let shaped = shaped_typed( + &["ts", "tstz"], + vec![DdlColType::Timestamp, DdlColType::Timestamptz], + obj(&[ + ("ts", Value::NaiveDateTime(at)), + ("tstz", Value::DateTime(at)), + ]), + ); + let formats = vec![FieldFormat::Binary; 2]; + let (response, _notice) = shaped_query_response(shaped, &formats); + let qr = query_of(response); + for f in qr.row_schema.iter() { + assert_eq!(f.format(), FieldFormat::Binary); + } + let rows = drain(qr).await; + let expected = (micros - 946_684_800_000_000).to_be_bytes().to_vec(); + assert_eq!(field_bytes(&rows[0], 0), Some(expected.clone())); + assert_eq!(field_bytes(&rows[0], 1), Some(expected)); + } + + #[tokio::test] + async fn notice_is_preserved_not_dropped() { + let mut shaped = make_shaped(&["a"], vec![obj(&[("a", text("x"))])]); + shaped.notice = Some("heads up".to_owned()); + let (_response, notice) = shaped_query_response(shaped, &[]); + assert_eq!(notice.as_deref(), Some("heads up")); + } + + /// A binary-format request for the supported scalar types encodes each + /// cell in its PostgreSQL binary wire form (big-endian), and the + /// RowDescription advertises `FieldFormat::Binary`. + #[tokio::test] + async fn binary_format_encodes_scalar_wire_bytes() { + let shaped = shaped_typed( + &["i", "f", "b", "t"], + vec![ + DdlColType::Int8, + DdlColType::Float8, + DdlColType::Bool, + DdlColType::Text, + ], + obj(&[ + ("i", Value::Integer(42)), + ("f", Value::Float(1.5)), + ("b", Value::Bool(true)), + ("t", text("hello")), + ]), + ); + let formats = vec![FieldFormat::Binary; 4]; + let (response, _notice) = shaped_query_response(shaped, &formats); + let qr = query_of(response); + // RowDescription advertises Binary for every column. + for f in qr.row_schema.iter() { + assert_eq!(f.format(), FieldFormat::Binary); + } + let rows = drain(qr).await; + // int8 -> 8-byte big-endian. + assert_eq!(field_bytes(&rows[0], 0), Some(42i64.to_be_bytes().to_vec())); + // float8 -> IEEE-754 big-endian bits. + assert_eq!( + field_bytes(&rows[0], 1), + Some(1.5f64.to_be_bytes().to_vec()) + ); + // bool -> single byte 0x01. + assert_eq!(field_bytes(&rows[0], 2), Some(vec![1u8])); + // text -> raw UTF-8 bytes (binary text is identical bytes). + assert_eq!(field_bytes(&rows[0], 3), Some(b"hello".to_vec())); + } + + /// A per-column format vector: only the columns whose format is Binary are + /// binary-encoded; the rest stay text. Mirrors an `Individual` Bind. + #[tokio::test] + async fn mixed_formats_are_per_column() { + let shaped = shaped_typed( + &["i", "j"], + vec![DdlColType::Int8, DdlColType::Int8], + obj(&[("i", Value::Integer(7)), ("j", Value::Integer(9))]), + ); + let formats = vec![FieldFormat::Binary, FieldFormat::Text]; + let (response, _notice) = shaped_query_response(shaped, &formats); + let rows = drain(query_of(response)).await; + // Column 0 binary: 8 raw bytes. + assert_eq!(field_bytes(&rows[0], 0), Some(7i64.to_be_bytes().to_vec())); + // Column 1 text: ASCII "9". + assert_eq!(field_text(&rows[0], 1).as_deref(), Some("9")); + } +} diff --git a/nodedb/src/control/server/pgwire/handler/stream_response.rs b/nodedb/src/control/server/pgwire/handler/stream_response.rs index 84103a59a..847242dfb 100644 --- a/nodedb/src/control/server/pgwire/handler/stream_response.rs +++ b/nodedb/src/control/server/pgwire/handler/stream_response.rs @@ -19,7 +19,7 @@ use crate::control::server::response_shape::types::DdlColType; use crate::control::server::result_stream::ResultStream; use crate::control::server::shared::metering::DetachedMeterGuard; use crate::control::state::SharedState; -use crate::data::executor::response_codec::decode_payload_to_json; +use crate::data::executor::response_codec::{decode_payload_to_json, decode_payload_value}; use super::super::ddl_encode::col_type_to_field_with_format; use super::super::types::{error_to_sqlstate, text_field}; @@ -214,8 +214,7 @@ pub(crate) fn streaming_shaped_response( break; } - let text = decode_payload_to_json(&batch.payload); - let value = sonic_rs::from_str::(&text).map_err(|e| { + let value = decode_payload_value(&batch.payload).map_err(|e| { PgWireError::UserError(Box::new(ErrorInfo::new( "ERROR".to_owned(), "XX000".to_owned(), @@ -225,7 +224,7 @@ pub(crate) fn streaming_shaped_response( // Resolved once before the first batch was pulled; this only // re-borrows it, so no batch can slip out ahead of the policy. let shaped = shape_decoded_rows( - &value, + value, Some(&schema_out), redaction.as_ref().map(|r| r.ctx(&state.redaction)), ) @@ -279,7 +278,7 @@ pub(crate) async fn streaming_star_response( ) -> Response { use futures::StreamExt; - let mut values: Vec = Vec::new(); + let mut values: Vec = Vec::new(); let mut batches = stream; while let Some(batch) = batches.next().await { let batch = match batch { @@ -298,9 +297,8 @@ pub(crate) async fn streaming_star_response( break; } - let text = decode_payload_to_json(&batch.payload); - match sonic_rs::from_str::(&text) { - Ok(serde_json::Value::Array(items)) => { + match decode_payload_value(&batch.payload) { + Ok(nodedb_types::Value::Array(items)) => { for item in items { if values.len() >= limit { break; @@ -312,7 +310,7 @@ pub(crate) async fn streaming_star_response( return single_pgwire_error(PgWireError::UserError(Box::new(ErrorInfo::new( "ERROR".to_owned(), "XX000".to_owned(), - "streamed batch payload was not a JSON array".to_owned(), + "streamed batch payload was not a row array".to_owned(), )))); } Err(e) => { @@ -340,7 +338,7 @@ pub(crate) async fn streaming_star_response( } let shaped = match shape_decoded_rows( - &serde_json::Value::Array(values), + nodedb_types::Value::Array(values), None, redaction.as_ref().map(|r| r.ctx(&state.redaction)), ) { diff --git a/nodedb/src/control/server/pgwire/numeric_narrow.rs b/nodedb/src/control/server/pgwire/numeric_narrow.rs index 01f3d2aa8..dffa427ee 100644 --- a/nodedb/src/control/server/pgwire/numeric_narrow.rs +++ b/nodedb/src/control/server/pgwire/numeric_narrow.rs @@ -32,9 +32,8 @@ use pgwire::error::PgWireResult; /// Range-check an integer cell against the width its column advertises, before /// it is narrowed for transmission. /// -/// `Ok(None)` means the cell is absent or not an integer and encodes as SQL -/// NULL, matching the wider `Int8` arm. `Ok(Some(n))` guarantees `n` fits -/// `width`, so the caller's narrowing cast is lossless by construction. +/// `Ok(n)` guarantees `n` fits `width`, so the caller's narrowing cast is +/// lossless by construction. /// /// An out-of-range value is a hard error rather than a truncation: the /// column's `RowDescription` already told the client to read two or four @@ -49,14 +48,11 @@ use pgwire::error::PgWireResult; /// exist: rows written before a column's width was declared, and rows /// arriving over non-SQL ingest paths, are not covered by that check. pub(in crate::control::server::pgwire) fn checked_narrow( - v: &serde_json::Value, + n: i64, width: IntWidth, -) -> PgWireResult> { - let Some(n) = v.as_i64() else { - return Ok(None); - }; +) -> PgWireResult { if width.contains(n) { - return Ok(Some(n)); + return Ok(n); } Err(out_of_range(format!( "value {n} is out of range for type {}", @@ -67,9 +63,6 @@ pub(in crate::control::server::pgwire) fn checked_narrow( /// Narrow a float cell to the `f32` a `real` column transmits, refusing the /// one narrowing that is not value-preserving. /// -/// `Ok(None)` means the cell is absent or not a number and encodes as SQL -/// NULL, matching the wider `Float8` arm. -/// /// This is deliberately *not* the float mirror of [`checked_narrow`]'s range /// constraint — see the module docs. Rounding is correct and never an error; /// only a finite `f64` overflowing to infinity is refused, under the same @@ -83,12 +76,7 @@ pub(in crate::control::server::pgwire) fn checked_narrow( /// exist, layered underneath rather than replaced by it: rows written before a /// column's width was declared, and rows arriving over non-SQL ingest paths, /// are not covered by that check. -pub(in crate::control::server::pgwire) fn checked_narrow_f32( - v: &serde_json::Value, -) -> PgWireResult> { - let Some(f) = v.as_f64() else { - return Ok(None); - }; +pub(in crate::control::server::pgwire) fn checked_narrow_f32(f: f64) -> PgWireResult { let narrowed = f as f32; if f.is_finite() && !narrowed.is_finite() { return Err(out_of_range(format!( @@ -96,7 +84,7 @@ pub(in crate::control::server::pgwire) fn checked_narrow_f32( FloatWidth::F32.pg_type_name() ))); } - Ok(Some(narrowed)) + Ok(narrowed) } /// A pgwire `22003` (`numeric_value_out_of_range`) error — the SQLSTATE @@ -113,7 +101,6 @@ fn out_of_range(message: String) -> pgwire::error::PgWireError { #[cfg(test)] mod tests { use super::*; - use serde_json::json; /// The SQLSTATE a guard raised, or a panic naming what it raised instead. fn sqlstate_of(err: pgwire::error::PgWireError) -> String { @@ -126,46 +113,28 @@ mod tests { #[test] fn integer_inside_declared_width_passes_through() { assert_eq!( - checked_narrow(&json!(i16::MAX as i64), IntWidth::I16).expect("in range"), - Some(i16::MAX as i64) - ); - assert_eq!( - checked_narrow(&json!(-1), IntWidth::I32).expect("in range"), - Some(-1) + checked_narrow(i16::MAX as i64, IntWidth::I16).expect("in range"), + i16::MAX as i64 ); + assert_eq!(checked_narrow(-1, IntWidth::I32).expect("in range"), -1); } #[test] fn integer_outside_declared_width_is_rejected() { - let err = checked_narrow(&json!(i16::MAX as i64 + 1), IntWidth::I16) + let err = checked_narrow(i16::MAX as i64 + 1, IntWidth::I16) .expect_err("one past the boundary must be refused"); assert_eq!(sqlstate_of(err), "22003"); } - #[test] - fn non_integer_cell_encodes_as_null() { - assert_eq!( - checked_narrow(&json!(null), IntWidth::I16).expect("null is not an error"), - None - ); - assert_eq!( - checked_narrow(&json!("x"), IntWidth::I16).expect("text is not an error"), - None - ); - } - /// Narrowing rounds, and rounding is not an error: PostgreSQL's `real` /// stores `1.1` as `1.10000002`. Rejecting that would make `REAL` columns /// unreadable for almost every value they hold. #[test] fn float_narrowing_rounds_without_erroring() { - for v in [json!(1.1), json!(0.0), json!(-2.5), json!(3.4e38)] { - let narrowed = checked_narrow_f32(&v) - .expect("an in-range value must narrow without error") - .expect("a JSON number must narrow to Some"); + for v in [1.1, 0.0, -2.5, 3.4e38] { + let narrowed = checked_narrow_f32(v).expect("an in-range value must narrow"); assert_eq!( - narrowed, - v.as_f64().expect("test value is a number") as f32, + narrowed, v as f32, "{v} must narrow by the ordinary rounding cast" ); } @@ -175,9 +144,9 @@ mod tests { /// number silently replaced by infinity on the wire. #[test] fn finite_float_overflowing_f32_is_rejected() { - for v in [json!(1e39), json!(-1e39), json!(f64::MAX), json!(f64::MIN)] { - let err = checked_narrow_f32(&v) - .expect_err("a finite value beyond f32 range must be refused"); + for v in [1e39, -1e39, f64::MAX, f64::MIN] { + let err = + checked_narrow_f32(v).expect_err("a finite value beyond f32 range must be refused"); assert_eq!(sqlstate_of(err), "22003", "{v} must report 22003"); } } @@ -186,8 +155,7 @@ mod tests { /// `f64` nodedb actually stores. #[test] fn float_overflow_error_names_the_declared_type() { - let err = - checked_narrow_f32(&json!(1e300)).expect_err("1e300 must overflow single precision"); + let err = checked_narrow_f32(1e300).expect_err("1e300 must overflow single precision"); let pgwire::error::PgWireError::UserError(info) = err else { panic!("expected a UserError carrying a SQLSTATE"); }; @@ -198,17 +166,18 @@ mod tests { ); } - /// A cell that is not a number at all encodes as SQL NULL, exactly as the - /// wider `Float8` arm does — never an error. + /// A value already infinite or NaN is representable in both widths and + /// passes through as itself, never as an overflow. #[test] - fn non_numeric_float_cell_encodes_as_null() { - assert_eq!( - checked_narrow_f32(&json!(null)).expect("null is not an error"), - None + fn non_finite_float_passes_through() { + assert!( + checked_narrow_f32(f64::NAN) + .expect("NaN is not an overflow") + .is_nan() ); assert_eq!( - checked_narrow_f32(&json!("Infinity")).expect("string is not an error"), - None + checked_narrow_f32(f64::INFINITY).expect("inf is not an overflow"), + f32::INFINITY ); } } diff --git a/nodedb/src/control/server/response_shape/cell.rs b/nodedb/src/control/server/response_shape/cell.rs new file mode 100644 index 000000000..c3c977d21 --- /dev/null +++ b/nodedb/src/control/server/response_shape/cell.rs @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Cell-level conversions a protocol entrypoint renders a shaped row through. +//! +//! The scalar JSON conversion, [`value_to_wire_json`], lives in +//! [`crate::util::wire_json`] — a neutral home so `control::security` +//! (which `response_shape` depends on for redaction) never has to depend +//! back on `control::server`. This module re-exports it and adds the +//! row-level helper, which depends on [`ShapedRow`], plus the two typed +//! readings pgwire renders a cell from: its PostgreSQL text form +//! ([`cell_text`]) and, for a timestamp column, the instant it denotes +//! ([`instant_of`]). + +use base64::Engine; +use nodedb_types::error::NodeDbError; +use nodedb_types::{NdbDateTime, Value}; + +use super::types::ShapedRow; + +pub use crate::util::wire_json::value_to_wire_json; + +/// Render one shaped row as a JSON object, cell by cell. +pub fn row_to_wire_json(row: &ShapedRow) -> serde_json::Map { + row.iter() + .map(|(k, v)| (k.clone(), value_to_wire_json(v))) + .collect() +} + +/// The PostgreSQL text form of one cell, or `None` for SQL NULL. +/// +/// `None` covers `Value::Null` and every shape with no JSON form: a +/// non-finite float, a range, a record. Scalars render directly — a string +/// verbatim, a bool as `t`/`f`, an integer as its decimal digits, a finite +/// float as its shortest round-trip JSON text (`0.0` stays `0.0`), an +/// instant as ISO-8601, bytes as unpadded standard base64. Every other shape +/// renders through [`value_to_wire_json`], so its text is the JSON a text +/// protocol emits for it. +pub fn cell_text(v: &Value) -> Option { + match v { + Value::Null => None, + Value::Bool(b) => Some(if *b { "t" } else { "f" }.to_owned()), + Value::Integer(i) => Some(i.to_string()), + Value::Float(f) => serde_json::Number::from_f64(*f).map(|n| n.to_string()), + Value::String(s) => Some(s.clone()), + Value::Bytes(bytes) => Some(base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes)), + Value::DateTime(at) | Value::NaiveDateTime(at) => Some(at.to_iso8601()), + Value::Array(_) + | Value::Object(_) + | Value::Uuid(_) + | Value::Ulid(_) + | Value::Duration(_) + | Value::Decimal(_) + | Value::Geometry(_) + | Value::Set(_) + | Value::Regex(_) + | Value::Range { .. } + | Value::Record { .. } + | Value::ArrayCell(_) + | Value::Vector(_) => wire_json_text(&value_to_wire_json(v)), + // `Value` is `#[non_exhaustive]`: a variant this crate cannot name + // renders through its wire JSON, like the composite arm above. + _ => wire_json_text(&value_to_wire_json(v)), + } +} + +/// The PostgreSQL text form of a wire-JSON cell, or `None` for JSON null. +/// +/// A string is verbatim, a bool is `t`/`f`, and every other JSON value is +/// its `Display` text. +fn wire_json_text(v: &serde_json::Value) -> Option { + match v { + serde_json::Value::Null => None, + serde_json::Value::String(s) => Some(s.clone()), + serde_json::Value::Bool(b) => Some(if *b { "t" } else { "f" }.to_owned()), + other => Some(other.to_string()), + } +} + +/// The instant a cell under a timestamp column denotes. +/// +/// A typed instant is itself. A string is the instant it parses to as +/// ISO-8601. Any other shape, including an integer, is an error: an integer +/// carries no unit, so reading it as any epoch scale would be a guess. The +/// error names `column` and the shape found. +pub fn instant_of(v: &Value, column: &str) -> Result { + match v { + Value::DateTime(at) | Value::NaiveDateTime(at) => Ok(*at), + Value::String(s) => NdbDateTime::parse(s).ok_or_else(|| { + NodeDbError::serialization( + "cell", + format!("column \"{column}\" holds text that is not a timestamp: {s:?}"), + ) + }), + other => Err(shape_mismatch(column, "a timestamp", other)), + } +} + +/// The error for a cell whose shape is not the one its column requires: +/// `column "ts" holds an integer where a timestamp is required`. +pub fn shape_mismatch(column: &str, expected: &str, found: &Value) -> NodeDbError { + NodeDbError::serialization( + "cell", + format!( + "column \"{column}\" holds {} where {expected} is required", + shape_name(found) + ), + ) +} + +/// The article-prefixed shape name an error names a cell by. +fn shape_name(v: &Value) -> &'static str { + match v { + Value::Null => "null", + Value::Bool(_) => "a bool", + Value::Integer(_) => "an integer", + Value::Float(_) => "a float", + Value::String(_) => "text", + Value::Bytes(_) => "bytes", + Value::Array(_) => "an array", + Value::Object(_) => "an object", + Value::Uuid(_) => "a uuid", + Value::Ulid(_) => "a ulid", + Value::DateTime(_) | Value::NaiveDateTime(_) => "a timestamp", + Value::Duration(_) => "a duration", + Value::Decimal(_) => "a decimal", + Value::Geometry(_) => "a geometry", + Value::Set(_) => "a set", + Value::Regex(_) => "a regex", + Value::Range { .. } => "a range", + Value::Record { .. } => "a record", + Value::ArrayCell(_) => "an array cell", + Value::Vector(_) => "a vector", + // `Value` is `#[non_exhaustive]`: a variant this crate cannot name. + _ => "an unsupported value", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The one instant these tests use: 2020-03-05T10:00:00Z. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + /// Every scalar renders the same text its wire JSON renders to, so the + /// direct arms and the JSON edge cannot drift apart. + #[test] + fn scalar_text_matches_the_wire_json_text() { + let at = NdbDateTime::from_micros(EARLY_MICROS); + let scalars = [ + Value::Null, + Value::Bool(true), + Value::Bool(false), + Value::Integer(42), + Value::Integer(-7), + Value::Float(0.0), + Value::Float(1.5), + Value::Float(1e20), + Value::Float(f64::NAN), + Value::Float(f64::INFINITY), + Value::String("hello".into()), + Value::Bytes(vec![0, 255, 7]), + Value::DateTime(at), + Value::NaiveDateTime(at), + Value::Uuid("550e8400-e29b-41d4-a716-446655440000".into()), + Value::Decimal(rust_decimal::Decimal::new(110, 2)), + Value::Duration(nodedb_types::NdbDuration::from_micros(1_500_000)), + Value::Array(vec![Value::Integer(1), Value::Bool(true)]), + Value::Range { + start: None, + end: None, + inclusive: false, + }, + ]; + for (index, v) in scalars.iter().enumerate() { + assert_eq!( + cell_text(v), + wire_json_text(&value_to_wire_json(v)), + "scalar {index} must render the same text either way" + ); + } + } + + /// The pinned text of the scalars pgwire renders most. + #[test] + fn scalar_text_is_the_postgres_text_form() { + let at = NdbDateTime::from_micros(EARLY_MICROS); + assert_eq!(cell_text(&Value::Null), None); + assert_eq!(cell_text(&Value::Bool(true)).as_deref(), Some("t")); + assert_eq!(cell_text(&Value::Bool(false)).as_deref(), Some("f")); + assert_eq!(cell_text(&Value::Integer(42)).as_deref(), Some("42")); + assert_eq!(cell_text(&Value::Float(0.0)).as_deref(), Some("0.0")); + assert_eq!(cell_text(&Value::Float(f64::NAN)), None); + assert_eq!(cell_text(&Value::String("x".into())).as_deref(), Some("x")); + assert_eq!( + cell_text(&Value::Bytes(vec![0, 255, 7])).as_deref(), + Some("AP8H") + ); + assert_eq!( + cell_text(&Value::NaiveDateTime(at)).as_deref(), + Some("2020-03-05T10:00:00.000000Z") + ); + } + + /// A typed instant is itself, whichever kind it is. + #[test] + fn instant_of_reads_a_typed_instant() { + let at = NdbDateTime::from_micros(EARLY_MICROS); + assert_eq!(instant_of(&Value::DateTime(at), "ts").expect("instant"), at); + assert_eq!( + instant_of(&Value::NaiveDateTime(at), "ts").expect("instant"), + at + ); + } + + /// An ISO-8601 string, with or without the `T` separator, is the instant + /// it denotes. + #[test] + fn instant_of_parses_iso8601_text() { + let at = NdbDateTime::from_micros(EARLY_MICROS); + for text in [ + "2020-03-05 10:00:00", + "2020-03-05T10:00:00Z", + "2020-03-05T10:00:00.000000Z", + ] { + assert_eq!( + instant_of(&Value::String(text.into()), "ts").expect("parses"), + at, + "{text} must parse to the instant" + ); + } + } + + /// An integer under a timestamp column is refused, and the error names + /// the column and the shape. + #[test] + fn instant_of_refuses_an_integer() { + let err = instant_of(&Value::Integer(EARLY_MICROS), "created_at") + .expect_err("an integer carries no unit"); + assert!( + err.message() + .contains("column \"created_at\" holds an integer where a timestamp is required"), + "message must name the column and the shape, got: {}", + err.message() + ); + } + + /// Text that is not a timestamp is refused, and the error names it. + #[test] + fn instant_of_refuses_text_that_does_not_parse() { + let err = instant_of(&Value::String("yesterday".into()), "ts") + .expect_err("free text is not an instant"); + assert!( + err.message() + .contains("column \"ts\" holds text that is not a timestamp"), + "message must name the column, got: {}", + err.message() + ); + assert!(err.message().contains("yesterday")); + } +} diff --git a/nodedb/src/control/server/response_shape/compose.rs b/nodedb/src/control/server/response_shape/compose.rs deleted file mode 100644 index 5e1286d97..000000000 --- a/nodedb/src/control/server/response_shape/compose.rs +++ /dev/null @@ -1,732 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Composed, protocol-neutral materialized response shaping. -//! -//! `shape_response_materialized` and `shape_decoded_rows` are the canonical -//! SELECT-read shaping used by every protocol entrypoint. `shape_response_materialized` -//! performs the full per-payload shaping order (`apply_kv_wrap` -> -//! `translate_search_response` -> decode -> scan-envelope unwrap -> optional -//! SELECT-list projection) as a single call, producing an already-shaped, -//! already-projected [`ShapeOutcome`]. Every SELECT-read producer — pgwire's -//! non-streaming dispatch, native's dispatch loop — calls this directly and -//! hands the resulting `ShapedRows` to its own protocol encoder; each -//! protocol then encodes those rows in its own wire format (pgwire's -//! RowDescription/DataRow, native's MessagePack, http's JSON). -//! -//! Producers with no `PhysicalPlan` in scope (ClusterArray, set-op merges, -//! gateway forwarding, clone merges) call [`shape_payload_no_plan`], which -//! skips the plan-dependent `apply_kv_wrap` / `translate_search_response` transforms -//! those callers never ran. The pure kernel [`shape_decoded_rows`] is shared -//! with per-batch lazy streaming callers, which have an already-decoded batch -//! and only need the envelope-unwrap + projection logic. - -use std::collections::HashSet; - -use serde_json::{Map, Value as JsonValue}; - -use crate::control::server::response_translate::dispatch::translate_search_response; -use crate::data::executor::response_codec::{ArraySliceResponse, decode_payload_to_json}; -use nodedb_types::NodeDbError; -use nodedb_types::columnar::schema::is_reserved_bitemporal_column; - -use super::kv::apply_kv_wrap; -use super::project::push_flat_rows; -use super::redaction::RedactionCtx; -use super::request::MaterializedShapeRequest; -use super::returning::shape_returning_rows; -use super::schema::OutputSchema; -use super::types::{DdlColType, PlanKind, ShapedRows}; - -/// NOTICE text for an `AS OF SYSTEM TIME` cutoff older than the oldest -/// retained tile version. This is the canonical definition, surfaced to -/// every protocol via [`ShapedRows::notice`]. -const TRUNCATED_BEFORE_HORIZON_NOTICE: &str = "AS OF SYSTEM TIME cutoff is older than the oldest retained tile version; \ - results may be incomplete"; - -/// Outcome of materialized response shaping. -/// -/// Row-producing plan kinds (`SingleDocument`, `MultiRow`, `ReturningRows`, -/// `ArraySlice`) yield `Rows`. Tag/execution kinds (`Execution`, -/// `DmlResult`) yield `Passthrough` — a `ShapedRows` cannot represent a bare -/// `CommandComplete` tag or affected-row count, so callers keep their -/// existing tag / `rows_affected` handling for those. -pub enum ShapeOutcome { - Rows(ShapedRows), - Passthrough, -} - -/// Shape a single Data-Plane payload into protocol-neutral rows, applying -/// the canonical shaping order: KV point-get wrap, vector surrogate->PK -/// translation, payload decode, scan-envelope unwrap, and (when -/// `projection` names columns) SELECT-list column selection. -pub fn shape_response_materialized( - request: MaterializedShapeRequest<'_>, -) -> Result { - let MaterializedShapeRequest { - payload, - plan, - plan_kind, - projection, - state, - database_id, - tenant_id, - redaction, - } = request; - - match plan_kind { - PlanKind::Execution | PlanKind::DmlResult(_) => return Ok(ShapeOutcome::Passthrough), - PlanKind::ArraySlice - | PlanKind::ReturningRows - | PlanKind::SingleDocument - | PlanKind::MultiRow => {} - } - - // Seam-1 order, exactly as pgwire's `dispatch_task_loop` applies it - // (apply_kv_wrap -> translate_search_response) before any decode/shape step. - let wrapped = apply_kv_wrap(plan, payload); - let translated = translate_search_response(&wrapped, plan, state, database_id, tenant_id); - - let shaped = match plan_kind { - PlanKind::ArraySlice => shape_array_slice(&translated, redaction)?, - // `RETURNING` rows are held to the columns already announced to the - // client, when any were — see `super::returning`. - PlanKind::ReturningRows => shape_returning_rows(&translated, projection, redaction)?, - PlanKind::SingleDocument | PlanKind::MultiRow => { - shape_generic_rows(&translated, projection, redaction)? - } - // Handled by the early return above; kept exhaustive (no catch-all, - // no panic) so a future PlanKind desync degrades to passthrough - // rather than crashing the connection. - PlanKind::Execution | PlanKind::DmlResult(_) => return Ok(ShapeOutcome::Passthrough), - }; - Ok(ShapeOutcome::Rows(shaped)) -} - -/// Shape a Data-Plane payload with no `PhysicalPlan` in scope. -/// -/// Producers that never had a plan to KV-wrap or vector-translate -/// (ClusterArray, set-op merges, gateway forwarding, clone merges) call this -/// instead of [`shape_response_materialized`]: it applies only the decode + -/// scan-envelope unwrap + optional SELECT-list projection steps, skipping the -/// plan-dependent `apply_kv_wrap` / `translate_search_response` transforms those -/// callers never ran. -pub fn shape_payload_no_plan( - payload: &[u8], - plan_kind: PlanKind, - projection: Option<&OutputSchema>, - redaction: Option>, -) -> Result { - Ok(match plan_kind { - PlanKind::Execution | PlanKind::DmlResult(_) => ShapeOutcome::Passthrough, - PlanKind::ArraySlice => ShapeOutcome::Rows(shape_array_slice(payload, redaction)?), - PlanKind::ReturningRows => { - ShapeOutcome::Rows(shape_returning_rows(payload, projection, redaction)?) - } - PlanKind::SingleDocument | PlanKind::MultiRow => { - ShapeOutcome::Rows(shape_generic_rows(payload, projection, redaction)?) - } - }) -} - -/// Shape an `ArrayOp::Slice` response: decode the `ArraySliceResponse` -/// envelope (falling back to a plain payload decode for legacy shapes), -/// unwrap the row envelope, and surface `truncated_before_horizon` as a -/// notice. -/// -/// Array slices never carry a SELECT-list projection today (matching the -/// pre-extraction behavior), so `shape_decoded_rows` is always called with -/// a `None` projection here — but redaction still applies to the cells. -fn shape_array_slice( - payload: &[u8], - redaction: Option>, -) -> crate::Result { - if payload.is_empty() { - return Ok(empty_shaped()); - } - let (rows_json, truncated) = - if let Ok(resp) = zerompk::from_msgpack::(payload) { - ( - decode_payload_to_json(&resp.rows_msgpack), - resp.truncated_before_horizon, - ) - } else { - (decode_payload_to_json(payload), false) - }; - let notice = truncated.then(|| TRUNCATED_BEFORE_HORIZON_NOTICE.to_string()); - - let mut shaped = match sonic_rs::from_str::(&rows_json) { - Ok(value) => shape_decoded_rows(&value, None, redaction)?, - Err(_) => empty_shaped(), - }; - shaped.notice = notice; - Ok(shaped) -} - -/// Shape a `SingleDocument` / `MultiRow` response: decode to JSON, then -/// hand the parsed value to the pure [`shape_decoded_rows`] core. -/// -/// Non-JSON scalar payloads (undecodable envelope) fall back to a single -/// "result" column, matching pgwire's single-row fallback. -fn shape_generic_rows( - payload: &[u8], - projection: Option<&OutputSchema>, - redaction: Option>, -) -> crate::Result { - if payload.is_empty() { - return Ok(empty_shaped()); - } - let text = decode_payload_to_json(payload); - match sonic_rs::from_str::(&text) { - Ok(value) => shape_decoded_rows(&value, projection, redaction), - Err(_) => Ok(single_result_row(text)), - } -} - -/// Pure shaping core: given an already-decoded Data-Plane JSON payload, -/// unwrap the `{id, data}` scan envelope via `push_flat_rows`, then either -/// select the named SELECT-list columns when a projection is given, or -/// derive the id-first column union across all rows when no named -/// projection applies. -/// -/// Callers needing the composed materialized-shaping order (KV wrap, vector -/// translation, payload decode) should use [`shape_response_materialized`]; -/// this function does none of that — it is the shared core both the -/// materialized path and a per-batch lazy streaming caller (native's -/// `emit_sql_stream`) call directly, since a streamed scan batch has no plan -/// to KV-wrap or vector-translate but still needs the same envelope-unwrap + -/// projection logic applied per batch. -pub fn shape_decoded_rows( - decoded: &JsonValue, - projection: Option<&OutputSchema>, - redaction: Option>, -) -> crate::Result { - let mut rows = Vec::new(); - push_flat_rows(decoded.clone(), &mut rows)?; - - // Column-level redaction runs on the flat row maps, AFTER the scan - // envelope is unwrapped and BEFORE any projection or column derivation. - // - // After projection, `SELECT email AS contact` would have renamed the - // field out from under its rule; after column derivation, a - // `RedactionMode::Null` column would be missing from a `SELECT *` result - // instead of present and null. Both orderings deliver data the policy - // says to withhold, so the hook belongs exactly here. - redact_rows(redaction.as_ref(), &mut rows); - - match projection { - Some(s) if !s.is_star && !s.columns.is_empty() => { - let lookup_keys: Vec = s.columns.iter().map(|c| c.lookup_key.clone()).collect(); - let display_names: Vec = - s.columns.iter().map(|c| c.display_name.clone()).collect(); - // Row cells are stored under per-column unique keys, not display - // names: duplicate display names (`SELECT w.id, b.id` → `id`, - // `id`) would collide in the row map and collapse both wire - // columns to the last value. Encoders re-derive the same keys via - // `cell_keys` when reading cells. - let keys = super::project::cell_keys(&display_names); - let projected_rows = rows - .iter() - .map(|row| project_row(row, &lookup_keys, &display_names, &keys)) - .collect(); - // Carry each projected column's real catalog type, aligned in - // order with `display_names`. Only the pgwire encoder consumes - // these — mapping them to typed RowDescription OIDs and rendering - // each cell in that type's PostgreSQL text form; native/http - // ignore column types entirely. - let column_types: Vec = s.columns.iter().map(|c| c.ty).collect(); - Ok(ShapedRows { - columns: display_names, - column_types, - rows: projected_rows, - notice: None, - }) - } - _ => { - // Star / derived columns come from JSON rows with no catalog type, - // so they stay TEXT — typing them would regress `SELECT *` on - // schemaless collections. - let columns = derive_columns(&rows); - let column_types = ShapedRows::text_types(columns.len()); - Ok(ShapedRows { - columns, - column_types, - rows, - notice: None, - }) - } - } -} - -/// Apply the statement's column-level redaction policy to every flat row. -/// -/// A `None` context means the producer has no requester identity in scope and -/// therefore no roles to evaluate a policy against. -pub(super) fn redact_rows( - redaction: Option<&RedactionCtx<'_>>, - rows: &mut [Map], -) { - let Some(ctx) = redaction else { - return; - }; - for row in rows.iter_mut() { - ctx.store - .apply_flat_row(ctx.tenant_id, ctx.roles, ctx.collections, row); - } -} - -/// Select and rename one flat row's fields per the projection lists, trying -/// each candidate key in order: the full lookup key, then the bare -/// (post-dot) column name, then the SELECT alias. -/// -/// Cells are inserted under `cell_keys` (unique per column, see -/// [`super::project::cell_keys`]) rather than the display names, which may -/// repeat across columns and would otherwise collapse in the output map. -pub(super) fn project_row( - row: &Map, - lookup_keys: &[String], - display_names: &[String], - cell_keys: &[String], -) -> Map { - let mut out = Map::new(); - for (i, lookup_key) in lookup_keys.iter().enumerate() { - let bare = lookup_key - .rfind('.') - .map(|dot_pos| &lookup_key[dot_pos + 1..]) - .unwrap_or(lookup_key.as_str()); - let display_name = display_names - .get(i) - .map(String::as_str) - .unwrap_or(lookup_key.as_str()); - let value = row - .get(lookup_key.as_str()) - .or_else(|| { - if bare != lookup_key { - row.get(bare) - } else { - None - } - }) - .or_else(|| { - if display_name != lookup_key.as_str() && display_name != bare { - row.get(display_name) - } else { - None - } - }) - .cloned() - .unwrap_or(JsonValue::Null); - let cell_key = cell_keys.get(i).map(String::as_str).unwrap_or(display_name); - out.insert(cell_key.to_string(), value); - } - out -} - -/// Derive the id-first column union across all rows: `id` first (if -/// present), then each row's remaining keys in first-seen order. -/// -/// The order is user-visible wire column order and is pinned by callers and -/// tests, so it stays exactly first-seen. Membership lives in a set beside the -/// vec rather than being answered by rescanning the vec: the vec alone makes -/// the union quadratic in the number of distinct columns, on a path that runs -/// once per result set. -fn derive_columns(rows: &[Map]) -> Vec { - let mut cols: Vec = Vec::new(); - let mut seen: HashSet<&str> = HashSet::new(); - if let Some(first) = rows.first() { - if first.contains_key("id") { - cols.push("id".to_string()); - seen.insert("id"); - } - for key in first.keys() { - if key != "id" && !is_reserved_bitemporal_column(key) { - cols.push(key.clone()); - seen.insert(key.as_str()); - } - } - } - for row in rows.iter().skip(1) { - for key in row.keys() { - if !is_reserved_bitemporal_column(key) && seen.insert(key.as_str()) { - cols.push(key.clone()); - } - } - } - cols -} - -fn empty_shaped() -> ShapedRows { - ShapedRows { - columns: Vec::new(), - column_types: Vec::new(), - rows: Vec::new(), - notice: None, - } -} - -pub(super) fn single_result_row(text: String) -> ShapedRows { - let mut map = Map::new(); - map.insert("result".to_string(), JsonValue::String(text)); - ShapedRows { - columns: vec!["result".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![map], - notice: None, - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use crate::wal::WalManager; - use nodedb_types::CrdtPreviewResult; - - use super::*; - use crate::bridge::dispatch::Dispatcher; - use crate::bridge::envelope::PhysicalPlan; - use crate::control::server::response_shape::types::describe_plan; - use crate::control::state::SharedState; - use nodedb_types::{DatabaseId, TenantId}; - - fn preview_plan() -> PhysicalPlan { - PhysicalPlan::Crdt(nodedb_physical::physical_plan::CrdtOp::PreviewApply { - collection: nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, "tasks"), - document_id: "task-1".to_string(), - delta: vec![0x92, 0x01], - }) - } - - fn preview_payload() -> (CrdtPreviewResult, Vec) { - let result = CrdtPreviewResult { - post_image_msgpack: vec![0xc0], - imported_ops: 17, - trimmed_ops: 0, - frontier_digest: [0x5a; 32], - }; - let payload = zerompk::to_msgpack_vec(&result).expect("preview result serializes"); - (result, payload) - } - - /// Build the minimum real shared state needed by the materialized entry - /// point. Execution plans return before consulting it, which is exactly - /// the property this test protects. - fn shared_state() -> Arc { - let directory = tempfile::tempdir().expect("temporary WAL directory"); - let wal = Arc::new( - WalManager::open_for_testing(&directory.path().join("response-shape.wal")) - .expect("test WAL"), - ); - let (dispatcher, _) = Dispatcher::new(1, 1); - SharedState::new(dispatcher, wal).expect("test shared state") - } - - #[tokio::test] - async fn crdt_preview_is_byte_preserving_through_both_shaping_entry_points() { - let plan = preview_plan(); - let kind = describe_plan(&plan); - assert!(matches!(kind, PlanKind::Execution)); - let (expected, payload) = preview_payload(); - let original_payload = payload.clone(); - - let state = shared_state(); - let materialized = shape_response_materialized(MaterializedShapeRequest { - payload: &payload, - plan: &plan, - plan_kind: kind, - projection: None, - state: &state, - database_id: DatabaseId::new(1), - tenant_id: TenantId::new(1), - redaction: None, - }) - .expect("execution plan passthrough"); - assert!(matches!(materialized, ShapeOutcome::Passthrough)); - assert_eq!( - payload, original_payload, - "materialized path must not rewrite bytes" - ); - assert_eq!( - zerompk::from_msgpack::(&payload) - .expect("materialized passthrough remains decodable"), - expected - ); - - let no_plan = shape_payload_no_plan(&payload, kind, None, None); - assert!(matches!(no_plan, Ok(ShapeOutcome::Passthrough))); - assert_eq!( - payload, original_payload, - "no-plan path must not rewrite bytes" - ); - assert_eq!( - zerompk::from_msgpack::(&payload) - .expect("no-plan passthrough remains decodable"), - expected - ); - } - - /// Two projected columns sharing the display name `id` (`SELECT w.id, - /// b.id`) must keep both values in the shaped row instead of collapsing - /// to the last table's value. - #[test] - fn project_row_keeps_both_columns_with_duplicate_display_names() { - let mut row = Map::new(); - row.insert("w.id".to_string(), JsonValue::String("w1".to_string())); - row.insert("b.id".to_string(), JsonValue::String("b1".to_string())); - - let lookup_keys = vec!["w.id".to_string(), "b.id".to_string()]; - let display_names = vec!["id".to_string(), "id".to_string()]; - let keys = super::super::project::cell_keys(&display_names); - - let out = project_row(&row, &lookup_keys, &display_names, &keys); - assert_eq!(out.len(), 2, "both cells must survive the projection"); - assert_eq!(out.get("id"), Some(&JsonValue::String("w1".to_string()))); - assert_eq!(out.get("id_1"), Some(&JsonValue::String("b1".to_string()))); - } - - // ── Column-level redaction ────────────────────────────────────────── - - use crate::control::security::redaction::{ - RedactionMode, RedactionPolicy, RedactionRule, RedactionStore, - }; - use crate::control::server::response_shape::schema::OutputColumn; - - fn policy(collection: &str, role: &str, field: &str, mode: RedactionMode) -> RedactionPolicy { - RedactionPolicy { - name: format!("{collection}_{role}_{field}"), - tenant_id: 1, - collection: collection.into(), - display_collection: collection.into(), - for_role: role.into(), - rules: vec![RedactionRule { - field: field.into(), - mode, - }], - } - } - - fn store_with(policies: Vec) -> RedactionStore { - let store = RedactionStore::new(); - for p in policies { - store.create_policy(p); - } - store - } - - fn ctx<'a>( - store: &'a RedactionStore, - roles: &'a [String], - collections: &'a [(String, String)], - ) -> RedactionCtx<'a> { - RedactionCtx { - store, - tenant_id: 1, - roles, - collections, - } - } - - fn named_projection(pairs: &[(&str, &str)]) -> OutputSchema { - OutputSchema { - columns: pairs - .iter() - .map(|(lookup, display)| OutputColumn { - display_name: (*display).to_string(), - lookup_key: (*lookup).to_string(), - ty: DdlColType::Text, - }) - .collect(), - is_star: false, - } - } - - fn one_row(fields: JsonValue) -> JsonValue { - JsonValue::Array(vec![fields]) - } - - /// A `Mask` rule redacts for the role that holds the policy. - #[test] - fn mask_rule_redacts_for_the_policy_role() { - let store = store_with(vec![policy( - "users", - "support", - "email", - RedactionMode::Mask("***".into()), - )]); - let roles = vec!["support".to_string()]; - let sources = vec![(String::new(), "users".to_string())]; - let decoded = one_row(serde_json::json!({"email": "a@b.c", "name": "Alice"})); - - let shaped = shape_decoded_rows(&decoded, None, Some(ctx(&store, &roles, &sources))) - .expect("shape rows"); - assert_eq!(shaped.rows[0]["email"], JsonValue::String("***".into())); - assert_eq!(shaped.rows[0]["name"], JsonValue::String("Alice".into())); - } - - /// A role with no policy sees the value in the clear, and the rows are - /// otherwise identical to the unredacted shaping. - #[test] - fn role_without_a_policy_passes_rows_through_unchanged() { - let store = store_with(vec![policy( - "users", - "support", - "email", - RedactionMode::Mask("***".into()), - )]); - let roles = vec!["analyst".to_string()]; - let sources = vec![(String::new(), "users".to_string())]; - let decoded = one_row(serde_json::json!({"email": "a@b.c", "name": "Alice"})); - - let baseline = shape_decoded_rows(&decoded, None, None).expect("shape rows"); - let shaped = shape_decoded_rows(&decoded, None, Some(ctx(&store, &roles, &sources))) - .expect("shape rows"); - assert_eq!(shaped.rows, baseline.rows); - assert_eq!(shaped.columns, baseline.columns); - } - - /// `SELECT email AS contact` must still be redacted: the rule names the - /// stored field, and redaction runs before the projection renames it. - #[test] - fn select_alias_does_not_escape_the_rule() { - let store = store_with(vec![policy( - "users", - "support", - "email", - RedactionMode::Mask("***".into()), - )]); - let roles = vec!["support".to_string()]; - let sources = vec![(String::new(), "users".to_string())]; - let decoded = one_row(serde_json::json!({"email": "a@b.c"})); - let projection = named_projection(&[("email", "contact")]); - - let shaped = shape_decoded_rows( - &decoded, - Some(&projection), - Some(ctx(&store, &roles, &sources)), - ) - .expect("shape rows"); - assert_eq!(shaped.columns, vec!["contact".to_string()]); - assert_eq!(shaped.rows[0]["contact"], JsonValue::String("***".into())); - } - - /// Two joined collections both carry `id`, but only the left side has a - /// rule. Matching the bare name would redact the right side too. - #[test] - fn join_redacts_only_the_side_the_rule_belongs_to() { - let store = store_with(vec![policy( - "workspaces", - "support", - "id", - RedactionMode::Mask("***".into()), - )]); - let roles = vec!["support".to_string()]; - let sources = vec![ - ("w".to_string(), "workspaces".to_string()), - ("b".to_string(), "boards".to_string()), - ]; - let decoded = one_row(serde_json::json!({"w.id": "w1", "b.id": "b1"})); - let projection = named_projection(&[("w.id", "id"), ("b.id", "id")]); - - let shaped = shape_decoded_rows( - &decoded, - Some(&projection), - Some(ctx(&store, &roles, &sources)), - ) - .expect("shape rows"); - // `cell_keys` suffixes the duplicate display name. - assert_eq!(shaped.rows[0]["id"], JsonValue::String("***".into())); - assert_eq!(shaped.rows[0]["id_1"], JsonValue::String("b1".into())); - } - - /// `RedactionMode::Null` must leave the column in a `SELECT *` result, - /// valued null — removing the key would drop it from the derived schema. - #[test] - fn star_keeps_a_null_redacted_column_in_the_schema() { - let store = store_with(vec![policy( - "users", - "support", - "email", - RedactionMode::Null, - )]); - let roles = vec!["support".to_string()]; - let sources = vec![(String::new(), "users".to_string())]; - let decoded = one_row(serde_json::json!({"id": "u1", "email": "a@b.c"})); - - let shaped = shape_decoded_rows(&decoded, None, Some(ctx(&store, &roles, &sources))) - .expect("shape rows"); - assert!( - shaped.columns.contains(&"email".to_string()), - "redacted column must stay in the derived SELECT * schema: {:?}", - shaped.columns - ); - assert_eq!(shaped.rows[0]["email"], JsonValue::Null); - } - - /// Duplicate-free projections still store cells under the display name - /// (`cell_keys` is the identity), so existing readers are unaffected. - #[test] - fn project_row_uses_display_names_when_unique() { - let mut row = Map::new(); - row.insert("w.id".to_string(), JsonValue::String("w1".to_string())); - row.insert("b.title".to_string(), JsonValue::String("t".to_string())); - - let lookup_keys = vec!["w.id".to_string(), "b.title".to_string()]; - let display_names = vec!["id".to_string(), "title".to_string()]; - let keys = super::super::project::cell_keys(&display_names); - - let out = project_row(&row, &lookup_keys, &display_names, &keys); - assert_eq!(out.get("id"), Some(&JsonValue::String("w1".to_string()))); - assert_eq!(out.get("title"), Some(&JsonValue::String("t".to_string()))); - } - - fn row_of(keys: &[&str]) -> Map { - let mut row = Map::new(); - for k in keys { - row.insert((*k).to_string(), JsonValue::String((*k).to_string())); - } - row - } - - /// Column order is user-visible: `id` first when the first row has it, - /// then every other column in the order it is first seen, scanning rows in - /// order. Overlapping and disjoint rows must not reorder or duplicate. - /// - /// Every `row_of` list here is already in ascending key order, so the - /// within-row iteration order is the same whichever map backs - /// `serde_json::Map`; what this pins is the cross-row order. - #[test] - fn derive_columns_pins_id_first_then_first_seen_order() { - let rows = vec![ - row_of(&["a", "b", "id"]), - row_of(&["a", "z"]), - row_of(&["b", "y"]), - row_of(&["q"]), - ]; - - assert_eq!( - derive_columns(&rows), - vec!["id", "a", "b", "z", "y", "q"], - "id leads; later rows append only their newly-seen columns" - ); - } - - #[test] - fn derive_columns_without_id_keeps_the_first_rows_columns_leading() { - let rows = vec![row_of(&["a", "b"]), row_of(&["c", "id"])]; - - assert_eq!( - derive_columns(&rows), - vec!["a", "b", "c", "id"], - "a late `id` appends where it is first seen; it is not hoisted" - ); - } - - #[test] - fn derive_columns_skips_reserved_bitemporal_columns() { - let rows = vec![ - row_of(&["__system_from_ms", "id"]), - row_of(&["__valid_from_ms", "name"]), - ]; - - assert_eq!(derive_columns(&rows), vec!["id", "name"]); - } -} diff --git a/nodedb/src/control/server/response_shape/compose/array_slice.rs b/nodedb/src/control/server/response_shape/compose/array_slice.rs new file mode 100644 index 000000000..e9d921ea6 --- /dev/null +++ b/nodedb/src/control/server/response_shape/compose/array_slice.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Shaping for an `ArrayOp::Slice` response. + +use crate::data::executor::response_codec::{ArraySliceResponse, decode_payload_value}; + +use super::super::redaction::RedactionCtx; +use super::super::types::ShapedRows; +use super::kernel::{empty_shaped, shape_decoded_rows}; + +/// NOTICE text for an `AS OF SYSTEM TIME` cutoff older than the oldest +/// retained tile version. This is the canonical definition, surfaced to +/// every protocol via [`ShapedRows::notice`]. +const TRUNCATED_BEFORE_HORIZON_NOTICE: &str = "AS OF SYSTEM TIME cutoff is older than the oldest retained tile version; \ + results may be incomplete"; + +/// Shape an `ArrayOp::Slice` response: decode the `ArraySliceResponse` +/// envelope (falling back to a plain payload decode for legacy shapes), +/// unwrap the row envelope, and surface `truncated_before_horizon` as a +/// notice. +/// +/// Array slices never carry a SELECT-list projection, so `shape_decoded_rows` +/// is always called with a `None` projection here — but redaction still +/// applies to the cells. A payload that decodes to no value shapes as an +/// empty result set. +pub(super) fn shape_array_slice( + payload: &[u8], + redaction: Option>, +) -> crate::Result { + if payload.is_empty() { + return Ok(empty_shaped()); + } + let (rows, truncated) = if let Ok(resp) = zerompk::from_msgpack::(payload) { + ( + decode_payload_value(&resp.rows_msgpack), + resp.truncated_before_horizon, + ) + } else { + (decode_payload_value(payload), false) + }; + let notice = truncated.then(|| TRUNCATED_BEFORE_HORIZON_NOTICE.to_string()); + + let mut shaped = match rows { + Ok(value) => shape_decoded_rows(value, None, redaction)?, + Err(_) => empty_shaped(), + }; + shaped.notice = notice; + Ok(shaped) +} diff --git a/nodedb/src/control/server/response_shape/compose/kernel.rs b/nodedb/src/control/server/response_shape/compose/kernel.rs new file mode 100644 index 000000000..d0c0942e3 --- /dev/null +++ b/nodedb/src/control/server/response_shape/compose/kernel.rs @@ -0,0 +1,492 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Pure shaping kernel over an already-decoded Data-Plane value. +//! +//! [`shape_decoded_rows`] unwraps the `{id, data}` scan envelope, redacts, +//! and either projects the SELECT list or derives the column union. It is +//! the shared core both the materialized path and every per-batch lazy +//! streaming caller (pgwire, native, http) call directly. + +use std::collections::HashSet; + +use nodedb_types::Value; +use nodedb_types::columnar::schema::is_reserved_bitemporal_column; + +use super::super::project::push_flat_rows; +use super::super::redaction::RedactionCtx; +use super::super::schema::OutputSchema; +use super::super::types::{DdlColType, ShapedRow, ShapedRows}; + +/// Pure shaping core: given an already-decoded Data-Plane value, unwrap the +/// `{id, data}` scan envelope via `push_flat_rows`, then either select the +/// named SELECT-list columns when a projection is given, or derive the +/// id-first column union across all rows when no named projection applies. +/// +/// Callers needing the composed materialized-shaping order (KV wrap, vector +/// translation, payload decode) use `shape_response_materialized`; this +/// function does none of that. A streamed scan batch has no plan to KV-wrap +/// or vector-translate but still needs the same envelope-unwrap + projection +/// logic applied per batch, so streaming callers call this directly. +pub fn shape_decoded_rows( + decoded: Value, + projection: Option<&OutputSchema>, + redaction: Option>, +) -> crate::Result { + let mut rows = Vec::new(); + push_flat_rows(decoded, &mut rows)?; + + // Column-level redaction runs on the flat row maps, AFTER the scan + // envelope is unwrapped and BEFORE any projection or column derivation. + // + // After projection, `SELECT email AS contact` would have renamed the + // field out from under its rule; after column derivation, a + // `RedactionMode::Null` column would be missing from a `SELECT *` result + // instead of present and null. Both orderings deliver data the policy + // says to withhold, so the hook belongs exactly here. + redact_rows(redaction.as_ref(), &mut rows); + + match projection { + Some(s) if !s.is_star && !s.columns.is_empty() => { + let lookup_keys: Vec = s.columns.iter().map(|c| c.lookup_key.clone()).collect(); + let display_names: Vec = + s.columns.iter().map(|c| c.display_name.clone()).collect(); + // Row cells are stored under per-column unique keys, not display + // names: duplicate display names (`SELECT w.id, b.id` → `id`, + // `id`) would collide in the row map and collapse both wire + // columns to the last value. Encoders re-derive the same keys via + // `cell_keys` when reading cells. + let keys = super::super::project::cell_keys(&display_names); + let projected_rows = rows + .iter() + .map(|row| project_row(row, &lookup_keys, &display_names, &keys)) + .collect(); + // Carry each projected column's real catalog type, aligned in + // order with `display_names`. Only the pgwire encoder consumes + // these — mapping them to typed RowDescription OIDs and rendering + // each cell in that type's PostgreSQL text form; native/http + // ignore column types entirely. + let column_types: Vec = s.columns.iter().map(|c| c.ty).collect(); + Ok(ShapedRows::from_rows( + display_names, + column_types, + projected_rows, + )) + } + _ => { + // Star / derived columns come from rows with no catalog type, so + // they stay TEXT — typing them would regress `SELECT *` on + // schemaless collections. + let columns = derive_columns(&rows); + let column_types = ShapedRows::text_types(columns.len()); + Ok(ShapedRows::from_rows(columns, column_types, rows)) + } + } +} + +/// Apply the statement's column-level redaction policy to every flat row. +/// +/// A `None` context means the producer has no requester identity in scope and +/// therefore no roles to evaluate a policy against. +pub(in crate::control::server::response_shape) fn redact_rows( + redaction: Option<&RedactionCtx<'_>>, + rows: &mut [ShapedRow], +) { + let Some(ctx) = redaction else { + return; + }; + for row in rows.iter_mut() { + ctx.store + .apply_flat_row_typed(ctx.tenant_id, ctx.roles, ctx.collections, row); + } +} + +/// Select and rename one flat row's fields per the projection lists, trying +/// each candidate key in order: the full lookup key, then the bare +/// (post-dot) column name, then the SELECT alias. +/// +/// Cells are inserted under `cell_keys` (unique per column, see +/// [`super::super::project::cell_keys`]) rather than the display names, which +/// may repeat across columns and would otherwise collapse in the output map. +pub(in crate::control::server::response_shape) fn project_row( + row: &ShapedRow, + lookup_keys: &[String], + display_names: &[String], + cell_keys: &[String], +) -> ShapedRow { + let mut out = ShapedRow::new(); + for (i, lookup_key) in lookup_keys.iter().enumerate() { + let bare = lookup_key + .rfind('.') + .map(|dot_pos| &lookup_key[dot_pos + 1..]) + .unwrap_or(lookup_key.as_str()); + let display_name = display_names + .get(i) + .map(String::as_str) + .unwrap_or(lookup_key.as_str()); + let value = row + .get(lookup_key.as_str()) + .or_else(|| { + if bare != lookup_key { + row.get(bare) + } else { + None + } + }) + .or_else(|| { + if display_name != lookup_key.as_str() && display_name != bare { + row.get(display_name) + } else { + None + } + }) + .cloned() + .unwrap_or(Value::Null); + let cell_key = cell_keys.get(i).map(String::as_str).unwrap_or(display_name); + out.insert(cell_key.to_string(), value); + } + out +} + +/// Derive the id-first column union across all rows: `id` first (if +/// present), then each row's remaining keys in first-seen order. +/// +/// The order is user-visible wire column order and is pinned by callers and +/// tests, so it stays exactly first-seen. Membership lives in a set beside the +/// vec rather than being answered by rescanning the vec: the vec alone makes +/// the union quadratic in the number of distinct columns, on a path that runs +/// once per result set. +fn derive_columns(rows: &[ShapedRow]) -> Vec { + let mut cols: Vec = Vec::new(); + let mut seen: HashSet<&str> = HashSet::new(); + if let Some(first) = rows.first() { + if first.contains_key("id") { + cols.push("id".to_string()); + seen.insert("id"); + } + for key in first.keys() { + if key != "id" && !is_reserved_bitemporal_column(key) { + cols.push(key.clone()); + seen.insert(key.as_str()); + } + } + } + for row in rows.iter().skip(1) { + for key in row.keys() { + if !is_reserved_bitemporal_column(key) && seen.insert(key.as_str()) { + cols.push(key.clone()); + } + } + } + cols +} + +/// A result set with no columns and no rows. +pub(in crate::control::server::response_shape) fn empty_shaped() -> ShapedRows { + ShapedRows::from_rows(Vec::new(), Vec::new(), Vec::new()) +} + +/// A single `result` text column holding one row of `text`. +pub(in crate::control::server::response_shape) fn single_result_row(text: String) -> ShapedRows { + let mut row = ShapedRow::new(); + row.insert("result".to_string(), Value::String(text)); + ShapedRows::from_rows( + vec!["result".to_string()], + ShapedRows::text_types(1), + vec![row], + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::security::redaction::{ + RedactionMode, RedactionPolicy, RedactionRule, RedactionStore, + }; + use crate::control::server::response_shape::schema::OutputColumn; + use nodedb_types::NdbDateTime; + + fn text(s: &str) -> Value { + Value::String(s.to_string()) + } + + fn row_of_pairs(pairs: &[(&str, Value)]) -> ShapedRow { + pairs + .iter() + .map(|(k, v)| ((*k).to_string(), v.clone())) + .collect() + } + + /// Two projected columns sharing the display name `id` (`SELECT w.id, + /// b.id`) must keep both values in the shaped row instead of collapsing + /// to the last table's value. + #[test] + fn project_row_keeps_both_columns_with_duplicate_display_names() { + let row = row_of_pairs(&[("w.id", text("w1")), ("b.id", text("b1"))]); + + let lookup_keys = vec!["w.id".to_string(), "b.id".to_string()]; + let display_names = vec!["id".to_string(), "id".to_string()]; + let keys = crate::control::server::response_shape::project::cell_keys(&display_names); + + let out = project_row(&row, &lookup_keys, &display_names, &keys); + assert_eq!(out.len(), 2, "both cells must survive the projection"); + assert_eq!(out.get("id"), Some(&text("w1"))); + assert_eq!(out.get("id_1"), Some(&text("b1"))); + } + + // ── Column-level redaction ────────────────────────────────────────── + + fn policy(collection: &str, role: &str, field: &str, mode: RedactionMode) -> RedactionPolicy { + RedactionPolicy { + name: format!("{collection}_{role}_{field}"), + tenant_id: 1, + collection: collection.into(), + display_collection: collection.into(), + for_role: role.into(), + rules: vec![RedactionRule { + field: field.into(), + mode, + }], + } + } + + fn store_with(policies: Vec) -> RedactionStore { + let store = RedactionStore::new(); + for p in policies { + store.create_policy(p); + } + store + } + + fn ctx<'a>( + store: &'a RedactionStore, + roles: &'a [String], + collections: &'a [(String, String)], + ) -> RedactionCtx<'a> { + RedactionCtx { + store, + tenant_id: 1, + roles, + collections, + } + } + + fn named_projection(pairs: &[(&str, &str)]) -> OutputSchema { + OutputSchema { + columns: pairs + .iter() + .map(|(lookup, display)| OutputColumn { + display_name: (*display).to_string(), + lookup_key: (*lookup).to_string(), + ty: DdlColType::Text, + }) + .collect(), + is_star: false, + } + } + + /// One decoded row: a one-element array holding the object `pairs`. + fn one_row(pairs: &[(&str, Value)]) -> Value { + let object = pairs + .iter() + .map(|(k, v)| ((*k).to_string(), v.clone())) + .collect(); + Value::Array(vec![Value::Object(object)]) + } + + /// A `Mask` rule redacts for the role that holds the policy. + #[test] + fn mask_rule_redacts_for_the_policy_role() { + let store = store_with(vec![policy( + "users", + "support", + "email", + RedactionMode::Mask("***".into()), + )]); + let roles = vec!["support".to_string()]; + let sources = vec![(String::new(), "users".to_string())]; + let decoded = one_row(&[("email", text("a@b.c")), ("name", text("Alice"))]); + + let shaped = shape_decoded_rows(decoded, None, Some(ctx(&store, &roles, &sources))) + .expect("shape rows"); + assert_eq!(shaped.rows[0]["email"], text("***")); + assert_eq!(shaped.rows[0]["name"], text("Alice")); + } + + /// A role with no policy sees the value in the clear, and the rows are + /// otherwise identical to the unredacted shaping. + #[test] + fn role_without_a_policy_passes_rows_through_unchanged() { + let store = store_with(vec![policy( + "users", + "support", + "email", + RedactionMode::Mask("***".into()), + )]); + let roles = vec!["analyst".to_string()]; + let sources = vec![(String::new(), "users".to_string())]; + let decoded = one_row(&[("email", text("a@b.c")), ("name", text("Alice"))]); + + let baseline = shape_decoded_rows(decoded.clone(), None, None).expect("shape rows"); + let shaped = shape_decoded_rows(decoded, None, Some(ctx(&store, &roles, &sources))) + .expect("shape rows"); + assert_eq!(shaped.rows, baseline.rows); + assert_eq!(shaped.columns, baseline.columns); + } + + /// `SELECT email AS contact` must still be redacted: the rule names the + /// stored field, and redaction runs before the projection renames it. + #[test] + fn select_alias_does_not_escape_the_rule() { + let store = store_with(vec![policy( + "users", + "support", + "email", + RedactionMode::Mask("***".into()), + )]); + let roles = vec!["support".to_string()]; + let sources = vec![(String::new(), "users".to_string())]; + let decoded = one_row(&[("email", text("a@b.c"))]); + let projection = named_projection(&[("email", "contact")]); + + let shaped = shape_decoded_rows( + decoded, + Some(&projection), + Some(ctx(&store, &roles, &sources)), + ) + .expect("shape rows"); + assert_eq!(shaped.columns, vec!["contact".to_string()]); + assert_eq!(shaped.rows[0]["contact"], text("***")); + } + + /// Two joined collections both carry `id`, but only the left side has a + /// rule. Matching the bare name would redact the right side too. + #[test] + fn join_redacts_only_the_side_the_rule_belongs_to() { + let store = store_with(vec![policy( + "workspaces", + "support", + "id", + RedactionMode::Mask("***".into()), + )]); + let roles = vec!["support".to_string()]; + let sources = vec![ + ("w".to_string(), "workspaces".to_string()), + ("b".to_string(), "boards".to_string()), + ]; + let decoded = one_row(&[("w.id", text("w1")), ("b.id", text("b1"))]); + let projection = named_projection(&[("w.id", "id"), ("b.id", "id")]); + + let shaped = shape_decoded_rows( + decoded, + Some(&projection), + Some(ctx(&store, &roles, &sources)), + ) + .expect("shape rows"); + // `cell_keys` suffixes the duplicate display name. + assert_eq!(shaped.rows[0]["id"], text("***")); + assert_eq!(shaped.rows[0]["id_1"], text("b1")); + } + + /// `RedactionMode::Null` must leave the column in a `SELECT *` result, + /// valued null — removing the key would drop it from the derived schema. + #[test] + fn star_keeps_a_null_redacted_column_in_the_schema() { + let store = store_with(vec![policy( + "users", + "support", + "email", + RedactionMode::Null, + )]); + let roles = vec!["support".to_string()]; + let sources = vec![(String::new(), "users".to_string())]; + let decoded = one_row(&[("id", text("u1")), ("email", text("a@b.c"))]); + + let shaped = shape_decoded_rows(decoded, None, Some(ctx(&store, &roles, &sources))) + .expect("shape rows"); + assert!( + shaped.columns.contains(&"email".to_string()), + "redacted column must stay in the derived SELECT * schema: {:?}", + shaped.columns + ); + assert_eq!(shaped.rows[0]["email"], Value::Null); + } + + /// Duplicate-free projections still store cells under the display name + /// (`cell_keys` is the identity), so existing readers are unaffected. + #[test] + fn project_row_uses_display_names_when_unique() { + let row = row_of_pairs(&[("w.id", text("w1")), ("b.title", text("t"))]); + + let lookup_keys = vec!["w.id".to_string(), "b.title".to_string()]; + let display_names = vec!["id".to_string(), "title".to_string()]; + let keys = crate::control::server::response_shape::project::cell_keys(&display_names); + + let out = project_row(&row, &lookup_keys, &display_names, &keys); + assert_eq!(out.get("id"), Some(&text("w1"))); + assert_eq!(out.get("title"), Some(&text("t"))); + } + + /// A typed instant cell passes through projection as itself; its + /// ISO-8601 text is the protocol edge's rendering, not the kernel's. + #[test] + fn an_instant_cell_survives_projection_as_a_typed_value() { + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + let decoded = one_row(&[("at", Value::NaiveDateTime(at)), ("id", text("r1"))]); + let projection = named_projection(&[("at", "at")]); + + let shaped = shape_decoded_rows(decoded, Some(&projection), None).expect("shape rows"); + assert_eq!(shaped.rows[0]["at"], Value::NaiveDateTime(at)); + assert_eq!( + crate::control::server::response_shape::cell::value_to_wire_json(&shaped.rows[0]["at"]), + serde_json::Value::String("2020-03-05T10:00:00.000000Z".into()) + ); + } + + fn row_of(keys: &[&str]) -> ShapedRow { + keys.iter().map(|k| ((*k).to_string(), text(k))).collect() + } + + /// Column order is user-visible: `id` first when the first row has it, + /// then every other column in the order it is first seen, scanning rows in + /// order. Overlapping and disjoint rows must not reorder or duplicate. + /// + /// Every `row_of` list here is already in ascending key order, so the + /// within-row iteration order is the sorted order a `ShapedRow` iterates + /// in; what this pins is the cross-row order. + #[test] + fn derive_columns_pins_id_first_then_first_seen_order() { + let rows = vec![ + row_of(&["a", "b", "id"]), + row_of(&["a", "z"]), + row_of(&["b", "y"]), + row_of(&["q"]), + ]; + + assert_eq!( + derive_columns(&rows), + vec!["id", "a", "b", "z", "y", "q"], + "id leads; later rows append only their newly-seen columns" + ); + } + + #[test] + fn derive_columns_without_id_keeps_the_first_rows_columns_leading() { + let rows = vec![row_of(&["a", "b"]), row_of(&["c", "id"])]; + + assert_eq!( + derive_columns(&rows), + vec!["a", "b", "c", "id"], + "a late `id` appends where it is first seen; it is not hoisted" + ); + } + + #[test] + fn derive_columns_skips_reserved_bitemporal_columns() { + let rows = vec![ + row_of(&["__system_from_ms", "id"]), + row_of(&["__valid_from_ms", "name"]), + ]; + + assert_eq!(derive_columns(&rows), vec!["id", "name"]); + } +} diff --git a/nodedb/src/control/server/response_shape/compose/materialized.rs b/nodedb/src/control/server/response_shape/compose/materialized.rs new file mode 100644 index 000000000..6c5b30142 --- /dev/null +++ b/nodedb/src/control/server/response_shape/compose/materialized.rs @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Composed, protocol-neutral materialized response shaping. +//! +//! `shape_response_materialized` is the canonical SELECT-read shaping used by +//! every protocol entrypoint. It performs the full per-payload shaping order +//! (`apply_kv_wrap` -> `translate_search_response` -> decode -> scan-envelope +//! unwrap -> optional SELECT-list projection) as a single call, producing an +//! already-shaped, already-projected [`ShapeOutcome`]. Every SELECT-read +//! producer — pgwire's non-streaming dispatch, native's dispatch loop — calls +//! this directly and hands the resulting `ShapedRows` to its own protocol +//! encoder; each protocol then encodes those rows in its own wire format +//! (pgwire's RowDescription/DataRow, native's MessagePack, http's JSON). +//! +//! Producers with no `PhysicalPlan` in scope (ClusterArray, set-op merges, +//! gateway forwarding, clone merges) call [`shape_payload_no_plan`], which +//! skips the plan-dependent `apply_kv_wrap` / `translate_search_response` +//! transforms those callers never ran. The pure kernel `shape_decoded_rows` +//! is shared with per-batch lazy streaming callers, which have an +//! already-decoded batch and only need the envelope-unwrap + projection logic. + +use crate::control::server::response_translate::dispatch::translate_search_response; +use crate::data::executor::response_codec::decode_payload_value; +use nodedb_types::NodeDbError; + +use super::super::kv::apply_kv_wrap; +use super::super::redaction::RedactionCtx; +use super::super::request::MaterializedShapeRequest; +use super::super::returning::shape_returning_rows; +use super::super::schema::OutputSchema; +use super::super::types::{PlanKind, ShapedRows}; +use super::array_slice::shape_array_slice; +use super::kernel::{empty_shaped, shape_decoded_rows, single_result_row}; + +/// Outcome of materialized response shaping. +/// +/// Row-producing plan kinds (`SingleDocument`, `MultiRow`, `ReturningRows`, +/// `ArraySlice`) yield `Rows`. Tag/execution kinds (`Execution`, +/// `DmlResult`) yield `Passthrough` — a `ShapedRows` cannot represent a bare +/// `CommandComplete` tag or affected-row count, so callers keep their +/// existing tag / `rows_affected` handling for those. +pub enum ShapeOutcome { + Rows(ShapedRows), + Passthrough, +} + +/// Shape a single Data-Plane payload into protocol-neutral rows, applying +/// the canonical shaping order: KV point-get wrap, vector surrogate->PK +/// translation, payload decode, scan-envelope unwrap, and (when +/// `projection` names columns) SELECT-list column selection. +pub fn shape_response_materialized( + request: MaterializedShapeRequest<'_>, +) -> Result { + let MaterializedShapeRequest { + payload, + plan, + plan_kind, + projection, + state, + database_id, + tenant_id, + redaction, + } = request; + + match plan_kind { + PlanKind::Execution | PlanKind::DmlResult(_) => return Ok(ShapeOutcome::Passthrough), + PlanKind::ArraySlice + | PlanKind::ReturningRows + | PlanKind::SingleDocument + | PlanKind::MultiRow => {} + } + + // Seam-1 order, exactly as pgwire's `dispatch_task_loop` applies it + // (apply_kv_wrap -> translate_search_response) before any decode/shape step. + let wrapped = apply_kv_wrap(plan, payload); + let translated = translate_search_response(&wrapped, plan, state, database_id, tenant_id); + + let shaped = match plan_kind { + PlanKind::ArraySlice => shape_array_slice(&translated, redaction)?, + // `RETURNING` rows are held to the columns already announced to the + // client, when any were — see `super::returning`. + PlanKind::ReturningRows => shape_returning_rows(&translated, projection, redaction)?, + PlanKind::SingleDocument | PlanKind::MultiRow => { + shape_generic_rows(&translated, projection, redaction)? + } + // Handled by the early return above; kept exhaustive (no catch-all, + // no panic) so a future PlanKind desync degrades to passthrough + // rather than crashing the connection. + PlanKind::Execution | PlanKind::DmlResult(_) => return Ok(ShapeOutcome::Passthrough), + }; + Ok(ShapeOutcome::Rows(shaped)) +} + +/// Shape a Data-Plane payload with no `PhysicalPlan` in scope. +/// +/// Producers that never had a plan to KV-wrap or vector-translate +/// (ClusterArray, set-op merges, gateway forwarding, clone merges) call this +/// instead of [`shape_response_materialized`]: it applies only the decode + +/// scan-envelope unwrap + optional SELECT-list projection steps, skipping the +/// plan-dependent `apply_kv_wrap` / `translate_search_response` transforms those +/// callers never ran. +pub fn shape_payload_no_plan( + payload: &[u8], + plan_kind: PlanKind, + projection: Option<&OutputSchema>, + redaction: Option>, +) -> Result { + Ok(match plan_kind { + PlanKind::Execution | PlanKind::DmlResult(_) => ShapeOutcome::Passthrough, + PlanKind::ArraySlice => ShapeOutcome::Rows(shape_array_slice(payload, redaction)?), + PlanKind::ReturningRows => { + ShapeOutcome::Rows(shape_returning_rows(payload, projection, redaction)?) + } + PlanKind::SingleDocument | PlanKind::MultiRow => { + ShapeOutcome::Rows(shape_generic_rows(payload, projection, redaction)?) + } + }) +} + +/// Shape a `SingleDocument` / `MultiRow` response: decode the payload to a +/// typed value, then hand it to the pure [`shape_decoded_rows`] core. +/// +/// A payload that decodes as neither msgpack nor JSON falls back to a single +/// "result" column holding its lossy UTF-8 text, matching pgwire's +/// single-row fallback. +fn shape_generic_rows( + payload: &[u8], + projection: Option<&OutputSchema>, + redaction: Option>, +) -> crate::Result { + if payload.is_empty() { + return Ok(empty_shaped()); + } + match decode_payload_value(payload) { + Ok(value) => shape_decoded_rows(value, projection, redaction), + Err(_) => Ok(single_result_row( + String::from_utf8_lossy(payload).into_owned(), + )), + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use crate::wal::WalManager; + use nodedb_types::CrdtPreviewResult; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::bridge::envelope::PhysicalPlan; + use crate::control::server::response_shape::types::describe_plan; + use crate::control::state::SharedState; + use nodedb_types::{DatabaseId, TenantId}; + + fn preview_plan() -> PhysicalPlan { + PhysicalPlan::Crdt(nodedb_physical::physical_plan::CrdtOp::PreviewApply { + collection: nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, "tasks"), + document_id: "task-1".to_string(), + delta: vec![0x92, 0x01], + }) + } + + fn preview_payload() -> (CrdtPreviewResult, Vec) { + let result = CrdtPreviewResult { + post_image_msgpack: vec![0xc0], + imported_ops: 17, + trimmed_ops: 0, + frontier_digest: [0x5a; 32], + }; + let payload = zerompk::to_msgpack_vec(&result).expect("preview result serializes"); + (result, payload) + } + + /// Build the minimum real shared state needed by the materialized entry + /// point. Execution plans return before consulting it, which is exactly + /// the property this test protects. + fn shared_state() -> Arc { + let directory = tempfile::tempdir().expect("temporary WAL directory"); + let wal = Arc::new( + WalManager::open_for_testing(&directory.path().join("response-shape.wal")) + .expect("test WAL"), + ); + let (dispatcher, _) = Dispatcher::new(1, 1); + SharedState::new(dispatcher, wal).expect("test shared state") + } + + #[tokio::test] + async fn crdt_preview_is_byte_preserving_through_both_shaping_entry_points() { + let plan = preview_plan(); + let kind = describe_plan(&plan); + assert!(matches!(kind, PlanKind::Execution)); + let (expected, payload) = preview_payload(); + let original_payload = payload.clone(); + + let state = shared_state(); + let materialized = shape_response_materialized(MaterializedShapeRequest { + payload: &payload, + plan: &plan, + plan_kind: kind, + projection: None, + state: &state, + database_id: DatabaseId::new(1), + tenant_id: TenantId::new(1), + redaction: None, + }) + .expect("execution plan passthrough"); + assert!(matches!(materialized, ShapeOutcome::Passthrough)); + assert_eq!( + payload, original_payload, + "materialized path must not rewrite bytes" + ); + assert_eq!( + zerompk::from_msgpack::(&payload) + .expect("materialized passthrough remains decodable"), + expected + ); + + let no_plan = shape_payload_no_plan(&payload, kind, None, None); + assert!(matches!(no_plan, Ok(ShapeOutcome::Passthrough))); + assert_eq!( + payload, original_payload, + "no-plan path must not rewrite bytes" + ); + assert_eq!( + zerompk::from_msgpack::(&payload) + .expect("no-plan passthrough remains decodable"), + expected + ); + } + + /// A msgpack `bin` cell reaches the shaped row as `Value::Bytes`; its + /// text form is decided at the protocol edge. + #[test] + fn a_byte_cell_shapes_as_bytes() { + let mut row = std::collections::HashMap::new(); + row.insert( + "blob".to_string(), + nodedb_types::Value::Bytes(vec![0, 255, 7]), + ); + let payload = nodedb_types::value_to_msgpack(&nodedb_types::Value::Array(vec![ + nodedb_types::Value::Object(row), + ])) + .expect("encode"); + + let ShapeOutcome::Rows(shaped) = + shape_payload_no_plan(&payload, PlanKind::MultiRow, None, None).expect("shape") + else { + panic!("multi-row plan must yield rows"); + }; + assert_eq!( + shaped.rows[0]["blob"], + nodedb_types::Value::Bytes(vec![0, 255, 7]) + ); + } +} diff --git a/nodedb/src/control/server/response_shape/compose/mod.rs b/nodedb/src/control/server/response_shape/compose/mod.rs new file mode 100644 index 000000000..bf76a3773 --- /dev/null +++ b/nodedb/src/control/server/response_shape/compose/mod.rs @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Composed, protocol-neutral response shaping: the materialized entry +//! points, the pure decoded-value kernel, and the array-slice envelope. + +pub mod array_slice; +pub mod kernel; +pub mod materialized; + +pub use kernel::shape_decoded_rows; +pub use materialized::{ShapeOutcome, shape_payload_no_plan, shape_response_materialized}; diff --git a/nodedb/src/control/server/response_shape/kv.rs b/nodedb/src/control/server/response_shape/kv.rs index 15efd5c29..b30e75dc5 100644 --- a/nodedb/src/control/server/response_shape/kv.rs +++ b/nodedb/src/control/server/response_shape/kv.rs @@ -4,12 +4,13 @@ //! into the stored value(s) before the protocol layer turns them into //! SQL rows. -use serde_json::{Map, Value as JsonValue}; +use std::collections::HashMap; use crate::bridge::envelope::PhysicalPlan; -use crate::data::executor::response_codec::decode_payload_to_json; +use crate::data::executor::response_codec::decode_payload_value; use nodedb_physical::physical_plan::KvOp; use nodedb_query::msgpack_scan; +use nodedb_types::Value; /// When `plan` is a KV point-get or batch-get, turn the engine's stored /// bytes into row-shaped msgpack. @@ -47,32 +48,70 @@ pub fn apply_kv_wrap(plan: &PhysicalPlan, payload: &[u8]) -> Vec { /// Zip `KvOp::BatchGet`'s `keys` with the Data Plane's positional /// `[value_or_null, ...]` array and wrap each pair into a `{key, value}` /// row, msgpack-encoded so the rest of the shaping pipeline -/// (`decode_payload_to_json` -> `push_flat_rows`) treats it exactly like +/// (`decode_payload_value` -> `push_flat_rows`) treats it exactly like /// any other row-array payload. /// /// Falls back to the raw payload (rather than panicking) if the Data /// Plane payload is not the expected JSON/msgpack array — a malformed -/// upstream payload degrades to the pre-fix (empty-looking) shape instead -/// of taking down the connection. +/// upstream payload degrades to an empty-looking shape instead of taking +/// down the connection. fn wrap_batch_get(keys: &[Vec], payload: &[u8]) -> Vec { - let decoded = decode_payload_to_json(payload); - let Ok(JsonValue::Array(values)) = sonic_rs::from_str::(&decoded) else { + let Ok(Value::Array(values)) = decode_payload_value(payload) else { return payload.to_vec(); }; - let rows: Vec = keys + let rows: Vec = keys .iter() .zip(values) .map(|(key, value)| { - let mut row = Map::new(); + let mut row = HashMap::with_capacity(2); row.insert( "key".to_string(), - JsonValue::String(String::from_utf8_lossy(key).into_owned()), + Value::String(String::from_utf8_lossy(key).into_owned()), ); row.insert("value".to_string(), value); - JsonValue::Object(row) + Value::Object(row) }) .collect(); - nodedb_types::json_to_msgpack(&JsonValue::Array(rows)).unwrap_or_else(|_| payload.to_vec()) + nodedb_types::value_to_msgpack(&Value::Array(rows)).unwrap_or_else(|_| payload.to_vec()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Each positional result is paired with its key into a `{key, value}` + /// row; a missing key stays `null` and the value keeps its type. + #[test] + fn wrap_batch_get_pairs_keys_with_positional_values() { + let keys = vec![b"k1".to_vec(), b"k2".to_vec()]; + let payload = nodedb_types::value_to_msgpack(&Value::Array(vec![ + Value::String("dmFs".into()), + Value::Null, + ])) + .expect("encode"); + + let wrapped = wrap_batch_get(&keys, &payload); + let Value::Array(rows) = decode_payload_value(&wrapped).expect("decode") else { + panic!("wrapped payload must be an array"); + }; + let Value::Object(first) = &rows[0] else { + panic!("row must be an object"); + }; + assert_eq!(first["key"], Value::String("k1".into())); + assert_eq!(first["value"], Value::String("dmFs".into())); + let Value::Object(second) = &rows[1] else { + panic!("row must be an object"); + }; + assert_eq!(second["key"], Value::String("k2".into())); + assert_eq!(second["value"], Value::Null); + } + + /// A payload that is not an array passes through unchanged. + #[test] + fn wrap_batch_get_passes_a_non_array_payload_through() { + let payload = nodedb_types::value_to_msgpack(&Value::Integer(1)).expect("encode"); + assert_eq!(wrap_batch_get(&[b"k".to_vec()], &payload), payload); + } } diff --git a/nodedb/src/control/server/response_shape/mod.rs b/nodedb/src/control/server/response_shape/mod.rs index e578e3ea7..01f1b6b08 100644 --- a/nodedb/src/control/server/response_shape/mod.rs +++ b/nodedb/src/control/server/response_shape/mod.rs @@ -2,6 +2,7 @@ //! Shared, protocol-neutral response shaping helpers. +pub mod cell; pub mod compose; pub mod kv; pub mod project; @@ -11,4 +12,5 @@ pub mod returning; pub mod schema; pub mod types; +pub use cell::{row_to_wire_json, value_to_wire_json}; pub use redaction::{redact_decoded_value, redact_envelope_row, redact_stored_value_bytes}; diff --git a/nodedb/src/control/server/response_shape/project.rs b/nodedb/src/control/server/response_shape/project.rs index 646868027..b976cac52 100644 --- a/nodedb/src/control/server/response_shape/project.rs +++ b/nodedb/src/control/server/response_shape/project.rs @@ -1,67 +1,78 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Pure JSON/plan projection and flattening helpers for SELECT responses. +//! Pure projection and flattening helpers for SELECT responses. //! -//! These operate purely on parsed SQL and `serde_json::Value` — no pgwire -//! wire types — so they are shared across any protocol-specific response -//! shaper. Protocol-specific encode glue that turns these into wire rows -//! (e.g. pgwire's `DataRow`) lives in each protocol's own handler code. +//! These operate on decoded `nodedb_types::Value` rows — no pgwire wire +//! types — so they are shared across any protocol-specific response shaper. +//! Protocol-specific encode glue that turns these into wire rows (e.g. +//! pgwire's `DataRow`) lives in each protocol's own handler code; the +//! per-cell text form lives in `response_shape::cell`. -/// Convert a JSON scalar value to its PostgreSQL text-format string. -/// -/// - `String` values are returned as-is (no extra quoting). -/// - `Bool` uses PostgreSQL text format: `t` for true, `f` for false. -/// - All other scalars (`Number`, `Array`, `Object`) use their JSON -/// `Display` representation; arrays/objects should not normally appear -/// as individual cell values but are rendered faithfully. -pub fn json_value_to_text(v: &serde_json::Value) -> String { - match v { - serde_json::Value::String(s) => s.clone(), - // PostgreSQL text format for boolean is `t`/`f`. - serde_json::Value::Bool(b) => if *b { "t" } else { "f" }.to_string(), - other => other.to_string(), - } -} +use std::collections::HashMap; + +use nodedb_types::Value; -/// Flatten a parsed JSON value into row objects. +use super::types::ShapedRow; + +/// Flatten a decoded Data-Plane value into typed row objects. /// /// The envelope `id` is a rendered [`StorageKey`](crate::engine::document::store::StorageKey) /// by construction. /// A value that fails `StorageKey::parse` is surfaced as `Err`, never accommodated. -pub fn push_flat_rows( - value: serde_json::Value, - out: &mut Vec>, -) -> crate::Result<()> { +pub fn push_flat_rows(value: Value, out: &mut Vec) -> crate::Result<()> { match value { - serde_json::Value::Array(items) => { + Value::Array(items) => { for item in items { push_flat_rows(item, out)?; } } - serde_json::Value::Object(mut map) => { + Value::Object(mut map) => { if is_scan_wrapper(&map) - && let Some(serde_json::Value::Object(mut inner)) = map.remove("data") + && let Some(Value::Object(inner)) = map.remove("data") { + let mut inner: ShapedRow = inner.into_iter().collect(); // The envelope carries the row's storage key, which is // internal. A body with no `id` field carries identity // nowhere else, so the key renders to an identity at this // boundary. `or_insert` leaves a declared primary key as the // authority. - if let Some(serde_json::Value::String(key)) = map.remove("id") { + if let Some(Value::String(key)) = map.remove("id") { let identity = crate::engine::document::store::StorageKey::parse(&key) .ok_or_else(|| crate::Error::Internal { detail: format!("scan envelope id is not a storage key: '{key}'"), })? .to_identity(); inner - .entry("id") - .or_insert(serde_json::Value::String(identity.into_string())); + .entry("id".to_string()) + .or_insert(Value::String(identity.into_string())); } out.push(inner); return Ok(()); } - out.push(map); + out.push(map.into_iter().collect()); } + // A scalar is not a row: the shaper only ever answers with objects. + Value::Null + | Value::Bool(_) + | Value::Integer(_) + | Value::Float(_) + | Value::String(_) + | Value::Bytes(_) + | Value::Uuid(_) + | Value::Ulid(_) + | Value::DateTime(_) + | Value::NaiveDateTime(_) + | Value::Duration(_) + | Value::Decimal(_) + | Value::Geometry(_) + | Value::Set(_) + | Value::Regex(_) + | Value::Range { .. } + | Value::Record { .. } + | Value::ArrayCell(_) + | Value::Vector(_) => {} + // `Value` is `#[non_exhaustive]`: a variant this crate cannot name is + // not a row either. _ => {} } Ok(()) @@ -70,7 +81,17 @@ pub fn push_flat_rows( /// The Data Plane's raw document-scan codec emits objects with exactly /// the keys `id` (string) and `data` (object). This is the wire shape /// we unwrap before column projection. -pub fn is_scan_wrapper(map: &serde_json::Map) -> bool { +pub fn is_scan_wrapper(map: &HashMap) -> bool { + map.len() == 2 + && matches!(map.get("id"), Some(Value::String(_))) + && matches!(map.get("data"), Some(Value::Object(_))) +} + +/// [`is_scan_wrapper`] for a row still held as JSON: the same two-key +/// `{id: string, data: object}` shape, read off a `serde_json::Map`. The +/// JSON-only redaction paths (`redact_envelope_row`) check the envelope +/// without lifting the row to a typed value. +pub fn is_scan_wrapper_json(map: &serde_json::Map) -> bool { map.len() == 2 && matches!(map.get("id"), Some(serde_json::Value::String(_))) && matches!(map.get("data"), Some(serde_json::Value::Object(_))) diff --git a/nodedb/src/control/server/response_shape/redaction/shapes.rs b/nodedb/src/control/server/response_shape/redaction/shapes.rs index 29a5b69dd..fc89c7a54 100644 --- a/nodedb/src/control/server/response_shape/redaction/shapes.rs +++ b/nodedb/src/control/server/response_shape/redaction/shapes.rs @@ -22,7 +22,7 @@ //! a decoded payload of unknown shape. use crate::control::security::redaction::RedactionStore; -use crate::control::server::response_shape::project::is_scan_wrapper; +use crate::control::server::response_shape::project::is_scan_wrapper_json; use super::query::QueryRedaction; @@ -49,7 +49,7 @@ pub fn redact_envelope_row( let Some(map) = item.as_object_mut() else { return; }; - let target = if is_scan_wrapper(map) { + let target = if is_scan_wrapper_json(map) { map.get_mut("data") .and_then(serde_json::Value::as_object_mut) } else { diff --git a/nodedb/src/control/server/response_shape/returning.rs b/nodedb/src/control/server/response_shape/returning.rs index 82af689ec..e9450ebed 100644 --- a/nodedb/src/control/server/response_shape/returning.rs +++ b/nodedb/src/control/server/response_shape/returning.rs @@ -3,7 +3,7 @@ //! Shaping for DML `RETURNING` responses. //! //! A `RETURNING` payload is a [`RowsPayload`]: the Data Plane's own column list -//! plus already-TEXT-formatted cells. For `RETURNING *` that column list is +//! plus typed `Value` cells. For `RETURNING *` that column list is //! derived from the STORED row, so a schemaless collection can carry fields no //! catalog column declares — the list is only knowable once the rows exist. //! @@ -34,17 +34,15 @@ //! a schemaless row carries fields no catalog column declares — which is the //! same answer `SELECT *` gives for the same row. -use serde_json::{Map, Value as JsonValue}; - -use nodedb_types::NodeDbError; +use nodedb_types::{NativeCell, NdbDateTime, NodeDbError, Value}; use crate::data::executor::response_codec::{RowsPayload, decode_payload_to_json}; -use super::compose::{project_row, redact_rows, single_result_row}; +use super::compose::kernel::{project_row, redact_rows, single_result_row}; use super::project::cell_keys; use super::redaction::RedactionCtx; use super::schema::OutputSchema; -use super::types::{DdlColType, ShapedRows}; +use super::types::{DdlColType, ShapedRow, ShapedRows}; /// Shape a DML-with-`RETURNING` response. /// @@ -91,20 +89,16 @@ pub fn shape_returning_rows( } }; - let mut rows = rows_keyed_by_column(&rp); + let RowsPayload { columns, rows } = rp; + let mut rows = rows_keyed_by_column(&columns, rows); // `RETURNING` delivers stored column values to the client just as a SELECT // does, so the same redaction applies — and it runs on the payload's own // names, before any projection renames or drops them. redact_rows(redaction.as_ref(), &mut rows); let Some(schema) = announced else { - let column_types = ShapedRows::text_types(rp.columns.len()); - return Ok(ShapedRows { - columns: rp.columns, - column_types, - rows, - notice: None, - }); + let column_types = ShapedRows::text_types(columns.len()); + return Ok(ShapedRows::from_rows(columns, column_types, rows)); }; Ok(project_onto_announced(schema, &rows)) } @@ -116,19 +110,16 @@ fn announced_columns(projection: Option<&OutputSchema>) -> Option<&OutputSchema> projection.filter(|schema| !schema.is_star && !schema.columns.is_empty()) } -/// Re-key each payload row from positional cells to a name-keyed map, with -/// JSON `null` for the cells the Data Plane marked SQL NULL. -fn rows_keyed_by_column(rp: &RowsPayload) -> Vec> { - rp.rows - .iter() +/// Re-key each payload row from positional typed cells to a name-keyed row. +/// A cell keeps its `Value` form: an instant stays an instant, `Value::Null` +/// is SQL NULL, numbers and booleans are themselves. Each protocol renders +/// the cell at its own edge. +fn rows_keyed_by_column(columns: &[String], rows: Vec>) -> Vec { + rows.into_iter() .map(|row_vals| { - let mut map = Map::new(); - for (col, cell) in rp.columns.iter().zip(row_vals.iter()) { - let v = match cell { - Some(s) => JsonValue::String(s.clone()), - None => JsonValue::Null, - }; - map.insert(col.clone(), v); + let mut map = ShapedRow::new(); + for (col, cell) in columns.iter().zip(row_vals) { + map.insert(col.clone(), cell.0); } map }) @@ -140,7 +131,7 @@ fn rows_keyed_by_column(rp: &RowsPayload) -> Vec> { /// Uses the same `project_row` + `cell_keys` pair the SELECT path uses, so a /// `RETURNING` result and a `SELECT` result of the same columns are laid out /// identically for the encoders. -fn project_onto_announced(schema: &OutputSchema, rows: &[Map]) -> ShapedRows { +fn project_onto_announced(schema: &OutputSchema, rows: &[ShapedRow]) -> ShapedRows { let lookup_keys: Vec = schema .columns .iter() @@ -167,45 +158,45 @@ fn project_onto_announced(schema: &OutputSchema, rows: &[Map] }) .collect(); - ShapedRows { - columns: display_names, - column_types, - rows: projected, - notice: None, - } + ShapedRows::from_rows(display_names, column_types, projected) } -/// Re-read a `RETURNING` cell's TEXT form as the column's announced type. +/// Re-read a text `RETURNING` cell as the column's announced type. /// -/// `RowsPayload` cells arrive already rendered as text, but the RowDescription -/// this response is held to announces each column's real catalog type — and a -/// client that asked for a column in BINARY result format is handed the -/// scalar's wire bytes, which have to come from a number or a bool, not from -/// the digits of its text form. Retyping here also makes a `RETURNING` -/// timestamp render in the same ISO-8601 text a `SELECT` of that column -/// renders, instead of raw epoch microseconds. +/// A typed cell is already a number, bool or instant and passes through. A +/// text cell under a numeric, bool or timestamp column is a stored string +/// (a schemaless row can hold `"42"` under a declared `INT`), and the +/// RowDescription this response is held to announces the catalog type — a +/// client that asked for BINARY result format is handed the scalar's wire +/// bytes, which have to come from a number, a bool or an instant, not from +/// the digits of its text form. /// -/// A cell that does not parse as its announced type is left as text, which -/// both encoders render verbatim. -fn retype_cell(ct: DdlColType, cell: &mut JsonValue) { - let JsonValue::String(text) = cell else { +/// A cell that does not parse as its announced type is left as text. The +/// numeric and bool encoders render such text verbatim; a timestamp column +/// refuses it, because a cell under a timestamp column renders only from an +/// instant. +fn retype_cell(ct: DdlColType, cell: &mut Value) { + let Value::String(text) = cell else { return; }; let retyped = match ct { - DdlColType::Int8 - | DdlColType::Int4 - | DdlColType::Int2 - // Epoch microseconds; the encoder formats the number as ISO-8601. - | DdlColType::Timestamp - | DdlColType::Timestamptz => text.parse::().ok().map(JsonValue::from), + DdlColType::Int8 | DdlColType::Int4 | DdlColType::Int2 => { + text.parse::().ok().map(Value::Integer) + } + // ISO-8601 text becomes the instant it denotes; an integer string + // carries no unit and stays text. + DdlColType::Timestamp => NdbDateTime::parse(text).map(Value::NaiveDateTime), + DdlColType::Timestamptz => NdbDateTime::parse(text).map(Value::DateTime), + // A float that parses but is not finite has no JSON form, so it stays + // text. DdlColType::Float8 | DdlColType::Float4 => text .parse::() .ok() - .and_then(serde_json::Number::from_f64) - .map(JsonValue::Number), + .filter(|f| f.is_finite()) + .map(Value::Float), DdlColType::Bool => match text.as_str() { - "t" | "true" | "TRUE" | "T" => Some(JsonValue::Bool(true)), - "f" | "false" | "FALSE" | "F" => Some(JsonValue::Bool(false)), + "t" | "true" | "TRUE" | "T" => Some(Value::Bool(true)), + "f" | "false" | "FALSE" | "F" => Some(Value::Bool(false)), _ => None, }, DdlColType::Text @@ -224,45 +215,66 @@ fn retype_cell(ct: DdlColType, cell: &mut JsonValue) { /// The announced columns with no rows — a write that matched nothing still /// answers with the result set the client was promised, just an empty one. fn empty_announced(schema: &OutputSchema) -> ShapedRows { - ShapedRows { - columns: schema + ShapedRows::from_rows( + schema .columns .iter() .map(|c| c.display_name.clone()) .collect(), - column_types: schema.columns.iter().map(|c| c.ty).collect(), - rows: Vec::new(), - notice: None, - } + schema.columns.iter().map(|c| c.ty).collect(), + Vec::new(), + ) } /// Single "result" column with zero rows, for an empty payload with nothing /// announced. fn single_result_column_empty() -> ShapedRows { - ShapedRows { - columns: vec!["result".to_string()], - column_types: ShapedRows::text_types(1), - rows: Vec::new(), - notice: None, - } + ShapedRows::from_rows( + vec!["result".to_string()], + ShapedRows::text_types(1), + Vec::new(), + ) } #[cfg(test)] mod tests { use super::*; + use crate::control::server::response_shape::cell::value_to_wire_json; use crate::control::server::response_shape::schema::OutputColumn; - fn payload(columns: &[&str], rows: &[&[Option<&str>]]) -> Vec { + fn text(s: &str) -> Value { + Value::String(s.to_string()) + } + + fn typed_payload(columns: &[&str], rows: Vec>) -> Vec { let rp = RowsPayload { columns: columns.iter().map(|c| (*c).to_string()).collect(), rows: rows - .iter() - .map(|row| row.iter().map(|cell| cell.map(|c| c.to_string())).collect()) + .into_iter() + .map(|row| row.into_iter().map(NativeCell).collect()) .collect(), }; zerompk::to_msgpack_vec(&rp).expect("encode RowsPayload") } + /// Text cells, the form a schemaless row's string fields arrive in; + /// `None` is SQL NULL. + fn payload(columns: &[&str], rows: &[&[Option<&str>]]) -> Vec { + typed_payload( + columns, + rows.iter() + .map(|row| { + row.iter() + .map(|cell| match cell { + Some(text) => Value::String((*text).to_string()), + None => Value::Null, + }) + .collect() + }) + .collect(), + ) + } + fn announced(columns: &[(&str, DdlColType)]) -> OutputSchema { OutputSchema { columns: columns @@ -305,8 +317,8 @@ mod tests { assert_eq!(shaped.column_types.len(), shaped.columns.len()); let keys = shaped.cell_keys(); assert_eq!(shaped.rows[0].len(), keys.len(), "one cell per column"); - assert_eq!(shaped.rows[0]["id"], JsonValue::String("a".into())); - assert_eq!(shaped.rows[0]["name"], JsonValue::String("x".into())); + assert_eq!(shaped.rows[0]["id"], text("a")); + assert_eq!(shaped.rows[0]["name"], text("x")); } /// An announced column the row does not carry is SQL NULL, in its @@ -321,9 +333,9 @@ mod tests { ]); let shaped = shape_returning_rows(&bytes, Some(&schema), None).expect("shape"); assert_eq!(shaped.columns, ["id", "name", "score"]); - assert_eq!(shaped.rows[0]["id"], JsonValue::String("a".into())); - assert_eq!(shaped.rows[0]["name"], JsonValue::Null); - assert_eq!(shaped.rows[0]["score"], JsonValue::from(7i64)); + assert_eq!(shaped.rows[0]["id"], text("a")); + assert_eq!(shaped.rows[0]["name"], Value::Null); + assert_eq!(shaped.rows[0]["score"], Value::Integer(7)); } /// Cells are re-read as the announced type so a binary-format request is @@ -341,11 +353,84 @@ mod tests { ("s", DdlColType::Text), ]); let shaped = shape_returning_rows(&bytes, Some(&schema), None).expect("shape"); - assert_eq!(shaped.rows[0]["i"], JsonValue::from(42i64)); - assert_eq!(shaped.rows[0]["f"], JsonValue::from(1.5f64)); - assert_eq!(shaped.rows[0]["b"], JsonValue::Bool(true)); + assert_eq!(shaped.rows[0]["i"], Value::Integer(42)); + assert_eq!(shaped.rows[0]["f"], Value::Float(1.5)); + assert_eq!(shaped.rows[0]["b"], Value::Bool(true)); // A TEXT column keeps its text even when it looks like a number. - assert_eq!(shaped.rows[0]["s"], JsonValue::String("42".into())); + assert_eq!(shaped.rows[0]["s"], text("42")); + } + + /// ISO-8601 text under a timestamp column becomes the instant it denotes, + /// naive under `TIMESTAMP` and UTC under `TIMESTAMPTZ`; an integer string + /// carries no unit and stays text. + #[test] + fn timestamp_text_is_retyped_to_an_instant() { + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + let bytes = payload( + &["ts", "tstz", "digits"], + &[&[ + Some("2020-03-05 10:00:00"), + Some("2020-03-05T10:00:00Z"), + Some("1583402400000000"), + ]], + ); + let schema = announced(&[ + ("ts", DdlColType::Timestamp), + ("tstz", DdlColType::Timestamptz), + ("digits", DdlColType::Timestamp), + ]); + let shaped = shape_returning_rows(&bytes, Some(&schema), None).expect("shape"); + assert_eq!(shaped.rows[0]["ts"], Value::NaiveDateTime(at)); + assert_eq!(shaped.rows[0]["tstz"], Value::DateTime(at)); + assert_eq!(shaped.rows[0]["digits"], text("1583402400000000")); + } + + /// A typed cell passes through as itself: an integer stays a number under + /// an INT column, an instant stays an instant (and renders at the edge as + /// the ISO-8601 text a `SELECT` of the same column renders), and SQL NULL + /// stays NULL. + #[test] + fn typed_cells_reach_the_row_as_themselves() { + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + let bytes = typed_payload( + &["n", "at", "f", "gone"], + vec![vec![ + Value::Integer(7), + Value::NaiveDateTime(at), + Value::Float(1.5), + Value::Null, + ]], + ); + let schema = announced(&[ + ("n", DdlColType::Int8), + ("at", DdlColType::Timestamp), + ("f", DdlColType::Float8), + ("gone", DdlColType::Text), + ]); + let shaped = shape_returning_rows(&bytes, Some(&schema), None).expect("shape"); + assert_eq!(shaped.rows[0]["n"], Value::Integer(7)); + assert_eq!(shaped.rows[0]["at"], Value::NaiveDateTime(at)); + assert_eq!( + value_to_wire_json(&shaped.rows[0]["at"]), + serde_json::Value::String("2020-03-05T10:00:00.000000Z".into()) + ); + assert_eq!(shaped.rows[0]["f"], Value::Float(1.5)); + assert_eq!(shaped.rows[0]["gone"], Value::Null); + } + + /// The same instant stays typed when nothing was announced, and renders + /// as ISO-8601 at the edge, so the simple-query protocol shows what the + /// extended one does. + #[test] + fn an_instant_cell_renders_iso8601_without_a_projection() { + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + let bytes = typed_payload(&["at"], vec![vec![Value::DateTime(at)]]); + let shaped = shape_returning_rows(&bytes, None, None).expect("shape"); + assert_eq!(shaped.rows[0]["at"], Value::DateTime(at)); + assert_eq!( + value_to_wire_json(&shaped.rows[0]["at"]), + serde_json::Value::String("2020-03-05T10:00:00.000000Z".into()) + ); } /// A value that does not parse as its announced type stays text rather @@ -355,10 +440,7 @@ mod tests { let bytes = payload(&["i"], &[&[Some("not a number")]]); let schema = announced(&[("i", DdlColType::Int8)]); let shaped = shape_returning_rows(&bytes, Some(&schema), None).expect("shape"); - assert_eq!( - shaped.rows[0]["i"], - JsonValue::String("not a number".into()) - ); + assert_eq!(shaped.rows[0]["i"], text("not a number")); } /// A write that matched nothing still answers with the announced result diff --git a/nodedb/src/control/server/response_shape/types/mod.rs b/nodedb/src/control/server/response_shape/types/mod.rs new file mode 100644 index 000000000..9b0a0fb2d --- /dev/null +++ b/nodedb/src/control/server/response_shape/types/mod.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Protocol-neutral plan classification and shaped row-set types. + +pub mod plan_kind; +pub mod shaped; + +pub use plan_kind::{PlanKind, describe_plan}; +pub use shaped::{DdlColType, ShapedRow, ShapedRows}; diff --git a/nodedb/src/control/server/response_shape/types.rs b/nodedb/src/control/server/response_shape/types/plan_kind.rs similarity index 77% rename from nodedb/src/control/server/response_shape/types.rs rename to nodedb/src/control/server/response_shape/types/plan_kind.rs index b70d97615..0f68cd00d 100644 --- a/nodedb/src/control/server/response_shape/types.rs +++ b/nodedb/src/control/server/response_shape/types/plan_kind.rs @@ -265,167 +265,11 @@ pub fn describe_plan(plan: &PhysicalPlan) -> PlanKind { // Bring the variant into scope for brevity in match arms above. use PlanKind::DmlResult; -/// Protocol-neutral SQL column type, mapped to each entrypoint's own wire -/// type. One variant per pgwire field-builder, so the mapping is lossless. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum DdlColType { - #[default] - Text, - Int8, - Int4, - Int2, - Float8, - Float4, - Bool, - Bytea, - Json, - Jsonb, - Timestamp, - Timestamptz, - Varchar, - Float4Array, - Float8Array, -} - -/// Protocol-neutral shaped row set: columns + row objects + an optional -/// client-facing notice. -#[derive(Debug, Clone)] -pub struct ShapedRows { - pub columns: Vec, - /// Per-column SQL type, parallel to `columns`. Only pgwire consumes this - /// (RowDescription OIDs); `Text` when the source type is unknown. - pub column_types: Vec, - /// One map per row, keyed by [`ShapedRows::cell_keys`] not `columns` — - /// SQL output names may repeat and a map can't hold two cells per key. - pub rows: Vec>, - pub notice: Option, -} - -impl ShapedRows { - /// Build a `column_types` vec of `n` `Text` entries, for non-DDL sites - /// whose consumers ignore column types. - pub fn text_types(n: usize) -> Vec { - vec![DdlColType::Text; n] - } - - /// Fold another shaped result into this one so N tasks answer with ONE result - /// set — some drivers reject multiple result sets. Columns are the union of - /// every contributor's; rows read by key so a missing column encodes NULL. - pub fn append(&mut self, other: ShapedRows) { - if self.notice.is_none() { - self.notice = other.notice; - } - if self.columns.is_empty() { - self.columns = other.columns; - self.column_types = other.column_types; - self.rows.extend(other.rows); - return; - } - for (index, name) in other.columns.iter().enumerate() { - if self.columns.iter().any(|existing| existing == name) { - continue; - } - self.columns.push(name.clone()); - self.column_types - .push(other.column_types.get(index).copied().unwrap_or_default()); - } - self.rows.extend(other.rows); - } - - /// Per-column keys for reading cells out of [`ShapedRows::rows`]. Identical - /// to `columns` unless names collide, then later duplicates take a `_` suffix - /// — visible in HTTP JSON, but pgwire/native stay positional. - pub fn cell_keys(&self) -> Vec { - super::project::cell_keys(&self.columns) - } -} - #[cfg(test)] mod tests { use super::*; use nodedb_types::{DatabaseId, QualifiedCollection}; - fn shaped(columns: &[&str], rows: &[&[(&str, &str)]]) -> ShapedRows { - ShapedRows { - columns: columns.iter().map(|c| (*c).to_string()).collect(), - column_types: ShapedRows::text_types(columns.len()), - rows: rows - .iter() - .map(|row| { - row.iter() - .map(|(k, v)| { - ( - (*k).to_string(), - serde_json::Value::String((*v).to_string()), - ) - }) - .collect() - }) - .collect(), - notice: None, - } - } - - /// A column only a later contributor carries must survive the fold — a key - /// absent from `columns` is never read by `cell_keys()`. - #[test] - fn append_unions_a_column_only_a_later_row_carries() { - let mut merged = shaped(&["id", "name"], &[&[("id", "r1"), ("name", "a")]]); - merged.append(shaped( - &["id", "name", "extra"], - &[&[("id", "r2"), ("name", "b"), ("extra", "x")]], - )); - - assert_eq!(merged.columns, vec!["id", "name", "extra"]); - assert_eq!( - merged.column_types.len(), - merged.columns.len(), - "column types must stay parallel to columns" - ); - - let keys = merged.cell_keys(); - assert_eq!(keys, vec!["id", "name", "extra"]); - assert_eq!( - merged.rows[1].get(keys[2].as_str()), - Some(&serde_json::Value::String("x".to_string())), - "the later row's extra value must be readable through the merged keys" - ); - assert!( - merged.rows[0].get(keys[2].as_str()).is_none(), - "the row that lacks the column encodes as NULL, not a shifted cell" - ); - } - - /// The first contributor's columns keep their positions and newly-seen - /// columns are appended in first-seen order, so a positional client never - /// sees a column move between rows. - #[test] - fn append_keeps_the_first_contributors_column_order_and_appends_the_rest() { - let mut merged = shaped(&["b", "a"], &[&[("b", "1"), ("a", "2")]]); - merged.append(shaped(&["a", "z"], &[&[("a", "3"), ("z", "4")]])); - merged.append(shaped(&["y", "b"], &[&[("y", "5"), ("b", "6")]])); - - assert_eq!( - merged.columns, - vec!["b", "a", "z", "y"], - "first contributor's positions are fixed; later columns append in \ - first-seen order" - ); - assert_eq!(merged.rows.len(), 3); - } - - /// A contributor with no columns at all — a task whose rows were entirely - /// removed by a read policy, which shapes as `RETURNING *` with an empty - /// column list — must not fix an empty shape for the statement. - #[test] - fn append_adopts_the_shape_of_the_first_contributor_that_has_columns() { - let mut merged = shaped(&[], &[]); - merged.append(shaped(&["id"], &[&[("id", "r1")]])); - - assert_eq!(merged.columns, vec!["id"]); - assert_eq!(merged.rows.len(), 1); - } - #[test] fn crdt_preview_is_an_opaque_execution_plan() { let plan = PhysicalPlan::Crdt(CrdtOp::PreviewApply { diff --git a/nodedb/src/control/server/response_shape/types/shaped.rs b/nodedb/src/control/server/response_shape/types/shaped.rs new file mode 100644 index 000000000..6774c8a0d --- /dev/null +++ b/nodedb/src/control/server/response_shape/types/shaped.rs @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Protocol-neutral shaped row set: typed cells keyed by column. + +use std::collections::BTreeMap; + +use nodedb_types::Value; + +/// Protocol-neutral SQL column type, mapped to each entrypoint's own wire +/// type. One variant per pgwire field-builder, so the mapping is lossless. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum DdlColType { + #[default] + Text, + Int8, + Int4, + Int2, + Float8, + Float4, + Bool, + Bytea, + Json, + Jsonb, + Timestamp, + Timestamptz, + Varchar, + Float4Array, + Float8Array, +} + +/// One shaped row: typed cells keyed by [`ShapedRows::cell_keys`]. +/// +/// A `BTreeMap` iterates in sorted key order, which is the order column +/// derivation reads a row's keys in. +pub type ShapedRow = BTreeMap; + +/// Protocol-neutral shaped row set: columns + row objects + an optional +/// client-facing notice. +#[derive(Debug, Clone)] +pub struct ShapedRows { + pub columns: Vec, + /// Per-column SQL type, parallel to `columns`. Only pgwire consumes this + /// (RowDescription OIDs); `Text` when the source type is unknown. + pub column_types: Vec, + /// One map per row, keyed by [`ShapedRows::cell_keys`] not `columns` — + /// SQL output names may repeat and a map can't hold two cells per key. + pub rows: Vec, + pub notice: Option, +} + +impl ShapedRows { + /// Build a `column_types` vec of `n` `Text` entries, for non-DDL sites + /// whose consumers ignore column types. + pub fn text_types(n: usize) -> Vec { + vec![DdlColType::Text; n] + } + + /// A result set from typed rows, with one catalog type per column and no + /// notice. + pub fn from_rows( + columns: Vec, + column_types: Vec, + rows: Vec, + ) -> Self { + Self { + columns, + column_types, + rows, + notice: None, + } + } + + /// A result set from decoded JSON rows, with one catalog type per column + /// and no notice. Each JSON cell becomes the matching [`Value`]: a JSON + /// string is a `Value::String`, never parsed further. + pub fn from_json_rows( + columns: Vec, + column_types: Vec, + rows: Vec>, + ) -> Self { + let rows = rows.into_iter().map(json_row_to_shaped).collect(); + Self::from_rows(columns, column_types, rows) + } + + /// A result set whose every column is `Text`: the shape a DDL or + /// inspection statement answers with. The type list is sized from + /// `columns`, so the two cannot disagree. + pub fn text_rows( + columns: Vec, + rows: Vec>, + ) -> Self { + let column_types = Self::text_types(columns.len()); + Self::from_json_rows(columns, column_types, rows) + } + + /// Attach a client-facing notice. + pub fn with_notice(mut self, notice: impl Into) -> Self { + self.notice = Some(notice.into()); + self + } + + /// Fold another shaped result into this one so N tasks answer with ONE result + /// set — some drivers reject multiple result sets. Columns are the union of + /// every contributor's; rows read by key so a missing column encodes NULL. + pub fn append(&mut self, other: ShapedRows) { + if self.notice.is_none() { + self.notice = other.notice; + } + if self.columns.is_empty() { + self.columns = other.columns; + self.column_types = other.column_types; + self.rows.extend(other.rows); + return; + } + for (index, name) in other.columns.iter().enumerate() { + if self.columns.iter().any(|existing| existing == name) { + continue; + } + self.columns.push(name.clone()); + self.column_types + .push(other.column_types.get(index).copied().unwrap_or_default()); + } + self.rows.extend(other.rows); + } + + /// Per-column keys for reading cells out of [`ShapedRows::rows`]. Identical + /// to `columns` unless names collide, then later duplicates take a `_` suffix + /// — visible in HTTP JSON, but pgwire/native stay positional. + pub fn cell_keys(&self) -> Vec { + crate::control::server::response_shape::project::cell_keys(&self.columns) + } +} + +/// Convert one JSON row map to a typed row, cell by cell. +fn json_row_to_shaped(row: serde_json::Map) -> ShapedRow { + row.into_iter().map(|(k, v)| (k, Value::from(v))).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn shaped(columns: &[&str], rows: &[&[(&str, &str)]]) -> ShapedRows { + ShapedRows { + columns: columns.iter().map(|c| (*c).to_string()).collect(), + column_types: ShapedRows::text_types(columns.len()), + rows: rows + .iter() + .map(|row| { + row.iter() + .map(|(k, v)| ((*k).to_string(), Value::String((*v).to_string()))) + .collect() + }) + .collect(), + notice: None, + } + } + + /// A column only a later contributor carries must survive the fold — a key + /// absent from `columns` is never read by `cell_keys()`. + #[test] + fn append_unions_a_column_only_a_later_row_carries() { + let mut merged = shaped(&["id", "name"], &[&[("id", "r1"), ("name", "a")]]); + merged.append(shaped( + &["id", "name", "extra"], + &[&[("id", "r2"), ("name", "b"), ("extra", "x")]], + )); + + assert_eq!(merged.columns, vec!["id", "name", "extra"]); + assert_eq!( + merged.column_types.len(), + merged.columns.len(), + "column types must stay parallel to columns" + ); + + let keys = merged.cell_keys(); + assert_eq!(keys, vec!["id", "name", "extra"]); + assert_eq!( + merged.rows[1].get(keys[2].as_str()), + Some(&Value::String("x".to_string())), + "the later row's extra value must be readable through the merged keys" + ); + assert!( + !merged.rows[0].contains_key(keys[2].as_str()), + "the row that lacks the column encodes as NULL, not a shifted cell" + ); + } + + /// The first contributor's columns keep their positions and newly-seen + /// columns are appended in first-seen order, so a positional client never + /// sees a column move between rows. + #[test] + fn append_keeps_the_first_contributors_column_order_and_appends_the_rest() { + let mut merged = shaped(&["b", "a"], &[&[("b", "1"), ("a", "2")]]); + merged.append(shaped(&["a", "z"], &[&[("a", "3"), ("z", "4")]])); + merged.append(shaped(&["y", "b"], &[&[("y", "5"), ("b", "6")]])); + + assert_eq!( + merged.columns, + vec!["b", "a", "z", "y"], + "first contributor's positions are fixed; later columns append in \ + first-seen order" + ); + assert_eq!(merged.rows.len(), 3); + } + + /// A contributor with no columns at all — a task whose rows were entirely + /// removed by a read policy, which shapes as `RETURNING *` with an empty + /// column list — must not fix an empty shape for the statement. + #[test] + fn append_adopts_the_shape_of_the_first_contributor_that_has_columns() { + let mut merged = shaped(&[], &[]); + merged.append(shaped(&["id"], &[&[("id", "r1")]])); + + assert_eq!(merged.columns, vec!["id"]); + assert_eq!(merged.rows.len(), 1); + } + + /// Every JSON cell kind lands as the matching typed cell; a JSON string + /// stays a string and is never parsed into an instant or a number. + #[test] + fn from_json_rows_maps_each_json_kind_to_the_matching_value() { + let row = match serde_json::json!({ + "n": 7, + "f": 1.5, + "s": "2020-03-05T10:00:00.000000Z", + "z": null, + "b": true, + "a": [1, "x"], + "o": {"k": false}, + }) { + serde_json::Value::Object(map) => map, + other => panic!("fixture must be an object, got {other}"), + }; + let columns: Vec = ["n", "f", "s", "z", "b", "a", "o"] + .iter() + .map(|c| (*c).to_string()) + .collect(); + let shaped = ShapedRows::from_json_rows( + columns.clone(), + ShapedRows::text_types(columns.len()), + vec![row], + ); + + let cells = &shaped.rows[0]; + assert_eq!(cells["n"], Value::Integer(7)); + assert_eq!(cells["f"], Value::Float(1.5)); + assert_eq!( + cells["s"], + Value::String("2020-03-05T10:00:00.000000Z".to_string()) + ); + assert_eq!(cells["z"], Value::Null); + assert_eq!(cells["b"], Value::Bool(true)); + assert_eq!( + cells["a"], + Value::Array(vec![Value::Integer(1), Value::String("x".to_string())]) + ); + assert_eq!( + cells["o"], + Value::Object(std::collections::HashMap::from([( + "k".to_string(), + Value::Bool(false) + )])) + ); + assert!(shaped.notice.is_none()); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/alert/show.rs b/nodedb/src/control/server/shared/ddl/neutral/alert/show.rs index 02a4c4b94..5720a65d7 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/alert/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/alert/show.rs @@ -107,12 +107,11 @@ pub fn show_alerts( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW ALERT STATUS ON — per-group active/cleared state. @@ -209,12 +208,11 @@ pub fn show_alert_status( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } fn format_duration_ms(ms: u64) -> String { diff --git a/nodedb/src/control/server/shared/ddl/neutral/apikey/create.rs b/nodedb/src/control/server/shared/ddl/neutral/apikey/create.rs index 28ced72dd..ba471fc00 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/apikey/create.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/apikey/create.rs @@ -13,7 +13,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::control::security::audit::AuditEvent; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use super::super::super::result::{DdlError, DdlResult}; @@ -146,10 +146,8 @@ pub fn create_api_key( let mut row = Map::new(); row.insert("api_key".to_string(), JsonValue::String(token)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["api_key".to_string()], - column_types: vec![DdlColType::Text], - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["api_key".to_string()], + vec![row], + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/apikey/manage.rs b/nodedb/src/control/server/shared/ddl/neutral/apikey/manage.rs index 035c04671..3069870d2 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/apikey/manage.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/apikey/manage.rs @@ -174,10 +174,9 @@ pub fn list_api_keys( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/auth_key.rs b/nodedb/src/control/server/shared/ddl/neutral/auth_key.rs index 7cafb8a7e..57db25aef 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/auth_key.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/auth_key.rs @@ -17,7 +17,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use super::super::result::{DdlError, DdlResult}; @@ -99,12 +99,10 @@ pub fn create_auth_key( let mut row = Map::new(); row.insert("auth_api_key".to_string(), JsonValue::String(token)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["auth_api_key".to_string()], - column_types: vec![DdlColType::Text], - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["auth_api_key".to_string()], + vec![row], + ))]) } /// ROTATE AUTH KEY '' [OVERLAP 24h] @@ -144,12 +142,10 @@ pub fn rotate_auth_key( let mut row = Map::new(); row.insert("new_auth_api_key".to_string(), JsonValue::String(new_token)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["new_auth_api_key".to_string()], - column_types: vec![DdlColType::Text], - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["new_auth_api_key".to_string()], + vec![row], + ))]) } /// LIST AUTH KEYS [FOR AUTH USER ''] @@ -179,7 +175,6 @@ pub fn list_auth_keys( "last_used_at".to_string(), "last_used_ip".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = keys .iter() @@ -219,10 +214,5 @@ pub fn list_auth_keys( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/auth_user.rs b/nodedb/src/control/server/shared/ddl/neutral/auth_user.rs index 34729cc3b..66e84db8f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/auth_user.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/auth_user.rs @@ -199,7 +199,6 @@ pub fn show_auth_users( "is_active".to_string(), "last_seen".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = users .iter() @@ -235,12 +234,7 @@ pub fn show_auth_users( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Public re-export of duration parser for use by other DDL modules. diff --git a/nodedb/src/control/server/shared/ddl/neutral/blacklist.rs b/nodedb/src/control/server/shared/ddl/neutral/blacklist.rs index 1e8203a16..5ce7301da 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/blacklist.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/blacklist.rs @@ -268,7 +268,6 @@ pub fn show_blacklist( "created_at".to_string(), "expires_at".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = entries .iter() @@ -297,12 +296,7 @@ pub fn show_blacklist( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Extract UNTIL timestamp from parts. Returns 0 (permanent) if not present. diff --git a/nodedb/src/control/server/shared/ddl/neutral/change_stream/show.rs b/nodedb/src/control/server/shared/ddl/neutral/change_stream/show.rs index 0b39f2319..1316326a7 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/change_stream/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/change_stream/show.rs @@ -61,11 +61,5 @@ pub fn show_change_streams( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/chunk_text.rs b/nodedb/src/control/server/shared/ddl/neutral/chunk_text.rs index 6872b14b4..5702298dc 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/chunk_text.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/chunk_text.rs @@ -10,7 +10,7 @@ use nodedb_sql::parser::preprocess::lex::find_ascii_case_insensitive; use serde_json::{Map, Value as JsonValue}; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use super::super::result::{DdlError, DdlResult}; @@ -135,13 +135,6 @@ pub fn execute_chunk_text(sql: &str) -> Result, DdlError> { "end".to_string(), "text".to_string(), ]; - let column_types = vec![ - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - ]; - let rows: Vec> = chunks .iter() .map(|c| { @@ -154,12 +147,7 @@ pub fn execute_chunk_text(sql: &str) -> Result, DdlError> { }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Build a [`DdlError`] from a SQLSTATE + message. diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/health.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/health.rs index 22f5375ec..4824b0371 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/health.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/health.rs @@ -103,10 +103,9 @@ pub fn show_peer_health( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/migration.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/migration.rs index f62d2e8db..8002aa2a3 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/migration.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/migration.rs @@ -76,10 +76,9 @@ pub fn show_migrations( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/raft.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/raft.rs index 014b8dbae..19c2977da 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/raft.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/raft.rs @@ -100,12 +100,11 @@ pub fn show_raft_groups( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW RAFT GROUP — detailed info for a specific Raft group. @@ -153,7 +152,6 @@ pub fn show_raft_group( }; let columns = vec!["property".to_string(), "value".to_string()]; - let column_types = vec![DdlColType::Text, DdlColType::Text]; let props = [ ("group_id", group.group_id.to_string()), @@ -203,12 +201,7 @@ pub fn show_raft_group( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// ALTER RAFT GROUP ADD|REMOVE NODE diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/ranges.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/ranges.rs index a3cc88651..c7de6a4b5 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/ranges.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/ranges.rs @@ -128,10 +128,9 @@ pub fn show_ranges( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/rebalance_cmd.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/rebalance_cmd.rs index cd2fe9fd5..593379f46 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/rebalance_cmd.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/rebalance_cmd.rs @@ -59,12 +59,10 @@ pub fn rebalance( "status".to_string(), JsonValue::String("cluster is balanced — no moves needed".to_string()), ); - return Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["status".to_string()], - column_types: vec![DdlColType::Text], - rows: vec![row], - notice: None, - })]); + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["status".to_string()], + vec![row], + ))]); } let columns = vec![ @@ -102,10 +100,9 @@ pub fn rebalance( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/routing_hint.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/routing_hint.rs index 1ac0dbe13..58291ac80 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/routing_hint.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/routing_hint.rs @@ -85,10 +85,9 @@ pub fn show_routing( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/schema_version.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/schema_version.rs index 99a94f561..47143629f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/schema_version.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/schema_version.rs @@ -11,7 +11,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use super::super::super::result::{DdlError, DdlResult}; @@ -31,7 +31,6 @@ pub fn show_schema_version( } let columns = vec!["property".to_string(), "value".to_string()]; - let column_types = vec![DdlColType::Text, DdlColType::Text]; let mut rows = Vec::new(); @@ -73,10 +72,5 @@ pub fn show_schema_version( ); rows.push(row); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/topology.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/topology.rs index 86093127d..e46c11976 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/topology.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/topology.rs @@ -92,12 +92,11 @@ pub fn show_nodes( } } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW NODE — detailed info for a specific node. @@ -124,7 +123,6 @@ pub fn show_node( .map_err(|_| ddl_err("42601", format!("invalid node_id: '{}'", parts[2])))?; let columns = vec!["property".to_string(), "value".to_string()]; - let column_types = vec![DdlColType::Text, DdlColType::Text]; let props = match &state.cluster_topology { Some(t) => { @@ -182,12 +180,7 @@ pub fn show_node( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// REMOVE NODE — mark a node as decommissioned. @@ -255,7 +248,6 @@ pub fn show_cluster( } let columns = vec!["property".to_string(), "value".to_string()]; - let column_types = vec![DdlColType::Text, DdlColType::Text]; let mut props = vec![("node_id", state.node_id.to_string())]; @@ -292,10 +284,5 @@ pub fn show_cluster( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/alter/add_column.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/alter/add_column.rs index 9a641a494..6976f6749 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/alter/add_column.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/alter/add_column.rs @@ -15,6 +15,9 @@ use nodedb_types::DatabaseId; use crate::control::security::audit::AuditEvent; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::shared::ddl::neutral::collection::helpers::parse_origin_column_def; +use crate::control::server::shared::ddl::neutral::column_default::{ + DeclaredColumn, validate_column_default, +}; use crate::control::server::shared::ddl::result::{DdlError, DdlResult}; use crate::control::state::SharedState; @@ -52,6 +55,18 @@ pub(super) async fn alter_table_add_column( ), )); } + // A DEFAULT passes the same gate `CREATE` applies: evaluable, and a + // literal the declared type can hold. + if let Some(expr) = &column.default { + validate_column_default( + &DeclaredColumn { + name: &column.name, + declared_type: &declared_type, + primary_key: column.primary_key, + }, + expr, + )?; + } let updated = { let catalog = state.credentials.catalog(); @@ -112,6 +127,7 @@ pub(super) async fn alter_table_add_column( super::super::register::dispatch_register_from_stored(state, coll) .await .map_err(|e| err("XX000", e.to_string()))?; + super::strict_schema::recompile_rls_policies(state, coll)?; } state.audit_record( diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/alter/strict_schema.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/alter/strict_schema.rs index e7cfa934d..2e48e4b01 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/alter/strict_schema.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/alter/strict_schema.rs @@ -108,7 +108,8 @@ pub(super) fn add_field(coll: &mut StoredCollection, column: &str, declared_type /// Replicate the mutated collection through the metadata raft group, /// refresh this node's Data Plane register so the in-memory shape -/// catches up with the new schema, then bump `schema_version`. +/// catches up with the new schema, recompile the collection's RLS +/// policies against it, then bump `schema_version`. pub(super) async fn persist_schema_change( state: &SharedState, updated: &StoredCollection, @@ -120,6 +121,28 @@ pub(super) async fn persist_schema_change( super::super::register::dispatch_register_from_stored(state, updated) .await .map_err(|e| err("XX000", e.to_string()))?; + recompile_rls_policies(state, updated)?; state.schema_version.bump(); Ok(()) } + +/// Recompile the RLS policies on `updated` against its new declared columns. +/// +/// A policy literal is typed against the column it compares with, so a +/// column the statement added, dropped, or renamed changes what the policy +/// compiles to. The Raft post-apply recompiles on every node in cluster mode; +/// this call covers the single-node path, where no post-apply runs. +pub(super) fn recompile_rls_policies( + state: &SharedState, + updated: &StoredCollection, +) -> Result<(), DdlError> { + state + .rls + .recompile_for_collection( + state.credentials.catalog(), + updated.database_id, + updated.tenant_id, + &updated.name, + ) + .map_err(|e| err("XX000", format!("rls recompile: {e}"))) +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs index a3ed56209..324cd03e6 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs @@ -87,9 +87,10 @@ pub async fn build_and_persist( )); } - // Refuse a DEFAULT the server cannot evaluate here, not at the first - // INSERT. It runs before any lifecycle guard or predecessor purge, so a - // rejected declaration leaves the existing state untouched. A SERIAL + // Refuse a DEFAULT the server cannot evaluate, or that the declared + // column type cannot hold, here, not at the first INSERT. It runs before + // any lifecycle guard or predecessor purge, so a rejected declaration + // leaves the existing state untouched. A SERIAL // column carries no DEFAULT text yet; the `nextval` this build generates // for it names a registered function and clears the same gate. validate_column_defaults(columns)?; diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/describe.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/describe.rs index 3c26f7d9e..62731497a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/describe.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/describe.rs @@ -66,8 +66,6 @@ pub fn describe_collection( "type".to_string(), "nullable".to_string(), ]; - let column_types = vec![DdlColType::Text, DdlColType::Text, DdlColType::Text]; - let mut rows = Vec::new(); // Synthesize the implicit 'id' field only when the collection does not @@ -147,12 +145,7 @@ pub fn describe_collection( } } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// SHOW COLLECTIONS @@ -242,10 +235,9 @@ pub fn show_collections( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/show_indexes.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/show_indexes.rs index a016807e9..199757094 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/show_indexes.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/show_indexes.rs @@ -16,7 +16,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::control::security::catalog::StoredIndexRecord; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::ddl::sql_parse::parse_ident_token; use crate::control::state::SharedState; use crate::types::DatabaseId; @@ -51,14 +51,6 @@ pub fn show_indexes( "fields".to_string(), "owner".to_string(), ]; - let column_types = vec![ - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - ]; - let mut records = state .credentials .catalog() @@ -105,10 +97,5 @@ pub fn show_indexes( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/vector_metadata.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/vector_metadata.rs index 9b23ad6c3..e77c3a0ba 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/vector_metadata.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/vector_metadata.rs @@ -218,12 +218,7 @@ pub fn handle_show_vector_models( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types: ShapedRows::text_types(6), - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Handle `SELECT VECTOR_METADATA('collection', 'column')` — return JSON. @@ -258,12 +253,10 @@ pub fn handle_vector_metadata_query( let mut row = Map::new(); row.insert("vector_metadata".to_string(), JsonValue::String(json)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["vector_metadata".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["vector_metadata".to_string()], + vec![row], + ))]) } /// Format a Unix timestamp (seconds) as an ISO-8601 UTC date string (YYYY-MM-DD). diff --git a/nodedb/src/control/server/shared/ddl/neutral/column_default.rs b/nodedb/src/control/server/shared/ddl/neutral/column_default.rs index 390409a66..d7c07eb6f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/column_default.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/column_default.rs @@ -14,31 +14,86 @@ //! The check runs `nodedb_sql`'s DEFAULT classifier, which consults the same //! `FunctionRegistry` the resolver's undefined-function gate consults. No //! second registry and no second name list exist here. +//! +//! A column `DEFAULT` that spells a literal is also checked against the +//! column's declared type, through the same coercion an INSERT applies to +//! the materialized value. `DEFAULT 'not a date'` on a `TIMESTAMP` column +//! and `DEFAULT 999999` on a `SMALLINT` column are refused at the +//! declaration, so no insert can ever materialize a value the column cannot +//! hold. use nodedb_sql::SqlError; use nodedb_sql::ddl_ast::collection_type::parse_column_type_str_full; +use nodedb_sql::planner::declared_type_coerce::coerce_write_literal; +use nodedb_sql::planner::defaults::{CompiledDefault, default_value_to_sql}; use nodedb_types::error::sqlstate; use super::super::result::DdlError; +use crate::control::planner::catalog_adapter::declared_column_info; + +/// One declared column as the DDL gate sees it. +pub(super) struct DeclaredColumn<'a> { + pub name: &'a str, + /// The declared type text, modifiers included (`SMALLINT NOT NULL`, + /// `TIMESTAMP TIME_KEY`). The bare type resolves out of it the same way + /// the catalog adapter resolves a tracked field. + pub declared_type: &'a str, + /// Whether the column is the primary key. The key keeps its literal as + /// written on every write path, so its DEFAULT is not re-typed either. + pub primary_key: bool, +} -/// Refuse a declared column `DEFAULT` the server cannot evaluate. +/// Refuse a declared column `DEFAULT` the server cannot evaluate, or that the +/// column's declared type cannot hold. /// /// Each pair carries a column name and its declared type text, and the /// `DEFAULT` clause is read out of that text. pub(super) fn validate_column_defaults(columns: &[(String, String)]) -> Result<(), DdlError> { for (column, type_str) in columns { - let (_, _, _, default_expr) = parse_column_type_str_full(type_str); + let (_, primary_key, _, default_expr) = parse_column_type_str_full(type_str); let Some(expr) = default_expr else { continue; }; - validate_column_default(column, &expr)?; + validate_column_default( + &DeclaredColumn { + name: column, + declared_type: type_str, + primary_key, + }, + &expr, + )?; } Ok(()) } -/// Refuse one declared column `DEFAULT` the server cannot evaluate. -pub(super) fn validate_column_default(column: &str, expr: &str) -> Result<(), DdlError> { - validate_clause_expr("DEFAULT", column, expr) +/// Refuse one declared column `DEFAULT` the server cannot evaluate, or that +/// the column's declared type cannot hold. +/// +/// The expression is classified and parsed, never evaluated, so a +/// `DEFAULT nextval('s')` column never advances its sequence at DDL time. A +/// literal is then coerced to the declared type and range-checked exactly as +/// an INSERT coerces the materialized value; a generator or an expression has +/// no value to check until it is evaluated. +/// +/// An unregistered function name raises SQLSTATE `42883`, a literal the +/// declared type cannot represent `42804`, a literal past the declared +/// numeric width `22003`, and every other rejection `42601`. +pub(super) fn validate_column_default( + column: &DeclaredColumn<'_>, + expr: &str, +) -> Result<(), DdlError> { + let compiled = CompiledDefault::declare(column.name, expr) + .map_err(|error| clause_error("DEFAULT", column.name, &error))?; + let Some(literal) = compiled.literal() else { + return Ok(()); + }; + let mut info = declared_column_info(column.name, column.declared_type); + info.is_primary_key = column.primary_key; + let value = default_value_to_sql(column.name, literal.clone()) + .map_err(|error| clause_error("DEFAULT", column.name, &error))?; + coerce_write_literal(&info, value) + .map(|_| ()) + .map_err(|error| clause_error("DEFAULT", column.name, &error)) } /// Refuse one declared value-producing clause the server cannot evaluate. @@ -61,6 +116,10 @@ pub(super) fn validate_clause_expr(clause: &str, owner: &str, expr: &str) -> Res fn clause_error(clause: &str, owner: &str, error: &SqlError) -> DdlError { let sqlstate = match error { SqlError::UndefinedFunction { .. } => sqlstate::UNDEFINED_FUNCTION, + SqlError::TypeMismatch { .. } => sqlstate::DATATYPE_MISMATCH, + SqlError::IntegerOutOfRange { .. } | SqlError::FloatOutOfRange { .. } => { + sqlstate::NUMERIC_VALUE_OUT_OF_RANGE + } _ => sqlstate::SYNTAX_ERROR, }; DdlError::new( @@ -68,3 +127,69 @@ fn clause_error(clause: &str, owner: &str, error: &SqlError) -> DdlError { format!("{clause} for '{owner}' is invalid: {error}"), ) } + +#[cfg(test)] +mod tests { + use super::*; + + fn declared(name: &str, declared_type: &str) -> (String, String) { + (name.to_string(), declared_type.to_string()) + } + + /// Text that spells no instant is refused on a TIMESTAMP column, naming + /// the column, with the datatype-mismatch SQLSTATE. + #[test] + fn a_text_default_with_no_instant_is_refused_on_a_timestamp_column() { + let error = validate_column_defaults(&[declared("at", "TIMESTAMP DEFAULT 'not a date'")]) + .expect_err("'not a date' is not an instant"); + assert_eq!(error.sqlstate, sqlstate::DATATYPE_MISMATCH); + assert!(error.message.contains("'at'"), "{}", error.message); + } + + /// An integer past the declared SMALLINT width is refused with the + /// out-of-range SQLSTATE, naming the column. + #[test] + fn an_out_of_range_default_is_refused_on_a_smallint_column() { + let error = validate_column_defaults(&[declared("s", "SMALLINT DEFAULT 999999")]) + .expect_err("999999 does not fit SMALLINT"); + assert_eq!(error.sqlstate, sqlstate::NUMERIC_VALUE_OUT_OF_RANGE); + assert!(error.message.contains("'s'"), "{}", error.message); + } + + /// A literal the column can hold is accepted, in every spelling the + /// INSERT coercion accepts: epoch milliseconds and ISO text on a + /// TIMESTAMP column, an in-range integer on SMALLINT, numeric text on a + /// FLOAT column, and a TIME_KEY modifier before the clause. + #[test] + fn a_representable_default_is_accepted() { + validate_column_defaults(&[ + declared("at", "TIMESTAMP DEFAULT 1583402400000"), + declared("at2", "TIMESTAMP DEFAULT '2020-03-05 10:00:00'"), + declared("ts", "TIMESTAMP TIME_KEY DEFAULT 1583402400000"), + declared("s", "SMALLINT DEFAULT 7"), + declared("r", "FLOAT DEFAULT '1.5'"), + declared("name", "TEXT DEFAULT 'h0'"), + ]) + .expect("every default is representable"); + } + + /// The primary key keeps its literal as written, so its DEFAULT is not + /// re-typed; a generator has no value to check at declaration. + #[test] + fn primary_key_and_generator_defaults_pass_the_type_gate() { + validate_column_defaults(&[ + declared("id", "INT PRIMARY KEY DEFAULT 'k'"), + declared("u", "TEXT DEFAULT UUID_V7"), + declared("n", "INT DEFAULT nextval('s')"), + ]) + .expect("no literal reaches the type gate"); + } + + /// An unregistered function name keeps its own SQLSTATE. + #[test] + fn an_unregistered_function_keeps_undefined_function() { + let error = validate_column_defaults(&[declared("a", "TEXT DEFAULT no_such_fn('x')")]) + .expect_err("unknown function refused"); + assert_eq!(error.sqlstate, sqlstate::UNDEFINED_FUNCTION); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/conflict_policy.rs b/nodedb/src/control/server/shared/ddl/neutral/conflict_policy.rs index 945b9132a..daaa8571f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/conflict_policy.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/conflict_policy.rs @@ -84,12 +84,10 @@ pub async fn alter_set_on_conflict( let mut row = Map::new(); row.insert("result".to_string(), JsonValue::String("OK".to_string())); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["result".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["result".to_string()], + vec![row], + ))]) } /// Handle `SHOW CONFLICT POLICY ON `. @@ -118,12 +116,10 @@ pub async fn show_conflict_policy( let mut row = Map::new(); row.insert("policy".to_string(), JsonValue::String(text)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["policy".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["policy".to_string()], + vec![row], + ))]) } fn resolve_policy_kind(kind: &ConflictPolicyKind) -> ConflictPolicy { diff --git a/nodedb/src/control/server/shared/ddl/neutral/constraint/show.rs b/nodedb/src/control/server/shared/ddl/neutral/constraint/show.rs index 5d418abe8..995fed066 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/constraint/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/constraint/show.rs @@ -109,13 +109,7 @@ pub fn show_constraints( rows.push(constraint_row(&cc.name, kind_str, "", &cc.check_sql)); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Build a single `SHOW CONSTRAINTS` row keyed by the four text columns. diff --git a/nodedb/src/control/server/shared/ddl/neutral/consumer_group/show.rs b/nodedb/src/control/server/shared/ddl/neutral/consumer_group/show.rs index ec2eb48c0..8b5f63cc5 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/consumer_group/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/consumer_group/show.rs @@ -72,13 +72,7 @@ pub fn show_consumer_groups( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Handle `SHOW PARTITIONS ON ` @@ -159,11 +153,5 @@ pub fn show_partitions( } } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/continuous_agg/show.rs b/nodedb/src/control/server/shared/ddl/neutral/continuous_agg/show.rs index 393dc5baf..2a4a098da 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/continuous_agg/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/continuous_agg/show.rs @@ -171,10 +171,9 @@ pub async fn show_continuous_aggregates( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs index f72364ef9..4e571907f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs @@ -14,7 +14,7 @@ use nodedb_sql::parser::preprocess::lex::{ use crate::control::server::shared::ddl::sql_parse::{parse_ident_token, split_values}; use super::super::super::result::DdlError; -use super::super::column_default::validate_column_default; +use super::super::column_default::{DeclaredColumn, validate_column_default}; use super::support::err; use super::type_map::sql_type_to_column_type; @@ -165,7 +165,14 @@ fn parse_column_defs(s: &str) -> Result, col = col.with_primary_key(); } if let Some(expr) = default_expr { - validate_column_default(&col.name, &expr)?; + validate_column_default( + &DeclaredColumn { + name: &col.name, + declared_type: &col_type, + primary_key, + }, + &expr, + )?; col = col.with_default(expr); } columns.push(col); diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs index 0bc1749cc..85fc6e48a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs @@ -6,7 +6,7 @@ //! reads the collection's active typeguards instead. use super::super::super::result::DdlError; -use super::super::column_default::validate_column_default; +use super::super::column_default::{DeclaredColumn, validate_column_default}; use super::support::err; use super::type_map::typeguard_type_to_column_type; @@ -50,7 +50,17 @@ pub(super) fn typeguards_to_column_defs( // A guard carries either DEFAULT or VALUE, never both. Strict schema // has one materialization slot, so both land on the column `DEFAULT`. if let Some(expr) = guard.default_expr.clone().or(guard.value_expr.clone()) { - validate_column_default(&col.name, &expr)?; + // The resolved type's own spelling stands in for the declaration: + // a guard names no numeric width, so the canonical name resolves + // to the same width-less type the column will carry. + validate_column_default( + &DeclaredColumn { + name: &col.name, + declared_type: &col.column_type.to_string(), + primary_key: col.primary_key, + }, + &expr, + )?; col.default = Some(expr); } columns.push(col); diff --git a/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs b/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs index 3627af386..832adb071 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs @@ -17,7 +17,7 @@ use crate::control::crdt_post_image_policy::ExternalCrdtPostImagePolicy; use crate::control::planner::sql_plan_convert::convert::db_qualified; use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::{AuthenticatedIdentity, Permission}; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::authorization::{authorize_collection, authorize_task_set}; use crate::control::server::shared::ddl::sql_parse::hex_decode; use crate::control::state::SharedState; @@ -99,27 +99,22 @@ pub async fn crdt_state( .map_err(|e| DdlError::new("XX000", e.to_string()))?; let columns = vec!["crdt_state".to_string()]; - let column_types = vec![DdlColType::Text]; if result.is_empty() { - return Ok(vec![DdlResult::Rows(ShapedRows { + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: Vec::new(), - notice: None, - })]); + Vec::new(), + ))]); } let text = String::from_utf8_lossy(&result).into_owned(); let mut row = Map::new(); row.insert("crdt_state".to_string(), JsonValue::String(text)); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } /// `SELECT crdt_apply('collection', 'doc_id', 'delta_hex')` @@ -234,14 +229,11 @@ pub async fn crdt_apply( .map_err(|e| DdlError::new("XX000", e.to_string()))?; let columns = vec!["result".to_string()]; - let column_types = vec![DdlColType::Text]; let mut row = Map::new(); row.insert("result".to_string(), JsonValue::String("OK".to_string())); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/custom_type.rs b/nodedb/src/control/server/shared/ddl/neutral/custom_type.rs index 69eb4f5f9..f23ea6683 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/custom_type.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/custom_type.rs @@ -272,13 +272,7 @@ pub fn show_types( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } // ── Helpers ─────────────────────────────────────────────────────────────── diff --git a/nodedb/src/control/server/shared/ddl/neutral/database/support.rs b/nodedb/src/control/server/shared/ddl/neutral/database/support.rs index 6d8c52d7d..04269fc67 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/database/support.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/database/support.rs @@ -3,7 +3,7 @@ //! Shared error / result constructors for the protocol-neutral database DDL //! handlers. -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use super::super::super::result::{DdlError, DdlResult}; @@ -32,11 +32,5 @@ pub(super) fn text_rows( columns: Vec, rows: Vec>, ) -> Vec { - let column_types = vec![DdlColType::Text; columns.len()]; - vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })] + vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))] } diff --git a/nodedb/src/control/server/shared/ddl/neutral/estimate_count.rs b/nodedb/src/control/server/shared/ddl/neutral/estimate_count.rs index e37d4026c..b1d278557 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/estimate_count.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/estimate_count.rs @@ -10,7 +10,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::bridge::envelope::PhysicalPlan; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use crate::types::{DatabaseId, TraceId}; use nodedb_physical::physical_plan::DocumentOp; @@ -72,18 +72,15 @@ pub async fn estimate_count( &resp.payload, ); let columns = vec!["estimate_count".to_string()]; - let column_types = vec![DdlColType::Text]; let mut row = Map::new(); row.insert( "estimate_count".to_string(), JsonValue::String(payload_text), ); - return Ok(vec![DdlResult::Rows(ShapedRows { + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]); + vec![row], + ))]); } Err(e) => { return Err(DdlError::new("XX000", e.to_string())); diff --git a/nodedb/src/control/server/shared/ddl/neutral/explain_ddl.rs b/nodedb/src/control/server/shared/ddl/neutral/explain_ddl.rs index 1bc1afaca..58d646541 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/explain_ddl.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/explain_ddl.rs @@ -110,12 +110,7 @@ pub fn explain_permission( ); rows.push(row); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types: ShapedRows::text_types(3), - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// EXPLAIN SCOPE FOR AUTH USER '' @@ -169,12 +164,7 @@ pub fn explain_scope( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types: ShapedRows::text_types(3), - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// `SELECT nodedb_assert_visible('', '', '')` @@ -211,12 +201,16 @@ pub fn assert_visible( ); let scope = RequestAuthScope::builder(&target_identity, state.auth_stores()).build(); - // Check if RLS policies would filter this user. - let rls_bytes = state.rls.combined_read_predicate_with_auth( - target_identity.tenant_id.as_u64(), - collection, - scope.auth(), - ); + // Check if RLS policies would filter this user. An unencodable policy + // is an error, not an answer. + let rls_bytes = state + .rls + .combined_read_predicate_with_auth( + target_identity.tenant_id.as_u64(), + collection, + scope.auth(), + ) + .map_err(|e| DdlError::new("XX000", format!("rls compile: {e}")))?; let visible = rls_bytes.is_some_and(|b| b.is_empty()); // No filters = visible. @@ -226,10 +220,8 @@ pub fn assert_visible( JsonValue::String(visible.to_string()), ); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["visible".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["visible".to_string()], + vec![row], + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/explain_tiers.rs b/nodedb/src/control/server/shared/ddl/neutral/explain_tiers.rs index 224e47953..cc9e2d414 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/explain_tiers.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/explain_tiers.rs @@ -8,7 +8,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::ddl::sql_parse::parse_ident_token; use crate::control::state::SharedState; use crate::types::DatabaseId; @@ -65,7 +65,6 @@ pub fn explain_tiers( crate::control::planner::auto_tier::explain_tier_selection(&policy, time_range); let columns = vec!["plan".to_string()]; - let column_types = vec![DdlColType::Text]; let mut rows = Vec::new(); for line in explanation.lines() { let mut row = Map::new(); @@ -73,12 +72,7 @@ pub fn explain_tiers( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Build a [`DdlError`] from a SQLSTATE + message. diff --git a/nodedb/src/control/server/shared/ddl/neutral/function/show.rs b/nodedb/src/control/server/shared/ddl/neutral/function/show.rs index 87b6ad54f..edb512dff 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/function/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/function/show.rs @@ -86,11 +86,5 @@ pub fn show_functions( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/algo.rs b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/algo.rs index bc7b4ab77..5826ce89f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/algo.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/algo.rs @@ -7,7 +7,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::bridge::envelope::PhysicalPlan; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::broadcast; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use crate::data::executor::response_codec; use crate::engine::graph::algo::GraphAlgorithm; @@ -235,15 +235,11 @@ fn algo_payload_to_rows( .iter() .map(|&(name, _)| name.to_string()) .collect(); - let column_types = vec![DdlColType::Text; columns.len()]; - if payload.is_empty() { - return Ok(vec![DdlResult::Rows(ShapedRows { + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: Vec::new(), - notice: None, - })]); + Vec::new(), + ))]); } let json_text = response_codec::decode_payload_to_json(payload); @@ -268,12 +264,10 @@ fn algo_payload_to_rows( shaped_rows.push(out); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: shaped_rows, - notice: None, - })]) + shaped_rows, + ))]) } #[cfg(test)] diff --git a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs index ec36332d6..2d5f1b1c9 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs @@ -16,7 +16,7 @@ use nodedb_sql::ddl_ast::GraphDirection; use crate::bridge::envelope::PhysicalPlan; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::ddl::user_dispatch; use crate::control::state::SharedState; use crate::data::executor::response_codec; @@ -140,10 +140,8 @@ pub async fn rag_fusion( let mut row = Map::new(); row.insert("result".to_string(), JsonValue::String(json_text)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["result".to_string()], - column_types: vec![DdlColType::Text], - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["result".to_string()], + vec![row], + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/response.rs b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/response.rs index d2c4e28b0..1056926ed 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/response.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/response.rs @@ -4,7 +4,7 @@ use serde_json::{Map, Value as JsonValue}; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::data::executor::response_codec; use super::super::super::result::DdlResult; @@ -16,25 +16,14 @@ use super::super::super::result::DdlResult; /// handler's empty `QueryResponse`. pub(super) fn payload_to_rows(payload: &crate::bridge::envelope::Payload) -> Vec { let columns = vec!["result".to_string()]; - let column_types = vec![DdlColType::Text]; if payload.is_empty() { - return vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows: Vec::new(), - notice: None, - })]; + return vec![DdlResult::Rows(ShapedRows::text_rows(columns, Vec::new()))]; } let json_text = response_codec::decode_payload_to_json(payload); let mut row = Map::new(); row.insert("result".to_string(), JsonValue::String(json_text)); - vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows: vec![row], - notice: None, - })] + vec![DdlResult::Rows(ShapedRows::text_rows(columns, vec![row]))] } diff --git a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/stats.rs b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/stats.rs index 24ce8f205..0f0e07a63 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/stats.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/stats.rs @@ -305,12 +305,11 @@ fn encode_compact_response(rows: Vec) -> Vec { data_rows.push(row); } - vec![DdlResult::Rows(ShapedRows { + vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, - rows: data_rows, - notice: None, - })] + data_rows, + ))] } fn encode_verbose_response(rows: Vec) -> Vec { @@ -338,10 +337,9 @@ fn encode_verbose_response(rows: Vec) -> Vec { } } - vec![DdlResult::Rows(ShapedRows { + vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, - rows: data_rows, - notice: None, - })] + data_rows, + ))] } diff --git a/nodedb/src/control/server/shared/ddl/neutral/impersonation.rs b/nodedb/src/control/server/shared/ddl/neutral/impersonation.rs index a0e860604..03a57094a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/impersonation.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/impersonation.rs @@ -236,11 +236,5 @@ pub fn show_delegations( }) .collect(); - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/inspect/grants.rs b/nodedb/src/control/server/shared/ddl/neutral/inspect/grants.rs index facd9a38a..621438822 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/inspect/grants.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/inspect/grants.rs @@ -11,7 +11,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use crate::types::DatabaseId; @@ -46,7 +46,6 @@ pub fn show_grants( }; let columns = vec!["username".to_string(), "role".to_string()]; - let column_types = vec![DdlColType::Text, DdlColType::Text]; let user = state.credentials.get_user(&target_user); let mut rows = Vec::new(); @@ -63,12 +62,7 @@ pub fn show_grants( } } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// `SHOW PERMISSIONS [ON ] [FOR ]` @@ -105,13 +99,6 @@ pub fn show_permissions( "target".to_string(), "type".to_string(), ]; - let column_types = vec![ - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - ]; - let mut rows = Vec::new(); if let Some(collection) = on_collection { @@ -224,10 +211,5 @@ pub fn show_permissions( } } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/inspect/tenants.rs b/nodedb/src/control/server/shared/ddl/neutral/inspect/tenants.rs index ae34b8769..8dad4a30e 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/inspect/tenants.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/inspect/tenants.rs @@ -33,12 +33,11 @@ pub fn show_tenants( } let (columns, column_types, rows) = tenant_rows(state, |_, _| true); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW TENANT — single-tenant introspection by identifier. @@ -71,12 +70,11 @@ pub fn show_tenant_by_identifier( return Err(ddl_err("42704", format!("tenant '{ident}' not found"))); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW TENANTS WITH NAME — filtered list form. Returns a row @@ -102,12 +100,11 @@ pub fn show_tenants_filtered_by_name( return Err(ddl_err("42704", format!("tenant '{name}' not found"))); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// Build the `(columns, column_types, rows)` triple shared by `SHOW TENANTS` diff --git a/nodedb/src/control/server/shared/ddl/neutral/inspect/trigger_dlq.rs b/nodedb/src/control/server/shared/ddl/neutral/inspect/trigger_dlq.rs index 4b66795b4..0a964091b 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/inspect/trigger_dlq.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/inspect/trigger_dlq.rs @@ -79,12 +79,11 @@ pub fn show_trigger_dlq( } } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: COLUMNS.iter().map(|c| (*c).to_owned()).collect(), - column_types: column_types(), + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( + COLUMNS.iter().map(|c| (*c).to_owned()).collect(), + column_types(), rows, - notice: None, - })]) + ))]) } /// REQUEUE TRIGGER DLQ — hand one dead-lettered action back to the diff --git a/nodedb/src/control/server/shared/ddl/neutral/inspect/users.rs b/nodedb/src/control/server/shared/ddl/neutral/inspect/users.rs index 8c6d4d71f..bfe7e0f56 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/inspect/users.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/inspect/users.rs @@ -68,12 +68,11 @@ pub fn show_users( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW ROLES — list all custom roles. Built-in role enum is fixed @@ -122,12 +121,11 @@ pub fn show_roles( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW SESSION — display current session identity. @@ -178,10 +176,9 @@ pub fn show_session(identity: &AuthenticatedIdentity) -> Result, JsonValue::String(if identity.is_superuser { "t" } else { "f" }.to_string()), ); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/inspect_audit.rs b/nodedb/src/control/server/shared/ddl/neutral/inspect_audit.rs index cb8ec5004..6c8aba052 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/inspect_audit.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/inspect_audit.rs @@ -111,12 +111,11 @@ pub fn show_audit_log( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW AUDIT WHERE event_type = '' @@ -206,12 +205,11 @@ pub fn show_audit_where( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// `SHOW AUDIT IN DATABASE [LIMIT ]` @@ -329,12 +327,11 @@ pub fn show_audit_in_database( } } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// Audit entries are read with a regular `SELECT` query against diff --git a/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs index b13a3f112..2316c3aa1 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs @@ -178,12 +178,7 @@ fn data_plane_error(code: Option) -> DdlErro pub(crate) fn single_text_col(col: &str, text: String) -> DdlResult { let mut row = Map::new(); row.insert(col.to_string(), JsonValue::String(text)); - DdlResult::Rows(ShapedRows { - columns: vec![col.to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - }) + DdlResult::Rows(ShapedRows::text_rows(vec![col.to_string()], vec![row])) } /// Parse function arguments from `SELECT FUNC_NAME(arg1, arg2, ...)`. diff --git a/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs index afd482069..66a91a2d2 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs @@ -205,12 +205,10 @@ pub(super) async fn dispatch_and_respond_json( let payload_text = crate::data::executor::response_codec::decode_payload_to_json(&resp.payload); let mut row = Map::new(); row.insert(col_name.to_string(), JsonValue::String(payload_text)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec![col_name.to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec![col_name.to_string()], + vec![row], + ))]) } /// Dispatch plan and return multi-row response (for TOPK, RANGE). @@ -240,10 +238,8 @@ pub(super) async fn dispatch_and_respond_rows( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["rank".to_string(), "key".to_string()], - column_types: ShapedRows::text_types(2), + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["rank".to_string(), "key".to_string()], rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/last_value.rs b/nodedb/src/control/server/shared/ddl/neutral/last_value.rs index dc26cef8c..9d12a5c17 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/last_value.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/last_value.rs @@ -80,16 +80,15 @@ pub async fn query_last_values( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec![ + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( + vec![ "series_id".to_string(), "timestamp_ms".to_string(), "value".to_string(), ], - column_types: vec![DdlColType::Int8, DdlColType::Int8, DdlColType::Text], + vec![DdlColType::Int8, DdlColType::Int8, DdlColType::Text], rows, - notice: None, - })]) + ))]) } /// `SELECT LAST_VALUE('', )` — returns single series value. @@ -143,12 +142,11 @@ pub async fn query_last_value( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["timestamp_ms".to_string(), "value".to_string()], - column_types: vec![DdlColType::Int8, DdlColType::Text], + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( + vec!["timestamp_ms".to_string(), "value".to_string()], + vec![DdlColType::Int8, DdlColType::Text], rows, - notice: None, - })]) + ))]) } fn ddl_err(sqlstate: &str, message: impl Into) -> DdlError { diff --git a/nodedb/src/control/server/shared/ddl/neutral/maintenance/storage_info.rs b/nodedb/src/control/server/shared/ddl/neutral/maintenance/storage_info.rs index 73124314a..1eb210070 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/maintenance/storage_info.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/maintenance/storage_info.rs @@ -13,7 +13,7 @@ use nodedb_types::DatabaseId; use serde_json::{Map, Value as JsonValue}; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::ddl::sql_parse::parse_ident_token; use crate::control::state::SharedState; @@ -65,7 +65,6 @@ pub fn handle_show_storage( "row_count".to_string(), "last_analyzed".to_string(), ]; - let column_types = vec![DdlColType::Text; 4]; let row_count = stats.first().map(|s| s.row_count).unwrap_or(0); let last_analyzed = stats @@ -97,12 +96,10 @@ pub fn handle_show_storage( JsonValue::String(last_analyzed), ); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } /// Handle `SHOW COMPACTION STATUS`. @@ -115,7 +112,6 @@ pub fn handle_show_compaction_status( "pending_jobs".to_string(), "compaction_debt".to_string(), ]; - let column_types = vec![DdlColType::Text; 3]; // Compaction runs automatically in the Data Plane. We report the current // state as "idle" — detailed stats require Data Plane query support. @@ -130,12 +126,10 @@ pub fn handle_show_compaction_status( JsonValue::String("0".to_string()), ); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } fn now_ms() -> u64 { diff --git a/nodedb/src/control/server/shared/ddl/neutral/maintenance/vector_index.rs b/nodedb/src/control/server/shared/ddl/neutral/maintenance/vector_index.rs index ea7306fc6..faf24988b 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/maintenance/vector_index.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/maintenance/vector_index.rs @@ -20,7 +20,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::bridge::envelope::PhysicalPlan; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use crate::types::DatabaseId; use crate::types::TraceId; @@ -72,7 +72,6 @@ pub async fn handle_show_vector_index( .map_err(|e| ddl_err("XX000", format!("decode vector stats: {e}")))?; let columns = vec!["property".to_string(), "value".to_string()]; - let column_types = vec![DdlColType::Text; 2]; let pairs: Vec<(&str, String)> = vec![ ("dimensions", stats.dimensions.to_string()), @@ -115,12 +114,7 @@ pub async fn handle_show_vector_index( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Handle `ALTER VECTOR INDEX ON collection.column SEAL`. diff --git a/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs b/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs index 35366d352..ab45188ef 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs @@ -232,15 +232,12 @@ fn match_payload_to_rows( column_names: &[String], ) -> Result, DdlError> { let columns = column_names.to_vec(); - let column_types = ShapedRows::text_types(column_names.len()); if payload.is_empty() { - return Ok(vec![DdlResult::Rows(ShapedRows { + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: Vec::new(), - notice: None, - })]); + Vec::new(), + ))]); } let json_text = response_codec::decode_payload_to_json(payload); @@ -257,12 +254,9 @@ fn match_payload_to_rows( out_rows.push(map); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows: out_rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + columns, out_rows, + ))]) } // Tenant-prefix stripping lives in the Data Plane, in diff --git a/nodedb/src/control/server/shared/ddl/neutral/materialized_view/show.rs b/nodedb/src/control/server/shared/ddl/neutral/materialized_view/show.rs index b119ee07d..b3f1db060 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/materialized_view/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/materialized_view/show.rs @@ -94,11 +94,5 @@ pub fn show_materialized_views( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/metering_ddl.rs b/nodedb/src/control/server/shared/ddl/neutral/metering_ddl.rs index db5f12990..fcd4e7d0c 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/metering_ddl.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/metering_ddl.rs @@ -115,12 +115,7 @@ pub fn show_usage( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types: ShapedRows::text_types(6), - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// SHOW QUOTA FOR AUTH USER '' / SHOW QUOTA FOR ORG '' @@ -183,12 +178,7 @@ pub fn show_quota( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types: ShapedRows::text_types(7), - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// SHOW USAGE FOR TENANT @@ -237,12 +227,7 @@ pub fn show_usage_for_tenant( } } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types: ShapedRows::text_types(2), - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// EXPORT USAGE FOR TENANT [PERIOD ''] FORMAT 'json' @@ -293,12 +278,10 @@ pub fn export_usage( let mut row = Map::new(); row.insert("usage_json".to_string(), JsonValue::String(json)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["usage_json".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["usage_json".to_string()], + vec![row], + ))]) } /// Parse FOR AUTH USER '' or FOR ORG '' from parts. diff --git a/nodedb/src/control/server/shared/ddl/neutral/observability.rs b/nodedb/src/control/server/shared/ddl/neutral/observability.rs index 8d66a2ed8..94cfa2a4f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/observability.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/observability.rs @@ -38,12 +38,10 @@ fn key_value_result(rows_in: Vec<(String, String)>) -> Result, Dd row.insert("value".to_string(), JsonValue::String(v)); rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["name".to_string(), "value".to_string()], - column_types: ShapedRows::text_types(2), + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["name".to_string(), "value".to_string()], rows, - notice: None, - })]) + ))]) } /// Build the canonical `(name, value)` rows for `SHOW STATS` and @@ -250,10 +248,9 @@ pub fn show_memory( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/oidc.rs b/nodedb/src/control/server/shared/ddl/neutral/oidc.rs index d2b63d366..44820fae6 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/oidc.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/oidc.rs @@ -354,11 +354,5 @@ pub fn show_oidc_providers( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/org_ddl.rs b/nodedb/src/control/server/shared/ddl/neutral/org_ddl.rs index 9f11bc426..2c4fb528a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/org_ddl.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/org_ddl.rs @@ -176,7 +176,6 @@ pub fn show_orgs( "tenant_id".to_string(), "status".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = orgs .iter() @@ -193,12 +192,7 @@ pub fn show_orgs( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// SHOW MEMBERS OF ORG '' @@ -223,7 +217,6 @@ pub fn show_members( "role".to_string(), "joined_at".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = members .iter() @@ -243,10 +236,5 @@ pub fn show_members( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/procedure/call.rs b/nodedb/src/control/server/shared/ddl/neutral/procedure/call.rs index 05713c84a..71bf5f7fe 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/procedure/call.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/procedure/call.rs @@ -130,13 +130,7 @@ fn build_out_response( row.insert(param.name.clone(), serde_json::Value::String(text)); } - let column_types = ShapedRows::text_types(columns.len()); - vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows: vec![row], - notice: None, - })] + vec![DdlResult::Rows(ShapedRows::text_rows(columns, vec![row]))] } /// Parse `CALL (arg1, arg2, ...)`. diff --git a/nodedb/src/control/server/shared/ddl/neutral/procedure/show.rs b/nodedb/src/control/server/shared/ddl/neutral/procedure/show.rs index 0f5de4b8c..169fed8ff 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/procedure/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/procedure/show.rs @@ -70,11 +70,5 @@ pub fn show_procedures( } } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/query_functions/helpers.rs b/nodedb/src/control/server/shared/ddl/neutral/query_functions/helpers.rs index 8cbfcb7e5..28cc868e6 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/query_functions/helpers.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/query_functions/helpers.rs @@ -9,9 +9,11 @@ //! of a pgwire `QueryResponse`. SQLSTATE codes and messages are unchanged. use nodedb_sql::parser::preprocess::lex::find_ascii_case_insensitive; +use nodedb_types::Value; use serde_json::{Map, Value as JsonValue}; -use crate::control::server::response_shape::project::{is_scan_wrapper, push_flat_rows}; +use crate::control::server::response_shape::cell::row_to_wire_json; +use crate::control::server::response_shape::project::{is_scan_wrapper_json, push_flat_rows}; use crate::control::server::response_shape::types::ShapedRows; use super::super::super::result::{DdlError, DdlResult}; @@ -73,12 +75,10 @@ pub fn json_to_decimal(v: &serde_json::Value) -> Option { pub fn single_result(value: &str) -> Vec { let mut row = Map::new(); row.insert("result".to_string(), JsonValue::String(value.to_string())); - vec![DdlResult::Rows(ShapedRows { - columns: vec!["result".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })] + vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["result".to_string()], + vec![row], + ))] } /// Unwrap the `DocumentOp::Scan` raw-passthrough envelope (`{"id": .., @@ -87,14 +87,16 @@ pub fn single_result(value: &str) -> Vec { /// /// Reuses `response_shape::project::push_flat_rows` — the same unwrap the /// pgwire/HTTP row shaper applies — so there is exactly one definition of -/// "unwrap a scan envelope" in the tree. Rows that are not `{id, data}` +/// "unwrap a scan envelope" in the tree. Each JSON document is lifted to a +/// typed value for the unwrap and its rows rendered back to JSON for the +/// callers, which read fields as JSON. Rows that are not `{id, data}` /// wrapped (already-flat producers) pass through unchanged. pub fn unwrap_scan_docs(docs: Vec) -> Result>, DdlError> { - let mut out = Vec::with_capacity(docs.len()); + let mut rows = Vec::with_capacity(docs.len()); for doc in docs { - push_flat_rows(doc, &mut out).map_err(|e| err("XX000", &e.to_string()))?; + push_flat_rows(Value::from(doc), &mut rows).map_err(|e| err("XX000", &e.to_string()))?; } - Ok(out) + Ok(rows.iter().map(row_to_wire_json).collect()) } /// Unwrap a `DocumentOp::Scan` envelope while also returning the row's wire @@ -104,15 +106,15 @@ pub fn unwrap_scan_docs(docs: Vec) -> Result (String, Map) { let JsonValue::Object(mut map) = doc else { return (String::new(), Map::new()); }; - if is_scan_wrapper(&map) { + if is_scan_wrapper_json(&map) { let id = map .get("id") .and_then(|v| v.as_str()) @@ -132,10 +134,8 @@ pub fn unwrap_scan_doc_with_id(doc: JsonValue) -> (String, Map Vec { - vec![DdlResult::Rows(ShapedRows { - columns: vec!["result".to_string()], - column_types: ShapedRows::text_types(1), - rows: Vec::new(), - notice: None, - })] + vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["result".to_string()], + Vec::new(), + ))] } diff --git a/nodedb/src/control/server/shared/ddl/neutral/quota_ddl.rs b/nodedb/src/control/server/shared/ddl/neutral/quota_ddl.rs index 02aec9bc7..566c09f7b 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/quota_ddl.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/quota_ddl.rs @@ -221,7 +221,6 @@ pub fn show_quotas( "enforcement".to_string(), "warning_threshold".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = state .quota_manager @@ -250,10 +249,5 @@ pub fn show_quotas( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/read_gate.rs b/nodedb/src/control/server/shared/ddl/neutral/read_gate.rs index 7d37e20ab..ded823621 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/read_gate.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/read_gate.rs @@ -50,6 +50,8 @@ const INSUFFICIENT_PRIVILEGE: &str = "42501"; const FEATURE_NOT_SUPPORTED: &str = "0A000"; /// SQLSTATE for a collection the catalog does not hold. const UNDEFINED_TABLE: &str = "42P01"; +/// SQLSTATE for a policy set that could not be compiled. +const INTERNAL_ERROR: &str = "XX000"; fn gate_err(sqlstate: &str, message: impl Into) -> DdlError { DdlError::new(sqlstate, message) @@ -217,6 +219,7 @@ impl<'a> CollectionReadGate<'a> { collection, self.scope.auth(), ) + .map_err(|e| gate_err(INTERNAL_ERROR, format!("rls compile: {e}")))? .is_some_and(|filters| filters.is_empty()); if unrestricted { return Ok(()); diff --git a/nodedb/src/control/server/shared/ddl/neutral/redaction/drop_show.rs b/nodedb/src/control/server/shared/ddl/neutral/redaction/drop_show.rs index fddabe4af..c5323cc87 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/redaction/drop_show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/redaction/drop_show.rs @@ -130,13 +130,7 @@ pub fn show_redaction_policies( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Render a redaction mode for `SHOW`. The mask literal is deliberately diff --git a/nodedb/src/control/server/shared/ddl/neutral/retention_policy/show.rs b/nodedb/src/control/server/shared/ddl/neutral/retention_policy/show.rs index b61a2b7fe..b67dfd866 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/retention_policy/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/retention_policy/show.rs @@ -118,12 +118,11 @@ pub fn show_retention_policy( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// Format tiers into a compact human-readable string. diff --git a/nodedb/src/control/server/shared/ddl/neutral/rls.rs b/nodedb/src/control/server/shared/ddl/neutral/rls.rs index 900010334..7cf67644d 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/rls.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/rls.rs @@ -19,8 +19,8 @@ use crate::control::security::catalog::StoredRlsPolicy; use crate::control::security::deny::{self, DenyMode}; use crate::control::security::identity::{AuthenticatedIdentity, Role}; use crate::control::security::predicate::RlsPredicate; -use crate::control::security::predicate_parser::{parse_predicate, validate_auth_refs}; use crate::control::security::rls::RlsPolicy; +use crate::control::security::rls::compile::{compile_policy_predicate, declared_columns}; use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use crate::types::DatabaseId; @@ -52,16 +52,38 @@ struct CompiledPredicate { on_deny: DenyMode, } -/// Compile a predicate string and optional `ON DENY` raw clause into a +/// Compile a predicate string against the target collection's declared +/// columns, and the optional `ON DENY` raw clause, into a /// `CompiledPredicate`. Called by [`create_rls_policy`] after the typed AST /// fields have been validated. +/// +/// The collection must exist: a literal compared against a declared +/// `TIMESTAMP` / `TIMESTAMPTZ` column is typed here by the rule the planner +/// applies to a query predicate, and a literal that type cannot read +/// refuses the statement naming the column. A policy that cannot be +/// enforced as written is never stored. fn compile_rls_predicate( + state: &SharedState, + database_id: DatabaseId, + tenant_id: u64, + collection: &str, predicate_str: &str, on_deny_raw: Option<&str>, ) -> Result { - let compiled = parse_predicate(predicate_str) - .map_err(|e| DdlError::new("42601", format!("predicate parse error: {e}")))?; - validate_auth_refs(&compiled).map_err(|e| DdlError::new("42601", e.to_string()))?; + let columns = declared_columns( + state.credentials.catalog(), + database_id, + tenant_id, + collection, + ) + .map_err(|e| match e { + crate::Error::CollectionNotFound { .. } => { + DdlError::new("42P01", format!("collection '{collection}' does not exist")) + } + other => DdlError::new("XX000", format!("catalog read: {other}")), + })?; + let compiled = compile_policy_predicate(predicate_str, &columns) + .map_err(|e| DdlError::new("42601", e.to_string()))?; let on_deny = if let Some(deny_text) = on_deny_raw { let deny_parts: Vec<&str> = deny_text.split_whitespace().collect(); @@ -166,7 +188,14 @@ pub fn create_rls_policy( crate::control::security::predicate::PolicyMode::Permissive }; - let compiled = compile_rls_predicate(predicate_raw, on_deny_raw)?; + let compiled = compile_rls_predicate( + state, + database_id, + tenant_id, + collection, + predicate_raw, + on_deny_raw, + )?; // Pre-check duplicate so the proposing node fails fast with a // clean SQLSTATE instead of going through raft only to be a @@ -198,7 +227,7 @@ pub fn create_rls_policy( .as_secs(), }; - let stored = StoredRlsPolicy::from_runtime(&policy) + let stored = StoredRlsPolicy::from_runtime(&policy, database_id, predicate_raw) .map_err(|e| DdlError::new("XX000", format!("rls serialize: {e}")))?; let entry = CatalogEntry::PutRlsPolicy(Box::new(stored.clone())); @@ -342,13 +371,7 @@ pub fn show_rls_policies( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } #[cfg(test)] diff --git a/nodedb/src/control/server/shared/ddl/neutral/schedule/show.rs b/nodedb/src/control/server/shared/ddl/neutral/schedule/show.rs index 500a6f431..4bd6dfc72 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/schedule/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/schedule/show.rs @@ -95,13 +95,7 @@ pub fn show_schedules( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Handle `SHOW SCHEDULE HISTORY name` @@ -164,13 +158,7 @@ pub fn show_schedule_history( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Format epoch seconds as ISO 8601 UTC string. diff --git a/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/grant.rs b/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/grant.rs index 39c6b3b6d..a746f6795 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/grant.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/grant.rs @@ -213,7 +213,6 @@ pub fn show_scope_grants( "conditions".to_string(), "granted_by".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = grants .iter() @@ -254,12 +253,7 @@ pub fn show_scope_grants( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } // ── Parse helpers for time-bound GRANT SCOPE syntax ──────────────── diff --git a/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/show.rs b/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/show.rs index 0711cb9a8..38d6ec78a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/show.rs @@ -22,7 +22,6 @@ pub fn show_scopes( let name = parts[2].trim_matches('\''); let resolved = state.scope_defs.resolve(name); let columns = vec!["permission".to_string(), "collection".to_string()]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = resolved .iter() .map(|(perm, coll)| { @@ -32,12 +31,7 @@ pub fn show_scopes( row }) .collect(); - return Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]); + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]); } // SHOW SCOPES — list all scope definitions. @@ -48,7 +42,6 @@ pub fn show_scopes( "includes".to_string(), "created_by".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = scopes .iter() @@ -76,10 +69,5 @@ pub fn show_scopes( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/scope_query_ddl.rs b/nodedb/src/control/server/shared/ddl/neutral/scope_query_ddl.rs index c9a29031b..178c66df8 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/scope_query_ddl.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/scope_query_ddl.rs @@ -129,7 +129,6 @@ pub fn show_my_scopes( let effective = state.scope_grants.effective_scopes(&user_id, &org_ids); let columns = vec!["scope".to_string(), "source".to_string()]; - let column_types = ShapedRows::text_types(columns.len()); let mut rows = Vec::new(); for scope_name in &effective { @@ -148,12 +147,7 @@ pub fn show_my_scopes( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// SHOW SCOPES FOR USER '' / SHOW SCOPES FOR ORG '' @@ -184,7 +178,6 @@ pub fn show_scopes_for( }; let columns = vec!["scope".to_string()]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = scopes .iter() .map(|s| { @@ -194,10 +187,5 @@ pub fn show_scopes_for( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/sequence.rs b/nodedb/src/control/server/shared/ddl/neutral/sequence.rs index b8f7995f0..6622fec5d 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/sequence.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/sequence.rs @@ -342,13 +342,7 @@ pub fn show_sequences( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Handle `DESCRIBE SEQUENCE `, resolved in the current database. @@ -408,11 +402,5 @@ pub fn describe_sequence( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/session_admin.rs b/nodedb/src/control/server/shared/ddl/neutral/session_admin.rs index c69b00dd8..ccc413664 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/session_admin.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/session_admin.rs @@ -145,13 +145,7 @@ pub fn show_sessions( }) .collect(); - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// KILL SESSION '' @@ -286,13 +280,10 @@ pub fn verify_audit_chain( "entries".to_string(), JsonValue::String(audit.len().to_string()), ); - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } Err(broken_seq) => Err(err( "XX001", diff --git a/nodedb/src/control/server/shared/ddl/neutral/show_changes.rs b/nodedb/src/control/server/shared/ddl/neutral/show_changes.rs index d57ab15f2..08a8a6e99 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/show_changes.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/show_changes.rs @@ -15,7 +15,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::control::change_stream::ReplayStart; use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::{AuthenticatedIdentity, Permission}; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::authorization::authorize_collection; use crate::control::state::SharedState; use crate::types::DatabaseId; @@ -95,13 +95,6 @@ pub fn show_changes( "timestamp_ms".to_string(), "lsn".to_string(), ]; - let column_types = vec![ - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - ]; let mut rows = Vec::with_capacity(changes.len()); for change in &changes { @@ -129,12 +122,7 @@ pub fn show_changes( rows.push(row); } - return Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]); + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]); } Err(DdlError::new( @@ -247,7 +235,7 @@ mod tests { .iter() .map(|row| { row.get("document_id") - .and_then(JsonValue::as_str) + .and_then(nodedb_types::Value::as_str) .expect("document id column") }) .collect::>(), @@ -295,7 +283,7 @@ mod tests { .iter() .map(|row| { row.get("document_id") - .and_then(JsonValue::as_str) + .and_then(nodedb_types::Value::as_str) .expect("document id column") }) .collect(); diff --git a/nodedb/src/control/server/shared/ddl/neutral/stream_select.rs b/nodedb/src/control/server/shared/ddl/neutral/stream_select.rs index 43a3f5bc6..1db3f4a29 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/stream_select.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/stream_select.rs @@ -146,13 +146,10 @@ pub async fn select_from_stream( Err(ConsumeError::BufferEmpty(_)) => { // Return empty result set. let columns = result_columns(); - let column_types = ShapedRows::text_types(columns.len()); - return Ok(vec![DdlResult::Rows(ShapedRows { + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: Vec::new(), - notice: None, - })]); + Vec::new(), + ))]); } Err(e) => { return Err(err("42704", e.to_string())); @@ -221,13 +218,7 @@ pub async fn select_from_stream( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Column schema for stream SELECT results. diff --git a/nodedb/src/control/server/shared/ddl/neutral/synonym_group/show.rs b/nodedb/src/control/server/shared/ddl/neutral/synonym_group/show.rs index c0334f06b..94ca9c7ae 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/synonym_group/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/synonym_group/show.rs @@ -36,11 +36,5 @@ pub fn show_synonym_groups( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/tenant/support.rs b/nodedb/src/control/server/shared/ddl/neutral/tenant/support.rs index 08e09c967..5d73fecf0 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/tenant/support.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/tenant/support.rs @@ -7,7 +7,7 @@ //! and `tenant_exists` are byte-identical except for the error type //! (`DdlError` instead of `PgWireError`). -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use crate::types::TenantId; @@ -31,13 +31,7 @@ pub(super) fn text_rows( columns: Vec, rows: Vec>, ) -> Vec { - let column_types = vec![DdlColType::Text; columns.len()]; - vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })] + vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))] } /// Resolve a tenant reference token to a [`TenantId`], accepting either a diff --git a/nodedb/src/control/server/shared/ddl/neutral/timeseries/show.rs b/nodedb/src/control/server/shared/ddl/neutral/timeseries/show.rs index 1680f047f..3035a4a50 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/timeseries/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/timeseries/show.rs @@ -91,8 +91,8 @@ pub fn show_partitions( } } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec![ + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( + vec![ "partition".to_string(), "min_ts".to_string(), "max_ts".to_string(), @@ -100,7 +100,7 @@ pub fn show_partitions( "size".to_string(), "state".to_string(), ], - column_types: vec![ + vec![ DdlColType::Text, DdlColType::Int8, DdlColType::Int8, @@ -109,6 +109,5 @@ pub fn show_partitions( DdlColType::Text, ], rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/topic/show.rs b/nodedb/src/control/server/shared/ddl/neutral/topic/show.rs index 5f398fe2d..b11ba143f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/topic/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/topic/show.rs @@ -57,11 +57,5 @@ pub fn show_topics( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/topic_subscribe.rs b/nodedb/src/control/server/shared/ddl/neutral/topic_subscribe.rs index f0c159ddc..dc058679a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/topic_subscribe.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/topic_subscribe.rs @@ -127,13 +127,10 @@ pub fn subscribe_to( JsonValue::String(backlog.len().to_string()), ); - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } #[cfg(test)] @@ -221,7 +218,10 @@ mod tests { let DdlResult::Rows(rows) = &result[0] else { panic!("expected rows"); }; - assert_eq!(rows.rows[0]["backlog"], JsonValue::String("1".into())); + assert_eq!( + rows.rows[0]["backlog"], + nodedb_types::Value::String("1".into()) + ); assert_eq!(state.ep_topic_registry.receiver_count(), 0); } } diff --git a/nodedb/src/control/server/shared/ddl/neutral/tree_ops/children.rs b/nodedb/src/control/server/shared/ddl/neutral/tree_ops/children.rs index 60fe6b0af..043920dea 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/tree_ops/children.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/tree_ops/children.rs @@ -96,10 +96,8 @@ pub async fn tree_children( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["child_id".to_string()], - column_types: ShapedRows::text_types(1), + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["child_id".to_string()], rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/tree_ops/create_index.rs b/nodedb/src/control/server/shared/ddl/neutral/tree_ops/create_index.rs index dce6deaf5..9c4862551 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/tree_ops/create_index.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/tree_ops/create_index.rs @@ -121,6 +121,7 @@ pub async fn create_graph_index( if state .rls .combined_read_predicate_with_auth(tenant_id.as_u64(), &collection, scope.auth()) + .map_err(|e| ddl_err("XX000", format!("rls compile: {e}")))? .is_none_or(|filters| !filters.is_empty()) { return Err(ddl_err( @@ -282,12 +283,10 @@ pub async fn create_graph_index( JsonValue::String(total_edges.to_string()), ); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["edges_created".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["edges_created".to_string()], + vec![row], + ))]) } /// Surface a build-time failure. diff --git a/nodedb/src/control/server/shared/ddl/neutral/tree_ops/sum.rs b/nodedb/src/control/server/shared/ddl/neutral/tree_ops/sum.rs index 982fda124..ed725078a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/tree_ops/sum.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/tree_ops/sum.rs @@ -179,10 +179,8 @@ pub async fn tree_sum( let mut row = Map::new(); row.insert("tree_sum".to_string(), JsonValue::String(total.to_string())); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["tree_sum".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["tree_sum".to_string()], + vec![row], + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/trigger/show.rs b/nodedb/src/control/server/shared/ddl/neutral/trigger/show.rs index f5302fbf6..c2d40371a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/trigger/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/trigger/show.rs @@ -90,11 +90,5 @@ pub fn show_triggers( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/typeguard/handlers.rs b/nodedb/src/control/server/shared/ddl/neutral/typeguard/handlers.rs index 19ab0c828..52e3fe66b 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/typeguard/handlers.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/typeguard/handlers.rs @@ -303,13 +303,7 @@ pub fn show_typeguard( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Handle `SHOW TYPEGUARDS` — list all collections with active type guards. @@ -343,11 +337,5 @@ pub fn show_typeguards( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/typeguard/validate.rs b/nodedb/src/control/server/shared/ddl/neutral/typeguard/validate.rs index 141675359..fe4c86bc2 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/typeguard/validate.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/typeguard/validate.rs @@ -52,13 +52,10 @@ pub async fn validate_typeguard( if coll.type_guards.is_empty() { // No type guards — return empty result. - let column_types = ShapedRows::text_types(columns.len()); - return Ok(vec![DdlResult::Rows(ShapedRows { + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: Vec::new(), - notice: None, - })]); + Vec::new(), + ))]); } let guards = coll.type_guards.clone(); @@ -163,11 +160,5 @@ pub async fn validate_typeguard( }) .collect(); - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/at_version.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/at_version.rs index 084ad94d6..bb215c239 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/at_version.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/at_version.rs @@ -71,16 +71,13 @@ pub async fn select_at_version( let text = String::from_utf8_lossy(&payload).into_owned(); let columns = vec!["document".to_string()]; - let column_types = ShapedRows::text_types(columns.len()); let mut row = Map::new(); row.insert("document".to_string(), JsonValue::String(text)); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } /// Resolve a checkpoint name to its version vector JSON. diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/diff.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/diff.rs index 23669abe6..7359d3042 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/diff.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/diff.rs @@ -113,12 +113,11 @@ pub async fn select_diff( ); row.insert("delta_hex".to_string(), JsonValue::String(hex)); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } /// Parse function arguments from `SELECT DIFF('a', 'b', 'c', 'd')`. diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/show_versions.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/show_versions.rs index 74ea7446b..ea6b97dcb 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/show_versions.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/show_versions.rs @@ -97,12 +97,11 @@ pub fn show_versions( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// Parse: SHOW VERSIONS OF collection WHERE id = 'doc-id' [LIMIT N] diff --git a/nodedb/src/control/server/shared/ddl/neutral/weighted_pick.rs b/nodedb/src/control/server/shared/ddl/neutral/weighted_pick.rs index 847cb1519..53edba9b0 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/weighted_pick.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/weighted_pick.rs @@ -220,12 +220,10 @@ pub async fn weighted_pick( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: pick_columns(), - column_types: ShapedRows::text_types(3), + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + pick_columns(), rows, - notice: None, - })]) + ))]) } // ── Helpers ──────────────────────────────────────────────────────────── @@ -325,12 +323,7 @@ fn pick_columns() -> Vec { /// Empty (no-rows) result set with the WEIGHTED_PICK schema. fn empty_pick_rows() -> DdlResult { - DdlResult::Rows(ShapedRows { - columns: pick_columns(), - column_types: ShapedRows::text_types(3), - rows: Vec::new(), - notice: None, - }) + DdlResult::Rows(ShapedRows::text_rows(pick_columns(), Vec::new())) } fn unquote(s: &str) -> String { diff --git a/nodedb/src/control/server/shared/ddl/result.rs b/nodedb/src/control/server/shared/ddl/result.rs index bc902d31c..3a135037c 100644 --- a/nodedb/src/control/server/shared/ddl/result.rs +++ b/nodedb/src/control/server/shared/ddl/result.rs @@ -179,6 +179,8 @@ pub fn code_for_sqlstate(sqlstate_str: &str) -> ErrorCode { // Invalid/incompatible object definition or a caller reaching a // dependent object still in use — all client-actionable, non-retriable. "42P17" | "42809" | "42P16" | "2BP01" => ErrorCode::BAD_REQUEST, + // A declared literal the column type cannot represent. + sqlstate::DATATYPE_MISMATCH => ErrorCode::BAD_REQUEST, // Default "object not in prerequisite state" meaning of `55006`; // `CLONE_DEPENDENCY` and `CLONE_WRITE_REQUIRES_MATERIALIZE` are // ambiguous-typed and cannot reach this function. diff --git a/nodedb/src/control/state/init_prod/bootstrap.rs b/nodedb/src/control/state/init_prod/bootstrap.rs index 36d26f531..8d130755d 100644 --- a/nodedb/src/control/state/init_prod/bootstrap.rs +++ b/nodedb/src/control/state/init_prod/bootstrap.rs @@ -158,27 +158,19 @@ pub(super) fn run( ep_topic_registry.load_from_catalog(catalog)?; mv_registry.load_from_catalog(catalog); sequence_registry.load_from_catalog(catalog); + // Every stored row is installed: a row that cannot be compiled + // against its collection installs as a restrictive deny-all + // (`StoredRlsPolicy::rehydrate`), never skipped. match catalog.load_all_rls_policies() { Ok(stored) => { - let mut loaded = 0usize; for s in &stored { - match s.to_runtime() { - Ok(p) => { - rls_store.install_replicated_policy(p); - loaded += 1; - } - Err(e) => { - tracing::warn!( - name = %s.name, - collection = %s.collection, - error = %e, - "boot replay: skipped invalid RLS policy" - ); - } - } + rls_store.install_replicated_policy(s.rehydrate(catalog)); } - if loaded > 0 { - tracing::info!(rls_policies = loaded, "loaded RLS policies from catalog"); + if !stored.is_empty() { + tracing::info!( + rls_policies = stored.len(), + "loaded RLS policies from catalog" + ); } } Err(e) => tracing::warn!(error = %e, "failed to load RLS policies"), diff --git a/nodedb/src/control/trigger/batch/collector.rs b/nodedb/src/control/trigger/batch/collector.rs index 7eda62a17..ae87e3d4e 100644 --- a/nodedb/src/control/trigger/batch/collector.rs +++ b/nodedb/src/control/trigger/batch/collector.rs @@ -20,6 +20,9 @@ use std::sync::OnceLock; use nodedb_types::Value; +use super::super::row_identity::inject_row_identity; +use crate::util::rmpv_value::rmpv_to_value; + /// A single row in a trigger batch. /// /// Stores raw MessagePack bytes from the WriteEvent and decodes directly @@ -176,42 +179,6 @@ fn decode_msgpack_to_value_map(bytes: &[u8], row_id: &str) -> Option Value { - match val { - rmpv::Value::Nil => Value::Null, - rmpv::Value::Boolean(b) => Value::Bool(*b), - rmpv::Value::Integer(i) => { - if let Some(n) = i.as_i64() { - Value::Integer(n) - } else if let Some(n) = i.as_u64() { - Value::Integer(n as i64) - } else { - Value::Null - } - } - rmpv::Value::F32(f) => Value::Float(*f as f64), - rmpv::Value::F64(f) => Value::Float(*f), - rmpv::Value::String(s) => Value::String(s.as_str().unwrap_or("").to_string()), - rmpv::Value::Binary(b) => Value::Bytes(b.clone()), - rmpv::Value::Array(arr) => Value::Array(arr.iter().map(rmpv_to_value).collect()), - rmpv::Value::Map(pairs) => { - let mut map = HashMap::new(); - for (k, v) in pairs { - let key = match k { - rmpv::Value::String(s) => s.as_str().unwrap_or("").to_string(), - other => format!("{other}"), - }; - map.insert(key, rmpv_to_value(v)); - } - Value::Object(map) - } - rmpv::Value::Ext(_, _) => Value::Null, - } -} - /// A complete batch of rows for trigger dispatch. #[derive(Debug)] pub struct TriggerBatch { diff --git a/nodedb/src/data/executor/columnar_checkpoint/geometry_restore.rs b/nodedb/src/data/executor/columnar_checkpoint/geometry_restore.rs index afd5ff1b0..86f592e92 100644 --- a/nodedb/src/data/executor/columnar_checkpoint/geometry_restore.rs +++ b/nodedb/src/data/executor/columnar_checkpoint/geometry_restore.rs @@ -199,7 +199,10 @@ impl CoreLoop { out.push( decoded_cols .iter() - .map(|dc| decoded_col_to_value(dc, row_idx)) + .zip(&schema.columns) + .map(|(dc, col_def)| { + decoded_col_to_value(dc, row_idx, &col_def.column_type) + }) .collect(), ); } diff --git a/nodedb/src/data/executor/core_loop/ts_declared_schema.rs b/nodedb/src/data/executor/core_loop/ts_declared_schema.rs index acad6049d..1b377db87 100644 --- a/nodedb/src/data/executor/core_loop/ts_declared_schema.rs +++ b/nodedb/src/data/executor/core_loop/ts_declared_schema.rs @@ -18,8 +18,9 @@ //! line's own timestamp as the time column. use nodedb_physical::physical_plan::TimeseriesSchema; +use nodedb_types::InstantKind; -use crate::engine::timeseries::columnar_memtable::{ColumnType, ColumnarSchema}; +use crate::engine::timeseries::columnar_memtable::{ColumnType, ColumnarSchema, TimeKind}; use crate::types::{DatabaseId, TenantId}; use super::state::CoreLoop; @@ -117,13 +118,33 @@ impl CoreLoop { }) } + /// Run `f` against the memtable schema that types this collection's + /// columns: the resident memtable's when one exists, else the declared + /// one. A memtable is created on first ingest, so a node serving only + /// flushed partitions after a restart has none, and the declared shape + /// is the same schema that memtable would have been built from. `None` + /// when the collection has neither. + fn with_ts_schema( + &self, + database_id: DatabaseId, + tid: TenantId, + collection: &str, + f: impl FnOnce(&ColumnarSchema) -> R, + ) -> Option { + let key = (database_id, tid, collection.to_string()); + if let Some(memtable) = self.columnar_memtables.get(&key) { + return Some(f(memtable.schema())); + } + self.declared_ts_memtable_schema(database_id, tid, collection) + .map(|schema| f(&schema)) + } + /// How each named GROUP BY column of a timeseries collection renders. /// /// A grouped column must carry the type it carries ungrouped, so this - /// resolves the same declared shape row emission reads. A declared - /// collection answers from its DDL; a measurement ingested over the raw - /// ILP protocol has no DDL, so its resident memtable schema answers. - /// A column present in neither renders as text. + /// answers from the memtable schema, whose column types carry the time + /// kind. A column absent from the schema, or a collection with no schema + /// at all, renders as text. pub(in crate::data::executor) fn ts_group_key_kinds( &self, database_id: DatabaseId, @@ -131,73 +152,43 @@ impl CoreLoop { collection: &str, group_by: &[String], ) -> Vec { - if let Some(declared) = self.declared_timeseries(database_id, tid, collection) { - let time_key_index = declared.time_key_index(); - return group_by + self.with_ts_schema(database_id, tid, collection, |schema| { + group_by .iter() .map(|name| { - let Some(index) = declared.columns.iter().position(|(c, _)| c == name) else { - return TsGroupKeyKind::Text; - }; - let declared_type = declared.columns[index].1.as_str(); - if declared_type_is_instant(declared_type) { - return TsGroupKeyKind::Instant; - } - kind_of_storage(memtable_column_type( - declared_type, - Some(index) == time_key_index, - )) + schema + .columns + .iter() + .find(|(c, _)| c == name) + .map(|(_, ty)| kind_of_storage(*ty)) + .unwrap_or(TsGroupKeyKind::Text) }) - .collect(); - } - - let key = (database_id, tid, collection.to_string()); - let Some(memtable) = self.columnar_memtables.get(&key) else { - return vec![TsGroupKeyKind::Text; group_by.len()]; - }; - let schema = memtable.schema(); - group_by - .iter() - .map(|name| { - schema - .columns - .iter() - .find(|(c, _)| c == name) - .map(|(_, ty)| kind_of_storage(*ty)) - .unwrap_or(TsGroupKeyKind::Text) - }) - .collect() + .collect() + }) + .unwrap_or_else(|| vec![TsGroupKeyKind::Text; group_by.len()]) } - /// Declared columns of a timeseries collection that carry an instant. + /// The time kind of a timeseries collection's designated time column. /// - /// A column declared `TIMESTAMP` or `TIMESTAMPTZ` is one. The memtable - /// keeps every timestamp column in epoch milliseconds, while a client - /// reads a `TIMESTAMP` cell as epoch microseconds, so row emission scales - /// exactly these columns. - /// - /// A `BIGINT TIME_KEY` shares the same millisecond column and is absent - /// from this list: its declared type is an integer, so it hands back the - /// number that was inserted. - /// - /// An undeclared measurement (raw ILP protocol ingest) has no entry and - /// yields an empty list — the planner types its columns as text, so no - /// cell of it is read as an instant. - pub(in crate::data::executor) fn ts_instant_columns( + /// A `time_bucket` result is derived from that column and renders with + /// its kind: a declared `TIMESTAMP` key yields an instant, a `BIGINT` + /// key yields the integer stored. A collection with no schema, or a + /// schema whose time index is out of range, renders as `Millis`. + pub(in crate::data::executor) fn ts_time_key_kind( &self, database_id: DatabaseId, tid: TenantId, collection: &str, - ) -> Vec { - let Some(declared) = self.declared_timeseries(database_id, tid, collection) else { - return Vec::new(); - }; - declared - .columns - .iter() - .filter(|(_, type_str)| declared_type_is_instant(type_str)) - .map(|(name, _)| name.clone()) - .collect() + ) -> TimeKind { + self.with_ts_schema(database_id, tid, collection, |schema| { + match schema.columns.get(schema.timestamp_idx) { + Some((_, ColumnType::Timestamp(kind))) => *kind, + Some((_, ColumnType::Int64 | ColumnType::Float64 | ColumnType::Symbol)) | None => { + TimeKind::Millis + } + } + }) + .unwrap_or(TimeKind::Millis) } } @@ -208,8 +199,8 @@ impl CoreLoop { /// timeseries column can take on the wire. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(in crate::data::executor) enum TsGroupKeyKind { - /// A declared `TIMESTAMP` / `TIMESTAMPTZ` column: epoch microseconds. - Instant, + /// A declared `TIMESTAMP` / `TIMESTAMPTZ` column: a typed instant. + Instant(InstantKind), /// An integer column, including a `BIGINT TIME_KEY`, in its stored unit. Integer, /// A floating-point column. @@ -220,43 +211,36 @@ pub(in crate::data::executor) enum TsGroupKeyKind { /// The wire shape a memtable storage type renders as. /// -/// `Timestamp` maps to `Integer` here: the instant case is decided from the -/// declared DDL type before this runs, so what reaches it is a `BIGINT` -/// time key or a system-time column, both of which render as the number -/// storage holds. +/// A time column answers from its own kind: an instant renders as one, a +/// `Millis` column (a `BIGINT` time key or the system-time column) renders +/// as the number storage holds. fn kind_of_storage(storage: ColumnType) -> TsGroupKeyKind { match storage { - ColumnType::Int64 | ColumnType::Timestamp => TsGroupKeyKind::Integer, + ColumnType::Timestamp(TimeKind::Instant(kind)) => TsGroupKeyKind::Instant(kind), + ColumnType::Int64 | ColumnType::Timestamp(TimeKind::Millis) => TsGroupKeyKind::Integer, ColumnType::Float64 => TsGroupKeyKind::Float, ColumnType::Symbol => TsGroupKeyKind::Text, } } -/// Whether a declared DDL type makes a column an instant on the wire. -/// -/// Both planes answer this from `nodedb_types::columnar::ColumnType`: the -/// Control Plane decides how a cell is READ, this decides the unit it is -/// WRITTEN in, and one classifier keeps them from naming different sets. -fn declared_type_is_instant(declared_type: &str) -> bool { - nodedb_types::columnar::ColumnType::from_declared_type(declared_type) - .is_some_and(|declared| declared.is_instant()) -} - /// Map a declared SQL type onto the memtable's storage type. /// -/// The designated time key is always the memtable's `Timestamp` column -/// regardless of how it was spelled — `TIMESTAMP`, `TIMESTAMPTZ`, and -/// `BIGINT` time keys all store epoch milliseconds. +/// The designated time key is always a memtable `Timestamp` column. Its +/// kind comes from the declared type: `TIMESTAMP` and `TIMESTAMPTZ` are +/// instants, while a `BIGINT` time key (or any other spelling) is `Millis`. +/// The engine-assigned system-time column is `Millis` too. fn memtable_column_type(declared_type: &str, is_time_key: bool) -> ColumnType { use nodedb_types::columnar::ColumnType as DeclaredType; - if is_time_key { - return ColumnType::Timestamp; - } match DeclaredType::from_declared_type(declared_type) { - Some( - DeclaredType::Timestamp | DeclaredType::Timestamptz | DeclaredType::SystemTimestamp, - ) => ColumnType::Timestamp, + Some(DeclaredType::Timestamp) => { + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive)) + } + Some(DeclaredType::Timestamptz) => { + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)) + } + Some(DeclaredType::SystemTimestamp) => ColumnType::Timestamp(TimeKind::Millis), + _ if is_time_key => ColumnType::Timestamp(TimeKind::Millis), Some(DeclaredType::Int64) => ColumnType::Int64, // The memtable has no boolean column; ILP ingest widens booleans to // f64, so a declared BOOLEAN lands in the same place. @@ -273,20 +257,47 @@ fn memtable_column_type(declared_type: &str, is_time_key: bool) -> ColumnType { mod tests { use super::*; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + const NAIVE: ColumnType = ColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive)); + const UTC: ColumnType = ColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)); + #[test] fn time_key_is_the_timestamp_column_whatever_its_declared_type() { + assert!(memtable_column_type("BIGINT TIME_KEY", true).is_time()); + assert!(memtable_column_type("TIMESTAMP TIME_KEY", true).is_time()); + assert!(memtable_column_type("TIMESTAMPTZ", true).is_time()); + assert!(memtable_column_type("TEXT", true).is_time()); + } + + /// The kind follows the declared type: a `BIGINT` time key reads back as + /// the integer stored, a `TIMESTAMP` as a naive instant, a `TIMESTAMPTZ` + /// as a UTC instant. The system-time column is engine-assigned and reads + /// back as milliseconds. + #[test] + fn time_kind_follows_the_declared_type() { + assert_eq!(memtable_column_type("BIGINT TIME_KEY", true), MILLIS); + assert_eq!(memtable_column_type("TIMESTAMP TIME_KEY", true), NAIVE); + assert_eq!(memtable_column_type("timestamp", true), NAIVE); + assert_eq!(memtable_column_type("TIMESTAMPTZ", true), UTC); + assert_eq!(memtable_column_type("TIMESTAMPTZ", false), UTC); + assert_eq!(memtable_column_type("SYSTEM_TIMESTAMP", false), MILLIS); + assert_eq!(memtable_column_type("SYSTEM_TIMESTAMP", true), MILLIS); + } + + #[test] + fn group_key_kind_comes_from_the_column_kind() { assert_eq!( - memtable_column_type("BIGINT TIME_KEY", true), - ColumnType::Timestamp - ); - assert_eq!( - memtable_column_type("TIMESTAMP TIME_KEY", true), - ColumnType::Timestamp + kind_of_storage(NAIVE), + TsGroupKeyKind::Instant(InstantKind::Naive) ); assert_eq!( - memtable_column_type("TIMESTAMPTZ", true), - ColumnType::Timestamp + kind_of_storage(UTC), + TsGroupKeyKind::Instant(InstantKind::Utc) ); + assert_eq!(kind_of_storage(MILLIS), TsGroupKeyKind::Integer); + assert_eq!(kind_of_storage(ColumnType::Int64), TsGroupKeyKind::Integer); + assert_eq!(kind_of_storage(ColumnType::Float64), TsGroupKeyKind::Float); + assert_eq!(kind_of_storage(ColumnType::Symbol), TsGroupKeyKind::Text); } #[test] @@ -308,25 +319,7 @@ mod tests { // Only the designated key drives partitioning, but a non-key // timestamp column keeps timestamp storage — its value comes from the // row, not from the ingest line's clock. - assert_eq!( - memtable_column_type("TIMESTAMP", false), - ColumnType::Timestamp - ); - } - - /// The declared types the planner reads back as instants are exactly the - /// ones emission scales. A `BIGINT` time key is stored in the same - /// millisecond column and must NOT be scaled. - #[test] - fn only_declared_timestamp_types_are_instants() { - assert!(declared_type_is_instant("TIMESTAMP TIME_KEY")); - assert!(declared_type_is_instant("TIMESTAMPTZ")); - assert!(declared_type_is_instant("timestamp")); - assert!(!declared_type_is_instant("BIGINT TIME_KEY")); - assert!(!declared_type_is_instant("INT")); - assert!(!declared_type_is_instant("TEXT")); - assert!(!declared_type_is_instant("SYSTEM_TIMESTAMP")); - assert!(!declared_type_is_instant("")); + assert_eq!(memtable_column_type("TIMESTAMP", false), NAIVE); } #[test] diff --git a/nodedb/src/data/executor/dispatch/meta_retention/columnar_plain.rs b/nodedb/src/data/executor/dispatch/meta_retention/columnar_plain.rs index 73e3fb9e2..075ee38af 100644 --- a/nodedb/src/data/executor/dispatch/meta_retention/columnar_plain.rs +++ b/nodedb/src/data/executor/dispatch/meta_retention/columnar_plain.rs @@ -93,9 +93,16 @@ impl CoreLoop { engine: "columnar".into(), detail: format!("read _ts_system: {e}"), })?; - let pk_cols: Vec = pk_indices + let pk_cols: Vec<( + nodedb_columnar::reader::DecodedColumn, + &nodedb_types::columnar::ColumnType, + )> = pk_indices .iter() - .map(|&i| reader.read_column(i)) + .map(|&i| { + reader + .read_column(i) + .map(|col| (col, &schema.columns[i].column_type)) + }) .collect::>() .map_err(|e| crate::Error::Storage { engine: "columnar".into(), @@ -201,13 +208,19 @@ fn int64_from_decoded(col: &nodedb_columnar::reader::DecodedColumn, row_idx: usi } } +/// Encode the primary key of one flushed row. Each cell is typed by its +/// declared column type, so the key bytes match what the memtable path +/// encodes for the same row: a `TIMESTAMP` key cell is an instant on both. fn encode_pk_from_decoded_cols( - cols: &[nodedb_columnar::reader::DecodedColumn], + cols: &[( + nodedb_columnar::reader::DecodedColumn, + &nodedb_types::columnar::ColumnType, + )], row_idx: usize, ) -> Vec { let values: Vec = cols .iter() - .map(|c| decoded_col_to_value(c, row_idx)) + .map(|(c, declared)| decoded_col_to_value(c, row_idx, declared)) .collect(); if values.len() == 1 { nodedb_columnar::pk_index::encode_pk(&values[0]) diff --git a/nodedb/src/data/executor/dispatch/query.rs b/nodedb/src/data/executor/dispatch/query.rs index cbe383a73..180a112d0 100644 --- a/nodedb/src/data/executor/dispatch/query.rs +++ b/nodedb/src/data/executor/dispatch/query.rs @@ -8,7 +8,6 @@ use nodedb_physical::physical_plan::{GroupKeySpec, QueryOp}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::handlers::join::{ HashJoinParams, JoinParams, NestedLoopJoinParams, ShuffleJoinInputs, SortMergeJoinParams, - instant_scale::JoinInstantSide, lateral::{LateralLoopParams, LateralTopKParams}, }; use crate::data::executor::task::ExecutionTask; @@ -111,34 +110,7 @@ impl CoreLoop { left_scan_filters, right_scan_filters, .. - } => { - // Only a side this join scans itself carries stored - // milliseconds. A side supplied by a sub-plan was already - // rescaled by the handler that read it, so listing it here - // would scale one instant twice. - let mut local_sides = Vec::with_capacity(2); - if left_input.is_none() { - local_sides.push(JoinInstantSide { - collection: left_collection.as_str(), - qualifier: left_alias - .as_deref() - .unwrap_or_else(|| left_collection.as_str()), - }); - } - if right_input.is_none() { - local_sides.push(JoinInstantSide { - collection: right_collection.as_str(), - qualifier: right_alias - .as_deref() - .unwrap_or_else(|| right_collection.as_str()), - }); - } - let instant_columns = self.join_instant_columns( - task.request.database_id, - crate::types::TenantId::new(tid), - &local_sides, - ); - self.execute_hash_join(HashJoinParams { + } => self.execute_hash_join(HashJoinParams { join: JoinParams { task, on, @@ -148,7 +120,6 @@ impl CoreLoop { computed_projection_bytes: computed_projection, join_filter_bytes: join_filters, post_filter_bytes: post_filters, - instant_columns: &instant_columns, }, tid, left_collection: left_collection.as_str(), @@ -163,8 +134,7 @@ impl CoreLoop { right_rls_filters, left_scan_filters, right_scan_filters, - }) - } + }), QueryOp::ShuffleJoinConsume { build_path, @@ -191,10 +161,6 @@ impl CoreLoop { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &[], - // A shuffle consumer's rows are gathered by the - // coordinator, not sent to a client, and its staged - // frames were produced by the sides' own scans. - instant_columns: &[], }; let inputs = ShuffleJoinInputs { build_path: std::path::PathBuf::from(build_path), @@ -216,34 +182,17 @@ impl CoreLoop { limit, left_rls_filters, right_rls_filters, - } => { - let instant_columns = self.join_instant_columns( - task.request.database_id, - crate::types::TenantId::new(tid), - &[ - JoinInstantSide { - collection: left_collection.as_str(), - qualifier: left_collection.as_str(), - }, - JoinInstantSide { - collection: right_collection.as_str(), - qualifier: right_collection.as_str(), - }, - ], - ); - self.execute_nested_loop_join(NestedLoopJoinParams { - task, - tid, - left_collection: left_collection.as_str(), - right_collection: right_collection.as_str(), - condition, - join_type, - limit: *limit, - left_rls_filters, - right_rls_filters, - instant_columns: &instant_columns, - }) - } + } => self.execute_nested_loop_join(NestedLoopJoinParams { + task, + tid, + left_collection: left_collection.as_str(), + right_collection: right_collection.as_str(), + condition, + join_type, + limit: *limit, + left_rls_filters, + right_rls_filters, + }), QueryOp::SortMergeJoin { left_collection, @@ -254,35 +203,18 @@ impl CoreLoop { pre_sorted, left_rls_filters, right_rls_filters, - } => { - let instant_columns = self.join_instant_columns( - task.request.database_id, - crate::types::TenantId::new(tid), - &[ - JoinInstantSide { - collection: left_collection.as_str(), - qualifier: left_collection.as_str(), - }, - JoinInstantSide { - collection: right_collection.as_str(), - qualifier: right_collection.as_str(), - }, - ], - ); - self.execute_sort_merge_join(SortMergeJoinParams { - task, - tid, - left_collection: left_collection.as_str(), - right_collection: right_collection.as_str(), - on, - join_type, - limit: *limit, - pre_sorted: *pre_sorted, - left_rls_filters, - right_rls_filters, - instant_columns: &instant_columns, - }) - } + } => self.execute_sort_merge_join(SortMergeJoinParams { + task, + tid, + left_collection: left_collection.as_str(), + right_collection: right_collection.as_str(), + on, + join_type, + limit: *limit, + pre_sorted: *pre_sorted, + left_rls_filters, + right_rls_filters, + }), QueryOp::RecursiveScan { collection, diff --git a/nodedb/src/data/executor/dispatch/timeseries.rs b/nodedb/src/data/executor/dispatch/timeseries.rs index 40fb260b4..5d1e0eb80 100644 --- a/nodedb/src/data/executor/dispatch/timeseries.rs +++ b/nodedb/src/data/executor/dispatch/timeseries.rs @@ -16,9 +16,13 @@ impl CoreLoop { op: &TimeseriesOp, ) -> Response { match op { + // `projection` is not destructured: the raw scan emits every + // stored column and the aggregate branch derives its own column + // set from GROUP BY, the aggregates, and the predicates. TimeseriesOp::Scan { collection, time_range, + projection: _, limit, filters, sort_keys, @@ -27,9 +31,9 @@ impl CoreLoop { aggregates, gap_fill, computed_columns, + rls_filters, system_time, valid_at_ms, - .. } => self.execute_timeseries_scan(TimeseriesScanParams { task, tid: task.request.tenant_id, @@ -37,6 +41,7 @@ impl CoreLoop { time_range: *time_range, limit: *limit, filters, + rls_filters, sort_keys, bucket_interval_ms: *bucket_interval_ms, group_by, @@ -130,17 +135,91 @@ mod tests { /// An autocommit ILP ingest exactly as the SQL and ILP planners build it: /// the plan carries NO LSN, the request envelope carries the minted one. fn autocommit_ingest_task(envelope_lsn: Option) -> ExecutionTask { - let plan = PhysicalPlan::Timeseries(TimeseriesOp::Ingest { - collection: nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, COLLECTION), - payload: format!("{COLLECTION},host=h0 value=1i\n").into_bytes(), - format: "ilp".to_string(), - wal_lsn: None, - surrogates: Vec::new(), - provenance: None, - rls_write_check: nodedb_types::RlsWriteCheck::NoPolicyApplies, - returning: None, - rls_filters: Vec::new(), - }); + ingest_task( + format!("{COLLECTION},host=h0 value=1i\n").into_bytes(), + envelope_lsn, + ) + } + + fn ingest_task(payload: Vec, envelope_lsn: Option) -> ExecutionTask { + task_for( + PhysicalPlan::Timeseries(TimeseriesOp::Ingest { + collection: nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, COLLECTION), + payload, + format: "ilp".to_string(), + wal_lsn: None, + surrogates: Vec::new(), + provenance: None, + rls_write_check: nodedb_types::RlsWriteCheck::NoPolicyApplies, + returning: None, + rls_filters: Vec::new(), + }), + envelope_lsn, + ) + } + + /// A read of every stored row, carrying `rls_filters` exactly as the + /// planner ships a read policy. `aggregates` empty is a raw scan. + fn scan_task(rls_filters: Vec, aggregates: Vec<(String, String)>) -> ExecutionTask { + task_for( + PhysicalPlan::Timeseries(TimeseriesOp::Scan { + collection: nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, COLLECTION), + time_range: nodedb_physical::physical_plan::UNBOUNDED_TIME_RANGE, + projection: Vec::new(), + limit: usize::MAX, + filters: Vec::new(), + sort_keys: Vec::new(), + bucket_interval_ms: 0, + group_by: Vec::new(), + aggregates, + gap_fill: String::new(), + computed_columns: Vec::new(), + rls_filters, + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + }), + None, + ) + } + + /// A read policy `owner = `, encoded the way the planner ships it. + fn policy_owner_eq(owner: &str) -> Vec { + let filter = crate::bridge::scan_filter::ScanFilter { + field: "owner".into(), + op: crate::bridge::scan_filter::FilterOp::Eq, + value: nodedb_types::Value::String(owner.into()), + clauses: vec![], + expr: None, + }; + zerompk::to_msgpack_vec(&vec![filter]).expect("encode policy") + } + + fn decode_rows(payload: &[u8]) -> Vec { + match nodedb_types::value_from_msgpack(payload).expect("decode scan payload") { + nodedb_types::Value::Array(rows) => rows, + other => panic!("scan payload is not an array: {other:?}"), + } + } + + /// Ingest one row owned by `mine` and two owned by `theirs`. + fn seed_owned_rows(h: &mut Harness) { + let task = ingest_task( + format!( + "{COLLECTION},owner=mine value=1i\n\ + {COLLECTION},owner=theirs value=2i\n\ + {COLLECTION},owner=theirs value=3i\n" + ) + .into_bytes(), + Some(1), + ); + let PhysicalPlan::Timeseries(op) = task.request.plan.clone() else { + panic!("timeseries plan"); + }; + let response = h.core.dispatch_timeseries(&task, &op); + assert_eq!(response.status, Status::Ok, "seed ingest must succeed"); + } + + fn task_for(plan: PhysicalPlan, envelope_lsn: Option) -> ExecutionTask { ExecutionTask::new(Request { request_id: RequestId::new(1), tenant_id: TenantId::new(TENANT), @@ -198,6 +277,113 @@ mod tests { ); } + /// A read policy governs a raw timeseries scan: only the rows it admits + /// come back, from the live memtable and from a flushed partition alike. + #[test] + fn a_read_policy_admits_only_matching_rows_from_memtable_and_partition() { + let mut h = make_core(); + seed_owned_rows(&mut h); + + let scan = scan_task(policy_owner_eq("mine"), Vec::new()); + let PhysicalPlan::Timeseries(op) = scan.request.plan.clone() else { + panic!("timeseries plan"); + }; + let live = h.core.dispatch_timeseries(&scan, &op); + assert_eq!(live.status, Status::Ok); + let rows = decode_rows(live.payload.as_bytes()); + assert_eq!(rows.len(), 1, "one live row is admitted: {rows:?}"); + + h.core + .flush_ts_collection(TenantId::new(TENANT), DatabaseId::DEFAULT, COLLECTION, 0) + .expect("flush"); + let flushed = h.core.dispatch_timeseries(&scan, &op); + assert_eq!(flushed.status, Status::Ok); + let rows = decode_rows(flushed.payload.as_bytes()); + assert_eq!(rows.len(), 1, "one flushed row is admitted: {rows:?}"); + } + + /// A governed `COUNT(*)` leaves the metadata fast path and counts only + /// the rows the policy admits. + #[test] + fn a_read_policy_governs_a_count_star_aggregate() { + let mut h = make_core(); + seed_owned_rows(&mut h); + + let scan = scan_task( + policy_owner_eq("mine"), + vec![("count".to_string(), "*".to_string())], + ); + let PhysicalPlan::Timeseries(op) = scan.request.plan.clone() else { + panic!("timeseries plan"); + }; + let response = h.core.dispatch_timeseries(&scan, &op); + assert_eq!(response.status, Status::Ok); + let rows = decode_rows(response.payload.as_bytes()); + let count_key = nodedb_query::agg_key::canonical_agg_key("count", "*"); + let counted = match rows.first() { + Some(nodedb_types::Value::Object(map)) => map.get(&count_key).cloned(), + other => panic!("expected one aggregate row, got {other:?}"), + }; + assert_eq!( + counted, + Some(nodedb_types::Value::Integer(1)), + "the policy admits one row for this caller: {rows:?}" + ); + } + + /// A read policy the grouped scan cannot lower fails the aggregate + /// instead of aggregating the rows the policy governs. + #[test] + fn an_unlowerable_read_policy_fails_an_aggregate() { + let mut h = make_core(); + seed_owned_rows(&mut h); + + let filter = crate::bridge::scan_filter::ScanFilter { + field: "owner".into(), + op: crate::bridge::scan_filter::FilterOp::Like, + value: nodedb_types::Value::String("mi%".into()), + clauses: vec![], + expr: None, + }; + let policy = zerompk::to_msgpack_vec(&vec![filter]).expect("encode policy"); + let scan = scan_task(policy, vec![("count".to_string(), "*".to_string())]); + let PhysicalPlan::Timeseries(op) = scan.request.plan.clone() else { + panic!("timeseries plan"); + }; + let response = h.core.dispatch_timeseries(&scan, &op); + assert_eq!( + response.status, + Status::Error, + "a policy the grouped scan cannot evaluate must not aggregate rows" + ); + assert!(response.payload.is_empty()); + } + + /// A read policy payload the Data Plane cannot decode fails the scan. + /// Treating it as "no policy" would return every row the policy hides. + #[test] + fn a_malformed_read_policy_payload_fails_the_scan() { + let mut h = make_core(); + seed_owned_rows(&mut h); + + // `0xC1` is the one byte MessagePack reserves and never emits. + let scan = scan_task(vec![0xC1], Vec::new()); + let PhysicalPlan::Timeseries(op) = scan.request.plan.clone() else { + panic!("timeseries plan"); + }; + let response = h.core.dispatch_timeseries(&scan, &op); + assert_eq!( + response.status, + Status::Error, + "an unreadable policy must not admit rows" + ); + assert!( + response.payload.is_empty(), + "a failed scan carries no rows, got {} bytes", + response.payload.len() + ); + } + /// Nothing minted an LSN, so nothing may be claimed as flushed: a stamp /// invented here would gate away records that are genuinely un-flushed. #[test] diff --git a/nodedb/src/data/executor/handlers/bulk_dml/delete.rs b/nodedb/src/data/executor/handlers/bulk_dml/delete.rs index 1c5d0e7b3..d4810a54e 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/delete.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/delete.rs @@ -225,7 +225,7 @@ impl CoreLoop { // replay soft-deletes the HNSW node through `apply_point_delete`. Only // populated when the collection has a vector index. let mut write_set: Vec = Vec::new(); - let mut returned_docs: Vec = if returning.is_some() { + let mut returned_docs: Vec = if returning.is_some() { Vec::with_capacity(apply_ids.len()) } else { Vec::new() @@ -241,33 +241,35 @@ impl CoreLoop { // will not decode is a different answer: it would silently drop out // of RETURNING and, worse, contribute no removed index tuples, so // its old secondary-index entries would survive the delete. - let pre_delete_doc: Option = if returning.is_some() - || !index_paths.is_empty() - { - match self - .sparse - .get( - task.request.database_id.as_u64(), - tid, - collection, - storage_key, - ) - .ok() - .flatten() - { - Some(bytes) => { - let identity = storage_key.to_identity(); - match returning_doc::from_stored(&bytes, &identity, strict_schema.as_ref()) - { - Ok(doc) => Some(doc), - Err(e) => return self.response_error(task, e), + let pre_delete_doc: Option = + if returning.is_some() || !index_paths.is_empty() { + match self + .sparse + .get( + task.request.database_id.as_u64(), + tid, + collection, + storage_key, + ) + .ok() + .flatten() + { + Some(bytes) => { + let identity = storage_key.to_identity(); + match returning_doc::from_stored_json( + &bytes, + &identity, + strict_schema.as_ref(), + ) { + Ok(doc) => Some(doc), + Err(e) => return self.response_error(task, e), + } } + None => None, } - None => None, - } - } else { - None - }; + } else { + None + }; // The removal and the materialized-sum deltas it owes share ONE // transaction, so a debited target row can never outlive a removal diff --git a/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs b/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs index 7e5c3949a..19f851396 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs @@ -51,7 +51,7 @@ impl CoreLoop { &mut self, cascade: BulkDeleteRowCascade<'_>, write_set: &mut Vec, - returned_docs: &mut Vec, + returned_docs: &mut Vec, ) { let BulkDeleteRowCascade { task, @@ -185,7 +185,7 @@ impl CoreLoop { Some(old_converted.as_deref().unwrap_or(deleted_bytes)), ); if returning && let Some(doc) = pre_delete_doc { - returned_docs.push(doc); + returned_docs.push(nodedb_types::Value::from(doc)); } } } diff --git a/nodedb/src/data/executor/handlers/bulk_dml/update.rs b/nodedb/src/data/executor/handlers/bulk_dml/update.rs index 12a0b029d..cbe7a232f 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/update.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/update.rs @@ -186,7 +186,7 @@ impl CoreLoop { // lets the Control Plane mint a durable `Put` redo per row. Only populated // when the collection has a vector index. let mut write_set: Vec = Vec::new(); - let mut returned_docs: Vec = if returning.is_some() { + let mut returned_docs: Vec = if returning.is_some() { Vec::with_capacity(apply_ids.len()) } else { Vec::new() @@ -230,7 +230,7 @@ impl CoreLoop { key: storage_key, current_bytes, old_doc: old_doc_json, - mut doc, + doc, updated_bytes, } = row; // Period lock, both images — matching `execute_point_update`: a @@ -391,8 +391,9 @@ impl CoreLoop { // `row_identity` only stands in as `id` for a row that // declares no primary key of its own — overwriting a // declared key would return a value the client never wrote. - returning_doc::attach_row_id(&mut doc, &row_identity); - returned_docs.push(doc); + let mut row = nodedb_types::Value::from(doc); + returning_doc::attach_row_id(&mut row, &row_identity); + returned_docs.push(row); } // Carry the surrogate + post-image back for a post-apply // `Put` redo. `updated_bytes` is moved as its last use; diff --git a/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs b/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs index 833476975..e2aed6754 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs @@ -100,7 +100,7 @@ impl CoreLoop { // decoded document's `id` is the row's client-visible // identity, not the storage key. let identity = key.to_identity(); - crate::data::executor::handlers::returning_doc::from_stored( + crate::data::executor::handlers::returning_doc::from_stored_json( ¤t_bytes, &identity, None, diff --git a/nodedb/src/data/executor/handlers/columnar_agg.rs b/nodedb/src/data/executor/handlers/columnar_agg.rs index 346dc2f48..a9f32fcfd 100644 --- a/nodedb/src/data/executor/handlers/columnar_agg.rs +++ b/nodedb/src/data/executor/handlers/columnar_agg.rs @@ -436,14 +436,14 @@ fn build_results_from_groups( mod tests { use super::*; use crate::engine::timeseries::columnar_memtable::{ - ColumnType, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, + ColumnType, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; use nodedb_types::timeseries::SeriesId; fn make_test_memtable() -> ColumnarMemtable { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), ("value".into(), ColumnType::Float64), ("qname".into(), ColumnType::Symbol), ("qtype".into(), ColumnType::Symbol), @@ -542,7 +542,7 @@ mod tests { let mt = make_test_memtable(); let filter = crate::bridge::scan_filter::ScanFilter { field: "value".into(), - op: "gt".into(), + op: crate::bridge::scan_filter::FilterOp::Gt, value: nodedb_types::Value::Float(5000.0), clauses: vec![], expr: None, diff --git a/nodedb/src/data/executor/handlers/columnar_agg_support.rs b/nodedb/src/data/executor/handlers/columnar_agg_support.rs index 7cf012d66..ec71cb21d 100644 --- a/nodedb/src/data/executor/handlers/columnar_agg_support.rs +++ b/nodedb/src/data/executor/handlers/columnar_agg_support.rs @@ -108,7 +108,7 @@ pub(in crate::data::executor::handlers) fn extract_group_key_part( GroupKeyPart::Null } } - ColumnType::Timestamp => { + ColumnType::Timestamp(_) => { if let ColumnData::Timestamp(vals) = col_data { GroupKeyPart::Int64(vals[row_idx]) } else { diff --git a/nodedb/src/data/executor/handlers/columnar_filter/eval.rs b/nodedb/src/data/executor/handlers/columnar_filter/eval.rs index 6c9c06cc9..9cde152fe 100644 --- a/nodedb/src/data/executor/handlers/columnar_filter/eval.rs +++ b/nodedb/src/data/executor/handlers/columnar_filter/eval.rs @@ -68,8 +68,8 @@ pub(crate) fn eval_filters_sparse( } } } - ColumnType::Timestamp => { - let fv = nodedb_query::scan_filter::value_as_timestamp_ms(&f.value)?; + ColumnType::Timestamp(kind) => { + let fv = kind.literal_ms(&f.value)?; if let ColumnData::Timestamp(vals) = col_data { for (mi, &idx) in indices.iter().enumerate() { if !mask[mi] { @@ -142,8 +142,8 @@ pub(crate) fn eval_filters_dense( } } } - ColumnType::Timestamp => { - let fv = nodedb_query::scan_filter::value_as_timestamp_ms(&f.value)?; + ColumnType::Timestamp(kind) => { + let fv = kind.literal_ms(&f.value)?; if let ColumnData::Timestamp(vals) = col_data { for i in 0..row_count { if !mask[i] { @@ -273,8 +273,8 @@ pub(crate) fn eval_filters_bitmask( _ => return None, } } - ColumnType::Timestamp => { - let fv = nodedb_query::scan_filter::value_as_timestamp_ms(&f.value)?; + ColumnType::Timestamp(kind) => { + let fv = kind.literal_ms(&f.value)?; let ColumnData::Timestamp(vals) = col_data else { return None; }; diff --git a/nodedb/src/data/executor/handlers/columnar_filter/memtable_source.rs b/nodedb/src/data/executor/handlers/columnar_filter/memtable_source.rs index 707a47a86..63e320566 100644 --- a/nodedb/src/data/executor/handlers/columnar_filter/memtable_source.rs +++ b/nodedb/src/data/executor/handlers/columnar_filter/memtable_source.rs @@ -22,9 +22,9 @@ impl ColumnarSource for ColumnarMemtable { #[cfg(test)] mod tests { use super::*; - use crate::bridge::scan_filter::ScanFilter; + use crate::bridge::scan_filter::{FilterOp, ScanFilter}; use crate::engine::timeseries::columnar_memtable::{ - ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, + ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; use nodedb_types::timeseries::SeriesId; @@ -33,7 +33,7 @@ mod tests { fn make_test_mt() -> ColumnarMemtable { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), ("value".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], @@ -62,7 +62,7 @@ mod tests { let mt = make_test_mt(); let f = ScanFilter { field: "value".into(), - op: "gt".into(), + op: FilterOp::Gt, value: nodedb_types::Value::Float(200.0), clauses: vec![], expr: None, @@ -78,7 +78,7 @@ mod tests { let indices: Vec = (0..30).collect(); let f = ScanFilter { field: "host".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("db-1".into()), clauses: vec![], expr: None, @@ -94,7 +94,7 @@ mod tests { let indices: Vec = (0..30).collect(); let f = ScanFilter { field: "host".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("nonexistent".into()), clauses: vec![], expr: None, @@ -111,14 +111,14 @@ mod tests { let filters = vec![ ScanFilter { field: "value".into(), - op: "gte".into(), + op: FilterOp::Gte, value: nodedb_types::Value::Float(100.0), clauses: vec![], expr: None, }, ScanFilter { field: "host".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("web-1".into()), clauses: vec![], expr: None, @@ -134,11 +134,11 @@ mod tests { let mt = make_test_mt(); let f = ScanFilter { field: "value".into(), - op: "or".into(), + op: FilterOp::Or, value: nodedb_types::Value::Null, clauses: vec![vec![ScanFilter { field: "value".into(), - op: "gt".into(), + op: FilterOp::Gt, value: nodedb_types::Value::Float(100.0), clauses: vec![], expr: None, diff --git a/nodedb/src/data/executor/handlers/columnar_filter/partition_source.rs b/nodedb/src/data/executor/handlers/columnar_filter/partition_source.rs index 0dc91dc15..b84bea1ca 100644 --- a/nodedb/src/data/executor/handlers/columnar_filter/partition_source.rs +++ b/nodedb/src/data/executor/handlers/columnar_filter/partition_source.rs @@ -28,7 +28,7 @@ impl ColumnarSource for PartitionColumns<'_> { #[cfg(test)] mod tests { use super::*; - use crate::bridge::scan_filter::ScanFilter; + use crate::bridge::scan_filter::{FilterOp, ScanFilter}; use nodedb_types::timeseries::SymbolDictionary; use super::super::{eval_filters_bitmask, eval_filters_dense, eval_filters_sparse}; @@ -60,7 +60,7 @@ mod tests { let f = ScanFilter { field: "host".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("web-1".into()), clauses: vec![], expr: None, @@ -96,7 +96,7 @@ mod tests { let f = ScanFilter { field: "col".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("z".into()), clauses: vec![], expr: None, @@ -130,7 +130,7 @@ mod tests { let f = ScanFilter { field: "host".into(), - op: "contains".into(), + op: FilterOp::Contains, value: nodedb_types::Value::String("web".into()), clauses: vec![], expr: None, @@ -165,7 +165,7 @@ mod tests { let f = ScanFilter { field: "tag".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("alpha".into()), clauses: vec![], expr: None, @@ -200,7 +200,7 @@ mod tests { let f = ScanFilter { field: "tag".into(), - op: "ne".into(), + op: FilterOp::Ne, value: nodedb_types::Value::String("y".into()), clauses: vec![], expr: None, @@ -234,7 +234,7 @@ mod tests { let f = ScanFilter { field: "col".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("a".into()), clauses: vec![], expr: None, @@ -248,7 +248,12 @@ mod tests { fn partition_columns_adapter() { // Simulate sealed partition data. let schema = vec![ - ("timestamp".into(), ColumnType::Timestamp), + ( + "timestamp".into(), + ColumnType::Timestamp( + crate::engine::timeseries::columnar_memtable::TimeKind::Millis, + ), + ), ("value".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ]; @@ -279,7 +284,7 @@ mod tests { // Filter: value > 500 let f = ScanFilter { field: "value".into(), - op: "gt".into(), + op: FilterOp::Gt, value: nodedb_types::Value::Float(500.0), clauses: vec![], expr: None, @@ -291,7 +296,7 @@ mod tests { // Filter: host = 'alpha' let f2 = ScanFilter { field: "host".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("alpha".into()), clauses: vec![], expr: None, diff --git a/nodedb/src/data/executor/handlers/columnar_read/convert.rs b/nodedb/src/data/executor/handlers/columnar_read/convert.rs index dd49e7298..6300f0bc5 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/convert.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/convert.rs @@ -1,54 +1,35 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Value conversions: engine Value → JSON for response encoding, and -//! timeseries columnar cell → raw msgpack for the timeseries scan path. - -/// Convert a `nodedb_types::Value` to `serde_json::Value` for response encoding. -pub(in crate::data::executor) fn value_to_json( - val: &nodedb_types::value::Value, -) -> serde_json::Value { - use nodedb_types::value::Value; - match val { - Value::Null => serde_json::Value::Null, - Value::Bool(b) => serde_json::Value::Bool(*b), - Value::Integer(i) => serde_json::Value::Number((*i).into()), - Value::Float(f) => serde_json::Number::from_f64(*f) - .map(serde_json::Value::Number) - .unwrap_or(serde_json::Value::Null), - Value::String(s) => serde_json::Value::String(s.clone()), - Value::DateTime(dt) | Value::NaiveDateTime(dt) => serde_json::Value::String(dt.to_string()), - Value::Decimal(d) => serde_json::Value::String(d.to_string()), - Value::Uuid(s) => serde_json::Value::String(s.clone()), - Value::Bytes(b) => { - use base64::Engine; - serde_json::Value::String(base64::engine::general_purpose::STANDARD.encode(b)) - } - Value::Array(arr) => serde_json::Value::Array(arr.iter().map(value_to_json).collect()), - Value::Geometry(g) => serde_json::to_value(g).unwrap_or(serde_json::Value::Null), - Value::Object(map) => { - let obj: serde_json::Map = map - .iter() - .map(|(k, v)| (k.clone(), value_to_json(v))) - .collect(); - serde_json::Value::Object(obj) - } - _ => serde_json::Value::Null, - } -} +//! Value conversions: a decoded columnar row → its projected response +//! object, and timeseries columnar cell → raw msgpack for the timeseries scan +//! path. + +use std::collections::HashMap; -/// Project a decoded columnar row into the scan's response JSON shape: -/// column projection, the forced `_ts_system` audit column, and computed -/// (scalar-expression) columns. Shared by the base memtable scan loop and -/// the in-transaction overlay merge (`merge_overlay_into_columnar_scan`) so -/// a staged row's JSON is built identically to a base row's. -pub(in crate::data::executor) fn row_to_projected_json( - row: &[nodedb_types::value::Value], +use nodedb_types::value::Value; + +/// Project a decoded columnar row into the scan's response object: column +/// projection, the forced `_ts_system` audit column, and computed +/// (scalar-expression) columns. Shared by the flushed-segment scan, the base +/// memtable scan loop, the in-transaction overlay merge +/// (`merge_overlay_into_columnar_scan`), and the predicate DML row read, so a +/// staged row's object is built identically to a base row's. +/// +/// Every cell is carried as the `Value` the row reader typed it as, so a +/// declared `TIMESTAMP` / `TIMESTAMPTZ` cell is an instant here and is +/// written as the instant ext by `value_to_msgpack`, from the live memtable +/// and from a flushed segment alike. The bitemporal `_ts_system`, +/// `_ts_valid_from`, `_ts_valid_until` columns are declared `Int64` and stay +/// integers: they hold epoch milliseconds with `i64::MIN` / `i64::MAX` as +/// the unbounded sentinels, which no instant can carry. +pub(in crate::data::executor) fn row_to_projected_value( + row: &[Value], schema: &nodedb_types::columnar::ColumnarSchema, projection: &[String], computed_cols: &[crate::bridge::expr_eval::ComputedColumn], all_versions: bool, -) -> crate::Result { - let mut obj = serde_json::Map::new(); +) -> crate::Result { + let mut obj: HashMap = HashMap::with_capacity(schema.columns.len()); for (i, col_def) in schema.columns.iter().enumerate() { let force_system_col = all_versions && col_def.name == nodedb_types::columnar::schema::TS_SYSTEM; @@ -59,22 +40,21 @@ pub(in crate::data::executor) fn row_to_projected_json( { continue; } - if i < row.len() { - obj.insert(col_def.name.clone(), value_to_json(&row[i])); + if let Some(cell) = row.get(i) { + obj.insert(col_def.name.clone(), cell.clone()); } } if !computed_cols.is_empty() { - let doc_val = nodedb_types::Value::from(serde_json::Value::Object(obj.clone())); + let doc_val = Value::Object(obj.clone()); for cc in computed_cols { - let existing = obj.get(&cc.alias); - if matches!(existing, Some(v) if !v.is_null()) { + if matches!(obj.get(&cc.alias), Some(v) if !matches!(v, Value::Null)) { continue; } // A computed column is projection-shaped: a division/modulo-by- // zero fails the whole scan rather than silently materializing // NULL into the response row. let v = cc.expr.eval(&doc_val)?; - obj.insert(cc.alias.clone(), serde_json::Value::from(v)); + obj.insert(cc.alias.clone(), v); } if !projection.is_empty() { obj.retain(|k, _| { @@ -84,7 +64,7 @@ pub(in crate::data::executor) fn row_to_projected_json( }); } } - Ok(serde_json::Value::Object(obj)) + Ok(Value::Object(obj)) } /// Write a timeseries columnar memtable cell value directly as msgpack bytes. @@ -92,6 +72,11 @@ pub(in crate::data::executor) fn row_to_projected_json( /// Encodes the column value at the given row index directly into `buf` /// without intermediate decoding. Used by timeseries raw_scan and aggregate /// handlers that still use the internal `ColumnarMemtable`. +/// +/// A time cell is written as its column's kind says: an instant column +/// yields a typed instant ext, a `Millis` column the integer stored. `Err` +/// when a stored millisecond count overflows the microsecond range an +/// instant carries; the cell is never written wrapped. pub(in crate::data::executor) fn emit_column_value( buf: &mut Vec, mt: &crate::engine::timeseries::columnar_memtable::ColumnarMemtable, @@ -99,13 +84,14 @@ pub(in crate::data::executor) fn emit_column_value( col_type: &crate::engine::timeseries::columnar_memtable::ColumnType, col_data: &crate::engine::timeseries::columnar_memtable::ColumnData, row_idx: usize, -) { +) -> crate::Result<()> { use crate::engine::timeseries::columnar_memtable::{ ColumnData as TsColumnData, ColumnType as TsColumnType, }; match col_type { - TsColumnType::Timestamp => { - nodedb_query::msgpack_scan::write_i64(buf, col_data.as_timestamps()[row_idx]); + TsColumnType::Timestamp(kind) => { + let millis = col_data.as_timestamps()[row_idx]; + write_time_cell(buf, *kind, millis)?; } TsColumnType::Float64 => { let v = col_data.as_f64()[row_idx]; @@ -135,4 +121,150 @@ pub(in crate::data::executor) fn emit_column_value( } } } + Ok(()) +} + +/// Epoch microseconds for a stored millisecond count. +/// +/// `Err` when `millis * 1000` overflows `i64`: the stored value cannot be +/// expressed as an instant, and wrapping it would hand back a different one. +fn instant_micros(millis: i64) -> crate::Result { + nodedb_types::NdbDateTime::from_millis(millis) + .map(|dt| dt.micros) + .map_err(|e| crate::Error::Internal { + detail: format!("timeseries time cell at {millis} ms: {e}"), + }) +} + +/// Write a stored millisecond time cell as the value its kind denotes. +pub(in crate::data::executor) fn write_time_cell( + buf: &mut Vec, + kind: crate::engine::timeseries::columnar_memtable::TimeKind, + millis: i64, +) -> crate::Result<()> { + use crate::engine::timeseries::columnar_memtable::TimeKind; + match kind { + TimeKind::Instant(k) => nodedb_types::write_instant(buf, k, instant_micros(millis)?), + TimeKind::Millis => nodedb_query::msgpack_scan::write_i64(buf, millis), + } + Ok(()) +} + +/// The rmpv cell for a stored millisecond time value, typed by its kind. +/// +/// An instant column yields the ten-byte instant ext, a `Millis` column the +/// integer stored. +pub(in crate::data::executor) fn rmpv_time_cell( + kind: crate::engine::timeseries::columnar_memtable::TimeKind, + millis: i64, +) -> crate::Result { + use crate::engine::timeseries::columnar_memtable::TimeKind; + Ok(match kind { + TimeKind::Instant(k) => { + rmpv::Value::Ext(k.ext_type(), instant_micros(millis)?.to_be_bytes().to_vec()) + } + TimeKind::Millis => rmpv::Value::Integer(millis.into()), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::engine::timeseries::columnar_memtable::{ + ColumnData, ColumnType, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, + }; + use nodedb_types::InstantKind; + + const MS: i64 = 1_583_402_400_000; + + fn memtable(kind: TimeKind) -> ColumnarMemtable { + let schema = ColumnarSchema { + columns: vec![ + ("ts".into(), ColumnType::Timestamp(kind)), + ("v".into(), ColumnType::Float64), + ], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 2], + }; + ColumnarMemtable::new(schema, ColumnarMemtableConfig::default()) + } + + fn emit(kind: TimeKind, millis: i64) -> crate::Result> { + let mt = memtable(kind); + let data = ColumnData::Timestamp(vec![millis]); + let mut buf = Vec::new(); + emit_column_value(&mut buf, &mt, 0, &ColumnType::Timestamp(kind), &data, 0)?; + Ok(buf) + } + + #[test] + fn an_instant_column_emits_a_typed_instant_ext() { + let buf = emit(TimeKind::Instant(InstantKind::Naive), MS).expect("emit"); + assert_eq!( + nodedb_types::read_instant(&buf, 0), + Some((InstantKind::Naive, MS * 1000)) + ); + let buf = emit(TimeKind::Instant(InstantKind::Utc), MS).expect("emit"); + assert_eq!( + nodedb_types::read_instant(&buf, 0), + Some((InstantKind::Utc, MS * 1000)) + ); + } + + #[test] + fn a_millis_column_emits_the_integer_stored() { + let buf = emit(TimeKind::Millis, MS).expect("emit"); + let mut expected = Vec::new(); + nodedb_query::msgpack_scan::write_i64(&mut expected, MS); + assert_eq!(buf, expected); + } + + /// A projected row carries a declared `TIMESTAMP` cell as the instant the + /// row reader typed it, and the `_ts_system` audit column as the integer + /// it is declared as. + #[test] + fn a_projected_row_keeps_instant_cells_and_integer_system_time() { + use nodedb_types::NdbDateTime; + use nodedb_types::columnar::schema::TS_SYSTEM; + use nodedb_types::columnar::{ColumnDef, ColumnType}; + + let schema = nodedb_types::columnar::ColumnarSchema::new(vec![ + ColumnDef::required("id", ColumnType::Int64).with_primary_key(), + ColumnDef::required("at", ColumnType::Timestamp), + ColumnDef::required(TS_SYSTEM, ColumnType::Int64), + ]) + .expect("valid schema"); + let at = Value::NaiveDateTime(NdbDateTime::from_micros(MS * 1000)); + let row = [Value::Integer(1), at.clone(), Value::Integer(MS)]; + + let Value::Object(all) = + row_to_projected_value(&row, &schema, &[], &[], false).expect("project") + else { + panic!("a projected row is an object"); + }; + assert_eq!(all.get("at"), Some(&at)); + assert_eq!(all.get(TS_SYSTEM), Some(&Value::Integer(MS))); + + let projection = ["at".to_string()]; + let Value::Object(audit) = + row_to_projected_value(&row, &schema, &projection, &[], true).expect("project") + else { + panic!("a projected row is an object"); + }; + assert_eq!(audit.get("at"), Some(&at)); + assert_eq!( + audit.get(TS_SYSTEM), + Some(&Value::Integer(MS)), + "an all-versions read forces the system-time column into the projection" + ); + assert!(!audit.contains_key("id")); + } + + #[test] + fn a_millisecond_count_past_the_microsecond_range_is_an_error() { + let err = emit(TimeKind::Instant(InstantKind::Naive), i64::MAX) + .expect_err("i64::MAX ms cannot be expressed in microseconds"); + assert!(err.to_string().contains("time cell"), "{err}"); + assert!(emit(TimeKind::Millis, i64::MAX).is_ok()); + } } diff --git a/nodedb/src/data/executor/handlers/columnar_read/filter.rs b/nodedb/src/data/executor/handlers/columnar_read/filter.rs index 66851b0e3..d15492c32 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/filter.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/filter.rs @@ -5,6 +5,8 @@ use nodedb_query::EvalError; use nodedb_query::scan_filter::{FilterOp, ScanFilter}; +use crate::bridge::scan_filter::decode_scan_filters; + /// Check whether a memtable row satisfies all filter predicates. /// /// Returns `Ok(true)` if every filter passes (AND semantics). Uses the full @@ -49,3 +51,34 @@ pub(in crate::data::executor) fn value_matches_filters( } Ok(true) } + +/// Decode the planner's row-level-security slot into predicates. +/// +/// Empty bytes mean no policy governs the caller on this collection and every +/// row is admitted. Bytes that do not decode as a MessagePack +/// `Vec` are an error that fails the scan: a policy the Data +/// Plane cannot read never admits the rows it governs. +pub(in crate::data::executor) fn decode_rls_filters( + bytes: &[u8], +) -> crate::Result> { + decode_scan_filters(bytes, "RLS filter") +} + +/// Whether a memtable row passes the query's WHERE predicates and then the +/// caller's read policy. +/// +/// The WHERE predicates run first so a policy predicate is only evaluated on +/// rows the query itself selects. Either set being empty is a pass for that +/// set. Errors follow [`row_matches_filters`]. +pub(in crate::data::executor) fn row_matches_filters_and_policy( + row: &[nodedb_types::value::Value], + schema: &nodedb_types::columnar::ColumnarSchema, + filters: &[ScanFilter], + rls_filters: &[ScanFilter], +) -> Result { + if filters.is_empty() && rls_filters.is_empty() { + return Ok(true); + } + let doc = crate::data::executor::handlers::columnar_write::row_values_to_object(schema, row); + Ok(value_matches_filters(&doc, filters)? && value_matches_filters(&doc, rls_filters)?) +} diff --git a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs index 9b5cdc96f..7220da011 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs @@ -166,7 +166,11 @@ impl CoreLoop { // Bitemporal system-time filter. if let (Some(ts_idx), Some(cutoff)) = (ts_system_idx, system_as_of_ms) { - let ts_val = decoded_col_to_value(&decoded_cols[ts_idx], row_idx); + let ts_val = decoded_col_to_value( + &decoded_cols[ts_idx], + row_idx, + &schema.columns[ts_idx].column_type, + ); if let Value::Integer(ts) = ts_val && ts > cutoff { @@ -177,7 +181,8 @@ impl CoreLoop { // Build a Value::Object for this row. let mut map = std::collections::HashMap::new(); for (col_idx, col_def) in schema.columns.iter().enumerate() { - let val = decoded_col_to_value(&decoded_cols[col_idx], row_idx); + let val = + decoded_col_to_value(&decoded_cols[col_idx], row_idx, &col_def.column_type); map.insert(col_def.name.clone(), val); } diff --git a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs index 862054d24..eca7e34a9 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs @@ -98,9 +98,9 @@ impl CoreLoop { } } - let value_bytes = match encode_ts_memtable_row(mt, col_count, row_idx, collection) { - Some(b) => b, - None => continue, + let value_bytes = match encode_ts_memtable_row(mt, col_count, row_idx) { + Ok(b) => b, + Err(e) => return self.response_error(task, e), }; // Surrogate: bit 31 set (memtable) | lower 31 bits = row_idx. @@ -223,10 +223,9 @@ impl CoreLoop { &col_data, &sym_dicts, row_idx, - collection, ) { - Some(b) => b, - None => continue, + Ok(b) => b, + Err(e) => return self.response_error(task, e), }; // Surrogate: (part_id & 0xFFFF) << 16 | (row_idx & 0xFFFF). @@ -293,35 +292,24 @@ pub(super) fn encode_ts_part_surrogate(part_id_1based: u32, row_idx: u32) -> u32 /// Encode a memtable row as msgpack `Value::Object` bytes. /// -/// Returns `None` if serialization fails (logged by caller). +/// A cell that cannot be read as its column's type, or a row that cannot +/// be encoded, fails the scan: a skipped row is a silently short result. fn encode_ts_memtable_row( mt: &crate::engine::timeseries::columnar_memtable::ColumnarMemtable, col_count: usize, row_idx: usize, - collection: &str, -) -> Option> { +) -> crate::Result> { let mut map: HashMap = HashMap::with_capacity(col_count); let schema = mt.schema(); for (col_idx, (col_name, col_type)) in schema.columns.iter().enumerate() { let col_data = mt.column(col_idx); - let val = memtable_col_to_value(col_data, col_type, col_idx, mt, row_idx); + let val = memtable_col_to_value(col_data, col_type, col_idx, mt, row_idx) + .map_err(|e| instant_read_error(col_name, e))?; map.insert(col_name.clone(), val); } - let ndb_val = Value::Object(map); - match nodedb_types::value_to_msgpack(&ndb_val) { - Ok(b) => Some(b), - Err(e) => { - tracing::warn!( - collection, - row_idx, - error = %e, - "ts_materialize_scan: memtable row encode failed; skipping" - ); - None - } - } + encode_row_map(map, row_idx) } /// Encode a partition row as msgpack `Value::Object` bytes. @@ -330,30 +318,33 @@ fn encode_ts_partition_row( col_data: &[Option], sym_dicts: &HashMap, row_idx: usize, - collection: &str, -) -> Option> { +) -> crate::Result> { let mut map: HashMap = HashMap::with_capacity(schema_columns.len()); for (col_i, (col_name, col_type)) in schema_columns.iter().enumerate() { let Some(data) = &col_data[col_i] else { continue; }; - let val = partition_col_to_value(data, col_type, col_i, sym_dicts, row_idx); + let val = partition_col_to_value(data, col_type, col_i, sym_dicts, row_idx) + .map_err(|e| instant_read_error(col_name, e))?; map.insert(col_name.clone(), val); } - let ndb_val = Value::Object(map); - match nodedb_types::value_to_msgpack(&ndb_val) { - Ok(b) => Some(b), - Err(e) => { - tracing::warn!( - collection, - row_idx, - error = %e, - "ts_materialize_scan: partition row encode failed; skipping" - ); - None - } + encode_row_map(map, row_idx) +} + +/// Serialize one row map to msgpack bytes. +fn encode_row_map(map: HashMap, row_idx: usize) -> crate::Result> { + nodedb_types::value_to_msgpack(&Value::Object(map)).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("timeseries materialize scan row {row_idx}: {e}"), + }) +} + +/// The error for a stored millisecond count that no instant can carry. +fn instant_read_error(column: &str, e: nodedb_types::NdbDateTimeError) -> crate::Error { + crate::Error::Internal { + detail: format!("timeseries column {column}: {e}"), } } @@ -368,9 +359,11 @@ fn memtable_col_to_value( col_idx: usize, mt: &crate::engine::timeseries::columnar_memtable::ColumnarMemtable, row_idx: usize, -) -> Value { - match col_type { - ColumnType::Timestamp => Value::Integer(col_data.as_timestamps()[row_idx]), +) -> Result { + let value = match col_type { + ColumnType::Timestamp(kind) => { + return kind.cell_value(col_data.as_timestamps()[row_idx]); + } ColumnType::Float64 => { let v = col_data.as_f64()[row_idx]; if v.is_nan() { @@ -387,7 +380,8 @@ fn memtable_col_to_value( .map(|s| Value::String(s.to_string())) .unwrap_or(Value::Null) } - } + }; + Ok(value) } /// Convert a partition column entry to `nodedb_types::Value`. @@ -397,9 +391,11 @@ fn partition_col_to_value( col_i: usize, sym_dicts: &HashMap, row_idx: usize, -) -> Value { - match col_type { - ColumnType::Timestamp => Value::Integer(data.as_timestamps()[row_idx]), +) -> Result { + let value = match col_type { + ColumnType::Timestamp(kind) => { + return kind.cell_value(data.as_timestamps()[row_idx]); + } ColumnType::Float64 => { let v = data.as_f64()[row_idx]; if v.is_nan() { @@ -426,7 +422,8 @@ fn partition_col_to_value( Value::Null } } - } + }; + Ok(value) } /// Extract a timestamp value from a column (for `_ts_system` filtering). diff --git a/nodedb/src/data/executor/handlers/columnar_read/mod.rs b/nodedb/src/data/executor/handlers/columnar_read/mod.rs index bb289151c..5fde1a0de 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/mod.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/mod.rs @@ -15,5 +15,5 @@ pub mod scan; pub mod scan_flushed; pub mod sort; -pub(in crate::data::executor) use convert::emit_column_value; +pub(in crate::data::executor) use convert::{emit_column_value, rmpv_time_cell}; pub(in crate::data::executor) use scan::ColumnarScanParams; diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan/block_skip.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/block_skip.rs new file mode 100644 index 000000000..b54c7acc6 --- /dev/null +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/block_skip.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Block-boundary surrogate prefilter for the live memtable. + +use nodedb_types::Surrogate; +use nodedb_types::surrogate_bitmap::SurrogateBitmap; + +/// Whether the whole live memtable can be skipped before any row decoding. +/// +/// True when the prefilter is empty, or when none of the memtable's recorded +/// surrogates fall within the bitmap's `[min, max]` range. A memtable with no +/// recorded surrogate at all cannot be block-skipped: the row-boundary check +/// decides each of its rows. +pub(super) fn memtable_block_skipped( + bitmap: &SurrogateBitmap, + surrogates: &[Option], +) -> bool { + if bitmap.is_empty() { + return true; + } + let (mt_min, mt_max) = surrogates + .iter() + .flatten() + .fold((u32::MAX, u32::MIN), |(lo, hi), s| { + (lo.min(s.0), hi.max(s.0)) + }); + if mt_min > mt_max { + return false; + } + // The bitmap is non-empty, so both bounds are `Some`; the `None` arm is + // the type-level fallback and never skips a block. + match (bitmap.0.min(), bitmap.0.max()) { + (Some(bm_min), Some(bm_max)) => bm_max < mt_min || bm_min > mt_max, + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn bitmap(ids: &[u32]) -> SurrogateBitmap { + let mut b = SurrogateBitmap::new(); + for id in ids { + b.insert(Surrogate(*id)); + } + b + } + + #[test] + fn an_empty_prefilter_skips_the_block() { + assert!(memtable_block_skipped(&bitmap(&[]), &[Some(Surrogate(1))])); + } + + #[test] + fn a_disjoint_range_skips_the_block() { + let mt = [Some(Surrogate(10)), None, Some(Surrogate(20))]; + assert!(memtable_block_skipped(&bitmap(&[1, 5]), &mt)); + assert!(memtable_block_skipped(&bitmap(&[21, 30]), &mt)); + assert!(!memtable_block_skipped(&bitmap(&[15]), &mt)); + } + + #[test] + fn a_memtable_without_surrogates_is_never_block_skipped() { + assert!(!memtable_block_skipped(&bitmap(&[1]), &[None, None])); + assert!(!memtable_block_skipped(&bitmap(&[1]), &[])); + } +} diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs similarity index 65% rename from nodedb/src/data/executor/handlers/columnar_read/scan.rs rename to nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs index 1ad97ce26..9787d637f 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/scan.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs @@ -1,59 +1,29 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Scan-params struct and the base scan entry point. +//! The columnar base scan entry point. use nodedb_types::columnar::schema::{TS_SYSTEM, TS_VALID_FROM, TS_VALID_UNTIL}; -use nodedb_types::surrogate_bitmap::SurrogateBitmap; +use nodedb_types::value::Value; use crate::bridge::envelope::{ErrorCode, Response}; use crate::bridge::expr_eval::ComputedColumn; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::columnar_read::bitemporal::bitemporal_row_visible; +use crate::data::executor::handlers::columnar_read::convert::row_to_projected_value; +use crate::data::executor::handlers::columnar_read::filter::{ + decode_rls_filters, row_matches_filters_and_policy, +}; +use crate::data::executor::handlers::columnar_read::scan_flushed::FlushedScanCtx; +use crate::data::executor::handlers::transaction::overlay::{ + ColumnarMatchedRow, ColumnarOverlayMergeParams, +}; use crate::data::executor::response_codec; use crate::data::executor::task::ExecutionTask; -use super::bitemporal::bitemporal_row_visible; -use super::filter::row_matches_filters; -use super::sort::{compare_sort_values, eval_row_sort_values}; - -/// Parameters for a columnar base scan. Bundled as a struct because the -/// raw parameter list exceeds the project's too-many-arguments bound. -pub(in crate::data::executor) struct ColumnarScanParams<'a> { - pub collection: &'a str, - pub projection: &'a [String], - pub limit: usize, - pub filters: &'a [u8], - /// RLS filter bytes — wiring is the responsibility of a separate - /// enforcement pass; the base scan handler itself does not consume - /// them (hence the `_` destructure). - #[allow(dead_code)] - pub rls_filters: &'a [u8], - pub sort_keys: &'a [nodedb_physical::physical_plan::SortKeySpec], - /// Bitemporal system-time selection. `Current` is a current-state read; - /// `AsOf(ms)` drops rows with `_ts_system > ms`; `AllVersions` emits every - /// `_ts_system` row ordered ascending (audit log), with the system-time - /// column projected. - pub system_time: nodedb_types::SystemTimeScope, - /// Bitemporal valid-time point: drop rows whose - /// `[_ts_valid_from, _ts_valid_until)` interval does not contain this - /// point. `None` skips valid-time filtering entirely. - pub valid_at_ms: Option, - /// Optional cross-engine surrogate prefilter. When `Some`, the scan - /// skips whole memtable blocks whose surrogate range does not intersect - /// the bitmap (block boundary) and skips individual rows whose surrogate - /// is absent from the bitmap (row boundary). `None` = no prefilter. - pub prefilter: Option<&'a SurrogateBitmap>, - /// MessagePack-serialized `Vec` for scalar projection - /// expressions such as JSON arrow operators. Empty slice means no - /// computed columns are requested. - pub computed_columns: &'a [u8], - /// The in-transaction identity of the caller, when the scan is issued - /// inside `BEGIN..COMMIT`. `Some` gates a post-scan overlay merge - /// (`merge_overlay_into_columnar_scan`) so the scan observes the - /// transaction's own staged, not-yet-durable `ColumnarOp::Insert` rows - /// (read-your-own-writes). `None` for autocommit reads. - pub txn_id: Option, -} +use super::block_skip::memtable_block_skipped; +use super::order::order_matched; +use super::params::ColumnarScanParams; impl CoreLoop { /// Execute a base columnar scan: flushed segments first, then the live @@ -64,6 +34,11 @@ impl CoreLoop { /// surrogate membership exactly like the live-memtable phase. See /// `scan_normalize::scan_columnar` for the parallel read path — keep both /// in sync on segment-iteration changes. + /// + /// Every result row is a `Value::Object` built by + /// `row_to_projected_value` and encoded by `encode_value_vec`, so a + /// declared `TIMESTAMP` / `TIMESTAMPTZ` cell reaches the client as a + /// typed instant from either phase. pub(in crate::data::executor) fn execute_columnar_scan( &mut self, task: &ExecutionTask, @@ -74,7 +49,7 @@ impl CoreLoop { projection, limit, filters, - rls_filters: _, + rls_filters, sort_keys, system_time, valid_at_ms, @@ -108,6 +83,13 @@ impl CoreLoop { } else { Vec::new() }; + // The read policy is decoded before any row is read: a payload the + // Data Plane cannot decode fails the statement instead of admitting + // the rows it governs. + let rls_predicates: Vec = match decode_rls_filters(rls_filters) { + Ok(p) => p, + Err(e) => return self.response_error(task, e), + }; // A no-LIMIT SQL `SELECT * FROM ` arrives as // `limit == usize::MAX`. Capture that before the `limit == 0` rewrite // so the budget bound applies only to the unbounded path. Spatial @@ -120,7 +102,11 @@ impl CoreLoop { // Bound the materialized row count to a ceiling derived from the // memory budget (+1 row to detect "more exist") so the scan does // not pull the whole memtable into the `matched` Vec. - super::super::scan_budget::fetch_limit_for(limit, 0, scan_budget_bytes) + crate::data::executor::handlers::scan_budget::fetch_limit_for( + limit, + 0, + scan_budget_bytes, + ) } else { limit }; @@ -142,7 +128,7 @@ impl CoreLoop { Some(e) => e, None => { // Empty result for missing collection. - return match response_codec::encode_json_vec_as_msgpack(&[]) { + return match response_codec::encode_value_vec(&[]) { Ok(payload) => self.response_with_payload(task, payload), Err(e) => self.response_error( task, @@ -172,21 +158,14 @@ impl CoreLoop { Vec::new() }; - // Collect matched rows as (row_values, json_object) pairs. We keep - // the raw `Vec` for sort-key comparison — the JSON form is - // emitted only after ORDER BY + limit are applied. When no sort - // is requested we short-circuit the limit enforcement inside the - // loop to avoid materialising the entire memtable. - let mut matched: Vec<( - Option, - Vec, - serde_json::Value, - )> = Vec::new(); - let scan_budget = if sort_keys.is_empty() { - limit.saturating_mul(10).max(limit) - } else { - usize::MAX - }; + // Collect matched rows as (surrogate, row_values, projected object) + // triples. The raw `Vec` is kept for sort-key comparison — the + // projected object is emitted only after ORDER BY + limit are + // applied. When no sort is requested the limit is enforced inside the + // loop so the whole memtable is not materialised. The limit counts + // rows that pass every predicate, so a row the WHERE clause or the + // read policy excludes never consumes a slot. + let mut matched: Vec = Vec::new(); // Resolve hidden bitemporal column positions once; `None` means // the collection is not bitemporal, so the per-row filter is a // no-op regardless of `system_as_of_ms` / `valid_at_ms` values. @@ -194,44 +173,10 @@ impl CoreLoop { let ts_valid_from_idx = schema.columns.iter().position(|c| c.name == TS_VALID_FROM); let ts_valid_until_idx = schema.columns.iter().position(|c| c.name == TS_VALID_UNTIL); - // Block-boundary prefilter: if a prefilter is present and none of - // the memtable's surrogates fall within the bitmap's [min, max] - // range, the entire memtable block can be skipped before any row - // decoding takes place. - let block_skipped = if let Some(bitmap) = prefilter { - if bitmap.is_empty() { - true - } else { - let surrogates = engine.memtable_surrogates(); - // Compute the surrogate range of non-None entries in the memtable. - let (mt_min, mt_max) = surrogates - .iter() - .flatten() - .fold((u32::MAX, u32::MIN), |(lo, hi), s| { - (lo.min(s.0), hi.max(s.0)) - }); - // If no surrogate was found (mt_min > mt_max) or the bitmap's - // range lies entirely outside the memtable range, skip. - if mt_min > mt_max { - // No surrogates in memtable — cannot apply block skip. - false - } else { - // The bitmap is non-empty (guarded above); min/max are - // always `Some`. Pattern-match to avoid `unwrap` in - // non-test code while keeping the compiler honest. - match (bitmap.0.min(), bitmap.0.max()) { - (Some(bm_min), Some(bm_max)) => { - // Disjoint ranges: bitmap entirely before or after memtable. - bm_max < mt_min || bm_min > mt_max - } - // Unreachable: `is_empty()` was already checked above. - _ => false, - } - } - } - } else { - false - }; + // Block-boundary prefilter: skip the whole live memtable before any + // row decoding when no recorded surrogate can be in the bitmap. + let block_skipped = prefilter + .is_some_and(|bitmap| memtable_block_skipped(bitmap, engine.memtable_surrogates())); // Deadline safe points for this scan: the phase boundary below, every // 1024th memtable row, and the stage boundary after the sort. The @@ -250,7 +195,7 @@ impl CoreLoop { // apply a per-row membership test below, mirroring the live-memtable // phase. See the method-level doc comment for the full rationale. if let Err(e) = self.scan_flushed_columnar_segments( - super::scan_flushed::FlushedScanCtx { + FlushedScanCtx { collection, engine_key: &engine_key, schema, @@ -258,6 +203,7 @@ impl CoreLoop { limit, sort_keys, filter_predicates: &filter_predicates, + rls_predicates: &rls_predicates, prefilter, computed_cols: &computed_cols, all_versions, @@ -285,20 +231,16 @@ impl CoreLoop { } // ── Phase 2: live memtable ────────────────────────────────────────── - // Rows still in the active memtable (not yet flushed). - // Reduce the over-fetch budget by however many rows flushed segments - // already contributed so we do not materialise more than needed. - let memtable_budget = scan_budget.saturating_sub(matched.len()); - if !block_skipped && memtable_budget > 0 { - for (row_surrogate, row) in engine - .scan_memtable_rows_with_surrogates() - .take(memtable_budget) - { + // Rows still in the active memtable (not yet flushed). Skipped when + // the flushed pass already filled an unsorted limit; otherwise the + // loop stops at the limit or the deadline, like the flushed pass. + if !block_skipped && (!sort_keys.is_empty() || matched.len() < limit) { + for (row_surrogate, row) in engine.scan_memtable_rows_with_surrogates() { // Row-boundary prefilter: skip this row when its surrogate is // absent from the bitmap. Rows without a recorded surrogate - // (legacy / test paths) are always included when no prefilter - // is active; when a prefilter is active they are excluded - // because the surrogate identity is unknown. + // are always included when no prefilter is active; when a + // prefilter is active they are excluded because the surrogate + // identity is unknown. if let Some(bitmap) = prefilter { match row_surrogate { Some(s) if bitmap.contains(s) => {} @@ -316,22 +258,27 @@ impl CoreLoop { ) { continue; } - if !filter_predicates.is_empty() { - match row_matches_filters(&row, schema, &filter_predicates) { - Ok(true) => {} - Ok(false) => continue, - // `row_matches_filters` returns `EvalError`, which - // has exactly one variant and no direct - // `Into` — mirrors - // `stage_columnar_dml.rs`'s identical call site, - // which hardcodes the same typed code rather than - // collapsing to `Internal`/`XX000`. - Err(_e) => { - return self.response_error(task, ErrorCode::DivisionByZero); - } + // The query's WHERE predicates, then the caller's read + // policy: a row the policy excludes is dropped here, before + // projection, sort, and limit, so a LIMIT counts admitted + // rows only. + match row_matches_filters_and_policy( + &row, + schema, + &filter_predicates, + &rls_predicates, + ) { + Ok(true) => {} + Ok(false) => continue, + // `EvalError` has exactly one variant and no direct + // `Into` — mirrors `stage_columnar_dml.rs`'s + // identical call site, which hardcodes the same typed + // code rather than collapsing to `Internal`/`XX000`. + Err(_e) => { + return self.response_error(task, ErrorCode::DivisionByZero); } } - let obj = match super::convert::row_to_projected_json( + let obj = match row_to_projected_value( &row, schema, projection, @@ -339,8 +286,8 @@ impl CoreLoop { all_versions, ) { Ok(v) => v, - // `row_to_projected_json` returns `crate::Result<_>` - // (unlike `row_matches_filters` above) — its only + // `row_to_projected_value` returns `crate::Result<_>` + // (unlike `row_matches_filters_and_policy` above) — its only // fallible step is a computed-column expression eval, // and `computed_cols` here is real, not `&[]` like the // DML staging path, so propagate the actual typed error @@ -383,12 +330,13 @@ impl CoreLoop { collection.to_string(), ); if let Err(e) = self.merge_overlay_into_columnar_scan( - crate::data::executor::handlers::transaction::overlay::ColumnarOverlayMergeParams { + ColumnarOverlayMergeParams { txn_id, coll_key: &coll_key, schema, projection, filter_predicates: &filter_predicates, + rls_predicates: &rls_predicates, computed_cols: &computed_cols, all_versions, }, @@ -405,43 +353,24 @@ impl CoreLoop { } } - if !sort_keys.is_empty() { - // Sort keys are expressions, so evaluating them can fail. Evaluate - // every row's keys first — `sort_by` has no way to report an error - // — then order by the results. - let mut keyed = Vec::with_capacity(matched.len()); - for (_, row, _) in &matched { - match eval_row_sort_values(row, schema, sort_keys) { - Ok(values) => keyed.push(values), - Err(e) => return self.response_error(task, e), - } - } - let mut order: Vec = (0..matched.len()).collect(); - order.sort_by(|&a, &b| compare_sort_values(&keyed[a], &keyed[b], sort_keys)); - let mut reordered: Vec<_> = order - .into_iter() - .map(|i| matched[i].clone()) - .collect::>(); - std::mem::swap(&mut matched, &mut reordered); - // Safe point: the sort is the one stage whose cost grows with the - // matched row count, and it has just finished. Nothing is emitted - // yet, so stopping here costs the client only the error. - if deadline.expired_now() { - return self.response_error(task, ErrorCode::DeadlineExceeded); - } - } else if all_versions { - // Audit-log order: ascending by system time. The hidden - // `_ts_system` column index was resolved above. - matched.sort_by(|(_, a, _), (_, b, _)| { - super::bitemporal::row_system_time(a, ts_system_idx) - .cmp(&super::bitemporal::row_system_time(b, ts_system_idx)) - }); + if let Err(e) = order_matched(&mut matched, schema, sort_keys, all_versions, ts_system_idx) + { + return self.response_error(task, e); + } + // Safe point: the sort is the one stage whose cost grows with the + // matched row count, and it has just finished. Nothing is emitted + // yet, so stopping here costs the client only the error. + if !sort_keys.is_empty() && deadline.expired_now() { + return self.response_error(task, ErrorCode::DeadlineExceeded); } - let results: Vec = - matched.into_iter().take(limit).map(|(_, _, j)| j).collect(); + let results: Vec = matched + .into_iter() + .take(limit) + .map(|(_, _, obj)| obj) + .collect(); - let payload = match response_codec::encode_json_vec_as_msgpack(&results) { + let payload = match response_codec::encode_value_vec(&results) { Ok(payload) => payload, Err(e) => { return self.response_error( @@ -458,7 +387,11 @@ impl CoreLoop { // surface a deterministic error if it exceeds the budget rather than // silently truncating. Spatial scans are bounded (finite limit) and so // skip this check. - if unbounded && super::super::scan_budget::budget_exceeded(payload.len(), scan_budget_bytes) + if unbounded + && crate::data::executor::handlers::scan_budget::budget_exceeded( + payload.len(), + scan_budget_bytes, + ) { return self.response_error(task, ErrorCode::ResourcesExhausted); } @@ -471,15 +404,14 @@ impl CoreLoop { mod tests { //! Cross-engine prefilter coverage for FLUSHED plain-columnar segments. //! - //! These tests prove the silent-wrong-results fix: rows that live only in a - //! flushed segment (drained out of the live memtable) are now visible to a - //! prefiltered scan and are filtered per-row by their cross-engine - //! surrogate, instead of being skipped wholesale. Each test forces a real - //! flush — drain the memtable, encode a `SegmentWriter` segment, push the - //! bytes + the captured surrogate sidecar in lockstep, then `on_memtable_flushed` - //! clears the memtable — so the rows truly exist only in the flushed segment - //! before the scan runs. This mirrors the production flush block in - //! `handlers/columnar_write/insert.rs`. + //! Rows that live only in a flushed segment (drained out of the live + //! memtable) are visible to a prefiltered scan and are filtered per-row by + //! their cross-engine surrogate, instead of being skipped wholesale. Each + //! test forces a real flush — drain the memtable, encode a `SegmentWriter` + //! segment, push the bytes + the captured surrogate sidecar in lockstep, + //! then `on_memtable_flushed` clears the memtable — so the rows truly exist + //! only in the flushed segment before the scan runs. This mirrors the + //! production flush block in `handlers/columnar_write/insert.rs`. use std::time::{Duration, Instant}; @@ -487,7 +419,7 @@ mod tests { use nodedb_columnar::MutationEngine; use nodedb_types::columnar::{ColumnDef, ColumnType, ColumnarSchema}; use nodedb_types::value::Value; - use nodedb_types::{Surrogate, SurrogateBitmap}; + use nodedb_types::{NdbDateTime, Surrogate, SurrogateBitmap}; use crate::bridge::dispatch::{BridgeRequest, BridgeResponse}; use crate::bridge::envelope::{PhysicalPlan, Priority, Request}; @@ -497,10 +429,13 @@ mod tests { use super::ColumnarScanParams; + const MICROS: i64 = 1_583_402_400_000_000; + fn schema() -> ColumnarSchema { ColumnarSchema::new(vec![ ColumnDef::required("id", ColumnType::Int64).with_primary_key(), ColumnDef::required("name", ColumnType::String), + ColumnDef::required("at", ColumnType::Timestamp), ]) .expect("valid schema") } @@ -548,27 +483,30 @@ mod tests { }) } - /// Insert `(id, name, surrogate)` rows into a fresh engine, then run the - /// EXACT production flush sequence so the rows end up only in a flushed - /// segment. Registers the engine + populates both lockstep maps on `core`. - /// Returns the engine key. - fn insert_and_flush( - core: &mut CoreLoop, - collection: &str, - rows: &[(i64, &str, Surrogate)], - ) -> (DatabaseId, TenantId, String) { - let key = ( + fn engine_key(collection: &str) -> (DatabaseId, TenantId, String) { + ( DatabaseId::DEFAULT, TenantId::new(1), collection.to_string(), - ); - let mut engine = MutationEngine::new(collection.to_string(), schema()); - for (id, name, surr) in rows { - engine - .insert_with_surrogate(&[Value::Integer(*id), Value::String((*name).into())], *surr) - .expect("insert_with_surrogate"); - } + ) + } + + fn row(id: i64, name: &str) -> [Value; 3] { + [ + Value::Integer(id), + Value::String(name.into()), + Value::NaiveDateTime(NdbDateTime::from_micros(MICROS)), + ] + } + /// Run the EXACT production flush sequence on `engine` so its memtable + /// rows end up only in a flushed segment, populating both lockstep maps + /// on `core`. + fn flush( + core: &mut CoreLoop, + key: &(DatabaseId, TenantId, String), + engine: &mut MutationEngine, + ) { // ── Mirror handlers/columnar_write/insert.rs flush block ──────────── let new_segment_id = engine.next_segment_id(); let (seg_schema, columns, row_count) = engine.memtable_mut().drain_optimized(); @@ -608,47 +546,106 @@ mod tests { 0, "memtable surrogates cleared after flush" ); - core.columnar_engines.insert(key.clone(), engine); - key } - /// Run a prefiltered scan and return the decoded result rows. - fn scan_with_prefilter( + /// Insert `(id, name, surrogate)` rows into a fresh engine, then flush so + /// the rows end up only in a flushed segment. Registers the engine on + /// `core`. Returns the engine key. + fn insert_and_flush( core: &mut CoreLoop, collection: &str, - prefilter: Option<&SurrogateBitmap>, - ) -> Vec { - let task = make_task(); - let params = ColumnarScanParams { + rows: &[(i64, &str, Surrogate)], + ) -> (DatabaseId, TenantId, String) { + let key = engine_key(collection); + let mut engine = MutationEngine::new(collection.to_string(), schema()); + for (id, name, surr) in rows { + engine + .insert_with_surrogate(&row(*id, name), *surr) + .expect("insert_with_surrogate"); + } + flush(core, &key, &mut engine); + core.columnar_engines.insert(key.clone(), engine); + key + } + + fn scan_params<'a>( + collection: &'a str, + prefilter: Option<&'a SurrogateBitmap>, + ) -> ColumnarScanParams<'a> { + scan_params_with_rls(collection, prefilter, &[]) + } + + fn scan_params_with_rls<'a>( + collection: &'a str, + prefilter: Option<&'a SurrogateBitmap>, + rls_filters: &'a [u8], + ) -> ColumnarScanParams<'a> { + ColumnarScanParams { collection, projection: &[], limit: 0, filters: &[], - rls_filters: &[], + rls_filters, sort_keys: &[], system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, prefilter, computed_columns: &[], txn_id: None, + } + } + + /// A read policy `name = `, encoded the way the planner ships it. + fn policy_name_eq(name: &str) -> Vec { + let filter = crate::bridge::scan_filter::ScanFilter { + field: "name".into(), + op: crate::bridge::scan_filter::FilterOp::Eq, + value: Value::String(name.into()), + clauses: vec![], + expr: None, }; - let resp = core.execute_columnar_scan(&task, params); - let decoded: Vec = - zerompk::from_msgpack(resp.payload.as_bytes()).expect("decode scan payload"); - decoded.into_iter().map(|j| j.0).collect() + zerompk::to_msgpack_vec(&vec![filter]).expect("encode policy") + } + + fn decode_rows(payload: &[u8]) -> Vec { + match nodedb_types::value_from_msgpack(payload).expect("decode scan payload") { + Value::Array(rows) => rows, + other => panic!("scan payload must be an array of rows: {other:?}"), + } + } + + /// Run a prefiltered scan and return the decoded result rows. + fn scan_with_prefilter( + core: &mut CoreLoop, + collection: &str, + prefilter: Option<&SurrogateBitmap>, + ) -> Vec { + let task = make_task(); + let resp = core.execute_columnar_scan(&task, scan_params(collection, prefilter)); + decode_rows(resp.payload.as_bytes()) } - fn ids(rows: &[serde_json::Value]) -> Vec { + fn field<'a>(row: &'a Value, name: &str) -> Option<&'a Value> { + match row { + Value::Object(map) => map.get(name), + _ => None, + } + } + + fn ids(rows: &[Value]) -> Vec { let mut v: Vec = rows .iter() - .filter_map(|r| r.get("id").and_then(|x| x.as_i64())) + .filter_map(|r| match field(r, "id") { + Some(Value::Integer(i)) => Some(*i), + _ => None, + }) .collect(); v.sort_unstable(); v } /// A prefilter that includes SOME flushed-row surrogates must return - /// exactly those rows — proving flushed rows are now prefiltered, not + /// exactly those rows — proving flushed rows are prefiltered, not /// skipped wholesale. #[test] fn flushed_rows_are_prefiltered_not_skipped() { @@ -698,8 +695,8 @@ mod tests { ); } - /// Sanity: with NO prefilter all flushed rows are returned (unchanged - /// behaviour — the gate only applies when a prefilter is present). + /// Sanity: with NO prefilter all flushed rows are returned (the gate only + /// applies when a prefilter is present). #[test] fn flushed_rows_no_prefilter_returns_all() { let (mut core, _dir) = make_core(); @@ -718,6 +715,103 @@ mod tests { assert_eq!(ids(&rows), vec![1, 2, 3]); } + /// A declared `TIMESTAMP` cell is the same naive instant whether the row + /// is read from the live memtable or from a flushed segment. + #[test] + fn a_timestamp_cell_is_the_same_instant_before_and_after_flush() { + let (mut core, _dir) = make_core(); + let coll = "cf_instant"; + let key = engine_key(coll); + let mut engine = MutationEngine::new(coll.to_string(), schema()); + engine + .insert_with_surrogate(&row(1, "live"), Surrogate(701)) + .expect("insert_with_surrogate"); + core.columnar_engines.insert(key.clone(), engine); + + let expected = Value::NaiveDateTime(NdbDateTime::from_micros(MICROS)); + let live = scan_with_prefilter(&mut core, coll, None); + assert_eq!(live.len(), 1); + assert_eq!(field(&live[0], "at"), Some(&expected), "live memtable cell"); + + let mut engine = core + .columnar_engines + .remove(&key) + .expect("engine registered"); + flush(&mut core, &key, &mut engine); + core.columnar_engines.insert(key.clone(), engine); + let flushed = scan_with_prefilter(&mut core, coll, None); + assert_eq!(flushed.len(), 1); + assert_eq!( + field(&flushed[0], "at"), + Some(&expected), + "flushed segment cell" + ); + } + + /// A read policy governs rows from the live memtable and from flushed + /// segments alike: only the rows it admits come back. + #[test] + fn a_read_policy_admits_only_matching_rows_from_both_phases() { + let (mut core, _dir) = make_core(); + let coll = "cf_rls_both"; + let key = insert_and_flush( + &mut core, + coll, + &[(1, "mine", Surrogate(801)), (2, "theirs", Surrogate(802))], + ); + let mut engine = core + .columnar_engines + .remove(&key) + .expect("engine registered"); + engine + .insert_with_surrogate(&row(3, "mine"), Surrogate(803)) + .expect("insert_with_surrogate"); + engine + .insert_with_surrogate(&row(4, "theirs"), Surrogate(804)) + .expect("insert_with_surrogate"); + core.columnar_engines.insert(key, engine); + + let policy = policy_name_eq("mine"); + let task = make_task(); + let resp = core.execute_columnar_scan(&task, scan_params_with_rls(coll, None, &policy)); + assert_eq!(resp.status, crate::bridge::envelope::Status::Ok); + assert_eq!( + ids(&decode_rows(resp.payload.as_bytes())), + vec![1, 3], + "one flushed and one live row are admitted; the others are excluded" + ); + } + + /// A read policy payload the Data Plane cannot decode fails the scan. + /// The alternative — treating it as "no policy" — would return every + /// row the policy exists to hide. + #[test] + fn a_malformed_read_policy_payload_fails_the_scan() { + let (mut core, _dir) = make_core(); + let coll = "cf_rls_malformed"; + insert_and_flush( + &mut core, + coll, + &[(1, "mine", Surrogate(901)), (2, "theirs", Surrogate(902))], + ); + + // `0xC1` is the one byte MessagePack reserves and never emits. + let malformed = [0xC1u8]; + let task = make_task(); + let resp = core.execute_columnar_scan(&task, scan_params_with_rls(coll, None, &malformed)); + + assert_eq!( + resp.status, + crate::bridge::envelope::Status::Error, + "an unreadable policy must not admit rows" + ); + assert!( + resp.payload.is_empty(), + "a failed scan carries no rows, got {} bytes", + resp.payload.len() + ); + } + /// Build a task whose deadline is already in the past. fn expired_task() -> ExecutionTask { let mut task = make_task(); @@ -730,22 +824,7 @@ mod tests { collection: &str, task: &ExecutionTask, ) -> crate::bridge::envelope::Response { - core.execute_columnar_scan( - task, - ColumnarScanParams { - collection, - projection: &[], - limit: 0, - filters: &[], - rls_filters: &[], - sort_keys: &[], - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - prefilter: None, - computed_columns: &[], - txn_id: None, - }, - ) + core.execute_columnar_scan(task, scan_params(collection, None)) } /// A statement over its deadline is stopped DURING execution, not merely @@ -796,9 +875,7 @@ mod tests { let resp = scan_response(&mut core, coll, &make_task()); assert_eq!(resp.status, crate::bridge::envelope::Status::Ok); - let decoded: Vec = - zerompk::from_msgpack(resp.payload.as_bytes()).expect("decode scan payload"); - let rows: Vec = decoded.into_iter().map(|j| j.0).collect(); + let rows = decode_rows(resp.payload.as_bytes()); assert_eq!(ids(&rows), vec![1, 2]); } diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan/mod.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/mod.rs new file mode 100644 index 000000000..1849f434c --- /dev/null +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Columnar base scan: parameters, the entry point, and its stages. + +mod block_skip; +mod execute; +mod order; +mod params; + +pub(in crate::data::executor) use params::ColumnarScanParams; diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan/order.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/order.rs new file mode 100644 index 000000000..023f22f5b --- /dev/null +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/order.rs @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Result ordering for a columnar base scan: `ORDER BY` keys, or audit-log +//! system-time order for an all-versions read. + +use nodedb_physical::physical_plan::SortKeySpec; +use nodedb_types::columnar::ColumnarSchema; + +use crate::data::executor::handlers::columnar_read::bitemporal::row_system_time; +use crate::data::executor::handlers::columnar_read::sort::{ + compare_sort_values, eval_row_sort_values, +}; +use crate::data::executor::handlers::transaction::overlay::ColumnarMatchedRow; + +/// Order `matched` in place. +/// +/// With sort keys, every row's keys are evaluated first — `sort_by` has no +/// way to report an error — then the rows are ordered by them. A computed +/// key that divides by zero fails the statement. Without sort keys, an +/// all-versions read orders ascending by `_ts_system` (audit-log order) and a +/// current read keeps scan order. +pub(super) fn order_matched( + matched: &mut Vec, + schema: &ColumnarSchema, + sort_keys: &[SortKeySpec], + all_versions: bool, + ts_system_idx: Option, +) -> crate::Result<()> { + if !sort_keys.is_empty() { + let mut keyed = Vec::with_capacity(matched.len()); + for (_, row, _) in matched.iter() { + keyed.push(eval_row_sort_values(row, schema, sort_keys)?); + } + let mut indexed: Vec<_> = keyed.into_iter().zip(matched.drain(..)).collect(); + indexed.sort_by(|a, b| compare_sort_values(&a.0, &b.0, sort_keys)); + matched.extend(indexed.into_iter().map(|(_, row)| row)); + } else if all_versions { + matched.sort_by(|(_, a, _), (_, b, _)| { + row_system_time(a, ts_system_idx).cmp(&row_system_time(b, ts_system_idx)) + }); + } + Ok(()) +} diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan/params.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/params.rs new file mode 100644 index 000000000..bceba5428 --- /dev/null +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/params.rs @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Parameters for a columnar base scan. + +use nodedb_types::surrogate_bitmap::SurrogateBitmap; + +/// Parameters for a columnar base scan. Bundled as a struct because the +/// raw parameter list exceeds the project's too-many-arguments bound. +pub(in crate::data::executor) struct ColumnarScanParams<'a> { + pub collection: &'a str, + pub projection: &'a [String], + pub limit: usize, + pub filters: &'a [u8], + /// The caller's read policy as a MessagePack `Vec`, injected + /// by the planner. Empty when no policy governs the caller. The scan + /// evaluates it per row after block pruning and `filters`, before + /// projection, sort, and limit, on the flushed-segment, live-memtable, + /// and in-transaction overlay rows alike. + pub rls_filters: &'a [u8], + pub sort_keys: &'a [nodedb_physical::physical_plan::SortKeySpec], + /// Bitemporal system-time selection. `Current` is a current-state read; + /// `AsOf(ms)` drops rows with `_ts_system > ms`; `AllVersions` emits every + /// `_ts_system` row ordered ascending (audit log), with the system-time + /// column projected. + pub system_time: nodedb_types::SystemTimeScope, + /// Bitemporal valid-time point: drop rows whose + /// `[_ts_valid_from, _ts_valid_until)` interval does not contain this + /// point. `None` skips valid-time filtering entirely. + pub valid_at_ms: Option, + /// Optional cross-engine surrogate prefilter. When `Some`, the scan + /// skips whole memtable blocks whose surrogate range does not intersect + /// the bitmap (block boundary) and skips individual rows whose surrogate + /// is absent from the bitmap (row boundary). `None` = no prefilter. + pub prefilter: Option<&'a SurrogateBitmap>, + /// MessagePack-serialized `Vec` for scalar projection + /// expressions such as JSON arrow operators. Empty slice means no + /// computed columns are requested. + pub computed_columns: &'a [u8], + /// The in-transaction identity of the caller, when the scan is issued + /// inside `BEGIN..COMMIT`. `Some` gates a post-scan overlay merge + /// (`merge_overlay_into_columnar_scan`) so the scan observes the + /// transaction's own staged, not-yet-durable `ColumnarOp::Insert` rows + /// (read-your-own-writes). `None` for autocommit reads. + pub txn_id: Option, +} diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan_flushed.rs b/nodedb/src/data/executor/handlers/columnar_read/scan_flushed.rs index aa559445b..eb7df962a 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/scan_flushed.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/scan_flushed.rs @@ -1,19 +1,16 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Phase-1 flushed-segment scanning, extracted from `scan.rs` to keep that -//! file within the 500-line non-test limit. Logic is verbatim from -//! `execute_columnar_scan`; only the parameterisation changes. - -use nodedb_types::columnar::schema::TS_SYSTEM; +//! The flushed-segment pass of the columnar base scan. use crate::bridge::expr_eval::ComputedColumn; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::transaction::overlay::ColumnarMatchedRow; use crate::data::executor::scan_normalize::decoded_col_to_value; use super::bitemporal::bitemporal_row_visible; -use super::convert::value_to_json; -use super::filter::row_matches_filters; +use super::convert::row_to_projected_value; +use super::filter::row_matches_filters_and_policy; /// Read-only context for the flushed-segment scan phase. All fields are /// borrowed from locals already computed in `execute_columnar_scan`. @@ -25,6 +22,8 @@ pub(in crate::data::executor) struct FlushedScanCtx<'a> { pub limit: usize, pub sort_keys: &'a [nodedb_physical::physical_plan::SortKeySpec], pub filter_predicates: &'a [ScanFilter], + /// The caller's decoded read policy; empty admits every row. + pub rls_predicates: &'a [ScanFilter], pub prefilter: Option<&'a nodedb_types::surrogate_bitmap::SurrogateBitmap>, pub computed_cols: &'a [ComputedColumn], pub all_versions: bool, @@ -40,18 +39,14 @@ impl CoreLoop { /// rows to `matched`. This is Phase 1 of `execute_columnar_scan`; Phase 2 /// (live memtable) continues in the caller with the same `matched` Vec. /// - /// The body is verbatim from the `// ── Phase 1: flushed segments ──` block - /// in `scan.rs`. Only local variables that were defined earlier in - /// `execute_columnar_scan` are now method parameters (borrowed read-only via - /// `FlushedScanCtx`) or the mutable `matched` accumulator. + /// Each cell is decoded by `decoded_col_to_value` with its declared column + /// type, and each row is projected by `row_to_projected_value` — the same + /// two steps the live-memtable phase applies — so a row reads identically + /// from a segment and from the memtable. pub(in crate::data::executor) fn scan_flushed_columnar_segments( &self, ctx: FlushedScanCtx<'_>, - matched: &mut Vec<( - Option, - Vec, - serde_json::Value, - )>, + matched: &mut Vec, ) -> crate::Result<()> { let FlushedScanCtx { collection, @@ -61,6 +56,7 @@ impl CoreLoop { limit, sort_keys, filter_predicates, + rls_predicates, prefilter, computed_cols, all_versions, @@ -179,10 +175,14 @@ impl CoreLoop { } } - // Build the row as Vec using the shared decoder. + // Build the row as Vec using the shared decoder, + // typing each cell by its declared column type. let row: Vec = decoded_cols .iter() - .map(|dc| decoded_col_to_value(dc, row_idx)) + .zip(&schema.columns) + .map(|(dc, col_def)| { + decoded_col_to_value(dc, row_idx, &col_def.column_type) + }) .collect(); if !bitemporal_row_visible( @@ -195,49 +195,28 @@ impl CoreLoop { ) { continue; } - if !filter_predicates.is_empty() - && !row_matches_filters(&row, schema, filter_predicates)? - { + // The query's WHERE predicates, then the caller's read + // policy, before projection so a limit counts admitted + // rows only. + if !row_matches_filters_and_policy( + &row, + schema, + filter_predicates, + rls_predicates, + )? { continue; } - let mut obj = serde_json::Map::new(); - for (i, col_def) in schema.columns.iter().enumerate() { - let force_system_col = all_versions && col_def.name == TS_SYSTEM; - if !projection.is_empty() - && !force_system_col - && !projection.iter().any(|p| p == &col_def.name) - && !computed_cols.iter().any(|cc| cc.alias == col_def.name) - { - continue; - } - if i < row.len() { - obj.insert(col_def.name.clone(), value_to_json(&row[i])); - } - } - if !computed_cols.is_empty() { - let doc_val = - nodedb_types::Value::from(serde_json::Value::Object(obj.clone())); - for cc in computed_cols { - let existing = obj.get(&cc.alias); - if matches!(existing, Some(v) if !v.is_null()) { - continue; - } - // Computed-column projection is - // projection-shaped: a division/modulo-by-zero - // fails the whole scan. - let v = cc.expr.eval(&doc_val)?; - obj.insert(cc.alias.clone(), serde_json::Value::from(v)); - } - if !projection.is_empty() { - obj.retain(|k, _| { - projection.iter().any(|p| p == k) - || computed_cols.iter().any(|cc| &cc.alias == k) - || (all_versions && k == TS_SYSTEM) - }); - } - } - matched.push((row_surrogate, row, serde_json::Value::Object(obj))); + // Computed-column projection is projection-shaped: a + // division/modulo-by-zero fails the whole scan. + let obj = row_to_projected_value( + &row, + schema, + projection, + computed_cols, + all_versions, + )?; + matched.push((row_surrogate, row, obj)); if sort_keys.is_empty() && matched.len() >= limit { break; } diff --git a/nodedb/src/data/executor/handlers/columnar_write/read_prior.rs b/nodedb/src/data/executor/handlers/columnar_write/read_prior.rs index 761748692..e10d77043 100644 --- a/nodedb/src/data/executor/handlers/columnar_write/read_prior.rs +++ b/nodedb/src/data/executor/handlers/columnar_write/read_prior.rs @@ -62,11 +62,12 @@ impl CoreLoop { usize::MAX, )?; let mut row = Vec::with_capacity(row_capacity); - for col_idx in 0..column_count { + for (col_idx, col_def) in schema.columns.iter().enumerate() { let decoded = reader.read_column(col_idx).ok()?; row.push(crate::data::executor::scan_normalize::decoded_col_to_value( &decoded, loc.row_index as usize, + &col_def.column_type, )); } Some(row) diff --git a/nodedb/src/data/executor/handlers/document/resolve/bulk.rs b/nodedb/src/data/executor/handlers/document/resolve/bulk.rs index e513ed705..cc4d2e7fa 100644 --- a/nodedb/src/data/executor/handlers/document/resolve/bulk.rs +++ b/nodedb/src/data/executor/handlers/document/resolve/bulk.rs @@ -103,7 +103,7 @@ impl CoreLoop { .map_err(ErrorCode::from)?; let mut mutations = Vec::with_capacity(projected.len()); - let mut returned_docs: Vec = Vec::new(); + let mut returned_docs: Vec = Vec::new(); for row in projected { let ProjectedUpdateRow { key: storage_key, @@ -129,7 +129,7 @@ impl CoreLoop { resolved_sum_targets, })); if returning.is_some() { - returned_docs.push(doc); + returned_docs.push(nodedb_types::Value::from(doc)); } } diff --git a/nodedb/src/data/executor/handlers/graph_edge_write.rs b/nodedb/src/data/executor/handlers/graph_edge_write.rs index a2a75ea80..7ef48c371 100644 --- a/nodedb/src/data/executor/handlers/graph_edge_write.rs +++ b/nodedb/src/data/executor/handlers/graph_edge_write.rs @@ -654,7 +654,7 @@ mod tests { fn owner_write_check(owner: &str) -> Vec { let filter = crate::bridge::scan_filter::ScanFilter { field: "owner".into(), - op: "eq".into(), + op: crate::bridge::scan_filter::FilterOp::Eq, value: nodedb_types::Value::String(owner.into()), clauses: Vec::new(), expr: None, diff --git a/nodedb/src/data/executor/handlers/join/instant_scale.rs b/nodedb/src/data/executor/handlers/join/instant_scale.rs deleted file mode 100644 index 4f2e41620..000000000 --- a/nodedb/src/data/executor/handlers/join/instant_scale.rs +++ /dev/null @@ -1,86 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Rescale the declared-instant cells a join materialized itself. -//! -//! A timeseries collection stores every timestamp column in epoch -//! milliseconds, and a client reads a `TIMESTAMP` cell as epoch microseconds. -//! A join scans its local sides through `scan_collection`, which hands back -//! the stored millisecond value, so the join rescales those cells as it emits -//! them — after every predicate has run, so the join itself still compares -//! milliseconds against milliseconds. -//! -//! Exactly once: a handler rescales ONLY the cells it read from a local -//! collection scan. Rows that arrive from a sub-plan response were already -//! rescaled by the handler that read them, and are never touched again. -//! -//! The rescale runs on the merged row, before any projection. A cell is named -//! by the key the join itself wrote, so a rename cannot hide it: the rename -//! happens strictly downstream, and a projected cell copies bytes the rescale -//! has already corrected. - -use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::handlers::timeseries::raw_scan::scale_instant_cells; -use crate::types::{DatabaseId, TenantId}; - -/// One locally-scanned join side: the collection its rows come from, and the -/// qualifier the merged row prefixes that side's keys with. -pub(in crate::data::executor) struct JoinInstantSide<'a> { - pub(in crate::data::executor) collection: &'a str, - pub(in crate::data::executor) qualifier: &'a str, -} - -impl CoreLoop { - /// The merged-row keys of the declared-instant cells `sides` contribute. - /// - /// A merged join row keys every cell `.`, so the - /// returned names are what the row carries before any projection runs. A - /// side that is not a timeseries collection contributes none, so a join - /// over ordinary collections returns an empty list and pays nothing. - pub(in crate::data::executor) fn join_instant_columns( - &self, - database_id: DatabaseId, - tid: TenantId, - sides: &[JoinInstantSide<'_>], - ) -> Vec { - let mut merged_keys = Vec::new(); - for side in sides { - for column in self.ts_instant_columns(database_id, tid, side.collection) { - merged_keys.push(format!("{}.{column}", side.qualifier)); - } - } - merged_keys - } -} - -/// Rescale the named cells of merged join rows, in place. -/// -/// Each row is a msgpack map. An empty `instant_columns` is a no-op, so a -/// join that touches no timeseries collection never decodes a row. -pub(in crate::data::executor) fn scale_join_instant_rows( - rows: &mut [Vec], - instant_columns: &[String], -) -> crate::Result<()> { - if instant_columns.is_empty() { - return Ok(()); - } - for row in rows.iter_mut() { - let mut decoded = [ - crate::util::bounded_msgpack::read_value(row.as_slice()).map_err(|e| { - crate::Error::Serialization { - format: "msgpack".into(), - detail: format!("decode join row for instant rescale: {e}"), - } - })?, - ]; - scale_instant_cells(&mut decoded, instant_columns)?; - let mut buf = Vec::with_capacity(row.len()); - rmpv::encode::write_value(&mut buf, &decoded[0]).map_err(|e| { - crate::Error::Serialization { - format: "msgpack".into(), - detail: format!("encode join row after instant rescale: {e}"), - } - })?; - *row = buf; - } - Ok(()) -} diff --git a/nodedb/src/data/executor/handlers/join/mod.rs b/nodedb/src/data/executor/handlers/join/mod.rs index a6dc9bff6..f9d4f8ad7 100644 --- a/nodedb/src/data/executor/handlers/join/mod.rs +++ b/nodedb/src/data/executor/handlers/join/mod.rs @@ -10,7 +10,6 @@ mod grace_repartition; mod grace_spill; pub mod hash; mod hash_handlers; -pub(in crate::data::executor) mod instant_scale; pub mod lateral; pub mod nested_loop; pub mod params; diff --git a/nodedb/src/data/executor/handlers/join/nested_loop.rs b/nodedb/src/data/executor/handlers/join/nested_loop.rs index d961bbc6a..e0e6db89a 100644 --- a/nodedb/src/data/executor/handlers/join/nested_loop.rs +++ b/nodedb/src/data/executor/handlers/join/nested_loop.rs @@ -30,7 +30,6 @@ impl CoreLoop { limit, left_rls_filters, right_rls_filters, - instant_columns, } = p; debug!( core = self.core_id, @@ -225,18 +224,6 @@ impl CoreLoop { return self.response_error(task, ErrorCode::ResourcesExhausted); } - // Last step before emission: the scans above compared the - // milliseconds storage holds, and the client reads microseconds. - if let Err(e) = super::instant_scale::scale_join_instant_rows(&mut results, instant_columns) - { - return self.response_error( - task, - ErrorCode::Internal { - detail: e.to_string(), - }, - ); - } - let payload = super::super::super::response_codec::encode_binary_rows(&results); self.response_with_payload(task, payload) } diff --git a/nodedb/src/data/executor/handlers/join/params.rs b/nodedb/src/data/executor/handlers/join/params.rs index 8b14df27f..c5620115e 100644 --- a/nodedb/src/data/executor/handlers/join/params.rs +++ b/nodedb/src/data/executor/handlers/join/params.rs @@ -17,11 +17,6 @@ pub(crate) struct JoinParams<'a> { pub computed_projection_bytes: &'a [u8], pub join_filter_bytes: &'a [u8], pub post_filter_bytes: &'a [u8], - /// Merged-row keys of the declared-instant cells this join reads from its - /// own local scans. Empty when no timeseries collection is scanned - /// locally — including for every side supplied by a sub-plan, whose rows - /// were already rescaled by the handler that read them. - pub instant_columns: &'a [String], } /// Hash join: scans both sides from storage or executes resolved child sub-plans. @@ -81,9 +76,6 @@ pub(crate) struct NestedLoopJoinParams<'a> { pub left_rls_filters: &'a [u8], /// Row-level-security filters for the locally-scanned right side. pub right_rls_filters: &'a [u8], - /// Merged-row keys of the declared-instant cells the two local scans - /// contribute. Empty when neither side is a timeseries collection. - pub instant_columns: &'a [String], } /// Sort-merge join: O((N+M)·log N) equi-join with optional pre-sorted inputs. @@ -103,9 +95,6 @@ pub(crate) struct SortMergeJoinParams<'a> { pub left_rls_filters: &'a [u8], /// Row-level-security filters for the locally-scanned right side. pub right_rls_filters: &'a [u8], - /// Merged-row keys of the declared-instant cells the two local scans - /// contribute. Empty when neither side is a timeseries collection. - pub instant_columns: &'a [String], } // ── Test helpers ───────────────────────────────────────────────────────────── @@ -202,11 +191,6 @@ impl JoinParams<'_> { } } - // Every predicate above compared the milliseconds storage holds, and - // the client reads microseconds. Scale here, on the keys the join - // wrote, so a projection that renames a cell copies a corrected value. - super::instant_scale::scale_join_instant_rows(results, self.instant_columns)?; - if !self.computed_projection_bytes.is_empty() { let computed: Vec = zerompk::from_msgpack(self.computed_projection_bytes).map_err(|e| { @@ -274,7 +258,6 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &[], - instant_columns: &[], }; let mut results = vec![vec![1u8, 2, 3], vec![4u8, 5, 6]]; assert!(params.filter_and_project(&mut results).is_ok()); @@ -297,7 +280,6 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: corrupt, - instant_columns: &[], }; let mut results = vec![vec![0u8; 8]]; // would be "leaked" under the old code let err = params.filter_and_project(&mut results); @@ -313,50 +295,6 @@ mod tests { ); } - /// A renaming projection cannot hide an instant cell from the rescale. - /// - /// The rescale names the merged key the join wrote, and the projection - /// renames the cell afterwards, so the emitted cell carries microseconds - /// under its output name. - #[test] - fn projection_rename_still_rescales_the_instant_cell() { - let task = make_dummy_task(); - let projection = vec![JoinProjection { - source: "e.captured_at".into(), - output: "ts".into(), - }]; - let params = JoinParams { - task: &task, - on: &[], - join_type: "inner", - limit: usize::MAX, - projection: &projection, - computed_projection_bytes: &[], - join_filter_bytes: &[], - post_filter_bytes: &[], - instant_columns: &["e.captured_at".to_string()], - }; - let mut results = vec![ - nodedb_types::json_to_msgpack( - &serde_json::json!({"e.captured_at": 1_583_402_400_000i64}), - ) - .expect("encode test row"), - ]; - params - .filter_and_project(&mut results) - .expect("filter_and_project"); - - let decoded = nodedb_types::value_from_msgpack(&results[0]).expect("decode emitted row"); - let nodedb_types::Value::Object(fields) = decoded else { - panic!("emitted row must be a map, got {decoded:?}"); - }; - assert_eq!( - fields.get("ts"), - Some(&nodedb_types::Value::Integer(1_583_402_400_000_000)), - "the renamed cell must carry epoch microseconds: {fields:?}" - ); - } - /// Valid encoded filters → matching rows retained, non-matching rows dropped /// (happy path unchanged). #[test] @@ -372,7 +310,6 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &filter_bytes, - instant_columns: &[], }; let mut results = vec![ row_with_score(99), // should be kept diff --git a/nodedb/src/data/executor/handlers/join/shuffle_join.rs b/nodedb/src/data/executor/handlers/join/shuffle_join.rs index 2cbe7abfd..d1fceefad 100644 --- a/nodedb/src/data/executor/handlers/join/shuffle_join.rs +++ b/nodedb/src/data/executor/handlers/join/shuffle_join.rs @@ -439,7 +439,6 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &[], - instant_columns: &[], }; let inputs = ShuffleJoinInputs { build_path, diff --git a/nodedb/src/data/executor/handlers/join/sort_merge.rs b/nodedb/src/data/executor/handlers/join/sort_merge.rs index c4803a8a5..286de26f6 100644 --- a/nodedb/src/data/executor/handlers/join/sort_merge.rs +++ b/nodedb/src/data/executor/handlers/join/sort_merge.rs @@ -51,7 +51,6 @@ impl CoreLoop { pre_sorted, left_rls_filters, right_rls_filters, - instant_columns, } = p; debug!( core = self.core_id, @@ -300,18 +299,6 @@ impl CoreLoop { return self.response_error(task, ErrorCode::ResourcesExhausted); } - // Last step before emission: the scans above compared the - // milliseconds storage holds, and the client reads microseconds. - if let Err(e) = super::instant_scale::scale_join_instant_rows(&mut results, instant_columns) - { - return self.response_error( - task, - ErrorCode::Internal { - detail: e.to_string(), - }, - ); - } - let payload = super::super::super::response_codec::encode_binary_rows(&results); self.response_with_payload(task, payload) } diff --git a/nodedb/src/data/executor/handlers/merge_orchestrated/apply/insert_rows.rs b/nodedb/src/data/executor/handlers/merge_orchestrated/apply/insert_rows.rs index e79998529..0946ea9bd 100644 --- a/nodedb/src/data/executor/handlers/merge_orchestrated/apply/insert_rows.rs +++ b/nodedb/src/data/executor/handlers/merge_orchestrated/apply/insert_rows.rs @@ -49,7 +49,7 @@ pub(super) struct InsertRowsTally<'a, 'p> { pub(super) put_events: &'a mut Vec>, pub(super) write_set: &'a mut Vec, pub(super) balanced_entries: &'a mut Vec, - pub(super) returned_docs: &'a mut Vec, + pub(super) returned_docs: &'a mut Vec, } impl CoreLoop { diff --git a/nodedb/src/data/executor/handlers/merge_orchestrated/apply/orchestrate.rs b/nodedb/src/data/executor/handlers/merge_orchestrated/apply/orchestrate.rs index 9129c39b8..b09af3f77 100644 --- a/nodedb/src/data/executor/handlers/merge_orchestrated/apply/orchestrate.rs +++ b/nodedb/src/data/executor/handlers/merge_orchestrated/apply/orchestrate.rs @@ -120,7 +120,7 @@ impl CoreLoop { // than carried in, because an attempt that ends in `OllpRetryRequired` // is fully re-resolved and re-applied by the orchestrator — rows from a // failed attempt describe a snapshot that never committed. - let mut returned_docs: Vec = Vec::new(); + let mut returned_docs: Vec = Vec::new(); if let Err(response) = self.apply_merge_update_arm( &txn, diff --git a/nodedb/src/data/executor/handlers/merge_orchestrated/apply/update_rows.rs b/nodedb/src/data/executor/handlers/merge_orchestrated/apply/update_rows.rs index 54024e2fe..54e897884 100644 --- a/nodedb/src/data/executor/handlers/merge_orchestrated/apply/update_rows.rs +++ b/nodedb/src/data/executor/handlers/merge_orchestrated/apply/update_rows.rs @@ -44,7 +44,7 @@ pub(super) struct UpdateRowsTally<'a, 'p> { pub(super) put_events: &'a mut Vec>, pub(super) write_set: &'a mut Vec, pub(super) balanced_entries: &'a mut Vec, - pub(super) returned_docs: &'a mut Vec, + pub(super) returned_docs: &'a mut Vec, } impl CoreLoop { diff --git a/nodedb/src/data/executor/handlers/merge_orchestrated/apply_support.rs b/nodedb/src/data/executor/handlers/merge_orchestrated/apply_support.rs index 1eb0bb74a..883d6c037 100644 --- a/nodedb/src/data/executor/handlers/merge_orchestrated/apply_support.rs +++ b/nodedb/src/data/executor/handlers/merge_orchestrated/apply_support.rs @@ -90,9 +90,9 @@ pub(super) fn gate_merge_arms( Ok(()) } -/// Decode one merge row body into the JSON document a RETURNING projection -/// reads. Same shape the point and bulk DML RETURNING paths emit, so a MERGE -/// row projects identically. +/// Decode one merge row body into the `Value` document a RETURNING +/// projection reads. Same shape the point and bulk DML RETURNING paths +/// emit, so a MERGE row projects identically. /// /// `key` is the row's storage key: every caller's `MergeUpdate::key`, /// `MergeDelete::key`, or a freshly minted insert key. This function converts @@ -102,7 +102,7 @@ pub(super) fn gate_merge_arms( /// bodies are MessagePack for BOTH storage modes (`collect_merge_plan` decodes /// a strict target's Binary Tuple and re-encodes the resolved row before the /// apply pass ever sees it), so the strict decoder would have nothing to read. -pub(super) fn returning_doc(body: &[u8], key: &StorageKey) -> crate::Result { +pub(super) fn returning_doc(body: &[u8], key: &StorageKey) -> crate::Result { let identity = key.to_identity(); super::super::returning_doc::from_stored(body, &identity, None) } diff --git a/nodedb/src/data/executor/handlers/merge_orchestrated/delete_arms.rs b/nodedb/src/data/executor/handlers/merge_orchestrated/delete_arms.rs index 81ce590c8..d8b1f6327 100644 --- a/nodedb/src/data/executor/handlers/merge_orchestrated/delete_arms.rs +++ b/nodedb/src/data/executor/handlers/merge_orchestrated/delete_arms.rs @@ -46,7 +46,7 @@ pub(super) struct MergeDeleteArms<'a> { pub(super) struct MergeDeleteTally<'a> { pub(super) affected: &'a mut u64, pub(super) write_set: &'a mut Vec, - pub(super) returned_docs: &'a mut Vec, + pub(super) returned_docs: &'a mut Vec, } impl CoreLoop { diff --git a/nodedb/src/data/executor/handlers/returning_doc.rs b/nodedb/src/data/executor/handlers/returning_doc.rs index 0a01f12c3..09586282b 100644 --- a/nodedb/src/data/executor/handlers/returning_doc.rs +++ b/nodedb/src/data/executor/handlers/returning_doc.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Construction of the JSON document a DML `RETURNING` projection reads. +//! Construction of the `Value` document a DML `RETURNING` projection reads. //! //! Every `RETURNING` path — point and bulk UPDATE/DELETE, `UPDATE ... FROM`, //! MERGE, CRDT document DML — builds its rows from a stored row body, and two @@ -16,6 +16,7 @@ //! key; overwriting it with the surrogate hex storage key returns a value the //! client never wrote and cannot use to address the row. +use nodedb_types::Value; use nodedb_types::columnar::StrictSchema; use crate::data::executor::doc_format; @@ -27,22 +28,19 @@ use crate::engine::document::store::RowIdentity; /// /// For callers that already hold the decoded document (the update paths /// re-project the image they just built rather than re-reading storage). -pub(in crate::data::executor) fn attach_row_id( - doc: &mut serde_json::Value, - identity: &RowIdentity, -) { - if let Some(obj) = doc.as_object_mut() +pub(in crate::data::executor) fn attach_row_id(doc: &mut Value, identity: &RowIdentity) { + if let Value::Object(obj) = doc && !obj.contains_key("id") { obj.insert( "id".to_string(), - serde_json::Value::String(identity.as_str().to_string()), + Value::String(identity.as_str().to_string()), ); } } /// Decode a STORED row body — a pre-image or a re-encoded post-image — into the -/// document a `RETURNING` projection reads. +/// typed document a `RETURNING` projection reads and the write gate judges. /// /// `strict_schema` is `Some` exactly when the collection stores Binary Tuples; /// passing `None` for a strict collection is the silent-misdecode failure this @@ -56,27 +54,58 @@ pub(in crate::data::executor) fn from_stored( body: &[u8], identity: &RowIdentity, strict_schema: Option<&StrictSchema>, -) -> crate::Result { +) -> crate::Result { let mut doc = match strict_schema { - Some(schema) => strict_format::binary_tuple_to_json(body, schema).ok_or_else(|| { - crate::Error::Serialization { - format: "binary_tuple".to_string(), - detail: format!( - "RETURNING row {identity}: stored body ({} bytes) is not a Binary Tuple \ - readable under the collection's strict schema", - body.len() - ), - } - })?, - // `inject_str_field` already honours an existing `id` and wraps a - // non-map body as `{id, value}`, which is the shape schemaless callers - // have always emitted for a body that is not a document map. - None => { - let with_id = - nodedb_query::msgpack_scan::inject_str_field(body, "id", identity.as_str()); - doc_format::decode_document(&with_id)? - } + Some(schema) => strict_format::binary_tuple_to_row_value(body, schema) + .ok_or_else(|| undecodable(identity, body.len()))?, + None => doc_format::decode_document_value(&with_identity(body, identity))?, }; attach_row_id(&mut doc, identity); Ok(doc) } + +/// [`from_stored`] for the JSON document the update pipeline patches and the +/// secondary-index diff reads. +/// +/// The index diff compares this image's rendered field values against the +/// entries the forward write path derived from the same JSON decoders, so a +/// pre-image must come from those decoders too — a binary field rendered by +/// a different converter would leave its index entry behind. +pub(in crate::data::executor) fn from_stored_json( + body: &[u8], + identity: &RowIdentity, + strict_schema: Option<&StrictSchema>, +) -> crate::Result { + let mut doc = match strict_schema { + Some(schema) => strict_format::binary_tuple_to_json(body, schema) + .ok_or_else(|| undecodable(identity, body.len()))?, + None => doc_format::decode_document(&with_identity(body, identity))?, + }; + if let serde_json::Value::Object(obj) = &mut doc + && !obj.contains_key("id") + { + obj.insert( + "id".to_string(), + serde_json::Value::String(identity.as_str().to_string()), + ); + } + Ok(doc) +} + +/// The schemaless body with the storage identity injected as `id`. +/// `inject_str_field` honours an existing `id` and wraps a non-map body as +/// `{id, value}`, which is the shape schemaless callers have always emitted +/// for a body that is not a document map. +fn with_identity(body: &[u8], identity: &RowIdentity) -> Vec { + nodedb_query::msgpack_scan::inject_str_field(body, "id", identity.as_str()) +} + +fn undecodable(identity: &RowIdentity, body_len: usize) -> crate::Error { + crate::Error::Serialization { + format: "binary_tuple".to_string(), + detail: format!( + "RETURNING row {identity}: stored body ({body_len} bytes) is not a Binary Tuple \ + readable under the collection's strict schema" + ), + } +} diff --git a/nodedb/src/data/executor/handlers/returning_rows.rs b/nodedb/src/data/executor/handlers/returning_rows.rs index 83aa32145..2a446c05f 100644 --- a/nodedb/src/data/executor/handlers/returning_rows.rs +++ b/nodedb/src/data/executor/handlers/returning_rows.rs @@ -15,8 +15,10 @@ use crate::data::executor::scan_normalize::{kv_row_to_doc, sparse_row_to_doc}; use crate::data::executor::sparse_body_format::SparseBodyFormatRef; use crate::data::executor::task::ExecutionTask; use crate::engine::document::store::RowIdentity; +use crate::util::rmpv_value::rmpv_to_value; use nodedb_physical::physical_plan::{ReturningColumns, ReturningSpec}; use nodedb_types::columnar::StrictSchema; +use nodedb_types::{NativeCell, Value}; /// Rows a write path hands back to a `RETURNING` projection: the row's /// client-visible identity paired with the exact bytes stored for it. @@ -78,11 +80,11 @@ pub(in crate::data::executor) fn kv_stored_rows_payload( rls_filters: &[u8], rows: &[KvStoredRow<'_>], ) -> crate::Result> { - let docs: Vec = rows + let docs: Vec = rows .iter() .map(|(key, value)| { let (_key_str, body) = kv_row_to_doc(key, value); - doc_format::decode_document(&body) + doc_format::decode_document_value(&body) }) .collect::>>()?; build_rows_payload(spec, rls_filters, &docs).map_err(|e| crate::Error::Internal { @@ -103,11 +105,9 @@ impl CoreLoop { schema: &nodedb_types::columnar::ColumnarSchema, rows: &[Vec], ) -> Response { - let docs: Vec = rows + let docs: Vec = rows .iter() - .map(|row| { - serde_json::Value::from(super::columnar_write::row_values_to_object(schema, row)) - }) + .map(|row| super::columnar_write::row_values_to_object(schema, row)) .collect(); match build_rows_payload(spec, rls_filters, &docs) { Ok(payload) => self.response_with_payload(task, payload), @@ -133,7 +133,7 @@ impl CoreLoop { rls_filters: &[u8], rows: &[rmpv::Value], ) -> Response { - let docs: Vec = rows.iter().map(rmpv_row_to_json).collect(); + let docs: Vec = rows.iter().map(rmpv_to_value).collect(); match build_rows_payload(spec, rls_filters, &docs) { Ok(payload) => self.response_with_payload(task, payload), Err(e) => self.response_error( @@ -164,7 +164,7 @@ impl CoreLoop { let (_id, mp) = sparse_row_to_doc(row_key, sidecar, SparseBodyFormatRef::VectorSidecar); // An empty row set here would report "the write affected nothing" for a // write that did land, so an unreadable sidecar fails the statement. - let docs: Vec = match doc_format::decode_document(&mp) { + let docs: Vec = match doc_format::decode_document_value(&mp) { Ok(doc) => vec![doc], Err(e) => return self.response_error(task, e), }; @@ -180,39 +180,6 @@ impl CoreLoop { } } -/// Re-key one scan-projected timeseries row into JSON for -/// `build_rows_payload`. A straight transcode: msgpack nil stays SQL NULL. -fn rmpv_row_to_json(row: &rmpv::Value) -> serde_json::Value { - let rmpv::Value::Map(fields) = row else { - return serde_json::Value::Null; - }; - let mut obj = serde_json::Map::with_capacity(fields.len()); - for (key, value) in fields { - let Some(name) = key.as_str() else { continue }; - let cell = match value { - rmpv::Value::Nil => serde_json::Value::Null, - rmpv::Value::Boolean(b) => serde_json::Value::Bool(*b), - rmpv::Value::Integer(n) => n - .as_i64() - .map(|i| serde_json::Value::Number(i.into())) - .unwrap_or(serde_json::Value::Null), - rmpv::Value::F64(f) => serde_json::Number::from_f64(*f) - .map(serde_json::Value::Number) - .unwrap_or(serde_json::Value::Null), - rmpv::Value::F32(f) => serde_json::Number::from_f64(f64::from(*f)) - .map(serde_json::Value::Number) - .unwrap_or(serde_json::Value::Null), - rmpv::Value::String(s) => s - .as_str() - .map(|text| serde_json::Value::String(text.to_string())) - .unwrap_or(serde_json::Value::Null), - _ => serde_json::Value::Null, - }; - obj.insert(name.to_string(), cell); - } - serde_json::Value::Object(obj) -} - /// Project the STORED post-images of freshly written rows into a /// `RowsPayload` — the write paths hold the exact bytes handed to storage, /// so `RETURNING` reports what landed rather than echoing the submitted @@ -224,27 +191,27 @@ pub(in crate::data::executor) fn build_stored_rows_payload( strict_schema: Option<&StrictSchema>, rows: &[StoredRow<'_>], ) -> crate::Result> { - let docs: Vec = rows + let docs: Vec = rows .iter() .map(|(doc_id, body)| returning_doc::from_stored(body, doc_id, strict_schema)) .collect::>>()?; build_rows_payload(spec, rls_filters, &docs) } -/// Project documents per `spec` (Star = insertion order, Named = spec order, -/// missing/null → `None`) into a `RowsPayload` msgpack blob. `rls_filters` -/// (the compiled read policy) is applied BEFORE projection — a predicate -/// often references a column `RETURNING` omits, so a post-projection check -/// would leak the row. Only the visible row set shrinks; the affected count -/// already counted the write. +/// Project documents per `spec` (Star = field name order, Named = spec +/// order, missing → `Value::Null`) into a `RowsPayload` msgpack blob. +/// `rls_filters` (the compiled read policy) is applied BEFORE projection — a +/// predicate often references a column `RETURNING` omits, so a +/// post-projection check would leak the row. Only the visible row set +/// shrinks; the affected count already counted the write. pub(super) fn build_rows_payload( spec: &ReturningSpec, rls_filters: &[u8], - docs: &[serde_json::Value], + docs: &[Value], ) -> crate::Result> { - let visible: Vec<&serde_json::Value> = docs + let visible: Vec<&Value> = docs .iter() - .filter(|doc| rls_eval::rls_check_document(rls_filters, doc)) + .filter(|doc| rls_eval::rls_check_value(rls_filters, doc)) .collect(); let (columns, source_names) = match &spec.columns { @@ -252,13 +219,14 @@ pub(super) fn build_rows_payload( if visible.is_empty() { return encode_empty(Vec::new()); } - // Derive column names from the first doc's keys; both output and - // source names are identical for `RETURNING *`. - let cols: Vec = visible - .first() - .and_then(|d| d.as_object()) - .map(|obj| obj.keys().cloned().collect()) - .unwrap_or_default(); + // Derive column names from the first doc's keys, sorted so the + // shape is deterministic; both output and source names are + // identical for `RETURNING *`. + let mut cols: Vec = match visible.first() { + Some(Value::Object(obj)) => obj.keys().cloned().collect(), + Some(_) | None => Vec::new(), + }; + cols.sort_unstable(); (cols.clone(), cols) } ReturningColumns::Named(items) => { @@ -271,7 +239,7 @@ pub(super) fn build_rows_payload( } }; - let rows: Vec>> = visible + let rows: Vec> = visible .iter() .map(|doc| project_row(doc, &source_names)) .collect(); @@ -292,31 +260,31 @@ fn encode_empty(columns: Vec) -> crate::Result> { }) } -/// Project a single document into one cell per source name. +/// Project a single document into one typed cell per source name. /// -/// Returns `None` for missing fields or JSON null, `Some(text)` otherwise. -fn project_row(doc: &serde_json::Value, source_names: &[String]) -> Vec> { - let obj = doc.as_object(); +/// A missing field is `Value::Null`; a document that is not an object has +/// every cell `Value::Null`. +fn project_row(doc: &Value, source_names: &[String]) -> Vec { + let fields = match doc { + Value::Object(obj) => Some(obj), + _ => None, + }; source_names .iter() - .map(|name| obj.and_then(|o| o.get(name)).and_then(value_to_text)) + .map(|name| { + let cell = match fields.and_then(|obj| obj.get(name)) { + Some(value) => value.clone(), + None => Value::Null, + }; + NativeCell(cell) + }) .collect() } -/// Convert a JSON value to its TEXT representation for pgwire. `None` for -/// JSON null (real SQL NULL); strings are as-is, other types use JSON text. -fn value_to_text(val: &serde_json::Value) -> Option { - match val { - serde_json::Value::Null => None, - serde_json::Value::String(s) => Some(s.clone()), - other => Some(other.to_string()), - } -} - #[cfg(test)] mod tests { use super::*; - use crate::bridge::scan_filter::ScanFilter; + use crate::bridge::scan_filter::{FilterOp, ScanFilter}; use nodedb_physical::physical_plan::ReturningItem; use serde_json::json; @@ -324,7 +292,7 @@ mod tests { fn owner_policy(value: &str) -> Vec { let filter = ScanFilter { field: "owner".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String(value.into()), clauses: Vec::new(), expr: None, @@ -349,10 +317,14 @@ mod tests { zerompk::from_msgpack(payload).expect("decode RowsPayload") } - fn docs() -> Vec { + fn cells(row: &[NativeCell]) -> Vec { + row.iter().map(|c| c.0.clone()).collect() + } + + fn docs() -> Vec { vec![ - json!({"id": "r1", "owner": "alice", "note": "hidden"}), - json!({"id": "r2", "owner": "bob", "note": "shown"}), + Value::from(json!({"id": "r1", "owner": "alice", "note": "hidden"})), + Value::from(json!({"id": "r2", "owner": "bob", "note": "shown"})), ] } @@ -368,7 +340,8 @@ mod tests { let payload = build_rows_payload(&named(&["id"]), &owner_policy("bob"), &docs()).expect("build"); let decoded = decode(&payload); - assert_eq!(decoded.rows, vec![vec![Some("r2".to_string())]]); + assert_eq!(decoded.rows.len(), 1); + assert_eq!(cells(&decoded.rows[0]), vec![Value::String("r2".into())]); } /// The predicate names a column the projection omits — the filter must @@ -379,7 +352,8 @@ mod tests { build_rows_payload(&named(&["note"]), &owner_policy("bob"), &docs()).expect("build"); let decoded = decode(&payload); assert_eq!(decoded.columns, vec!["note".to_string()]); - assert_eq!(decoded.rows, vec![vec![Some("shown".to_string())]]); + assert_eq!(decoded.rows.len(), 1); + assert_eq!(cells(&decoded.rows[0]), vec![Value::String("shown".into())]); } /// `RETURNING *` derives its column list from the first VISIBLE row, so a @@ -387,8 +361,8 @@ mod tests { #[test] fn star_columns_come_from_the_first_visible_row() { let docs = vec![ - json!({"id": "r1", "owner": "alice", "secret": "hidden"}), - json!({"id": "r2", "owner": "bob"}), + Value::from(json!({"id": "r1", "owner": "alice", "secret": "hidden"})), + Value::from(json!({"id": "r2", "owner": "bob"})), ]; let spec = ReturningSpec { columns: ReturningColumns::Star, @@ -405,4 +379,52 @@ mod tests { let payload = build_rows_payload(&named(&["id"]), &[0xFF, 0xFE], &docs()).expect("build"); assert!(decode(&payload).rows.is_empty()); } + + /// Cells keep their stored type: an instant stays an instant, a missing + /// field is SQL NULL, a nested object stays nested. + #[test] + fn cells_keep_their_stored_type() { + let at = nodedb_types::NdbDateTime::from_micros(1_583_402_400_000_000); + let mut doc = std::collections::HashMap::new(); + doc.insert("at".to_string(), Value::NaiveDateTime(at)); + doc.insert("n".to_string(), Value::Integer(7)); + doc.insert( + "meta".to_string(), + Value::Object(std::collections::HashMap::from([( + "k".to_string(), + Value::Bool(true), + )])), + ); + let payload = build_rows_payload( + &named(&["at", "n", "meta", "absent"]), + &[], + &[Value::Object(doc)], + ) + .expect("build"); + let decoded = decode(&payload); + assert_eq!( + cells(&decoded.rows[0]), + vec![ + Value::NaiveDateTime(at), + Value::Integer(7), + Value::Object(std::collections::HashMap::from([( + "k".to_string(), + Value::Bool(true), + )])), + Value::Null, + ] + ); + } + + /// A timeseries row's instant ext crosses into the payload typed. + #[test] + fn a_timeseries_instant_cell_stays_typed() { + let at = nodedb_types::NdbDateTime::from_micros(42); + let row = crate::util::rmpv_value::value_to_rmpv(&Value::Object( + std::collections::HashMap::from([("t".to_string(), Value::DateTime(at))]), + )); + let docs = vec![rmpv_to_value(&row)]; + let payload = build_rows_payload(&named(&["t"]), &[], &docs).expect("build"); + assert_eq!(cells(&decode(&payload).rows[0]), vec![Value::DateTime(at)]); + } } diff --git a/nodedb/src/data/executor/handlers/rls_eval.rs b/nodedb/src/data/executor/handlers/rls_eval.rs index 87dc2e136..5bdbb985e 100644 --- a/nodedb/src/data/executor/handlers/rls_eval.rs +++ b/nodedb/src/data/executor/handlers/rls_eval.rs @@ -10,38 +10,36 @@ //! not pass all filters, the handler MUST return `NOT_FOUND` (no info leak). //! Empty `rls_filters` means no RLS policies apply — allow unconditionally. +use nodedb_types::Value; + use crate::bridge::scan_filter::ScanFilter; -/// Evaluate RLS filters against a document. +/// Evaluate RLS filters against a JSON document. /// /// Returns `true` if the document passes all RLS filters (or if no filters). /// Returns `false` if any filter rejects the document (caller must deny). /// -/// Used by point-get and key-get handlers after fetching the raw document. +/// Used by the write gate over the JSON post-images the update paths build. pub fn rls_check_document(rls_filters: &[u8], doc: &serde_json::Value) -> bool { if rls_filters.is_empty() { return true; } + check_encoded(rls_filters, &nodedb_types::json_to_msgpack_or_empty(doc)) +} - let filters: Vec = match zerompk::from_msgpack(rls_filters) { - Ok(f) => f, - Err(_) => { - // Deserialization failure → deny (fail-closed). - tracing::warn!("RLS filter deserialization failed — denying access"); - return false; - } - }; - - let msgpack = nodedb_types::json_to_msgpack_or_empty(doc); - // RLS is a security boundary: fail closed on a division/modulo-by-zero - // in a filter, exactly like the deserialization failure above — deny - // rather than propagate a query error, so a - // malformed/adversarial RLS predicate can never be used to distinguish - // "row exists but errors" from "row doesn't exist". - match ScanFilter::all_match_binary(&filters, &msgpack) { - Ok(pass) => pass, +/// Evaluate RLS filters against a typed `Value` document. +/// +/// Same contract as [`rls_check_document`]. A document that does not encode +/// is denied: a row the policy could not be evaluated against is not a row +/// the policy admitted. +pub fn rls_check_value(rls_filters: &[u8], doc: &Value) -> bool { + if rls_filters.is_empty() { + return true; + } + match nodedb_types::value_to_msgpack(doc) { + Ok(msgpack) => check_encoded(rls_filters, &msgpack), Err(e) => { - tracing::warn!(error = %e, "RLS filter evaluation failed — denying access"); + tracing::warn!(error = %e, "RLS document encode failed — denying access"); false } } @@ -55,7 +53,17 @@ pub fn rls_check_msgpack_bytes(rls_filters: &[u8], doc_bytes: &[u8]) -> bool { if rls_filters.is_empty() { return true; } + // Ensure bytes are standard msgpack for matches_binary. + let mp = super::super::doc_format::json_to_msgpack(doc_bytes); + check_encoded(rls_filters, &mp) +} +/// Decode the compiled filters and match them against one standard msgpack +/// document. RLS is a security boundary, so it fails closed: an undecodable +/// filter payload or a division/modulo-by-zero inside a filter denies rather +/// than propagating a query error, so a malformed or adversarial predicate +/// can never distinguish "row exists but errors" from "row doesn't exist". +fn check_encoded(rls_filters: &[u8], msgpack: &[u8]) -> bool { let filters: Vec = match zerompk::from_msgpack(rls_filters) { Ok(f) => f, Err(_) => { @@ -63,11 +71,7 @@ pub fn rls_check_msgpack_bytes(rls_filters: &[u8], doc_bytes: &[u8]) -> bool { return false; } }; - - // Ensure bytes are standard msgpack for matches_binary. - let mp = super::super::doc_format::json_to_msgpack(doc_bytes); - // RLS is a security boundary: fail closed — see `rls_check_document`. - match ScanFilter::all_match_binary(&filters, &mp) { + match ScanFilter::all_match_binary(&filters, msgpack) { Ok(pass) => pass, Err(e) => { tracing::warn!(error = %e, "RLS filter evaluation failed — denying access"); @@ -79,12 +83,13 @@ pub fn rls_check_msgpack_bytes(rls_filters: &[u8], doc_bytes: &[u8]) -> bool { #[cfg(test)] mod tests { use super::*; + use crate::bridge::scan_filter::FilterOp; use serde_json::json; - fn make_rls_bytes(field: &str, op: &str, value: nodedb_types::Value) -> Vec { + fn make_rls_bytes(field: &str, op: FilterOp, value: nodedb_types::Value) -> Vec { let filter = ScanFilter { field: field.into(), - op: op.into(), + op, value, clauses: Vec::new(), expr: None, @@ -100,25 +105,51 @@ mod tests { #[test] fn matching_filter_allows() { - let rls = make_rls_bytes("user_id", "eq", nodedb_types::Value::String("42".into())); + let rls = make_rls_bytes( + "user_id", + FilterOp::Eq, + nodedb_types::Value::String("42".into()), + ); let doc = json!({"user_id": "42", "name": "alice"}); assert!(rls_check_document(&rls, &doc)); } #[test] fn non_matching_filter_denies() { - let rls = make_rls_bytes("user_id", "eq", nodedb_types::Value::String("42".into())); + let rls = make_rls_bytes( + "user_id", + FilterOp::Eq, + nodedb_types::Value::String("42".into()), + ); let doc = json!({"user_id": "99", "name": "bob"}); assert!(!rls_check_document(&rls, &doc)); } #[test] fn missing_field_denies() { - let rls = make_rls_bytes("user_id", "eq", nodedb_types::Value::String("42".into())); + let rls = make_rls_bytes( + "user_id", + FilterOp::Eq, + nodedb_types::Value::String("42".into()), + ); let doc = json!({"name": "alice"}); assert!(!rls_check_document(&rls, &doc)); } + #[test] + fn typed_documents_evaluate_the_same_filters() { + let rls = make_rls_bytes( + "user_id", + FilterOp::Eq, + nodedb_types::Value::String("42".into()), + ); + let ok = Value::from(json!({"user_id": "42"})); + let bad = Value::from(json!({"user_id": "99"})); + assert!(rls_check_value(&rls, &ok)); + assert!(!rls_check_value(&rls, &bad)); + assert!(rls_check_value(&[], &bad)); + } + #[test] fn corrupt_filters_deny() { let corrupt = vec![0xFF, 0xFE, 0xFD]; @@ -131,14 +162,14 @@ mod tests { let filters = vec![ ScanFilter { field: "user_id".into(), - op: "eq".into(), + op: crate::bridge::scan_filter::FilterOp::Eq, value: nodedb_types::Value::String("42".into()), clauses: Vec::new(), expr: None, }, ScanFilter { field: "status".into(), - op: "eq".into(), + op: crate::bridge::scan_filter::FilterOp::Eq, value: nodedb_types::Value::String("active".into()), clauses: Vec::new(), expr: None, diff --git a/nodedb/src/data/executor/handlers/rls_write_gate.rs b/nodedb/src/data/executor/handlers/rls_write_gate.rs index bba77abdd..211369b15 100644 --- a/nodedb/src/data/executor/handlers/rls_write_gate.rs +++ b/nodedb/src/data/executor/handlers/rls_write_gate.rs @@ -37,7 +37,8 @@ use super::columnar_write::row_values_to_object; use super::returning_doc; use super::rls_eval; -/// Decide one already-decoded row image against the compiled write policy. +/// Decide one already-decoded JSON row image against the compiled write +/// policy. /// /// Fails closed: an undecodable filter payload or an evaluation error denies, /// so an adversarial predicate cannot be turned into an admitted write. @@ -46,11 +47,42 @@ pub(in crate::data::executor) fn admit_row( image: &serde_json::Value, tid: u64, collection: &str, +) -> crate::Result<()> { + decide(rls_write_check, tid, collection, |bytes| { + rls_eval::rls_check_document(bytes, image) + }) +} + +/// Decide one already-decoded typed DOCUMENT row image against the compiled +/// write policy. +/// +/// The document read side tests the stored msgpack body, so the image is +/// encoded to msgpack and tested by the same evaluator — one compiled +/// predicate means the same thing on both sides. Columnar rows go through +/// [`admit_value_row`], whose evaluator is the columnar read side's. +pub(in crate::data::executor) fn admit_document_value( + rls_write_check: &RlsWriteCheck, + image: &nodedb_types::Value, + tid: u64, + collection: &str, +) -> crate::Result<()> { + decide(rls_write_check, tid, collection, |bytes| { + rls_eval::rls_check_value(bytes, image) + }) +} + +/// Map the gate's decision to a result, running `passes` on the compiled +/// predicate only when there is one to evaluate. +fn decide( + rls_write_check: &RlsWriteCheck, + tid: u64, + collection: &str, + passes: impl FnOnce(&[u8]) -> bool, ) -> crate::Result<()> { match rls_write_check.decision() { WriteGateDecision::AdmitAll => Ok(()), WriteGateDecision::Evaluate(bytes) => { - if rls_eval::rls_check_document(bytes, image) { + if passes(bytes) { return Ok(()); } Err(crate::Error::RejectedAuthz { @@ -100,7 +132,7 @@ pub(in crate::data::executor) fn admit_stored_row( }), WriteGateDecision::Evaluate(_) => { match returning_doc::from_stored(body, identity, strict_schema) { - Ok(image) => admit_row(rls_write_check, &image, tid, collection), + Ok(image) => admit_document_value(rls_write_check, &image, tid, collection), Err(e) => Err(crate::Error::RejectedAuthz { tenant_id: crate::types::TenantId::new(tid), resource: format!( @@ -175,30 +207,15 @@ pub(in crate::data::executor) fn admit_value_row( tid: u64, collection: &str, ) -> crate::Result<()> { - let bytes = match rls_write_check.decision() { - WriteGateDecision::AdmitAll => return Ok(()), - WriteGateDecision::DenyNotInjected => { - return Err(crate::Error::Internal { - detail: format!( - "write plan for '{collection}' reached the Data Plane RLS write gate before \ - RLS injection ran; this is an internal invariant break, not a policy \ - rejection" - ), - }); - } - WriteGateDecision::Evaluate(bytes) => bytes, - }; - let admitted = match zerompk::from_msgpack::>(bytes) { - Ok(filters) => value_matches_filters(image, &filters).unwrap_or(false), - Err(_) => false, - }; - if admitted { - return Ok(()); - } - Err(crate::Error::RejectedAuthz { - tenant_id: crate::types::TenantId::new(tid), - resource: format!("RLS write policy on '{collection}' rejected the row"), - }) + decide( + rls_write_check, + tid, + collection, + |bytes| match zerompk::from_msgpack::>(bytes) { + Ok(filters) => value_matches_filters(image, &filters).unwrap_or(false), + Err(_) => false, + }, + ) } /// Decide one schema-ordered columnar row — the values about to be written, or @@ -226,7 +243,7 @@ mod tests { fn owner_policy(value: &str) -> Vec { let filter = ScanFilter { field: "owner".into(), - op: "eq".into(), + op: crate::bridge::scan_filter::FilterOp::Eq, value: nodedb_types::Value::String(value.into()), clauses: Vec::new(), expr: None, diff --git a/nodedb/src/data/executor/handlers/spatial/full_scan.rs b/nodedb/src/data/executor/handlers/spatial/full_scan.rs new file mode 100644 index 000000000..5da855a10 --- /dev/null +++ b/nodedb/src/data/executor/handlers/spatial/full_scan.rs @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: BUSL-1.1 + +use tracing::debug; + +use crate::bridge::envelope::{ErrorCode, Response}; +use crate::bridge::scan_filter::ScanFilter; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::doc_format; +use crate::data::executor::handlers::spatial_refine::{ + apply_predicate, extract_geometry, project_doc, +}; +use crate::data::executor::handlers::transaction::overlay::SpatialOverlayMergeParams; +use crate::data::executor::response_codec; +use crate::data::executor::task::ExecutionTask; +use nodedb_physical::physical_plan::SpatialPredicate; +use nodedb_types::SurrogateBitmap; + +use super::prefilter::prefilter_admits; + +/// Parameters for [`CoreLoop::spatial_full_scan`]. +pub(super) struct SpatialFullScanParams<'a> { + pub task: &'a ExecutionTask, + pub tid: u64, + pub collection: &'a str, + pub field: &'a str, + pub predicate: &'a SpatialPredicate, + pub query_geom: &'a nodedb_types::geometry::Geometry, + pub distance_meters: f64, + pub limit: usize, + pub projection: &'a [String], + pub attr_filters: &'a [ScanFilter], + pub rls_filters: &'a [ScanFilter], + pub prefilter: Option<&'a SurrogateBitmap>, +} + +impl CoreLoop { + /// Full scan when no R-tree exists for the field. + pub(super) fn spatial_full_scan(&self, params: SpatialFullScanParams<'_>) -> Response { + let SpatialFullScanParams { + task, + tid, + collection, + field, + predicate, + query_geom, + distance_meters, + limit, + projection, + attr_filters, + rls_filters, + prefilter, + } = params; + debug!(core = self.core_id, %collection, "spatial full scan (no R-tree index yet)"); + + let scan_limit = limit * 10; + let entries = match self.scan_collection( + task.request.database_id.as_u64(), + tid, + collection, + scan_limit, + ) { + Ok(e) => e, + Err(e) => { + return self.response_error( + task, + ErrorCode::Internal { + detail: e.to_string(), + }, + ); + } + }; + + let mut results = Vec::new(); + for (doc_id, doc_bytes) in &entries { + if results.len() >= limit { + break; + } + + // Prefilter: skip non-members before geometry evaluation. + if let Some(bitmap) = prefilter + && !prefilter_admits(bitmap, doc_id) + { + continue; + } + + // A row skipped here silently drops out of the spatial result set, + // which reads as "no row matched the geometry" rather than "a row + // could not be read". + let doc = match doc_format::decode_document_value(doc_bytes) { + Ok(d) => d, + Err(e) => return self.response_error(task, e), + }; + + let doc_geom = match extract_geometry(&doc, field) { + Some(g) => g, + None => continue, + }; + + if !apply_predicate(predicate, query_geom, &doc_geom, distance_meters) { + continue; + } + + match ScanFilter::all_match_value(attr_filters, &doc) { + Ok(true) => {} + Ok(false) => continue, + Err(_e) => { + return self.response_error(task, ErrorCode::DivisionByZero); + } + } + match ScanFilter::all_match_value(rls_filters, &doc) { + Ok(true) => {} + Ok(false) => continue, + Err(_e) => { + return self.response_error(task, ErrorCode::DivisionByZero); + } + } + + results.push(project_doc(&doc, doc_id, projection)); + } + + if let Some(txn_id) = task.request.txn_id { + let coll_key = ( + task.request.database_id, + crate::types::TenantId::new(tid), + collection.to_string(), + ); + if let Err(e) = self.merge_overlay_into_spatial_scan( + SpatialOverlayMergeParams { + txn_id, + coll_key: &coll_key, + field, + predicate, + query_geom, + distance_meters, + projection, + attr_filters, + row_level_filters: rls_filters, + }, + &mut results, + ) { + return self.response_error(task, e); + } + } + + match response_codec::encode_value_vec(&results) { + Ok(payload) => self.response_with_payload(task, payload), + Err(e) => self.response_error( + task, + ErrorCode::Internal { + detail: e.to_string(), + }, + ), + } + } +} diff --git a/nodedb/src/data/executor/handlers/spatial/mod.rs b/nodedb/src/data/executor/handlers/spatial/mod.rs new file mode 100644 index 000000000..ce5c3d708 --- /dev/null +++ b/nodedb/src/data/executor/handlers/spatial/mod.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Spatial query handler: R-tree index scan with predicate refinement. +//! +//! Documents with geometry fields are auto-indexed into per-field R-trees +//! on insert (see `handlers/point.rs`). Spatial queries use the R-tree for +//! fast bbox candidate selection, then refine with exact predicates. +//! +//! Internal document representation: `nodedb_types::Value` (no JSON intermediary). + +mod full_scan; +mod prefilter; +mod rtree_scan; + +pub(in crate::data::executor) use rtree_scan::SpatialScanParams; diff --git a/nodedb/src/data/executor/handlers/spatial/prefilter.rs b/nodedb/src/data/executor/handlers/spatial/prefilter.rs new file mode 100644 index 000000000..ebeb85f16 --- /dev/null +++ b/nodedb/src/data/executor/handlers/spatial/prefilter.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: BUSL-1.1 + +use nodedb_types::SurrogateBitmap; + +/// Whether `doc_id`'s surrogate is a member of `prefilter`. +/// +/// `doc_id` is the hex-encoded surrogate for a document-collection row, or +/// a columnar-family user `id` that never parses as a storage key — the +/// latter is never admitted, matching a sparse miss on a parsed key. +pub(super) fn prefilter_admits(prefilter: &SurrogateBitmap, doc_id: &str) -> bool { + match nodedb_types::StorageKey::parse(doc_id) { + Some(key) => prefilter.contains(key.surrogate()), + None => false, + } +} + +#[cfg(test)] +mod tests { + use super::prefilter_admits; + use nodedb_types::{Surrogate, SurrogateBitmap}; + + fn doc_id(surrogate: u32) -> String { + nodedb_types::StorageKey::for_surrogate(Surrogate::new(surrogate)).to_string() + } + + #[test] + fn prefilter_skips_non_member_doc_ids() { + // Direct unit on the production prefilter check (`prefilter_admits`), + // not a re-implementation of it. + let mut bitmap = SurrogateBitmap::new(); + bitmap.insert(Surrogate(2)); + + let candidate_doc_ids = [doc_id(1), doc_id(2), doc_id(3)]; + + let kept: Vec<_> = candidate_doc_ids + .iter() + .filter(|doc_id| prefilter_admits(&bitmap, doc_id)) + .cloned() + .collect(); + + assert_eq!(kept.len(), 1); + assert_eq!(kept[0], doc_id(2)); + } +} diff --git a/nodedb/src/data/executor/handlers/spatial.rs b/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs similarity index 74% rename from nodedb/src/data/executor/handlers/spatial.rs rename to nodedb/src/data/executor/handlers/spatial/rtree_scan.rs index cd90daece..1c2b09e53 100644 --- a/nodedb/src/data/executor/handlers/spatial.rs +++ b/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs @@ -1,36 +1,23 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Spatial query handler: R-tree index scan with predicate refinement. -//! -//! Documents with geometry fields are auto-indexed into per-field R-trees -//! on insert (see `handlers/point.rs`). Spatial queries use the R-tree for -//! fast bbox candidate selection, then refine with exact predicates. -//! -//! Internal document representation: `nodedb_types::Value` (no JSON intermediary). - use tracing::debug; -use super::super::response_codec; use crate::bridge::envelope::{ErrorCode, Response}; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::doc_format; +use crate::data::executor::handlers::columnar_read::filter::decode_rls_filters; +use crate::data::executor::handlers::spatial_refine::{ + apply_predicate, expand_bbox, extract_geometry, project_doc, +}; +use crate::data::executor::handlers::transaction::overlay::SpatialOverlayMergeParams; +use crate::data::executor::response_codec; use crate::data::executor::task::ExecutionTask; use nodedb_physical::physical_plan::SpatialPredicate; use nodedb_types::SurrogateBitmap; -use super::spatial_refine::{apply_predicate, expand_bbox, extract_geometry, project_doc}; - -/// Whether `doc_id`'s surrogate is a member of `prefilter`. -/// -/// `doc_id` is the hex-encoded surrogate for a document-collection row, or -/// a columnar-family user `id` that never parses as a storage key — the -/// latter is never admitted, matching a sparse miss on a parsed key. -fn prefilter_admits(prefilter: &SurrogateBitmap, doc_id: &str) -> bool { - match nodedb_types::StorageKey::parse(doc_id) { - Some(key) => prefilter.contains(key.surrogate()), - None => false, - } -} +use super::full_scan::SpatialFullScanParams; +use super::prefilter::prefilter_admits; /// Parameters for [`CoreLoop::execute_spatial_scan`]. pub(in crate::data::executor) struct SpatialScanParams<'a> { @@ -48,22 +35,6 @@ pub(in crate::data::executor) struct SpatialScanParams<'a> { pub prefilter: Option<&'a SurrogateBitmap>, } -/// Parameters for [`CoreLoop::spatial_full_scan`]. -struct SpatialFullScanParams<'a> { - task: &'a ExecutionTask, - tid: u64, - collection: &'a str, - field: &'a str, - predicate: &'a SpatialPredicate, - query_geom: &'a nodedb_types::geometry::Geometry, - distance_meters: f64, - limit: usize, - projection: &'a [String], - attr_filters: &'a [ScanFilter], - rls_filters: &'a [ScanFilter], - prefilter: Option<&'a SurrogateBitmap>, -} - impl CoreLoop { /// Execute a spatial scan using the R-tree index. /// @@ -106,16 +77,27 @@ impl CoreLoop { // The query geometry was parsed and validated on the Control Plane. let query_geom = query_geometry; - // 2. Deserialize attribute and RLS filters. + // 2. Deserialize attribute and RLS filters. A payload that does not + // decode fails the scan: an unreadable predicate never admits the + // rows it would have excluded. let attr_filters: Vec = if attribute_filters.is_empty() { Vec::new() } else { - zerompk::from_msgpack(attribute_filters).unwrap_or_default() + match zerompk::from_msgpack(attribute_filters) { + Ok(f) => f, + Err(e) => { + return self.response_error( + task, + ErrorCode::Internal { + detail: format!("malformed spatial attribute filters: {e}"), + }, + ); + } + } }; - let row_level_filters: Vec = if rls_filters.is_empty() { - Vec::new() - } else { - zerompk::from_msgpack(rls_filters).unwrap_or_default() + let row_level_filters: Vec = match decode_rls_filters(rls_filters) { + Ok(f) => f, + Err(e) => return self.response_error(task, e), }; // 3. Compute search bbox (expand by distance for ST_DWithin). @@ -260,7 +242,7 @@ impl CoreLoop { // A candidate skipped here silently drops out of the // spatial result set, which reads as "no row matched the // geometry" rather than "a row could not be read". - match super::super::doc_format::decode_document_value(&doc_mp) { + match doc_format::decode_document_value(&doc_mp) { Ok(d) => d, Err(e) => return self.response_error(task, e), } @@ -288,7 +270,7 @@ impl CoreLoop { let Some(doc_mp) = columnar_docs.as_ref().and_then(|m| m.get(&doc_id)) else { continue; }; - match super::super::doc_format::decode_document_value(doc_mp) { + match doc_format::decode_document_value(doc_mp) { Ok(d) => d, Err(e) => return self.response_error(task, e), } @@ -333,7 +315,7 @@ impl CoreLoop { if let Some(txn_id) = task.request.txn_id && let Err(e) = self.merge_overlay_into_spatial_scan( - super::transaction::overlay::SpatialOverlayMergeParams { + SpatialOverlayMergeParams { txn_id, coll_key: &coll_key, field, @@ -360,130 +342,11 @@ impl CoreLoop { ), } } - - /// Full scan when no R-tree exists for the field. - fn spatial_full_scan(&self, params: SpatialFullScanParams<'_>) -> Response { - let SpatialFullScanParams { - task, - tid, - collection, - field, - predicate, - query_geom, - distance_meters, - limit, - projection, - attr_filters, - rls_filters, - prefilter, - } = params; - debug!(core = self.core_id, %collection, "spatial full scan (no R-tree index yet)"); - - let scan_limit = limit * 10; - let entries = match self.scan_collection( - task.request.database_id.as_u64(), - tid, - collection, - scan_limit, - ) { - Ok(e) => e, - Err(e) => { - return self.response_error( - task, - ErrorCode::Internal { - detail: e.to_string(), - }, - ); - } - }; - - let mut results = Vec::new(); - for (doc_id, doc_bytes) in &entries { - if results.len() >= limit { - break; - } - - // Prefilter: skip non-members before geometry evaluation. - if let Some(bitmap) = prefilter - && !prefilter_admits(bitmap, doc_id) - { - continue; - } - - // A row skipped here silently drops out of the spatial result set, - // which reads as "no row matched the geometry" rather than "a row - // could not be read". - let doc = match super::super::doc_format::decode_document_value(doc_bytes) { - Ok(d) => d, - Err(e) => return self.response_error(task, e), - }; - - let doc_geom = match extract_geometry(&doc, field) { - Some(g) => g, - None => continue, - }; - - if !apply_predicate(predicate, query_geom, &doc_geom, distance_meters) { - continue; - } - - match ScanFilter::all_match_value(attr_filters, &doc) { - Ok(true) => {} - Ok(false) => continue, - Err(_e) => { - return self.response_error(task, ErrorCode::DivisionByZero); - } - } - match ScanFilter::all_match_value(rls_filters, &doc) { - Ok(true) => {} - Ok(false) => continue, - Err(_e) => { - return self.response_error(task, ErrorCode::DivisionByZero); - } - } - - results.push(project_doc(&doc, doc_id, projection)); - } - - if let Some(txn_id) = task.request.txn_id { - let coll_key = ( - task.request.database_id, - crate::types::TenantId::new(tid), - collection.to_string(), - ); - if let Err(e) = self.merge_overlay_into_spatial_scan( - super::transaction::overlay::SpatialOverlayMergeParams { - txn_id, - coll_key: &coll_key, - field, - predicate, - query_geom, - distance_meters, - projection, - attr_filters, - row_level_filters: rls_filters, - }, - &mut results, - ) { - return self.response_error(task, e); - } - } - - match response_codec::encode_value_vec(&results) { - Ok(payload) => self.response_with_payload(task, payload), - Err(e) => self.response_error( - task, - ErrorCode::Internal { - detail: e.to_string(), - }, - ), - } - } } #[cfg(test)] mod tests { - use super::{SpatialScanParams, prefilter_admits}; + use super::SpatialScanParams; use crate::bridge::envelope::{PhysicalPlan, Priority, Request, Status}; use crate::data::executor::task::ExecutionTask; use crate::engine::spatial::RTreeEntry; @@ -622,29 +485,6 @@ mod tests { }) } - fn doc_id(surrogate: u32) -> String { - nodedb_types::StorageKey::for_surrogate(Surrogate::new(surrogate)).to_string() - } - - #[test] - fn prefilter_skips_non_member_doc_ids() { - // Direct unit on the production prefilter check (`prefilter_admits`), - // not a re-implementation of it. - let mut bitmap = SurrogateBitmap::new(); - bitmap.insert(Surrogate(2)); - - let candidate_doc_ids = [doc_id(1), doc_id(2), doc_id(3)]; - - let kept: Vec<_> = candidate_doc_ids - .iter() - .filter(|doc_id| prefilter_admits(&bitmap, doc_id)) - .cloned() - .collect(); - - assert_eq!(kept.len(), 1); - assert_eq!(kept[0], doc_id(2)); - } - // Note: the R-tree-branch by-surrogate candidate hydration // (execute_spatial_scan → sparse.get → sparse_row_to_doc) is covered // end-to-end by `tests/engine_surface_spatial.rs:: diff --git a/nodedb/src/data/executor/handlers/timeseries/aggregate.rs b/nodedb/src/data/executor/handlers/timeseries/aggregate.rs index 7ee65f367..023bb735c 100644 --- a/nodedb/src/data/executor/handlers/timeseries/aggregate.rs +++ b/nodedb/src/data/executor/handlers/timeseries/aggregate.rs @@ -6,10 +6,16 @@ //! for low-cardinality symbol GROUP BY, parallel partition processing via //! std::thread::scope, sparse index block-level skip, and single metadata //! read per partition. +//! +//! The caller's read policy travels as part of `filter_predicates`: the +//! grouped scan lowers every predicate onto typed column vectors before any +//! row is aggregated, and a predicate it cannot lower fails the statement. +//! An unlowerable policy therefore never aggregates the rows it governs. use crate::bridge::envelope::Response; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; +use crate::engine::timeseries::grouped_filter::UnsupportedPredicate; use crate::engine::timeseries::grouped_scan::{ GroupedAggResult, PartitionAggParams, aggregate_memtable, aggregate_partition, }; @@ -23,6 +29,9 @@ pub(in crate::data::executor) struct TsAggregateParams<'a> { pub collection: &'a str, pub time_range: (i64, i64), pub limit: usize, + /// The query's WHERE predicates followed by the caller's read policy. + /// Every predicate is pushed into the grouped scan, so the policy + /// excludes rows before they reach an accumulator. pub filter_predicates: &'a [crate::bridge::scan_filter::ScanFilter], pub bucket_interval_ms: i64, pub group_by: &'a [String], @@ -62,15 +71,17 @@ impl CoreLoop { let mut merged = if let Some(mt) = self.columnar_memtables.get(&key) && !mt.is_empty() { - aggregate_memtable( + match aggregate_memtable( mt, group_by, aggregates, filter_predicates, time_range, bucket_interval_ms, - ) - .unwrap_or_else(|| GroupedAggResult::new(num_aggs)) + ) { + Ok(result) => result.unwrap_or_else(|| GroupedAggResult::new(num_aggs)), + Err(e) => return self.response_error(task, crate::Error::from(e)), + } } else { GroupedAggResult::new(num_aggs) }; @@ -98,7 +109,7 @@ impl CoreLoop { let io_priority = Some(task.request.priority); let io_metrics: &crate::data::io::IoMetrics = &self.io_metrics; for dir in &partition_dirs { - if let Some(part_result) = aggregate_partition(PartitionAggParams { + match aggregate_partition(PartitionAggParams { partition_dir: dir, group_by, aggregates, @@ -110,7 +121,9 @@ impl CoreLoop { io_priority, io_metrics: Some(io_metrics), }) { - merged.merge(&part_result); + Ok(Some(part_result)) => merged.merge(&part_result), + Ok(None) => {} + Err(e) => return self.response_error(task, crate::Error::from(e)), } } } else { @@ -127,41 +140,48 @@ impl CoreLoop { let thread_count = available.min(partition_dirs.len()).min(8); let chunk_size = partition_dirs.len().div_ceil(thread_count); - let partition_results: Vec = std::thread::scope(|s| { - let handles: Vec<_> = partition_dirs - .chunks(chunk_size) - .map(|chunk| { - let gb = &group_by_owned; - let ag = &agg_owned; - let fl = &filters_owned; - let nc = &needed_owned; - s.spawn(move || { - let mut local = GroupedAggResult::new(ag.len()); - for dir in chunk { - // Parallel threads: no io_uring (fadvise fallback). - if let Some(r) = aggregate_partition(PartitionAggParams { - partition_dir: dir, - group_by: gb, - aggregates: ag, - filters: fl, - time_range, - needed_columns: nc, - bucket_interval_ms, - uring_reader: None, - io_priority: None, - io_metrics: None, - }) { - local.merge(&r); + let partition_results: Result, UnsupportedPredicate> = + std::thread::scope(|s| { + let handles: Vec<_> = partition_dirs + .chunks(chunk_size) + .map(|chunk| { + let gb = &group_by_owned; + let ag = &agg_owned; + let fl = &filters_owned; + let nc = &needed_owned; + s.spawn(move || -> Result<_, UnsupportedPredicate> { + let mut local = GroupedAggResult::new(ag.len()); + for dir in chunk { + // Parallel threads: no io_uring (fadvise fallback). + if let Some(r) = + aggregate_partition(PartitionAggParams { + partition_dir: dir, + group_by: gb, + aggregates: ag, + filters: fl, + time_range, + needed_columns: nc, + bucket_interval_ms, + uring_reader: None, + io_priority: None, + io_metrics: None, + })? + { + local.merge(&r); + } } - } - local + Ok(local) + }) }) - }) - .collect(); + .collect(); - handles.into_iter().filter_map(|h| h.join().ok()).collect() - }); + handles.into_iter().filter_map(|h| h.join().ok()).collect() + }); + let partition_results = match partition_results { + Ok(results) => results, + Err(e) => return self.response_error(task, crate::Error::from(e)), + }; for r in &partition_results { merged.merge(r); } @@ -188,6 +208,7 @@ impl CoreLoop { // encoder is told each key column's declared shape. let group_key_kinds = self.ts_group_key_kinds(task.request.database_id, tid, collection, group_by); + let bucket_kind = self.ts_time_key_kind(task.request.database_id, tid, collection); let payload = match super::encode::encode_grouped_results( &merged, group_by, @@ -195,7 +216,10 @@ impl CoreLoop { limit, bucket_interval_ms, sort_keys, - &group_key_kinds, + super::encode::GroupedKeyTypes { + group_key_kinds: &group_key_kinds, + bucket_kind, + }, ) { Ok(p) => p, Err(e) => return self.response_error(task, e), diff --git a/nodedb/src/data/executor/handlers/timeseries/encode.rs b/nodedb/src/data/executor/handlers/timeseries/encode.rs index 922b4dfdc..5d0d9a269 100644 --- a/nodedb/src/data/executor/handlers/timeseries/encode.rs +++ b/nodedb/src/data/executor/handlers/timeseries/encode.rs @@ -5,13 +5,24 @@ use nodedb_query::agg_key::canonical_agg_key; use crate::data::executor::core_loop::TsGroupKeyKind; +use crate::data::executor::handlers::columnar_read::rmpv_time_cell; +use crate::engine::timeseries::columnar_memtable::TimeKind; + +/// The wire types of a grouped result's key columns. +pub(in crate::data::executor) struct GroupedKeyTypes<'a> { + /// One kind per GROUP BY column, in `group_by` order. + pub group_key_kinds: &'a [TsGroupKeyKind], + /// The kind of the collection's time key, which the `bucket` column + /// derives from. + pub bucket_kind: TimeKind, +} /// Render one GROUP BY key part with the type its column carries ungrouped. /// /// The grouped scan reduces every key to a string, so the column's own type /// is put back here. An empty part is SQL NULL. A declared instant is stored -/// in milliseconds and read in microseconds, exactly as row emission reads -/// it, so the two routes to one stored instant render it identically. +/// in milliseconds and rendered as a typed instant, exactly as row emission +/// renders it, so the two routes to one stored instant render it identically. /// /// A part that does not parse as its column's type falls back to the text it /// holds: the key is data the scan produced, and dropping the group would @@ -21,15 +32,8 @@ fn group_key_value(part: Option<&&str>, kind: TsGroupKeyKind) -> crate::Result match text.parse::() { - Ok(millis) => { - let micros = nodedb_types::NdbDateTime::from_millis(millis) - .map_err(|e| crate::Error::Internal { - detail: format!("grouped timeseries key at {millis} ms: {e}"), - })? - .micros; - rmpv::Value::Integer(micros.into()) - } + TsGroupKeyKind::Instant(k) => match text.parse::() { + Ok(millis) => rmpv_time_cell(TimeKind::Instant(k), millis)?, Err(_) => rmpv::Value::String((*text).into()), }, TsGroupKeyKind::Integer => match text.parse::() { @@ -54,6 +58,9 @@ fn group_key_value(part: Option<&&str>, kind: TsGroupKeyKind) -> crate::Result, ) -> crate::Result> { + let GroupedKeyTypes { + group_key_kinds, + bucket_kind, + } = key_types; let has_bucket = bucket_interval_ms > 0; // An ordered query has to see every group before cutting to `limit`: // groups arrive in hash-map order, so the first `limit` of them are an @@ -100,7 +111,7 @@ pub(in crate::data::executor) fn encode_grouped_results( .unwrap_or(0); fields.push(( rmpv::Value::String("bucket".into()), - rmpv::Value::Integer(bucket_ts.into()), + rmpv_time_cell(bucket_kind, bucket_ts)?, )); for (i, field) in group_by.iter().enumerate() { @@ -152,3 +163,84 @@ pub(in crate::data::executor) fn encode_grouped_results( rmpv::encode::write_value(&mut buf, &array).unwrap_or(()); Ok(buf) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::engine::timeseries::grouped_scan::GroupedAggResult; + use nodedb_types::InstantKind; + + const BUCKET_MS: i64 = 1_583_402_400_000; + + fn one_bucket() -> GroupedAggResult { + let mut result = GroupedAggResult::new(1); + result + .groups + .insert(BUCKET_MS.to_string(), vec![Default::default()]); + result + } + + fn bucket_cell(bucket_kind: TimeKind) -> rmpv::Value { + let bytes = encode_grouped_results( + &one_bucket(), + &[], + &[("count".to_string(), "v".to_string())], + usize::MAX, + 60_000, + &[], + GroupedKeyTypes { + group_key_kinds: &[], + bucket_kind, + }, + ) + .expect("encode"); + let rmpv::Value::Array(rows) = + crate::util::bounded_msgpack::read_value(&bytes).expect("decode") + else { + panic!("not an array"); + }; + let rmpv::Value::Map(fields) = &rows[0] else { + panic!("not a map"); + }; + fields + .iter() + .find(|(k, _)| k.as_str() == Some("bucket")) + .map(|(_, v)| v.clone()) + .expect("bucket column") + } + + #[test] + fn bucket_over_an_instant_time_key_is_a_typed_instant() { + assert_eq!( + bucket_cell(TimeKind::Instant(InstantKind::Naive)), + rmpv::Value::Ext( + InstantKind::Naive.ext_type(), + (BUCKET_MS * 1000).to_be_bytes().to_vec() + ) + ); + } + + #[test] + fn bucket_over_a_millis_time_key_is_the_integer_stored() { + assert_eq!( + bucket_cell(TimeKind::Millis), + rmpv::Value::Integer(BUCKET_MS.into()) + ); + } + + #[test] + fn an_instant_group_key_is_a_typed_instant() { + let cell = group_key_value( + Some(&BUCKET_MS.to_string().as_str()), + TsGroupKeyKind::Instant(InstantKind::Utc), + ) + .expect("group key"); + assert_eq!( + cell, + rmpv::Value::Ext( + InstantKind::Utc.ext_type(), + (BUCKET_MS * 1000).to_be_bytes().to_vec() + ) + ); + } +} diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest.rs b/nodedb/src/data/executor/handlers/timeseries/ingest.rs index 1135c32a6..39f825d98 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest.rs @@ -15,9 +15,7 @@ use crate::bridge::envelope::{ErrorCode, Payload, Response, Status}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::response_codec; -use crate::engine::timeseries::columnar_memtable::{ - ColumnType, ColumnarMemtable, ColumnarMemtableConfig, -}; +use crate::engine::timeseries::columnar_memtable::{ColumnarMemtable, ColumnarMemtableConfig}; use crate::engine::timeseries::ilp; use crate::engine::timeseries::ilp_ingest; @@ -327,22 +325,19 @@ impl CoreLoop { // missing float field is stored as NaN and both paths render it as SQL // NULL, which a hand-written projection over the ingest values would // have printed as "NaN". - let mut returned_rows: Vec = match returning { + let returned_rows: Vec = match returning { Some(_) => match self.columnar_memtables.get(&key) { Some(mt) => { - super::raw_scan::emit_memtable_rows_at(mt, &outcome.accepted_row_indices) + match super::raw_scan::emit_memtable_rows_at(mt, &outcome.accepted_row_indices) + { + Ok(rows) => rows, + Err(e) => return self.response_error(task, e), + } } None => Vec::new(), }, None => Vec::new(), }; - // Same scan-unit rule `SELECT` applies: a declared `TIMESTAMP` cell - // leaves the engine as epoch microseconds, not the milliseconds - // storage holds. - let instant_columns = self.ts_instant_columns(task.request.database_id, tid, collection); - if let Err(e) = super::raw_scan::scale_instant_cells(&mut returned_rows, &instant_columns) { - return self.response_error(task, e); - } if accepted > 0 && let Some(lsn) = wal_lsn @@ -402,15 +397,7 @@ impl CoreLoop { .schema() .columns .iter() - .map(|(name, col_type)| { - let type_str = match col_type { - ColumnType::Timestamp => "TIMESTAMP", - ColumnType::Float64 => "FLOAT", - ColumnType::Int64 => "BIGINT", - ColumnType::Symbol => "VARCHAR", - }; - serde_json::json!([name, type_str]) - }) + .map(|(name, col_type)| serde_json::json!([name, col_type.ddl_type_name()])) .collect(); serde_json::json!({ "accepted": accepted, diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest_formats.rs b/nodedb/src/data/executor/handlers/timeseries/ingest_formats.rs index 410cd6740..813a64957 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest_formats.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest_formats.rs @@ -125,7 +125,18 @@ impl CoreLoop { .declared_ts_time_key(task.request.database_id, tid, collection) .map(str::to_string); - let ilp_buf = normalize::msgpack_rows_to_ilp(&rows, measurement, time_key.as_deref()); + let ilp_buf = match normalize::msgpack_rows_to_ilp(&rows, measurement, time_key.as_deref()) + { + Ok(buf) => buf, + Err(error) => { + return self.response_error( + task, + ErrorCode::RejectedPrevalidation { + reason: format!("timeseries ingest: {error}"), + }, + ); + } + }; if ilp_buf.is_empty() { return self.response_error( diff --git a/nodedb/src/data/executor/handlers/timeseries/msgpack_decode.rs b/nodedb/src/data/executor/handlers/timeseries/msgpack_decode.rs index 2b5437a6e..83976ff9f 100644 --- a/nodedb/src/data/executor/handlers/timeseries/msgpack_decode.rs +++ b/nodedb/src/data/executor/handlers/timeseries/msgpack_decode.rs @@ -3,13 +3,22 @@ //! Lightweight msgpack decoder for timeseries ingest rows. //! //! Decodes a msgpack array of maps into `Vec>`. -//! Only supports the value types produced by `row_to_msgpack` in the planner. +//! Supports the value types the planner's row writer and the native `Value` +//! encoder produce for a timeseries row: scalars, strings, and the `fixext8` +//! instant ext (`nodedb_types::read_instant`). A `bin` value decodes as +//! `Null`. Any other ext, array, or map value is an error. + +use nodedb_types::json_msgpack::INSTANT_EXT_LEN; +use nodedb_types::read_instant; pub(super) enum MsgpackValue { Int(i64), Float(f64), Str(String), Bool(bool), + /// A typed instant, as epoch microseconds. The column's declared kind, + /// not the cell's tag, decides how the engine stores and reads it. + Instant(i64), Null, } @@ -165,7 +174,19 @@ fn read_value(buf: &[u8], pos: &mut usize) -> Result let bytes = read_bytes::<8>(buf, pos)?; Ok(MsgpackValue::Int(u64::from_be_bytes(bytes) as i64)) } - // Skip bin/ext/array/map values we don't use for timeseries fields + // fixext8: an instant of either kind. Any other ext type is + // unsupported, like every other ext marker. + 0xD7 => { + let start = *pos - 1; + if buf.len() < start + INSTANT_EXT_LEN { + return Err("unexpected EOF in fixext8"); + } + let (_, micros) = + read_instant(buf, start).ok_or("unsupported msgpack ext type in timeseries row")?; + *pos = start + INSTANT_EXT_LEN; + Ok(MsgpackValue::Instant(micros)) + } + // Skip bin values; ext/array/map values are unsupported. _ => { skip_msgpack_value(b, buf, pos)?; Ok(MsgpackValue::Null) @@ -233,6 +254,59 @@ fn read_be_f64(buf: &[u8], pos: &mut usize) -> Result { #[cfg(test)] mod tests { use super::*; + use nodedb_types::InstantKind; + + /// `2020-03-05T10:00:00Z` in epoch microseconds. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + /// A row map `{"captured_at": }` for the given kind. + fn row_with_instant(kind: InstantKind) -> Vec { + let mut payload = vec![0x91, 0x81]; + payload.extend_from_slice(&[0xAB]); + payload.extend_from_slice(b"captured_at"); + nodedb_types::write_instant(&mut payload, kind, EARLY_MICROS); + payload + } + + /// A `fixext8` instant of either kind decodes as the typed instant. + #[test] + fn a_fixext8_instant_decodes_as_a_typed_instant() { + for kind in [InstantKind::Naive, InstantKind::Utc] { + let rows = decode_msgpack_rows(&row_with_instant(kind)).expect("decode"); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].len(), 1); + assert_eq!(rows[0][0].0, "captured_at"); + assert!( + matches!( + rows[0][0].1, + MsgpackValue::Instant(micros) if micros == EARLY_MICROS + ), + "instant of kind {kind:?} must decode typed" + ); + } + } + + /// A `fixext8` of a non-instant type is unsupported, like every other ext. + #[test] + fn a_fixext8_of_another_type_is_rejected() { + let mut payload = vec![0x91, 0x81, 0xA1, b'k']; + payload.extend_from_slice(&[0xD7, 0x09, 0, 0, 0, 0, 0, 0, 0, 1]); + assert!(matches!( + decode_msgpack_rows(&payload), + Err("unsupported msgpack ext type in timeseries row") + )); + } + + /// A truncated instant is an EOF error, never a partial value. + #[test] + fn a_truncated_instant_is_rejected() { + let mut payload = row_with_instant(InstantKind::Utc); + payload.truncate(payload.len() - 1); + assert!(matches!( + decode_msgpack_rows(&payload), + Err("unexpected EOF in fixext8") + )); + } #[test] fn huge_array_count_with_tiny_payload_is_rejected_without_reservation() { diff --git a/nodedb/src/data/executor/handlers/timeseries/normalize.rs b/nodedb/src/data/executor/handlers/timeseries/normalize.rs index fb1e6ee7f..a16ce4d6c 100644 --- a/nodedb/src/data/executor/handlers/timeseries/normalize.rs +++ b/nodedb/src/data/executor/handlers/timeseries/normalize.rs @@ -6,6 +6,7 @@ //! Anything reasoning about the persisted row (RLS gate, resolve pass) must //! go through here, not the submitted values. +use nodedb_types::datetime::NdbDateTime; use sonic_rs::{JsonContainerTrait, JsonValueTrait}; use super::msgpack_decode::MsgpackValue; @@ -15,6 +16,16 @@ use crate::engine::timeseries::ilp::{self, IlpError}; /// stored time column is milliseconds. const NANOS_PER_MILLI: i64 = 1_000_000; +/// Nanoseconds per microsecond — a typed instant carries epoch microseconds. +const NANOS_PER_MICRO: i64 = 1_000; + +/// The ISO 8601 text of a typed instant, for a non-time column that stores +/// it: the ILP line carries it as a string field, the same way a client +/// that spells the instant as text sends it. +fn instant_to_iso8601(micros: i64) -> String { + NdbDateTime::from_micros(micros).to_iso8601() +} + /// Is `column` the time column of the row being ingested? Matches only the /// declared `TIME_KEY` when DDL exists; falls back to conventional names /// (`ts`/`timestamp`/`time`) only for a measurement with no DDL behind it. @@ -68,30 +79,25 @@ fn push_line(buf: &mut String, measurement: &str, fields: &[String], timestamp_n } /// Normalize decoded MessagePack rows into line protocol. +/// +/// A row whose time column is absent or NULL carries no timestamp, and the +/// ingest clock stamps it. A time column that holds a value the line cannot +/// carry — text that is not a datetime, a boolean, or an instant past the +/// nanosecond range — is an error: stamping such a row with the clock would +/// store a point at a time the client never wrote. pub(in crate::data::executor) fn msgpack_rows_to_ilp( rows: &[Vec<(String, MsgpackValue)>], measurement: &str, time_key: Option<&str>, -) -> String { +) -> Result { let mut ilp_buf = String::new(); - for row in rows { + for (line_number, row) in rows.iter().enumerate() { let mut fields = Vec::new(); let mut timestamp_ns: Option = None; for (key, val) in row { if is_time_column(key, time_key) { - match val { - MsgpackValue::Str(s) => { - timestamp_ns = parse_ts_string_to_nanos(s); - } - MsgpackValue::Int(n) => { - timestamp_ns = Some(*n * NANOS_PER_MILLI); - } - MsgpackValue::Float(f) => { - timestamp_ns = Some(*f as i64 * NANOS_PER_MILLI); - } - _ => {} - } + timestamp_ns = time_column_nanos(val, key, line_number + 1)?; continue; } @@ -111,14 +117,58 @@ pub(in crate::data::executor) fn msgpack_rows_to_ilp( fields.push(format!("{key}=\"{}\"", s.replace('\"', "\\\""))); } } + // An instant in a non-time column is carried as ISO 8601 text, + // the form a text-spelled instant field takes above. + MsgpackValue::Instant(micros) => { + fields.push(format!("{key}=\"{}\"", instant_to_iso8601(*micros))); + } MsgpackValue::Bool(b) => fields.push(format!("{key}={b}")), - _ => {} + MsgpackValue::Null => {} } } push_line(&mut ilp_buf, measurement, &fields, timestamp_ns); } - ilp_buf + Ok(ilp_buf) +} + +/// The nanosecond timestamp a time-column cell denotes: `None` when the cell +/// is NULL (the clock stamps the row), an error when the cell holds a value +/// no timestamp can carry. +fn time_column_nanos( + val: &MsgpackValue, + column: &str, + line_number: usize, +) -> Result, IlpError> { + let invalid = |detail: &str| { + IlpError::new( + line_number, + &format!("{column}={detail}"), + 0..0, + ilp::IlpErrorKind::InvalidTimestamp, + ) + }; + match val { + MsgpackValue::Null => Ok(None), + MsgpackValue::Str(s) => parse_ts_string_to_nanos(s) + .map(Some) + .ok_or_else(|| invalid(&format!("\"{s}\""))), + MsgpackValue::Int(n) => n + .checked_mul(NANOS_PER_MILLI) + .map(Some) + .ok_or_else(|| invalid(&n.to_string())), + MsgpackValue::Float(f) => (*f as i64) + .checked_mul(NANOS_PER_MILLI) + .map(Some) + .ok_or_else(|| invalid(&f.to_string())), + // A typed instant of either kind: the stored time column is the + // instant's millisecond count whatever the kind. + MsgpackValue::Instant(micros) => micros + .checked_mul(NANOS_PER_MICRO) + .map(Some) + .ok_or_else(|| invalid(&instant_to_iso8601(*micros))), + MsgpackValue::Bool(b) => Err(invalid(&b.to_string())), + } } /// Normalize decoded JSON rows into line protocol. The JSON value model @@ -195,6 +245,84 @@ pub(in crate::data::executor) fn stamp_timestamps( mod tests { use super::*; + /// `2020-03-05T10:00:00Z` in epoch microseconds. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + fn instant() -> MsgpackValue { + MsgpackValue::Instant(EARLY_MICROS) + } + + /// A typed instant under the declared time key is the line's timestamp — + /// the same nanosecond count its text spelling gives. + #[test] + fn a_typed_instant_time_key_is_the_line_timestamp() { + let rows = vec![vec![ + ("captured_at".to_string(), instant()), + ("v".to_string(), MsgpackValue::Float(1.5)), + ]]; + let ilp = msgpack_rows_to_ilp(&rows, "m", Some("captured_at")).expect("ilp"); + assert_eq!(ilp, "m v=1.5 1583402400000000000\n"); + let text = vec![vec![ + ( + "captured_at".to_string(), + MsgpackValue::Str("2020-03-05 10:00:00".into()), + ), + ("v".to_string(), MsgpackValue::Float(1.5)), + ]]; + assert_eq!( + msgpack_rows_to_ilp(&text, "m", Some("captured_at")).expect("ilp"), + "m v=1.5 1583402400000000000\n" + ); + } + + /// A typed instant in a non-time column is a string field carrying its + /// ISO 8601 text. + #[test] + fn a_typed_instant_field_is_iso8601_text() { + let rows = vec![vec![ + ("ts".to_string(), MsgpackValue::Int(7)), + ("seen".to_string(), instant()), + ]]; + assert_eq!( + msgpack_rows_to_ilp(&rows, "m", Some("ts")).expect("ilp"), + "m seen=\"2020-03-05T10:00:00.000000Z\" 7000000\n" + ); + } + + /// A time-column value the nanosecond timestamp cannot carry is an + /// error naming the column: the row is neither wrapped nor stamped with + /// the clock. + #[test] + fn a_time_key_the_line_cannot_carry_is_an_error() { + for bad in [ + MsgpackValue::Int(i64::MAX), + MsgpackValue::Str("not a date".into()), + MsgpackValue::Bool(true), + ] { + let rows = vec![vec![ + ("ts".to_string(), bad), + ("v".to_string(), MsgpackValue::Int(1)), + ]]; + let err = msgpack_rows_to_ilp(&rows, "m", Some("ts")).expect_err("refused"); + assert_eq!(err.kind, ilp::IlpErrorKind::InvalidTimestamp); + assert!(err.raw.starts_with("ts="), "{err:?}"); + } + } + + /// A NULL time column carries no timestamp, so the ingest clock stamps + /// the row. + #[test] + fn a_null_time_key_carries_no_timestamp() { + let rows = vec![vec![ + ("ts".to_string(), MsgpackValue::Null), + ("v".to_string(), MsgpackValue::Int(1)), + ]]; + assert_eq!( + msgpack_rows_to_ilp(&rows, "m", Some("ts")).expect("ilp"), + "m v=1i\n" + ); + } + /// A line without a timestamp takes the batch default; one that carries its /// own keeps it byte for byte. #[test] diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs index 9b346ff12..081da03b5 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs @@ -8,5 +8,5 @@ pub mod partition_scan; pub mod row_emit; pub mod scan; -pub(in crate::data::executor) use row_emit::{emit_memtable_rows_at, scale_instant_cells}; +pub(in crate::data::executor) use row_emit::emit_memtable_rows_at; pub(in crate::data::executor) use scan::RawScanParams; diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/partition_scan.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/partition_scan.rs index 4f3eeabc1..135c66dfb 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/partition_scan.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/partition_scan.rs @@ -4,25 +4,39 @@ use std::collections::HashMap; +use crate::bridge::scan_filter::ScanFilter; use crate::engine::timeseries::columnar_agg::timestamp_range_filter; use crate::engine::timeseries::columnar_memtable::{ColumnData, ColumnType}; use crate::engine::timeseries::columnar_segment::ColumnarSegmentReader; -use super::row_emit::{emit_partition_row, extract_timestamp}; +use super::row_emit::{emit_partition_row, extract_timestamp, row_admitted}; /// Scan disk partitions in parallel, returning rmpv rows sorted by timestamp. +/// +/// `rls_predicates` is the caller's read policy: every row is checked +/// against it before it counts toward `limit`. `Err` when a stored time +/// cell cannot be read as its column's instant or a predicate expression +/// divides by zero. pub(super) fn scan_partitions_parallel( partition_dirs: &[std::path::PathBuf], time_range: (i64, i64), limit: usize, - filter_predicates: &[crate::bridge::scan_filter::ScanFilter], + filter_predicates: &[ScanFilter], has_filters: bool, -) -> Vec { + rls_predicates: &[ScanFilter], +) -> crate::Result> { if partition_dirs.len() <= 1 { - return partition_dirs - .first() - .map(|dir| scan_one_partition(dir, time_range, limit, filter_predicates, has_filters)) - .unwrap_or_default(); + return match partition_dirs.first() { + Some(dir) => scan_one_partition( + dir, + time_range, + limit, + filter_predicates, + has_filters, + rls_predicates, + ), + None => Ok(Vec::new()), + }; } #[cfg(not(target_arch = "wasm32"))] @@ -39,11 +53,13 @@ pub(super) fn scan_partitions_parallel( limit, filter_predicates, has_filters, + rls_predicates, ); } let chunk_size = partition_dirs.len().div_ceil(thread_count); let filters_ref = filter_predicates; + let rls_ref = rls_predicates; let mut thread_results: Vec> = std::thread::scope(|s| { let handles: Vec<_> = partition_dirs @@ -56,13 +72,17 @@ pub(super) fn scan_partitions_parallel( limit, filters_ref, has_filters, + rls_ref, ) }) }) .collect(); - handles.into_iter().filter_map(|h| h.join().ok()).collect() - }); + handles + .into_iter() + .filter_map(|h| h.join().ok()) + .collect::>>() + })?; // Merge: each thread's results are already time-sorted (partitions are // time-ordered). Flatten, sort globally, truncate to limit. @@ -74,7 +94,7 @@ pub(super) fn scan_partitions_parallel( // Sort by timestamp (first field in each row map). merged.sort_by_key(extract_timestamp); merged.truncate(limit); - merged + Ok(merged) } #[cfg(target_arch = "wasm32")] @@ -85,6 +105,7 @@ pub(super) fn scan_partitions_parallel( limit, filter_predicates, has_filters, + rls_predicates, ) } } @@ -93,33 +114,48 @@ pub(super) fn scan_partitions_sequential( partition_dirs: &[std::path::PathBuf], time_range: (i64, i64), limit: usize, - filter_predicates: &[crate::bridge::scan_filter::ScanFilter], + filter_predicates: &[ScanFilter], has_filters: bool, -) -> Vec { + rls_predicates: &[ScanFilter], +) -> crate::Result> { let mut results = Vec::new(); for dir in partition_dirs { if results.len() >= limit { break; } let remaining = limit - results.len(); - let rows = scan_one_partition(dir, time_range, remaining, filter_predicates, has_filters); + let rows = scan_one_partition( + dir, + time_range, + remaining, + filter_predicates, + has_filters, + rls_predicates, + )?; results.extend(rows); } results.truncate(limit); - results + Ok(results) } /// Scan a single disk partition, returning rmpv rows. +/// +/// The WHERE predicates run on the typed columns when the evaluator can +/// lower them and per row otherwise. The read policy runs per row after +/// them, before the row counts toward `limit`. `Err` when a stored time +/// cell cannot be read as its column's instant or a predicate expression +/// divides by zero. pub(super) fn scan_one_partition( part_dir: &std::path::Path, time_range: (i64, i64), limit: usize, - filter_predicates: &[crate::bridge::scan_filter::ScanFilter], + filter_predicates: &[ScanFilter], has_filters: bool, -) -> Vec { + rls_predicates: &[ScanFilter], +) -> crate::Result> { let schema = match ColumnarSegmentReader::read_schema(part_dir, None) { Ok(s) => s, - Err(_) => return Vec::new(), + Err(_) => return Ok(Vec::new()), }; // Prefetch all column files into page cache before reading. @@ -146,7 +182,7 @@ pub(super) fn scan_one_partition( let ts_col = col_data.get(schema.timestamp_idx).and_then(|d| d.as_ref()); let Some(ts_col) = ts_col else { - return Vec::new(); + return Ok(Vec::new()); }; let timestamps = ts_col.as_timestamps(); let indices = timestamp_range_filter(timestamps, time_range.0, time_range.1); @@ -158,8 +194,11 @@ pub(super) fn scan_one_partition( sym_dicts: &sym_dicts, }; + // `row_filters` carries the WHERE predicates only when the typed-column + // evaluator could not lower them, so they still apply per row instead + // of being dropped. let row_count = timestamps.len(); - let filtered_indices = if has_filters { + let (filtered_indices, row_filters): (Vec, &[ScanFilter]) = if has_filters { if let Some(bitmask) = crate::data::executor::handlers::columnar_filter::eval_filters_bitmask( &part_src, @@ -167,21 +206,22 @@ pub(super) fn scan_one_partition( row_count, ) { - nodedb_query::simd_filter::bitmask_to_indices(&bitmask) + (nodedb_query::simd_filter::bitmask_to_indices(&bitmask), &[]) } else { match crate::data::executor::handlers::columnar_filter::eval_filters_sparse( &part_src, filter_predicates, &indices, ) { - Some(mask) => { - crate::data::executor::handlers::columnar_filter::apply_mask(&indices, &mask) - } - None => indices, + Some(mask) => ( + crate::data::executor::handlers::columnar_filter::apply_mask(&indices, &mask), + &[], + ), + None => (indices, filter_predicates), } } } else { - indices + (indices, &[]) }; let mut rows = Vec::with_capacity(filtered_indices.len().min(limit)); @@ -189,12 +229,15 @@ pub(super) fn scan_one_partition( if rows.len() >= limit { break; } - let row = emit_partition_row(&schema_vec, &col_data, &sym_dicts, idx as usize); + let row = emit_partition_row(&schema_vec, &col_data, &sym_dicts, idx as usize)?; + if !row_admitted(&row, row_filters, rls_predicates)? { + continue; + } rows.push(row); } // Release page cache for this partition. crate::data::io::fadvise::release_partition_columns(part_dir, &all_col_names); - rows + Ok(rows) } diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs index 63180c16b..8fe5f2100 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs @@ -1,12 +1,46 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Row emission helpers — build `rmpv::Value` directly, plus value conversions. +//! Row emission helpers — build `rmpv::Value` directly. use std::collections::HashMap; use nodedb_types::columnar::schema::TS_SYSTEM; +use crate::bridge::scan_filter::ScanFilter; +use crate::data::executor::handlers::columnar_read::filter::value_matches_filters; +use crate::data::executor::handlers::columnar_read::{emit_column_value, rmpv_time_cell}; use crate::engine::timeseries::columnar_memtable::{ColumnData, ColumnType}; +use crate::util::rmpv_value::{rmpv_to_value, value_to_rmpv}; + +/// Whether an emitted row passes `where_predicates` and then the caller's +/// read policy. +/// +/// Both sets empty admits the row without touching it. Otherwise the row is +/// converted once and both sets are evaluated by the predicate matcher the +/// columnar read path and the write gate use, so a policy means one thing +/// on every engine. `where_predicates` carries the WHERE clause only when +/// the typed-column evaluator could not lower it. `Err` when a predicate +/// expression divides by zero. +pub(super) fn row_admitted( + row: &rmpv::Value, + where_predicates: &[ScanFilter], + rls_predicates: &[ScanFilter], +) -> crate::Result { + if where_predicates.is_empty() && rls_predicates.is_empty() { + return Ok(true); + } + let doc = rmpv_to_value(row); + Ok(value_matches_filters(&doc, where_predicates)? + && value_matches_filters(&doc, rls_predicates)?) +} + +/// [`row_admitted`] for a row whose WHERE clause was already applied. +pub(super) fn row_admitted_by_policy( + row: &rmpv::Value, + rls_predicates: &[ScanFilter], +) -> crate::Result { + row_admitted(row, &[], rls_predicates) +} /// Extract the `_ts_system` value from an rmpv-encoded row for audit-log /// ordering. Rows without the column sort first (treated as `i64::MIN`). @@ -39,10 +73,12 @@ pub(super) fn rmpv_system_time(row: &rmpv::Value) -> i64 { /// An index past the memtable's row count is skipped rather than panicking: /// the caller reads indices recorded before a flush, and a flush in between /// would invalidate them. Callers must project before flushing. +/// +/// `Err` when a stored time cell cannot be read as its column's instant. pub(in crate::data::executor) fn emit_memtable_rows_at( mt: &crate::engine::timeseries::columnar_memtable::ColumnarMemtable, row_indices: &[usize], -) -> Vec { +) -> crate::Result> { let schema = mt.schema().clone(); let columns: Vec<_> = schema .columns @@ -59,30 +95,34 @@ pub(in crate::data::executor) fn emit_memtable_rows_at( } /// Emit a single row from the memtable as rmpv::Value::Map. +/// +/// Every cell is written by [`emit_column_value`], so a time cell carries +/// the type its column kind declares. `Err` when a stored time cell cannot +/// be read as its column's instant. pub(super) fn emit_memtable_row( mt: &crate::engine::timeseries::columnar_memtable::ColumnarMemtable, columns: &[(usize, &String, &ColumnType, &ColumnData)], idx: usize, -) -> rmpv::Value { +) -> crate::Result { // Build raw msgpack bytes, then decode to rmpv::Value. let mut buf = Vec::with_capacity(columns.len() * 32); nodedb_query::msgpack_scan::write_map_header(&mut buf, columns.len()); for (col_idx, col_name, col_type, col_data) in columns { nodedb_query::msgpack_scan::write_str(&mut buf, col_name); - crate::data::executor::handlers::columnar_read::emit_column_value( - &mut buf, mt, *col_idx, col_type, col_data, idx, - ); + emit_column_value(&mut buf, mt, *col_idx, col_type, col_data, idx)?; } - crate::util::bounded_msgpack::read_value(&buf).unwrap_or(rmpv::Value::Nil) + Ok(crate::util::bounded_msgpack::read_value(&buf).unwrap_or(rmpv::Value::Nil)) } /// Emit a single row from a disk partition as rmpv::Value::Map. +/// +/// `Err` when a stored time cell cannot be read as its column's instant. pub(super) fn emit_partition_row( schema: &[(String, ColumnType)], col_data: &[Option], sym_dicts: &HashMap, idx: usize, -) -> rmpv::Value { +) -> crate::Result { let mut fields: Vec<(rmpv::Value, rmpv::Value)> = Vec::with_capacity(schema.len()); for (col_i, (col_name, col_type)) in schema.iter().enumerate() { // A column whose file could not be read is emitted as NULL, never @@ -99,7 +139,7 @@ pub(super) fn emit_partition_row( continue; }; let val = match col_type { - ColumnType::Timestamp => rmpv::Value::Integer(data.as_timestamps()[idx].into()), + ColumnType::Timestamp(kind) => rmpv_time_cell(*kind, data.as_timestamps()[idx])?, ColumnType::Float64 => { let v = data.as_f64()[idx]; if v.is_nan() { @@ -129,15 +169,27 @@ pub(super) fn emit_partition_row( }; fields.push((rmpv::Value::String(col_name.as_str().into()), val)); } - rmpv::Value::Map(fields) + Ok(rmpv::Value::Map(fields)) } -/// Extract timestamp from a row (first integer field) for sort-merge. +/// Extract the sort key of a partition row for the merge across partitions: +/// the first time-shaped field, an integer or an instant ext, as `i64`. +/// +/// Every row of one collection carries the same kind in that field, so the +/// key is comparable across the rows being merged. pub(super) fn extract_timestamp(row: &rmpv::Value) -> i64 { if let rmpv::Value::Map(fields) = row { for (_, v) in fields { - if let rmpv::Value::Integer(n) = v { - return n.as_i64().unwrap_or(0); + match v { + rmpv::Value::Integer(n) => return n.as_i64().unwrap_or(0), + rmpv::Value::Ext(ext_type, payload) => { + if let Some((_, micros)) = + nodedb_types::json_msgpack::instant_from_ext(*ext_type, payload) + { + return micros; + } + } + _ => {} } } } @@ -154,7 +206,7 @@ pub(super) fn apply_computed_columns_rmpv( row: rmpv::Value, computed_cols: &[crate::bridge::expr_eval::ComputedColumn], ) -> crate::Result { - let doc = rmpv_to_nodedb_value(&row); + let doc = rmpv_to_value(&row); let mut fields: Vec<(rmpv::Value, rmpv::Value)> = Vec::with_capacity(computed_cols.len()); for cc in computed_cols { // A computed column is projection-shaped: a division/modulo-by-zero @@ -163,175 +215,8 @@ pub(super) fn apply_computed_columns_rmpv( let result = cc.expr.eval(&doc)?; fields.push(( rmpv::Value::String(cc.alias.as_str().into()), - nodedb_value_to_rmpv(&result), + value_to_rmpv(&result), )); } Ok(rmpv::Value::Map(fields)) } - -/// Convert rmpv row to nodedb_types::Value for expression evaluation. -pub(super) fn rmpv_to_nodedb_value(row: &rmpv::Value) -> nodedb_types::Value { - match row { - rmpv::Value::Map(fields) => { - let mut map = std::collections::HashMap::new(); - for (k, v) in fields { - let key = match k { - rmpv::Value::String(s) => s.as_str().unwrap_or("").to_string(), - _ => continue, - }; - let val = match v { - rmpv::Value::Integer(n) => { - nodedb_types::Value::Integer(n.as_i64().unwrap_or(0)) - } - rmpv::Value::F64(f) => nodedb_types::Value::Float(*f), - rmpv::Value::String(s) => { - nodedb_types::Value::String(s.as_str().unwrap_or("").to_string()) - } - rmpv::Value::Nil => nodedb_types::Value::Null, - rmpv::Value::Boolean(b) => nodedb_types::Value::Bool(*b), - _ => nodedb_types::Value::Null, - }; - map.insert(key, val); - } - nodedb_types::Value::Object(map) - } - _ => nodedb_types::Value::Null, - } -} - -/// Convert nodedb_types::Value back to rmpv::Value for response encoding. -pub(super) fn nodedb_value_to_rmpv(v: &nodedb_types::Value) -> rmpv::Value { - match v { - nodedb_types::Value::Integer(n) => rmpv::Value::Integer((*n).into()), - nodedb_types::Value::Float(f) => rmpv::Value::F64(*f), - nodedb_types::Value::String(s) => rmpv::Value::String(s.as_str().into()), - nodedb_types::Value::Bool(b) => rmpv::Value::Boolean(*b), - nodedb_types::Value::Null => rmpv::Value::Nil, - _ => rmpv::Value::Nil, - } -} - -/// Rescale every declared-instant cell of `rows` from the milliseconds the -/// memtable and the partitions store to the epoch microseconds a `TIMESTAMP` -/// cell carries on the wire. -/// -/// The engine's own unit stays milliseconds: partition ranges, retention, -/// `time_bucket` and every scan predicate read it. The scale therefore runs -/// once, as rows leave the scan — after filtering, sorting and computed -/// columns — so nothing inside the engine sees the wire unit. -/// -/// `instant_columns` comes from `CoreLoop::ts_instant_columns`, which lists -/// the columns declared `TIMESTAMP` or `TIMESTAMPTZ`. A `BIGINT TIME_KEY` -/// lives in the same millisecond column and is not in that list, so it keeps -/// the integer the client inserted. -/// -/// SQL NULL cells pass through untouched. A stored value that cannot be -/// expressed in microseconds fails the read rather than wrapping. -pub(in crate::data::executor) fn scale_instant_cells( - rows: &mut [rmpv::Value], - instant_columns: &[String], -) -> crate::Result<()> { - if instant_columns.is_empty() { - return Ok(()); - } - for row in rows.iter_mut() { - let rmpv::Value::Map(fields) = row else { - continue; - }; - for (key, value) in fields.iter_mut() { - let Some(name) = key.as_str() else { continue }; - if !instant_columns.iter().any(|c| c == name) { - continue; - } - let rmpv::Value::Integer(stored) = value else { - continue; - }; - let millis = stored.as_i64().ok_or_else(|| crate::Error::Internal { - detail: format!( - "timeseries column {name} holds {stored}, which is not a millisecond \ - count an instant can be read from" - ), - })?; - let micros = nodedb_types::NdbDateTime::from_millis(millis) - .map_err(|e| crate::Error::Internal { - detail: format!("timeseries column {name} at {millis} ms: {e}"), - })? - .micros; - *value = rmpv::Value::Integer(micros.into()); - } - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::scale_instant_cells; - - fn row(cells: &[(&str, i64)]) -> rmpv::Value { - rmpv::Value::Map( - cells - .iter() - .map(|(k, v)| { - ( - rmpv::Value::String((*k).into()), - rmpv::Value::Integer((*v).into()), - ) - }) - .collect(), - ) - } - - fn cell(row: &rmpv::Value, name: &str) -> Option { - let rmpv::Value::Map(fields) = row else { - return None; - }; - fields - .iter() - .find(|(k, _)| k.as_str() == Some(name)) - .and_then(|(_, v)| v.as_i64()) - } - - /// A declared `TIMESTAMP` column is read as epoch microseconds, so the - /// millisecond value storage holds is scaled on the way out. 2020-03-05 - /// stays 2020-03-05 instead of landing 50 years earlier. - #[test] - fn a_declared_instant_column_leaves_the_scan_in_microseconds() { - let mut rows = vec![row(&[("captured_at", 1_583_402_400_000)])]; - scale_instant_cells(&mut rows, &["captured_at".to_string()]).expect("scale"); - assert_eq!(cell(&rows[0], "captured_at"), Some(1_583_402_400_000_000)); - } - - /// A `BIGINT TIME_KEY` shares the millisecond storage column but is not a - /// declared instant, so its value is handed back exactly as inserted. - #[test] - fn a_column_that_is_not_a_declared_instant_keeps_its_value() { - let mut rows = vec![row(&[("ts", 1000), ("n", 7)])]; - scale_instant_cells(&mut rows, &["other".to_string()]).expect("scale"); - assert_eq!(cell(&rows[0], "ts"), Some(1000)); - assert_eq!(cell(&rows[0], "n"), Some(7)); - } - - /// A NULL instant cell stays NULL — there is no instant to scale. - #[test] - fn a_null_instant_cell_passes_through() { - let mut rows = vec![rmpv::Value::Map(vec![( - rmpv::Value::String("captured_at".into()), - rmpv::Value::Nil, - )])]; - scale_instant_cells(&mut rows, &["captured_at".to_string()]).expect("scale"); - assert_eq!(cell(&rows[0], "captured_at"), None); - } - - /// A stored millisecond count past the microsecond range fails the read. - /// Wrapping it would hand back an instant that is not the stored one. - #[test] - fn a_millisecond_value_beyond_the_microsecond_range_fails_the_read() { - let mut rows = vec![row(&[("captured_at", i64::MAX)])]; - let err = scale_instant_cells(&mut rows, &["captured_at".to_string()]) - .expect_err("i64::MAX ms cannot be expressed in microseconds"); - assert!( - err.to_string().contains("captured_at"), - "the error must name the column: {err}" - ); - } -} diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs index a51828b8f..824d7f204 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs @@ -3,12 +3,15 @@ //! Raw scan entry point: `RawScanParams` and `execute_ts_raw_scan`. use crate::bridge::envelope::{ErrorCode, Response}; +use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; use crate::engine::timeseries::columnar_agg::timestamp_range_filter; use super::partition_scan::scan_partitions_parallel; -use super::row_emit::{apply_computed_columns_rmpv, emit_memtable_row, rmpv_system_time}; +use super::row_emit::{ + apply_computed_columns_rmpv, emit_memtable_row, rmpv_system_time, row_admitted_by_policy, +}; /// Parameters for a timeseries raw scan (no aggregation). pub(in crate::data::executor) struct RawScanParams<'a> { @@ -17,8 +20,13 @@ pub(in crate::data::executor) struct RawScanParams<'a> { pub collection: &'a str, pub time_range: (i64, i64), pub limit: usize, - pub filter_predicates: &'a [crate::bridge::scan_filter::ScanFilter], + pub filter_predicates: &'a [ScanFilter], pub has_filters: bool, + /// The caller's decoded read policy; empty admits every row. Evaluated + /// per row after `filter_predicates` and before a row is gathered, so a + /// row the policy excludes never counts toward `limit`, on memtable, + /// partition, and in-transaction overlay rows alike. + pub rls_predicates: &'a [ScanFilter], pub computed_columns: &'a [u8], /// `AS OF SYSTEM TIME NULL`: emit every `_ts_system` version ordered /// ascending by system time (audit-log semantics). @@ -49,6 +57,7 @@ impl CoreLoop { limit, filter_predicates, has_filters, + rls_predicates, computed_columns: computed_columns_bytes, all_versions, txn_id, @@ -144,7 +153,10 @@ impl CoreLoop { if results.len() >= gather_limit { break; } - let row = emit_memtable_row(mt, &columns, idx as usize); + let row = match emit_memtable_row(mt, &columns, idx as usize) { + Ok(row) => row, + Err(e) => return self.response_error(task, e), + }; if need_json_filter { // Encode rmpv row to msgpack bytes for binary filter eval. let mut buf = Vec::new(); @@ -160,6 +172,14 @@ impl CoreLoop { } } } + // The caller's read policy, after the WHERE predicates and + // before the row is gathered, so the limit counts admitted + // rows only. + match row_admitted_by_policy(&row, rls_predicates) { + Ok(true) => {} + Ok(false) => continue, + Err(e) => return self.response_error(task, e), + } results.push(row); } } @@ -186,13 +206,17 @@ impl CoreLoop { let remaining = gather_limit.saturating_sub(results.len()); if remaining > 0 && !partition_dirs.is_empty() { - let partition_rows = scan_partitions_parallel( + let partition_rows = match scan_partitions_parallel( &partition_dirs, time_range, remaining, filter_predicates, has_filters, - ); + rls_predicates, + ) { + Ok(rows) => rows, + Err(e) => return self.response_error(task, e), + }; results.extend(partition_rows); results.truncate(gather_limit); } @@ -214,6 +238,7 @@ impl CoreLoop { time_range, filter_predicates, has_filters, + rls_predicates, limit: gather_limit, }, &mut results, @@ -271,15 +296,6 @@ impl CoreLoop { } results.truncate(limit); - // The engine stores its timestamp columns in milliseconds; a client - // reads a `TIMESTAMP` cell as epoch microseconds. Scale here, once - // every predicate, sort and computed column has run against the - // engine's own unit. - let instant_columns = self.ts_instant_columns(task.request.database_id, tid, collection); - if let Err(e) = super::row_emit::scale_instant_cells(&mut results, &instant_columns) { - return self.response_error(task, e); - } - let array = rmpv::Value::Array(results); let mut buf = Vec::new(); rmpv::encode::write_value(&mut buf, &array).unwrap_or(()); diff --git a/nodedb/src/data/executor/handlers/timeseries/resolve_ingest.rs b/nodedb/src/data/executor/handlers/timeseries/resolve_ingest.rs index 201bbc371..60dac3c85 100644 --- a/nodedb/src/data/executor/handlers/timeseries/resolve_ingest.rs +++ b/nodedb/src/data/executor/handlers/timeseries/resolve_ingest.rs @@ -150,7 +150,11 @@ impl CoreLoop { reason: format!("timeseries resolve: msgpack decode error: {error}"), } })?; - Ok(normalize::msgpack_rows_to_ilp(&rows, measurement, time_key)) + normalize::msgpack_rows_to_ilp(&rows, measurement, time_key).map_err(|error| { + ErrorCode::RejectedPrevalidation { + reason: format!("timeseries resolve: {error}"), + } + }) } "json" => { let rows: sonic_rs::Array = sonic_rs::from_slice(payload).map_err(|error| { diff --git a/nodedb/src/data/executor/handlers/timeseries/rls_gate.rs b/nodedb/src/data/executor/handlers/timeseries/rls_gate.rs index 030ccbfcd..f249fdc82 100644 --- a/nodedb/src/data/executor/handlers/timeseries/rls_gate.rs +++ b/nodedb/src/data/executor/handlers/timeseries/rls_gate.rs @@ -63,7 +63,8 @@ pub(in crate::data::executor) fn admit_msgpack_rows( ), }; let rows = msgpack_decode::decode_msgpack_rows(payload).map_err(|_| undecodable())?; - let batch = normalize::msgpack_rows_to_ilp(&rows, measurement, time_key); + let batch = + normalize::msgpack_rows_to_ilp(&rows, measurement, time_key).map_err(|_| undecodable())?; let parsed = ilp::parse_batch(&batch).map_err(|_| undecodable())?; admit_ilp_lines( rls_write_check, @@ -114,12 +115,12 @@ fn field_value(value: &FieldValue<'_>) -> Value { #[cfg(test)] mod tests { use super::*; - use crate::bridge::scan_filter::ScanFilter; + use crate::bridge::scan_filter::{FilterOp, ScanFilter}; fn owner_policy(owner: &str) -> Vec { let filter = ScanFilter { field: "owner".into(), - op: "eq".into(), + op: FilterOp::Eq, value: Value::String(owner.into()), clauses: Vec::new(), expr: None, @@ -242,7 +243,7 @@ mod tests { fn a_msgpack_row_is_decided_on_its_normalized_values() { let filter = ScanFilter { field: "reading".into(), - op: "eq".into(), + op: FilterOp::Eq, value: Value::Float(1.5), clauses: Vec::new(), expr: None, diff --git a/nodedb/src/data/executor/handlers/timeseries/scan.rs b/nodedb/src/data/executor/handlers/timeseries/scan.rs index 942d89e17..ccbb8b3e1 100644 --- a/nodedb/src/data/executor/handlers/timeseries/scan.rs +++ b/nodedb/src/data/executor/handlers/timeseries/scan.rs @@ -3,7 +3,9 @@ //! Data Plane timeseries scan parameters and execution. use crate::bridge::envelope::{Payload, Response, Status}; +use crate::bridge::scan_filter::{ScanFilter, decode_scan_filters}; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::columnar_read::filter::decode_rls_filters; use crate::data::executor::task::ExecutionTask; use nodedb_query::agg_key::canonical_agg_key; use nodedb_types::columnar::schema::{TS_SYSTEM, TS_VALID_FROM, TS_VALID_UNTIL}; @@ -18,6 +20,14 @@ pub(in crate::data::executor) struct TimeseriesScanParams<'a> { pub time_range: (i64, i64), pub limit: usize, pub filters: &'a [u8], + /// The caller's read policy as a MessagePack `Vec`, injected + /// by the planner. Empty when no policy governs the caller. The raw scan + /// evaluates it per row after `filters`, before computed columns, sort, + /// and limit, on memtable, partition, and in-transaction overlay rows + /// alike. The aggregate branch pushes it into the grouped scan so it + /// excludes rows before aggregation, and fails when the grouped scan + /// cannot lower it. + pub rls_filters: &'a [u8], /// `ORDER BY` keys as `(column, ascending)`. Applied to the materialized /// result before `limit`, on both the raw and the aggregate branch. pub sort_keys: &'a [nodedb_physical::physical_plan::SortKeySpec], @@ -54,6 +64,7 @@ impl CoreLoop { time_range, limit, filters, + rls_filters, sort_keys, bucket_interval_ms, group_by, @@ -71,6 +82,7 @@ impl CoreLoop { // projection pushdown. Resolved once here and threaded through both // branches; nothing downstream guesses it from a column name. let time_key = self.ts_time_column(task.request.database_id, tid, collection); + let time_key_kind = self.ts_time_key_kind(task.request.database_id, tid, collection); // Lazy-load partition registry from disk if not yet loaded. if let Err(e) = self.ensure_ts_registry(tid, task.request.database_id, collection) { @@ -82,12 +94,20 @@ impl CoreLoop { ); } - let mut filter_predicates: Vec = - if filters.is_empty() { - Vec::new() - } else { - zerompk::from_msgpack(filters).unwrap_or_default() + // Both predicate sets are decoded before any row is read. A payload + // the Data Plane cannot decode fails the statement: for the policy + // that is the difference between hiding governed rows and returning + // them, and for the WHERE clause it is the difference between the + // asked-for rows and every row. + let mut filter_predicates: Vec = + match decode_scan_filters(filters, "timeseries filter") { + Ok(p) => p, + Err(e) => return self.response_error(task, e), }; + let rls_predicates: Vec = match decode_rls_filters(rls_filters) { + Ok(p) => p, + Err(e) => return self.response_error(task, e), + }; // Bitemporal cutoffs: translate to column-level predicates on // `_ts_system` / `_ts_valid_from` / `_ts_valid_until`. The // segment reader's block-skip infrastructure applies these @@ -123,18 +143,21 @@ impl CoreLoop { let time_range = super::time_range::narrow_time_range( time_range, &filter_predicates, - Some(time_key.as_str()), + Some((time_key.as_str(), time_key_kind)), ); let has_filters = !filter_predicates.is_empty(); let is_aggregate = !aggregates.is_empty(); let has_time_range = time_range.0 > 0 || time_range.1 < i64::MAX; - // Fast path: COUNT(*) with no GROUP BY, no filters. + // Fast path: COUNT(*) with no GROUP BY, no filters, no read policy. + // The metadata count covers every stored row, so a governed read + // takes the grouped scan, where the policy excludes rows first. if is_aggregate && bucket_interval_ms == 0 && group_by.is_empty() && !has_filters + && rls_predicates.is_empty() && !has_time_range && aggregates.len() == 1 && aggregates[0].0 == "count" @@ -158,7 +181,7 @@ impl CoreLoop { needed.push(field.clone()); } } - for fp in &filter_predicates { + for fp in filter_predicates.iter().chain(&rls_predicates) { if !needed.contains(&fp.field) { needed.push(fp.field.clone()); } @@ -170,13 +193,23 @@ impl CoreLoop { // Mode dispatch. if is_aggregate || bucket_interval_ms > 0 { + // The policy joins the WHERE predicates so the grouped scan + // excludes governed rows before any accumulator sees them. The + // grouped scan fails on a predicate it cannot lower, so a policy + // shape it does not support fails the statement rather than + // aggregating ungoverned rows. + let governed_predicates: Vec = filter_predicates + .iter() + .chain(&rls_predicates) + .cloned() + .collect(); self.execute_ts_aggregate(aggregate::TsAggregateParams { task, tid, collection, time_range, limit, - filter_predicates: &filter_predicates, + filter_predicates: &governed_predicates, bucket_interval_ms, group_by, aggregates, @@ -202,6 +235,7 @@ impl CoreLoop { limit, filter_predicates: &filter_predicates, has_filters, + rls_predicates: &rls_predicates, computed_columns, all_versions, txn_id: overlay_txn, diff --git a/nodedb/src/data/executor/handlers/timeseries/sort.rs b/nodedb/src/data/executor/handlers/timeseries/sort.rs index 8e8f650cd..3284d1268 100644 --- a/nodedb/src/data/executor/handlers/timeseries/sort.rs +++ b/nodedb/src/data/executor/handlers/timeseries/sort.rs @@ -11,6 +11,9 @@ use std::cmp::Ordering; use nodedb_physical::physical_plan::SortKeySpec; +use nodedb_types::json_msgpack::instant_from_ext; + +use crate::util::rmpv_value::{rmpv_to_value, value_to_rmpv}; /// Sort materialized result rows by the planner's ORDER BY terms. /// @@ -70,54 +73,6 @@ fn eval_row_keys(row: &rmpv::Value, sort_keys: &[SortKeySpec]) -> crate::Result< Ok(out) } -fn rmpv_to_value(row: &rmpv::Value) -> nodedb_types::Value { - let rmpv::Value::Map(entries) = row else { - return nodedb_types::Value::Null; - }; - let mut map = std::collections::HashMap::with_capacity(entries.len()); - for (key, value) in entries { - if let rmpv::Value::String(name) = key - && let Some(name) = name.as_str() - { - map.insert(name.to_string(), rmpv_value_to_value(value)); - } - } - nodedb_types::Value::Object(map) -} - -fn rmpv_value_to_value(value: &rmpv::Value) -> nodedb_types::Value { - match value { - rmpv::Value::Nil => nodedb_types::Value::Null, - rmpv::Value::Boolean(b) => nodedb_types::Value::Bool(*b), - rmpv::Value::Integer(n) => n - .as_i64() - .map(nodedb_types::Value::Integer) - .or_else(|| n.as_f64().map(nodedb_types::Value::Float)) - .unwrap_or(nodedb_types::Value::Null), - rmpv::Value::F32(f) => nodedb_types::Value::Float(*f as f64), - rmpv::Value::F64(f) => nodedb_types::Value::Float(*f), - rmpv::Value::String(s) => s - .as_str() - .map(|s| nodedb_types::Value::String(s.to_string())) - .unwrap_or(nodedb_types::Value::Null), - rmpv::Value::Array(items) => { - nodedb_types::Value::Array(items.iter().map(rmpv_value_to_value).collect()) - } - _ => nodedb_types::Value::Null, - } -} - -fn value_to_rmpv(value: &nodedb_types::Value) -> rmpv::Value { - match value { - nodedb_types::Value::Null => rmpv::Value::Nil, - nodedb_types::Value::Bool(b) => rmpv::Value::Boolean(*b), - nodedb_types::Value::Integer(n) => rmpv::Value::Integer((*n).into()), - nodedb_types::Value::Float(f) => rmpv::Value::F64(*f), - nodedb_types::Value::String(s) => rmpv::Value::String(s.clone().into()), - other => rmpv::Value::String(format!("{other:?}").into()), - } -} - fn compare_key_rows(a: &[rmpv::Value], b: &[rmpv::Value], sort_keys: &[SortKeySpec]) -> Ordering { for (idx, key) in sort_keys.iter().enumerate() { let av = a.get(idx).filter(|v| !matches!(v, rmpv::Value::Nil)); @@ -167,6 +122,15 @@ fn compare_values(a: Option<&rmpv::Value>, b: Option<&rmpv::Value>) -> Ordering x.as_str().unwrap_or("").cmp(y.as_str().unwrap_or("")) } (rmpv::Value::Boolean(x), rmpv::Value::Boolean(y)) => x.cmp(y), + // A declared instant column, and a computed key that evaluates to an + // instant, arrive as the instant ext; two instants order by their + // epoch microseconds. + (rmpv::Value::Ext(tx, px), rmpv::Value::Ext(ty, py)) => { + match (instant_from_ext(*tx, px), instant_from_ext(*ty, py)) { + (Some((_, x)), Some((_, y))) => x.cmp(&y), + _ => Ordering::Equal, + } + } // Exotic shapes never appear in a timeseries result row; keep the // order stable rather than inventing one. _ => Ordering::Equal, diff --git a/nodedb/src/data/executor/handlers/timeseries/time_range.rs b/nodedb/src/data/executor/handlers/timeseries/time_range.rs index ac14e16a6..35dbf67b7 100644 --- a/nodedb/src/data/executor/handlers/timeseries/time_range.rs +++ b/nodedb/src/data/executor/handlers/timeseries/time_range.rs @@ -17,20 +17,22 @@ //! is still evaluated per row afterwards. A too-wide envelope costs I/O; a //! too-narrow one loses rows. -use nodedb_query::scan_filter::value_as_timestamp_ms; - use crate::bridge::scan_filter::{FilterOp, ScanFilter}; +use crate::engine::timeseries::columnar_memtable::TimeKind; -/// Narrow `plan_range` with every bound the query places on `time_key`. +/// Narrow `plan_range` with every bound the query places on the time key +/// named by `time_key`, lowering each literal to stored milliseconds by the +/// key's kind. /// /// Returns `plan_range` unchanged when the collection has no declared time -/// key, or when no predicate references it. +/// key, or when no predicate references it. A literal the kind cannot lower +/// narrows nothing: the exact predicate still decides each row afterwards. pub(in crate::data::executor) fn narrow_time_range( plan_range: (i64, i64), filters: &[ScanFilter], - time_key: Option<&str>, + time_key: Option<(&str, TimeKind)>, ) -> (i64, i64) { - let Some(time_key) = time_key else { + let Some((time_key, kind)) = time_key else { return plan_range; }; let (mut min_ts, mut max_ts) = plan_range; @@ -38,7 +40,7 @@ pub(in crate::data::executor) fn narrow_time_range( if !filter.field.eq_ignore_ascii_case(time_key) { continue; } - let Some(ms) = value_as_timestamp_ms(&filter.value) else { + let Some(ms) = kind.literal_ms(&filter.value) else { continue; }; match filter.op { @@ -69,9 +71,11 @@ pub(in crate::data::executor) fn narrow_time_range( #[cfg(test)] mod tests { use super::*; - use nodedb_types::Value; + use nodedb_types::{InstantKind, Value}; const UNBOUNDED: (i64, i64) = (i64::MIN, i64::MAX); + const MILLIS: TimeKind = TimeKind::Millis; + const NAIVE: TimeKind = TimeKind::Instant(InstantKind::Naive); fn filter(field: &str, op: FilterOp, value: Value) -> ScanFilter { ScanFilter { @@ -96,21 +100,38 @@ mod tests { filter("captured_at", FilterOp::Lt, Value::Integer(900)), ]; assert_eq!( - narrow_time_range(UNBOUNDED, &filters, Some("captured_at")), + narrow_time_range(UNBOUNDED, &filters, Some(("captured_at", MILLIS))), (100, 900) ); } + /// A declared instant key lowers a typed instant, and text that spells + /// one, to stored milliseconds. #[test] - fn a_datetime_literal_bound_is_understood() { - let filters = vec![filter( - "captured_at", - FilterOp::Lt, - Value::String("2020-03-05 10:00:00".into()), - )]; + fn an_instant_key_lowers_instant_literals() { + let at = nodedb_types::NdbDateTime::from_micros(1_583_402_400_000_000); + let filters = vec![ + filter("captured_at", FilterOp::Gte, Value::NaiveDateTime(at)), + filter( + "captured_at", + FilterOp::Lt, + Value::String("2020-03-05 11:00:00".into()), + ), + ]; assert_eq!( - narrow_time_range(UNBOUNDED, &filters, Some("captured_at")), - (i64::MIN, 1_583_402_400_000) + narrow_time_range(UNBOUNDED, &filters, Some(("captured_at", NAIVE))), + (1_583_402_400_000, 1_583_406_000_000) + ); + } + + /// A bare integer against an instant key carries no unit, so it narrows + /// nothing; the per-row predicate still decides. + #[test] + fn an_integer_against_an_instant_key_narrows_nothing() { + let filters = vec![filter("captured_at", FilterOp::Gt, Value::Integer(100))]; + assert_eq!( + narrow_time_range(UNBOUNDED, &filters, Some(("captured_at", NAIVE))), + UNBOUNDED ); } @@ -123,7 +144,7 @@ mod tests { filter("value", FilterOp::Lt, Value::Float(1.0)), ]; assert_eq!( - narrow_time_range(UNBOUNDED, &filters, Some("captured_at")), + narrow_time_range(UNBOUNDED, &filters, Some(("captured_at", MILLIS))), UNBOUNDED ); } @@ -132,7 +153,7 @@ mod tests { fn equality_pins_both_ends() { let filters = vec![filter("ts", FilterOp::Eq, Value::Integer(4_200))]; assert_eq!( - narrow_time_range(UNBOUNDED, &filters, Some("ts")), + narrow_time_range(UNBOUNDED, &filters, Some(("ts", MILLIS))), (4_200, 4_200) ); } @@ -144,7 +165,7 @@ mod tests { filter("ts", FilterOp::Lte, Value::Integer(10_000)), ]; assert_eq!( - narrow_time_range((100, 900), &filters, Some("ts")), + narrow_time_range((100, 900), &filters, Some(("ts", MILLIS))), (100, 900) ); } @@ -153,7 +174,7 @@ mod tests { fn case_differences_in_the_predicate_still_match_the_key() { let filters = vec![filter("TS", FilterOp::Gt, Value::Integer(7))]; assert_eq!( - narrow_time_range(UNBOUNDED, &filters, Some("ts")), + narrow_time_range(UNBOUNDED, &filters, Some(("ts", MILLIS))), (7, i64::MAX) ); } diff --git a/nodedb/src/data/executor/handlers/transaction/overlay/columnar_merge.rs b/nodedb/src/data/executor/handlers/transaction/overlay/columnar_merge.rs index 365e91fb4..69252f328 100644 --- a/nodedb/src/data/executor/handlers/transaction/overlay/columnar_merge.rs +++ b/nodedb/src/data/executor/handlers/transaction/overlay/columnar_merge.rs @@ -29,17 +29,17 @@ use nodedb_types::value::Value; use crate::bridge::expr_eval::ComputedColumn; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::handlers::columnar_read::convert::row_to_projected_json; -use crate::data::executor::handlers::columnar_read::filter::row_matches_filters; +use crate::data::executor::handlers::columnar_read::convert::row_to_projected_value; +use crate::data::executor::handlers::columnar_read::filter::row_matches_filters_and_policy; use crate::data::executor::handlers::transaction::overlay::Staged; use crate::types::{DatabaseId, TenantId, TxnId}; /// One matched columnar row: its cross-engine surrogate (when known), the /// decoded schema-ordered column values, and the already-projected response -/// JSON. Shared shape between the base scan (`execute_columnar_scan`) and -/// this overlay merge. -pub(in crate::data::executor) type ColumnarMatchedRow = - (Option, Vec, serde_json::Value); +/// object (`Value::Object`). Shared shape between the base scan +/// (`execute_columnar_scan`), the predicate DML row read, and this overlay +/// merge. +pub(in crate::data::executor) type ColumnarMatchedRow = (Option, Vec, Value); /// Inputs for [`CoreLoop::merge_overlay_into_columnar_scan`]. pub(in crate::data::executor) struct ColumnarOverlayMergeParams<'a> { @@ -48,6 +48,11 @@ pub(in crate::data::executor) struct ColumnarOverlayMergeParams<'a> { pub schema: &'a ColumnarSchema, pub projection: &'a [String], pub filter_predicates: &'a [ScanFilter], + /// The caller's decoded read policy. A staged row the policy excludes is + /// dropped from the result exactly like a base row; empty admits every + /// row. The predicate DML row read passes an empty slice because its + /// plan carries a write check, not a read policy. + pub rls_predicates: &'a [ScanFilter], pub computed_cols: &'a [ComputedColumn], pub all_versions: bool, } @@ -82,6 +87,7 @@ impl CoreLoop { schema, projection, filter_predicates, + rls_predicates, computed_cols, all_versions, } = params; @@ -94,10 +100,7 @@ impl CoreLoop { }; let predicate = |row: &[Value]| -> Result { - if filter_predicates.is_empty() { - return Ok(true); - } - row_matches_filters(row, schema, filter_predicates) + row_matches_filters_and_policy(row, schema, filter_predicates, rls_predicates) }; // Surrogates already represented in the base result. Additions @@ -109,7 +112,7 @@ impl CoreLoop { .collect(); // Base-minus-superseded: a tombstoned row is dropped; a staged put - // replaces the row's decoded values + JSON and is re-checked against + // replaces the row's decoded values + object and is re-checked against // the scan predicate (an update may have moved the row out of the // result). A row with no recorded surrogate has no overlay identity // to resolve and is left untouched, matching the base scan's own @@ -121,7 +124,7 @@ impl CoreLoop { // checked once the retain pass finishes, aborting the merge before // the overlay-addition pass below runs. let mut first_err: Option = None; - matched.retain_mut(|(surrogate, row, json)| { + matched.retain_mut(|(surrogate, row, obj)| { if first_err.is_some() { return true; } @@ -140,14 +143,14 @@ impl CoreLoop { return true; } } - match row_to_projected_json( + match row_to_projected_value( &new_row, schema, projection, computed_cols, all_versions, ) { - Ok(v) => *json = v, + Ok(v) => *obj = v, Err(e) => { first_err = Some(e); return true; @@ -183,9 +186,9 @@ impl CoreLoop { if !predicate(&new_row)? { continue; } - let json = - row_to_projected_json(&new_row, schema, projection, computed_cols, all_versions)?; - matched.push((Some(Surrogate::new(surrogate)), new_row, json)); + let obj = + row_to_projected_value(&new_row, schema, projection, computed_cols, all_versions)?; + matched.push((Some(Surrogate::new(surrogate)), new_row, obj)); seen.insert(surrogate); } Ok(()) diff --git a/nodedb/src/data/executor/handlers/transaction/overlay/mod.rs b/nodedb/src/data/executor/handlers/transaction/overlay/mod.rs index f8e5719ff..66bd40c12 100644 --- a/nodedb/src/data/executor/handlers/transaction/overlay/mod.rs +++ b/nodedb/src/data/executor/handlers/transaction/overlay/mod.rs @@ -10,7 +10,9 @@ mod staged; mod timeseries_merge; mod vector_merge; -pub(in crate::data::executor) use columnar_merge::{ColumnarOverlayMergeParams, decode_staged_row}; +pub(in crate::data::executor) use columnar_merge::{ + ColumnarMatchedRow, ColumnarOverlayMergeParams, decode_staged_row, +}; pub(in crate::data::executor) use fts_merge::FtsMergeParams; pub use graph_staged::{GraphCollKey, GraphTxnOverlay, NodeLabelDelta}; pub(in crate::data::executor) use merge::IndexOverlayMergeParams; diff --git a/nodedb/src/data/executor/handlers/transaction/overlay/spatial_merge.rs b/nodedb/src/data/executor/handlers/transaction/overlay/spatial_merge.rs index 34c32a810..8520fe9eb 100644 --- a/nodedb/src/data/executor/handlers/transaction/overlay/spatial_merge.rs +++ b/nodedb/src/data/executor/handlers/transaction/overlay/spatial_merge.rs @@ -37,7 +37,7 @@ use nodedb_types::value::Value; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::handlers::columnar_read::convert::row_to_projected_json; +use crate::data::executor::handlers::columnar_read::convert::row_to_projected_value; use crate::data::executor::handlers::spatial_refine::{ apply_predicate, extract_geometry, project_doc, }; @@ -64,20 +64,17 @@ pub(in crate::data::executor) struct SpatialOverlayMergeParams<'a> { /// staged row whose collection has no known columnar schema (defensively /// treated as "does not match" rather than surfacing a panic). Returns /// `Err` only when the row *does* decode but its computed-column projection -/// hits a division/modulo-by-zero — `row_to_projected_json` is called with +/// hits a division/modulo-by-zero — `row_to_projected_value` is called with /// no computed columns here (`&[]`), so this is currently unreachable, but /// the `Result` return keeps the signature honest about what -/// `row_to_projected_json` can do. +/// `row_to_projected_value` can do. fn decode_staged_spatial_row( body: &[u8], schema: Option<&ColumnarSchema>, ) -> crate::Result> { Ok(match nodedb_types::value_from_msgpack(body).ok() { Some(Value::Array(row)) => match schema { - Some(schema) => { - let json = row_to_projected_json(&row, schema, &[], &[], false)?; - Some(Value::from(json)) - } + Some(schema) => Some(row_to_projected_value(&row, schema, &[], &[], false)?), None => None, }, Some(obj @ Value::Object(_)) => Some(obj), diff --git a/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs b/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs index 3d725774d..29bf7095e 100644 --- a/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs +++ b/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs @@ -24,8 +24,11 @@ use nodedb_types::value::Value; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::columnar_read::filter::value_matches_filters; use crate::data::executor::handlers::transaction::overlay::Staged; +use crate::engine::timeseries::columnar_memtable::TimeKind; use crate::types::{DatabaseId, TenantId, TxnId}; +use crate::util::rmpv_value::value_to_rmpv; /// Inputs for [`CoreLoop::merge_overlay_into_timeseries_scan`]. pub(in crate::data::executor) struct TimeseriesOverlayMergeParams<'a> { @@ -36,6 +39,9 @@ pub(in crate::data::executor) struct TimeseriesOverlayMergeParams<'a> { pub time_range: (i64, i64), pub filter_predicates: &'a [ScanFilter], pub has_filters: bool, + /// The caller's decoded read policy. A staged row the policy excludes is + /// dropped exactly like a base row; empty admits every row. + pub rls_predicates: &'a [ScanFilter], /// Row ceiling for the whole scan (base + staged). Staged rows are only /// appended while the result is below this bound, so the merge never /// exceeds the SQL `LIMIT`. @@ -57,39 +63,25 @@ fn decode_staged_row(body: &[u8]) -> Option { /// /// A staged row keeps the INSERT's own column names, so the lookup is by the /// collection's declared time column — the same name the base scan prunes on. -/// A row that carries no readable instant under that name falls outside every -/// bounded range and is treated as non-matching by the caller. -fn row_timestamp_ms(row: &Value, time_column: &str) -> Option { +/// The cell was staged typed by the column's kind, so it lowers to stored +/// milliseconds by that same kind. A row that carries no such cell under +/// that name falls outside every bounded range and is treated as +/// non-matching by the caller. +fn row_timestamp_ms(row: &Value, time_column: &str, kind: TimeKind) -> Option { let Value::Object(map) = row else { return None; }; map.iter() .find(|(key, _)| key.eq_ignore_ascii_case(time_column)) - .and_then(|(_, value)| nodedb_query::scan_filter::value_as_timestamp_ms(value)) + .and_then(|(_, value)| kind.literal_ms(value)) } /// Convert a decoded staged row (`Value::Object`) into the `rmpv::Value::Map` /// row shape the base raw scan emits, so a merged staged row is /// indistinguishable from a base row downstream (computed columns, encoding). fn staged_row_to_rmpv(row: &Value) -> rmpv::Value { - let Value::Object(map) = row else { - return rmpv::Value::Nil; - }; - let fields: Vec<(rmpv::Value, rmpv::Value)> = map - .iter() - .map(|(k, v)| (rmpv::Value::String(k.as_str().into()), scalar_to_rmpv(v))) - .collect(); - rmpv::Value::Map(fields) -} - -/// Scalar `nodedb_types::Value` → `rmpv::Value`, matching the raw scan's own -/// value emission (`row_emit::nodedb_value_to_rmpv`). -fn scalar_to_rmpv(v: &Value) -> rmpv::Value { - match v { - Value::Integer(n) => rmpv::Value::Integer((*n).into()), - Value::Float(f) => rmpv::Value::F64(*f), - Value::String(s) => rmpv::Value::String(s.as_str().into()), - Value::Bool(b) => rmpv::Value::Boolean(*b), + match row { + Value::Object(_) => value_to_rmpv(row), _ => rmpv::Value::Nil, } } @@ -97,8 +89,8 @@ fn scalar_to_rmpv(v: &Value) -> rmpv::Value { impl CoreLoop { /// Append this transaction's staged `TimeseriesOp::Ingest` rows to /// `results` (base raw-scan rows), each subject to the scan's time-range, - /// WHERE predicate, and row limit. No-op when the transaction has no - /// overlay entries for this collection. + /// WHERE predicate, read policy, and row limit. No-op when the + /// transaction has no overlay entries for this collection. pub(in crate::data::executor) fn merge_overlay_into_timeseries_scan( &self, params: TimeseriesOverlayMergeParams<'_>, @@ -110,10 +102,12 @@ impl CoreLoop { time_range, filter_predicates, has_filters, + rls_predicates, limit, } = params; let time_column = self.ts_time_column(coll_key.0, coll_key.1, &coll_key.2); + let time_kind = self.ts_time_key_kind(coll_key.0, coll_key.1, &coll_key.2); // Read-your-own-writes refreshes the lease (see the reaper). self.touch_overlay(txn_id); @@ -137,7 +131,7 @@ impl CoreLoop { // Time-range prune, mirroring the base memtable scan's // `timestamp_range_filter` (inclusive bounds). - match row_timestamp_ms(&row, &time_column) { + match row_timestamp_ms(&row, &time_column, time_kind) { Some(ts) if ts >= time_range.0 && ts <= time_range.1 => {} _ => continue, } @@ -148,6 +142,11 @@ impl CoreLoop { if has_filters && !ScanFilter::all_match_binary(filter_predicates, body)? { continue; } + // The caller's read policy, on the decoded row, after the WHERE + // predicate and before the row takes a limit slot. + if !value_matches_filters(&row, rls_predicates)? { + continue; + } results.push(staged_row_to_rmpv(&row)); } diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_columnar_dml.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_columnar_dml.rs index 6c9e09b5d..f55d34856 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_columnar_dml.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_columnar_dml.rs @@ -50,9 +50,11 @@ use nodedb_types::{RowIdentity, Surrogate, value_to_pk_string}; use crate::bridge::envelope::{ErrorCode, Response}; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::handlers::columnar_read::convert::row_to_projected_json; +use crate::data::executor::handlers::columnar_read::convert::row_to_projected_value; use crate::data::executor::handlers::columnar_read::filter::row_matches_filters; -use crate::data::executor::handlers::transaction::overlay::ColumnarOverlayMergeParams; +use crate::data::executor::handlers::transaction::overlay::{ + ColumnarMatchedRow, ColumnarOverlayMergeParams, +}; use crate::data::executor::response_codec; use crate::data::executor::task::ExecutionTask; use crate::types::{TenantId, TxnId}; @@ -297,7 +299,7 @@ impl CoreLoop { // shared `ColumnarMatchedRow` tuple the overlay merge consumes. A // missing engine means the only affected rows are overlay-only staged // inserts, which the merge appends below. - let mut matched: Vec<(Option, Vec, serde_json::Value)> = Vec::new(); + let mut matched: Vec = Vec::new(); if let Some(engine) = self.columnar_engines.get(&coll_key) { for (surrogate, row) in engine.scan_memtable_rows_with_surrogates() { if !filter_predicates.is_empty() { @@ -311,15 +313,15 @@ impl CoreLoop { } // No computed columns on this path (`&[]` below), so this // can never actually raise `DivisionByZero` today — handled - // uniformly with every other `row_to_projected_json` caller + // uniformly with every other `row_to_projected_value` caller // instead of assuming that invariant with an `unwrap`. - let json = match row_to_projected_json(&row, &schema, &[], &[], false) { + let obj = match row_to_projected_value(&row, &schema, &[], &[], false) { Ok(v) => v, Err(_e) => { return Err(self.response_error(task, ErrorCode::DivisionByZero)); } }; - matched.push((surrogate, row, json)); + matched.push((surrogate, row, obj)); } } @@ -335,6 +337,7 @@ impl CoreLoop { schema: &schema, projection: &[], filter_predicates: &filter_predicates, + rls_predicates: &[], computed_cols: &[], all_versions: false, }, diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs index 5fb54863f..95731299a 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs @@ -338,12 +338,19 @@ impl CoreLoop { // A staged row is read back by name, so the line's timestamp must be // keyed under the collection's own time column — the same name the - // base scan and the overlay merge resolve. + // base scan and the overlay merge resolve — and typed by that + // column's kind, so a predicate literal coerced for the column + // compares against the staged cell exactly as against a base cell. let time_column = self.ts_time_column( task.request.database_id, crate::types::TenantId::new(tid), collection, ); + let time_kind = self.ts_time_key_kind( + task.request.database_id, + crate::types::TenantId::new(tid), + collection, + ); // Encode every row before mutating an overlay. let mut images = Vec::with_capacity(lines.len()); @@ -374,7 +381,21 @@ impl CoreLoop { object.insert(name.to_string(), value); } if let Some(timestamp) = row.timestamp_ns { - object.insert(time_column.clone(), Value::Integer(timestamp / 1_000_000)); + let cell = match time_kind.cell_value(timestamp / 1_000_000) { + Ok(cell) => cell, + Err(error) => { + return self.response_error( + task, + ErrorCode::Internal { + detail: format!( + "canonical ILP overlay row time cell {}: {error}", + timestamp / 1_000_000 + ), + }, + ); + } + }; + object.insert(time_column.clone(), cell); } images.push(Value::Object(object)); } diff --git a/nodedb/src/data/executor/handlers/transaction/undo/apply.rs b/nodedb/src/data/executor/handlers/transaction/undo/apply.rs index 6c756e434..5425640a9 100644 --- a/nodedb/src/data/executor/handlers/transaction/undo/apply.rs +++ b/nodedb/src/data/executor/handlers/transaction/undo/apply.rs @@ -465,7 +465,7 @@ mod tests { use super::*; use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; use crate::engine::timeseries::columnar_memtable::{ - ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, + ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; use crate::engine::timeseries::last_value_cache::LastValueCache; use crate::types::{DatabaseId, TenantId}; @@ -486,7 +486,7 @@ mod tests { ColumnarMemtable::new( ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), ("value".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], diff --git a/nodedb/src/data/executor/handlers/update_from_join_write.rs b/nodedb/src/data/executor/handlers/update_from_join_write.rs index 01fe76e60..cd5f57afd 100644 --- a/nodedb/src/data/executor/handlers/update_from_join_write.rs +++ b/nodedb/src/data/executor/handlers/update_from_join_write.rs @@ -27,7 +27,7 @@ pub(in crate::data::executor) struct UpdateFromJoinWriteOutcome { pub write_set: Vec, /// Post-image JSON per affected row, populated only when the caller asked /// for `RETURNING`. - pub returned_docs: Vec, + pub returned_docs: Vec, } /// Everything the write pass needs about the statement, gathered once by the @@ -73,7 +73,7 @@ impl CoreLoop { ); let mut affected = 0u64; let mut write_set: Vec = Vec::new(); - let mut returned_docs: Vec = if want_returning { + let mut returned_docs: Vec = if want_returning { Vec::with_capacity(rows.len()) } else { Vec::new() @@ -84,7 +84,7 @@ impl CoreLoop { key: storage_key, body: updated_bytes, old_body, - mut doc, + doc, } = row; // Period lock, both images — matching `execute_point_update`: a @@ -239,8 +239,9 @@ impl CoreLoop { // `row_identity` only stands in as `id` for a row that // declares no primary key of its own — overwriting a // declared key would return a value the client never wrote. - returning_doc::attach_row_id(&mut doc, &row_identity); - returned_docs.push(doc); + let mut row = nodedb_types::Value::from(doc); + returning_doc::attach_row_id(&mut row, &row_identity); + returned_docs.push(row); } } } diff --git a/nodedb/src/data/executor/response_codec/encode.rs b/nodedb/src/data/executor/response_codec/encode.rs index d563cbf42..25cff1599 100644 --- a/nodedb/src/data/executor/response_codec/encode.rs +++ b/nodedb/src/data/executor/response_codec/encode.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: BUSL-1.1 //! Generic encoders for Data Plane response payloads, plus the -//! `decode_payload` / `decode_payload_to_json` counterparts used at the Control -//! Plane boundary. +//! `decode_payload` / `decode_payload_to_json` / `decode_payload_value` +//! counterparts used at the Control Plane boundary. //! //! # Every encoder here emits MessagePack //! @@ -140,26 +140,55 @@ pub fn decode_payload(payload: &[u8]) -> crate::R /// to JSON text via streaming transcoder (no intermediate `serde_json::Value`). /// If already JSON (starts with `[` or `{`), returns as-is. pub fn decode_payload_to_json(payload: &[u8]) -> String { - if payload.is_empty() { + let Some(&first) = payload.first() else { return String::new(); + }; + + if looks_like_json(first) { + return String::from_utf8_lossy(payload).into_owned(); + } + + nodedb_types::msgpack_to_json_string(payload) + .unwrap_or_else(|_| String::from_utf8_lossy(payload).into_owned()) +} + +/// Decode a MessagePack or JSON payload to a typed [`nodedb_types::Value`]. +/// +/// The counterpart of [`decode_payload_to_json`] for the shaping kernel, +/// with the same format sniff: JSON text parses through `serde_json::Value` +/// (so a JSON string is a `Value::String`, never parsed further), anything +/// else reads as msgpack (a `bin` is `Value::Bytes`, an instant ext is a +/// `Value::DateTime` / `Value::NaiveDateTime`). An empty payload is +/// `Value::Null`. A non-empty payload that decodes as neither is an error. +pub fn decode_payload_value(payload: &[u8]) -> crate::Result { + let Some(&first) = payload.first() else { + return Ok(nodedb_types::Value::Null); + }; + + if looks_like_json(first) { + return sonic_rs::from_slice::(payload) + .map(nodedb_types::Value::from) + .map_err(|e| crate::Error::Codec { + detail: format!("response payload is not JSON text: {e}"), + }); } - let first = payload[0]; + nodedb_types::value_from_msgpack(payload).map_err(|e| crate::Error::Codec { + detail: format!("response payload is not msgpack: {e}"), + }) +} - let is_likely_json = first == b'[' +/// True when `first` can open JSON text: an array, object, string, number or +/// one of the `true` / `false` / `null` literals. No msgpack marker for a map +/// or array shares these bytes. +fn looks_like_json(first: u8) -> bool { + first == b'[' || first == b'{' || first == b'"' || first.is_ascii_digit() || first == b't' || first == b'f' - || first == b'n'; - - if is_likely_json { - return String::from_utf8_lossy(payload).into_owned(); - } - - nodedb_types::msgpack_to_json_string(payload) - .unwrap_or_else(|_| String::from_utf8_lossy(payload).into_owned()) + || first == b'n' } #[cfg(test)] @@ -210,6 +239,34 @@ mod tests { ); } + /// `decode_payload_value` sniffs the same way `decode_payload_to_json` + /// does: JSON text and msgpack both land as the typed value, a msgpack + /// `bin` as bytes, and an empty payload as NULL. + #[test] + fn decode_payload_value_reads_json_text_and_msgpack_alike() { + use nodedb_types::Value; + + let from_json = decode_payload_value(br#"[{"id":1,"s":"x"}]"#).unwrap(); + let mut row = std::collections::HashMap::new(); + row.insert("id".to_string(), Value::Integer(1)); + row.insert("s".to_string(), Value::String("x".into())); + assert_eq!(from_json, Value::Array(vec![Value::Object(row.clone())])); + + let msgpack = + nodedb_types::value_to_msgpack(&Value::Array(vec![Value::Object(row)])).unwrap(); + assert_eq!(decode_payload_value(&msgpack).unwrap(), from_json); + + let bin = nodedb_types::value_to_msgpack(&Value::Bytes(vec![0, 255])).unwrap(); + assert_eq!( + decode_payload_value(&bin).unwrap(), + Value::Bytes(vec![0, 255]) + ); + + assert_eq!(decode_payload_value(&[]).unwrap(), Value::Null); + assert!(decode_payload_value(&[0xC1]).is_err()); + assert!(decode_payload_value(b"true-ish").is_err()); + } + /// `TOPK` / `RANGE` rows — `encode_json_vec_as_msgpack`. #[test] fn decode_payload_reads_back_json_vec_rows() { diff --git a/nodedb/src/data/executor/response_codec/hits.rs b/nodedb/src/data/executor/response_codec/hits.rs index a93312f83..77891681d 100644 --- a/nodedb/src/data/executor/response_codec/hits.rs +++ b/nodedb/src/data/executor/response_codec/hits.rs @@ -9,6 +9,7 @@ //! score under a dynamic key. Without this the alias would be lost and the //! response would always carry `rrf_score` regardless of what the SQL named it. +use nodedb_types::NativeCell; use serde::Serialize; use crate::data::executor::handlers::hybrid_key::HybridFusionKey; @@ -86,15 +87,19 @@ pub(in crate::data::executor) struct GraphRagResult { /// Carries one entry per affected row, with the projected column values. /// The Control Plane decodes this to build a multi-column pgwire QueryResponse /// (one pgwire field per entry in `columns`). -#[derive(Serialize, serde::Deserialize, zerompk::ToMessagePack, zerompk::FromMessagePack)] +/// +/// Cells are typed `Value`s in plain msgpack (`NativeCell`), so an instant +/// crosses as the instant ext and the JSON transcoder renders the payload as +/// `{"columns": [...], "rows": [[cell, ...], ...]}` with plain cells. +#[derive(zerompk::ToMessagePack, zerompk::FromMessagePack)] #[msgpack(map)] pub(crate) struct RowsPayload { /// Projected column names (output names, respecting AS aliases). pub columns: Vec, /// One inner Vec per affected row; each inner Vec has one cell per - /// column in the same order as `columns`. `None` denotes SQL NULL - /// (missing field or JSON null); `Some` carries the TEXT representation. - pub rows: Vec>>, + /// column in the same order as `columns`. `Value::Null` denotes SQL NULL + /// (missing field or a stored null). + pub rows: Vec>, } /// Carries the row payload alongside a flag that signals whether the @@ -215,4 +220,33 @@ mod tests { assert!(json.contains("\"id\"")); assert!(json.contains("\"distance\"")); } + + /// A `RowsPayload` transcodes to JSON with plain cells: a typed instant + /// renders as ISO-8601, an integer as a number, SQL NULL as `null`. + #[test] + fn rows_payload_cells_transcode_plain() { + use nodedb_types::{NdbDateTime, Value}; + + let payload = RowsPayload { + columns: vec!["id".into(), "n".into(), "at".into(), "gone".into()], + rows: vec![vec![ + NativeCell(Value::String("r1".into())), + NativeCell(Value::Integer(7)), + NativeCell(Value::NaiveDateTime(NdbDateTime::from_micros( + 1_583_402_400_000_000, + ))), + NativeCell(Value::Null), + ]], + }; + let bytes = zerompk::to_msgpack_vec(&payload).unwrap(); + let json = decode_payload_to_json(&bytes); + assert_eq!( + json, + "{\"columns\":[\"id\",\"n\",\"at\",\"gone\"],\"rows\":[[\"r1\",7,\"2020-03-05T10:00:00.000000Z\",null]]}" + ); + + let back: RowsPayload = zerompk::from_msgpack(&bytes).unwrap(); + assert_eq!(back.columns, payload.columns); + assert_eq!(back.rows, payload.rows); + } } diff --git a/nodedb/src/data/executor/response_codec/mod.rs b/nodedb/src/data/executor/response_codec/mod.rs index 1cd862b07..82a38892a 100644 --- a/nodedb/src/data/executor/response_codec/mod.rs +++ b/nodedb/src/data/executor/response_codec/mod.rs @@ -13,7 +13,8 @@ //! Split by concern: //! //! - `encode` — generic encoders + the `decode_payload` / -//! `decode_payload_to_json` counterparts used at the Control Plane boundary. +//! `decode_payload_to_json` / `decode_payload_value` counterparts used at +//! the Control Plane boundary. //! - `decode` — payload→docs decoders for inline sub-plans (e.g. multi-way //! joins consuming an inner-join Response). //! - `raw` — raw-msgpack passthrough encoders (`encode_raw_document_rows`, @@ -30,7 +31,7 @@ mod raw; pub use arrow::encode_as_arrow_ipc; pub(in crate::data::executor) use decode::decode_response_to_docs; -pub use encode::{decode_payload, decode_payload_to_json}; +pub use encode::{decode_payload, decode_payload_to_json, decode_payload_value}; pub(in crate::data::executor) use encode::{ encode, encode_count, encode_json_as_msgpack, encode_json_vec_as_msgpack, encode_serde, encode_value_vec, diff --git a/nodedb/src/data/executor/row_shape.rs b/nodedb/src/data/executor/row_shape.rs index 16e89fa9d..e56a8a494 100644 --- a/nodedb/src/data/executor/row_shape.rs +++ b/nodedb/src/data/executor/row_shape.rs @@ -84,18 +84,32 @@ pub(in crate::data::executor) fn sparse_row_to_doc( /// Convert a single row from a `DecodedColumn` to a `nodedb_types::value::Value`. /// +/// `declared` is the column's declared type from the collection schema. It +/// decides how an eight-byte integer cell is typed, because the segment +/// reader infers a column's physical kind from its codec and decodes every +/// time column as `DecodedColumn::Int64`: a `Timestamp` column yields +/// `Value::NaiveDateTime`, a `Timestamptz` column `Value::DateTime`, both from +/// the epoch microseconds the segment stores. Every other declared type +/// yields the integer stored: a `SystemTimestamp` column is an engine-assigned +/// system-time count, and the bitemporal `_ts_system`, `_ts_valid_from`, +/// `_ts_valid_until` columns are declared `Int64` and hold epoch milliseconds +/// with `i64::MIN` / `i64::MAX` as the unbounded sentinels, which no instant +/// can carry. The live memtable applies the same rule, so a row reads +/// identically before and after a flush. +/// /// Returns `Value::Null` if the row index is out of range or the validity bit is false. pub(in crate::data::executor) fn decoded_col_to_value( col: &nodedb_columnar::reader::DecodedColumn, row_idx: usize, + declared: &nodedb_types::columnar::ColumnType, ) -> nodedb_types::value::Value { use nodedb_columnar::reader::DecodedColumn; use nodedb_types::value::Value; match col { - DecodedColumn::Int64 { values, valid } => { + DecodedColumn::Int64 { values, valid } | DecodedColumn::Timestamp { values, valid } => { if row_idx < valid.len() && valid[row_idx] { - Value::Integer(values[row_idx]) + declared.time_cell(values[row_idx]) } else { Value::Null } @@ -107,14 +121,6 @@ pub(in crate::data::executor) fn decoded_col_to_value( Value::Null } } - DecodedColumn::Timestamp { values, valid } => { - if row_idx < valid.len() && valid[row_idx] { - // Represent as integer microseconds (same as Value::Integer for timestamps). - Value::Integer(values[row_idx]) - } else { - Value::Null - } - } DecodedColumn::Bool { values, valid } => { if row_idx < valid.len() && valid[row_idx] { Value::Bool(values[row_idx]) @@ -160,13 +166,94 @@ pub(in crate::data::executor) fn decoded_col_to_value( Value::Null } } - _ => Value::Null, } } #[cfg(test)] mod tests { - use super::{kv_row_to_doc, msgpack_scan}; + use nodedb_columnar::reader::DecodedColumn; + use nodedb_types::columnar::ColumnType; + use nodedb_types::value::Value; + use nodedb_types::{InstantKind, NdbDateTime}; + + use super::{decoded_col_to_value, kv_row_to_doc, msgpack_scan}; + + const MICROS: i64 = 1_583_402_400_000_000; + + /// A time column as the segment reader decodes it: the reader infers the + /// physical kind from the codec, so a time column arrives as `Int64`. + fn time_column() -> DecodedColumn { + DecodedColumn::Int64 { + values: vec![MICROS, 0], + valid: vec![true, false], + } + } + + /// A `TIMESTAMP` column reads back as a naive instant, a `TIMESTAMPTZ` + /// column as a UTC instant, each carrying the stored microseconds, + /// whether the reader decoded the column as `Int64` or as `Timestamp`. + #[test] + fn a_declared_instant_column_reads_back_as_its_instant_variant() { + let col = time_column(); + assert_eq!( + decoded_col_to_value(&col, 0, &ColumnType::Timestamp), + Value::NaiveDateTime(NdbDateTime::from_micros(MICROS)) + ); + assert_eq!( + decoded_col_to_value(&col, 0, &ColumnType::Timestamptz), + Value::DateTime(NdbDateTime::from_micros(MICROS)) + ); + assert_eq!( + decoded_col_to_value(&col, 0, &ColumnType::Timestamp), + InstantKind::Naive.from_micros(MICROS) + ); + let typed = DecodedColumn::Timestamp { + values: vec![MICROS], + valid: vec![true], + }; + assert_eq!( + decoded_col_to_value(&typed, 0, &ColumnType::Timestamptz), + Value::DateTime(NdbDateTime::from_micros(MICROS)) + ); + } + + /// A system-time column and a duration column share time storage but are + /// not instants: they read back as the integer stored. + #[test] + fn a_non_instant_time_column_reads_back_as_the_integer_stored() { + let col = time_column(); + assert_eq!( + decoded_col_to_value(&col, 0, &ColumnType::SystemTimestamp), + Value::Integer(MICROS) + ); + assert_eq!( + decoded_col_to_value(&col, 0, &ColumnType::Duration), + Value::Integer(MICROS) + ); + } + + /// The declared type never changes how a non-time column reads, and an + /// invalid or out-of-range row is NULL for every declared type. + #[test] + fn a_null_time_cell_is_null_for_every_declared_type() { + let col = time_column(); + for ty in [ + ColumnType::Timestamp, + ColumnType::Timestamptz, + ColumnType::SystemTimestamp, + ] { + assert_eq!(decoded_col_to_value(&col, 1, &ty), Value::Null); + assert_eq!(decoded_col_to_value(&col, 2, &ty), Value::Null); + } + let ints = DecodedColumn::Int64 { + values: vec![7], + valid: vec![true], + }; + assert_eq!( + decoded_col_to_value(&ints, 0, &ColumnType::Int64), + Value::Integer(7) + ); + } /// A raw (non-msgpack) KV value must be wrapped as a msgpack STRING, not /// appended verbatim. diff --git a/nodedb/src/data/executor/scan_normalize.rs b/nodedb/src/data/executor/scan_normalize.rs index 4fc173ed9..564516c58 100644 --- a/nodedb/src/data/executor/scan_normalize.rs +++ b/nodedb/src/data/executor/scan_normalize.rs @@ -55,10 +55,7 @@ impl CoreLoop { return Ok(docs); } - let filters: Vec = - zerompk::from_msgpack(filter_bytes).map_err(|e| crate::Error::PlanError { - detail: format!("{context} deserialization failed: {e}"), - })?; + let filters = crate::bridge::scan_filter::decode_scan_filters(filter_bytes, context)?; let mut kept = Vec::with_capacity(docs.len()); for (id, bytes) in docs { @@ -83,10 +80,7 @@ impl CoreLoop { if rls_filters.is_empty() { return Ok(true); } - let filters: Vec = - zerompk::from_msgpack(rls_filters).map_err(|e| crate::Error::PlanError { - detail: format!("RLS filter deserialization failed: {e}"), - })?; + let filters = crate::bridge::scan_filter::decode_scan_filters(rls_filters, "RLS filter")?; Ok(crate::bridge::scan_filter::ScanFilter::all_match_binary( &filters, row, )?) @@ -116,7 +110,7 @@ impl CoreLoop { } // 2. Columnar memtable - let col_docs = self.scan_columnar(did, tid, collection, limit); + let col_docs = self.scan_columnar(did, tid, collection, limit)?; if !col_docs.is_empty() { return Ok(col_docs); } @@ -182,7 +176,7 @@ impl CoreLoop { // 2. Columnar — materializes internally; iterate the batch per-row. // columnar stays materialized — per-row segment streaming is a separate // follow-up (flushed-segment decode). - let col_docs = self.scan_columnar(did, tid, collection, usize::MAX); + let col_docs = self.scan_columnar(did, tid, collection, usize::MAX)?; if !col_docs.is_empty() { for (id, bytes) in &col_docs { f(id, bytes)?; @@ -238,13 +232,16 @@ impl CoreLoop { } /// Scan columnar rows → standard msgpack. + /// + /// A timeseries memtable's time cells are typed by their column kind. + /// `Err` when a stored time cell cannot be read as its column's instant. fn scan_columnar( &self, database_id: u64, tid: u64, collection: &str, limit: usize, - ) -> Vec<(String, Vec)> { + ) -> crate::Result)>> { let columnar_key = ( nodedb_types::DatabaseId::new(database_id), crate::types::TenantId::new(tid), @@ -282,15 +279,15 @@ impl CoreLoop { } super::handlers::columnar_read::emit_column_value( &mut mp, mt, *col_idx, col_type, col_data, idx, - ); + )?; } results.push((id, mp)); } - return results; + return Ok(results); } let Some(engine) = self.columnar_engines.get(&columnar_key) else { - return Vec::new(); + return Ok(Vec::new()); }; let schema = engine.schema(); @@ -348,7 +345,11 @@ impl CoreLoop { let mut map = std::collections::HashMap::new(); let mut id = String::new(); for (col_idx, col_def) in schema.columns.iter().enumerate() { - let val = decoded_col_to_value(&decoded_cols[col_idx], row_idx); + let val = decoded_col_to_value( + &decoded_cols[col_idx], + row_idx, + &col_def.column_type, + ); if col_def.name == "id" && let nodedb_types::value::Value::String(s) = &val { @@ -386,7 +387,7 @@ impl CoreLoop { } } - results + Ok(results) } /// Scan sparse/document engine → standard msgpack. diff --git a/nodedb/src/data/executor/strict_format/coerce.rs b/nodedb/src/data/executor/strict_format/coerce.rs index d2ec8eec1..4ebba44fd 100644 --- a/nodedb/src/data/executor/strict_format/coerce.rs +++ b/nodedb/src/data/executor/strict_format/coerce.rs @@ -268,8 +268,10 @@ pub fn value_to_json(val: &Value) -> serde_json::Value { &base64::engine::general_purpose::STANDARD, b, )), - Value::DateTime(dt) | Value::NaiveDateTime(dt) => serde_json::json!(dt.micros / 1000), - Value::Duration(d) => serde_json::json!(d.as_millis()), + Value::DateTime(dt) | Value::NaiveDateTime(dt) => { + serde_json::Value::String(dt.to_iso8601()) + } + Value::Duration(d) => serde_json::Value::String(d.to_string()), Value::Decimal(d) => serde_json::Value::String(d.to_string()), Value::Array(arr) => serde_json::Value::Array(arr.iter().map(value_to_json).collect()), Value::Object(map) => { diff --git a/nodedb/src/data/executor/strict_format/decode.rs b/nodedb/src/data/executor/strict_format/decode.rs index 3dfa174e5..848a37b61 100644 --- a/nodedb/src/data/executor/strict_format/decode.rs +++ b/nodedb/src/data/executor/strict_format/decode.rs @@ -81,26 +81,46 @@ pub fn undecodable_strict_row(collection: &str, identity: &str) -> crate::Error } } +/// Decode a Binary Tuple to the row a projection reads: every declared +/// column, in schema order, with the reserved bitemporal bookkeeping columns +/// dropped. A column the tuple does not carry is `Value::Null`. +/// +/// The walk is shared by [`binary_tuple_to_json`], so the JSON and `Value` +/// forms of a row can never disagree about which columns it has. +pub fn binary_tuple_to_row_value(tuple_bytes: &[u8], schema: &StrictSchema) -> Option { + let Value::Object(mut map) = binary_tuple_to_value(tuple_bytes, schema)? else { + return None; + }; + let mut row = std::collections::HashMap::with_capacity(map.len()); + for col in projected_columns(schema) { + let v = map.remove(&col.name).unwrap_or(Value::Null); + row.insert(col.name.clone(), v); + } + Some(Value::Object(row)) +} + /// Decode a Binary Tuple to a JSON object using the schema (for pgwire output). pub fn binary_tuple_to_json( tuple_bytes: &[u8], schema: &StrictSchema, ) -> Option { - // Delegate to binary_tuple_to_value (which handles version-aware decoding) - // then convert Value → JSON. - let val = binary_tuple_to_value(tuple_bytes, schema)?; - match val { - Value::Object(map) => { - let mut obj = serde_json::Map::with_capacity(map.len()); - for col in &schema.columns { - if is_reserved_bitemporal_column(&col.name) { - continue; - } - let v = map.get(&col.name).unwrap_or(&Value::Null); - obj.insert(col.name.clone(), value_to_json(v)); - } - Some(serde_json::Value::Object(obj)) - } - _ => None, + let Value::Object(map) = binary_tuple_to_row_value(tuple_bytes, schema)? else { + return None; + }; + let mut obj = serde_json::Map::with_capacity(map.len()); + for col in projected_columns(schema) { + let v = map.get(&col.name).unwrap_or(&Value::Null); + obj.insert(col.name.clone(), value_to_json(v)); } + Some(serde_json::Value::Object(obj)) +} + +/// The schema's columns a projection sees, in declaration order. +fn projected_columns( + schema: &StrictSchema, +) -> impl Iterator { + schema + .columns + .iter() + .filter(|col| !is_reserved_bitemporal_column(&col.name)) } diff --git a/nodedb/src/data/executor/strict_format/mod.rs b/nodedb/src/data/executor/strict_format/mod.rs index f2dc6e30a..2b4625d8d 100644 --- a/nodedb/src/data/executor/strict_format/mod.rs +++ b/nodedb/src/data/executor/strict_format/mod.rs @@ -10,7 +10,8 @@ mod decode; mod encode; pub(crate) use decode::{ - binary_tuple_to_json, binary_tuple_to_msgpack, binary_tuple_to_value, undecodable_strict_row, + binary_tuple_to_json, binary_tuple_to_msgpack, binary_tuple_to_row_value, + binary_tuple_to_value, undecodable_strict_row, }; pub(super) use encode::{ bytes_to_binary_tuple, bytes_to_binary_tuple_bitemporal, value_to_binary_tuple, diff --git a/nodedb/src/engine/graph/pattern/executor/predicates.rs b/nodedb/src/engine/graph/pattern/executor/predicates.rs index 89a1c6b27..7c1f39030 100644 --- a/nodedb/src/engine/graph/pattern/executor/predicates.rs +++ b/nodedb/src/engine/graph/pattern/executor/predicates.rs @@ -293,7 +293,7 @@ fn check_property( op: &ComparisonOp, expected_value: &nodedb_types::Value, ) -> Result { - use nodedb_query::value_ops::{coerced_eq, compare_values}; + use nodedb_query::value_ops::{coerced_eq, partial_compare_values}; use std::cmp::Ordering; let collection = props.collection.ok_or_else(|| crate::Error::BadRequest { @@ -323,18 +323,22 @@ fn check_property( let result = match op { ComparisonOp::Eq => coerced_eq(field_value, expected_value), ComparisonOp::Neq => !coerced_eq(field_value, expected_value), - ComparisonOp::Lt => compare_values(field_value, expected_value) == Ordering::Less, + ComparisonOp::Lt => { + partial_compare_values(field_value, expected_value) == Some(Ordering::Less) + } ComparisonOp::Lte => { matches!( - compare_values(field_value, expected_value), - Ordering::Less | Ordering::Equal + partial_compare_values(field_value, expected_value), + Some(Ordering::Less | Ordering::Equal) ) } - ComparisonOp::Gt => compare_values(field_value, expected_value) == Ordering::Greater, + ComparisonOp::Gt => { + partial_compare_values(field_value, expected_value) == Some(Ordering::Greater) + } ComparisonOp::Gte => { matches!( - compare_values(field_value, expected_value), - Ordering::Greater | Ordering::Equal + partial_compare_values(field_value, expected_value), + Some(Ordering::Greater | Ordering::Equal) ) } }; diff --git a/nodedb/src/engine/spatial/mod.rs b/nodedb/src/engine/spatial/mod.rs index 6dfdf3ef3..83959cb00 100644 --- a/nodedb/src/engine/spatial/mod.rs +++ b/nodedb/src/engine/spatial/mod.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 // Re-export shared spatial engine from nodedb-spatial crate. -// Origin's spatial handlers (data/executor/handlers/spatial.rs) and +// Origin's spatial handlers (data/executor/handlers/spatial/) and // checkpoint logic (data/executor/spatial_checkpoint/) use these directly. pub use nodedb_spatial::GeohashIndex; pub use nodedb_spatial::RTree; diff --git a/nodedb/src/engine/timeseries/columnar_bridge.rs b/nodedb/src/engine/timeseries/columnar_bridge.rs index 567b160d3..d8ad1c782 100644 --- a/nodedb/src/engine/timeseries/columnar_bridge.rs +++ b/nodedb/src/engine/timeseries/columnar_bridge.rs @@ -16,10 +16,11 @@ use nodedb_columnar::predicate::ScanPredicate; use nodedb_columnar::reader::{DecodedColumn, SegmentReader}; use nodedb_columnar::writer::SegmentWriter; use nodedb_mem::ScopedMemory; +use nodedb_types::InstantKind; use nodedb_types::columnar::{ColumnDef, ColumnType as SharedColumnType, ColumnarSchema}; use super::columnar_memtable::{ - ColumnData as TsColumnData, ColumnType as TsColumnType, ColumnarDrainResult, + ColumnData as TsColumnData, ColumnType as TsColumnType, ColumnarDrainResult, TimeKind, }; /// Convert a timeseries ColumnarSchema to a shared ColumnarSchema. @@ -46,9 +47,19 @@ pub fn ts_schema_to_shared(ts_schema: &super::columnar_memtable::ColumnarSchema) } /// Map a timeseries ColumnType to a shared ColumnType. +/// +/// A time column maps by its kind: a naive instant is `Timestamp`, a UTC +/// instant is `Timestamptz`, and a `Millis` column is the `Int64` its +/// declaration named. All three share `i64` storage on both sides. fn ts_column_type_to_shared(ts_type: TsColumnType) -> SharedColumnType { match ts_type { - TsColumnType::Timestamp => SharedColumnType::Timestamp, + TsColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive)) => { + SharedColumnType::Timestamp + } + TsColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)) => { + SharedColumnType::Timestamptz + } + TsColumnType::Timestamp(TimeKind::Millis) => SharedColumnType::Int64, TsColumnType::Float64 => SharedColumnType::Float64, TsColumnType::Int64 => SharedColumnType::Int64, // Symbol columns store u32 IDs — represented as Int64 in shared format. @@ -79,10 +90,18 @@ fn ts_column_to_shared( row_count: usize, ) -> SharedColumnData { match (ts_col, ts_type) { - (TsColumnData::Timestamp(values), TsColumnType::Timestamp) => SharedColumnData::Timestamp { - values: values.clone(), - valid: None, // Timeseries columns are non-nullable. - }, + (TsColumnData::Timestamp(values), TsColumnType::Timestamp(TimeKind::Instant(_))) => { + SharedColumnData::Timestamp { + values: values.clone(), + valid: None, // Timeseries columns are non-nullable. + } + } + (TsColumnData::Timestamp(values), TsColumnType::Timestamp(TimeKind::Millis)) => { + SharedColumnData::Int64 { + values: values.clone(), + valid: None, + } + } (TsColumnData::Float64(values), TsColumnType::Float64) => SharedColumnData::Float64 { values: values.clone(), valid: None, @@ -297,20 +316,34 @@ mod tests { fn schema_conversion() { let ts_schema = super::super::columnar_memtable::ColumnarSchema { columns: vec![ - ("timestamp".into(), TsColumnType::Timestamp), + ( + "timestamp".into(), + TsColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive)), + ), ("value".into(), TsColumnType::Float64), ("host".into(), TsColumnType::Symbol), + ( + "seen_at".into(), + TsColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)), + ), + ( + "_ts_system".into(), + TsColumnType::Timestamp(TimeKind::Millis), + ), ], timestamp_idx: 0, - codecs: vec![nodedb_codec::ColumnCodec::Auto; 3], + codecs: vec![nodedb_codec::ColumnCodec::Auto; 5], }; let shared = ts_schema_to_shared(&ts_schema); - assert_eq!(shared.columns.len(), 3); + assert_eq!(shared.columns.len(), 5); assert_eq!(shared.columns[0].column_type, SharedColumnType::Timestamp); assert_eq!(shared.columns[1].column_type, SharedColumnType::Float64); // Symbol → Int64 in shared format. assert_eq!(shared.columns[2].column_type, SharedColumnType::Int64); + assert_eq!(shared.columns[3].column_type, SharedColumnType::Timestamptz); + // A millisecond integer time column is the integer it was declared as. + assert_eq!(shared.columns[4].column_type, SharedColumnType::Int64); } #[test] diff --git a/nodedb/src/engine/timeseries/columnar_memtable/memtable.rs b/nodedb/src/engine/timeseries/columnar_memtable/memtable.rs deleted file mode 100644 index 3307a91a2..000000000 --- a/nodedb/src/engine/timeseries/columnar_memtable/memtable.rs +++ /dev/null @@ -1,952 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! `ColumnarMemtable` — per-column ingest buffer for timeseries data. -//! -//! NOT thread-safe — lives on a single Data Plane core (!Send by design). - -use std::collections::HashMap; - -use nodedb_types::timeseries::{IngestResult, MetricSample, SeriesId, SymbolDictionary}; - -use super::snapshot::{MemtableSnapshot, column_to_snapshot, rebuild_columns}; -use super::types::{ - ColumnData, ColumnType, ColumnValue, ColumnarDrainResult, ColumnarFlushView, - ColumnarMemtableConfig, ColumnarSchema, max_system_ts_of, -}; - -/// Columnar memtable: per-column vectors instead of per-series hash maps. -/// -/// Each row is a flat tuple of (timestamp, value, tag1, tag2, ...). -/// Series identity is derived from the tag columns at query time. -/// This layout is SIMD-friendly: aggregation functions operate on -/// contiguous `&[f64]` or `&[i64]` slices. -pub struct ColumnarMemtable { - schema: ColumnarSchema, - columns: Vec, - /// Per-series row count for quick cardinality checks. - series_row_counts: HashMap, - /// Per-tag-column symbol dictionary. - symbol_dicts: HashMap, - row_count: u64, - memory_bytes: usize, - config: ColumnarMemtableConfig, - min_ts: i64, - max_ts: i64, -} - -impl ColumnarMemtable { - /// Create a new columnar memtable with the given schema. - pub fn new(schema: ColumnarSchema, config: ColumnarMemtableConfig) -> Self { - let columns: Vec = schema - .columns - .iter() - .map(|(_, ty)| ColumnData::new(*ty)) - .collect(); - - // Initialize symbol dicts for tag columns. - let mut symbol_dicts = HashMap::new(); - for (i, (_, ty)) in schema.columns.iter().enumerate() { - if *ty == ColumnType::Symbol { - symbol_dicts.insert(i, SymbolDictionary::new()); - } - } - - Self { - schema, - columns, - series_row_counts: HashMap::new(), - symbol_dicts, - row_count: 0, - memory_bytes: 0, - config, - min_ts: i64::MAX, - max_ts: i64::MIN, - } - } - - /// Create a simple metrics memtable (timestamp + f64 value, no tags). - pub fn new_metric(config: ColumnarMemtableConfig) -> Self { - Self::new(ColumnarSchema::metric_default(), config) - } - - /// Ingest a metric sample into the default (timestamp, value) layout. - /// - /// For the simple 2-column schema. For multi-column schemas with tags, - /// use `ingest_row()` instead. - /// - /// Does NOT enforce `hard_memory_limit`, for the same reason `ingest_row` - /// does not (see its doc): the sole production caller is WAL replay - /// (`replay_timeseries_payload`), and a sample reaching here belongs to a - /// record that has ALREADY COMMITTED, so refusing it is not backpressure — - /// it is silent loss of a durable write. Replay must take the record whole; - /// the ceiling lives at the record boundary in the live ingest handler's - /// admission gate, never here. NOTE: if a structured-`TimeseriesWalBatch` - /// ingest producer is ever added, its replay must gain that same - /// record-boundary flush, or a mid-record replay flush would stamp the - /// partition with an LSN covering rows it does not hold. - /// `replay_timeseries_wal` advances `ts_max_ingested_lsn` only AFTER a - /// record is applied, which is what keeps that stamp honest. - pub fn ingest_metric(&mut self, series_id: SeriesId, sample: MetricSample) -> IngestResult { - // Push to timestamp column. - if let ColumnData::Timestamp(ref mut v) = self.columns[self.schema.timestamp_idx] { - v.push(sample.timestamp_ms); - } - - // Push to value column (assume index 1 for default schema). - if self.columns.len() > 1 - && let ColumnData::Float64(ref mut v) = self.columns[1] - { - v.push(sample.value); - } - - self.update_stats(series_id, sample.timestamp_ms, 16); - self.check_flush_state() - } - - /// Ingest a row with explicit column values. - /// - /// `values` must match the schema length. Tag string values are resolved - /// to symbol IDs via the per-column dictionary. - /// - /// ## Why this does NOT enforce `hard_memory_limit` - /// - /// It used to: a row arriving above the ceiling came back - /// `Ok(IngestResult::Rejected)`. But every row reaching here belongs to a - /// WAL record that has ALREADY COMMITTED, so refusing it is not - /// backpressure — it is silent loss of a durable write. The caller then - /// tried to rescue the refusal by flushing and re-ingesting mid-record, - /// which stamped the flushed partition with the PREVIOUS record's LSN - /// while it held part of the current one; replay, gated on that stamp, - /// re-appended the whole record on top. - /// - /// The ceiling now lives at the record boundary instead (the ingest - /// handler's admission gate flushes BEFORE a record when the memtable is - /// at or over it), so a flush always lands on a whole-record prefix and - /// the partition's stamp is true of every row in it. Errors returned here - /// are therefore genuine per-row data faults — bad arity, type mismatch, - /// exhausted tag dictionary — never "come back later". - pub fn ingest_row( - &mut self, - series_id: SeriesId, - values: &[ColumnValue], - ) -> crate::Result { - let col_types: Vec<(String, ColumnType)> = self.schema.columns.clone(); - - if values.len() != col_types.len() { - return Err(crate::Error::BadRequest { - detail: format!("expected {} columns, got {}", col_types.len(), values.len()), - }); - } - - let mut ts = 0i64; - let mut row_bytes = 0usize; - let max_card = self.config.max_tag_cardinality; - - for (i, (val, (col_name, col_type))) in values.iter().zip(col_types.iter()).enumerate() { - match (val, col_type) { - (ColumnValue::Timestamp(t), ColumnType::Timestamp) => { - if let ColumnData::Timestamp(ref mut v) = self.columns[i] { - v.push(*t); - } - ts = *t; - row_bytes += 8; - } - (ColumnValue::Float64(f), ColumnType::Float64) => { - if let ColumnData::Float64(ref mut v) = self.columns[i] { - v.push(*f); - } - row_bytes += 8; - } - (ColumnValue::Int64(n), ColumnType::Int64) => { - if let ColumnData::Int64(ref mut v) = self.columns[i] { - v.push(*n); - } - row_bytes += 8; - } - (ColumnValue::Symbol(s), ColumnType::Symbol) => { - let dict = - self.symbol_dicts - .get_mut(&i) - .ok_or_else(|| crate::Error::BadRequest { - detail: format!( - "internal error: symbol dict missing for column {i}" - ), - })?; - match dict.resolve(s, max_card) { - Some(sym_id) => { - if let ColumnData::Symbol(ref mut v) = self.columns[i] { - v.push(sym_id); - } - } - None => { - self.rollback_partial_row(i); - return Err(crate::Error::BadRequest { - detail: format!( - "tag cardinality limit ({max_card}) exceeded for column '{col_name}'" - ), - }); - } - } - row_bytes += 4; - } - _ => { - self.rollback_partial_row(i); - return Err(crate::Error::BadRequest { - detail: format!("type mismatch at column {i}: expected {col_type:?}"), - }); - } - } - } - - self.update_stats(series_id, ts, row_bytes); - Ok(self.check_flush_state()) - } - - /// Roll back a partially written row (called on error during `ingest_row`). - fn rollback_partial_row(&mut self, columns_written: usize) { - for col in self.columns.iter_mut().take(columns_written) { - match col { - ColumnData::Timestamp(v) => { - v.pop(); - } - ColumnData::Float64(v) => { - v.pop(); - } - ColumnData::Int64(v) => { - v.pop(); - } - ColumnData::Symbol(v) => { - v.pop(); - } - ColumnData::DictEncoded { ids, valid, .. } => { - ids.pop(); - valid.pop(); - } - } - } - } - - fn update_stats(&mut self, series_id: SeriesId, ts: i64, row_bytes: usize) { - *self.series_row_counts.entry(series_id).or_insert(0) += 1; - self.row_count += 1; - self.memory_bytes += row_bytes; - if ts < self.min_ts { - self.min_ts = ts; - } - if ts > self.max_ts { - self.max_ts = ts; - } - } - - fn check_flush_state(&self) -> IngestResult { - if self.memory_bytes >= self.config.max_memory_bytes { - IngestResult::FlushNeeded - } else { - IngestResult::Ok - } - } - - /// Borrow this memtable's live rows as the payload a flush would write. - /// - /// The read-only half of a flush: a segment can be encoded and landed from - /// this view, and only then does [`Self::drain`] take the rows out. Nothing - /// leaves memory before it is durable, so a failed segment write leaves the - /// memtable exactly as it was. - pub fn flush_view(&self) -> ColumnarFlushView<'_> { - ColumnarFlushView { - columns: &self.columns, - schema: &self.schema, - symbol_dicts: &self.symbol_dicts, - row_count: self.row_count, - min_ts: self.min_ts, - max_ts: self.max_ts, - max_system_ts: max_system_ts_of(&self.schema, &self.columns), - } - } - - /// Drain all data from the memtable, resetting it for reuse. - /// - /// Returns the column data, schema, symbol dicts, and stats. - /// - /// Callers that flush must write the segment from [`Self::flush_view`] FIRST - /// and drain only once that write has committed — the rows here have no - /// other copy but the WAL, and the checkpoint that calls the flush is what - /// authorises deleting it. - pub fn drain(&mut self) -> ColumnarDrainResult { - let mut drained_columns = Vec::with_capacity(self.columns.len()); - for col in &mut self.columns { - // DictEncoded columns are drained by swapping in a fresh Symbol - // placeholder. The flusher converts Symbol → DictEncoded during - // segment encoding once it has enough cardinality data. - let col_type = match col { - ColumnData::Timestamp(_) => ColumnType::Timestamp, - ColumnData::Float64(_) => ColumnType::Float64, - ColumnData::Int64(_) => ColumnType::Int64, - ColumnData::Symbol(_) => ColumnType::Symbol, - ColumnData::DictEncoded { .. } => ColumnType::Symbol, - }; - let mut empty = ColumnData::new(col_type); - std::mem::swap(col, &mut empty); - drained_columns.push(empty); - } - - let drained_dicts = std::mem::take(&mut self.symbol_dicts); - // Reinitialize symbol dicts. - for (i, (_, ty)) in self.schema.columns.iter().enumerate() { - if *ty == ColumnType::Symbol { - self.symbol_dicts.insert(i, SymbolDictionary::new()); - } - } - - // Scan `_ts_system` column (if present) for retention's system-time axis. - let max_system_ts = max_system_ts_of(&self.schema, &drained_columns); - - let result = ColumnarDrainResult { - columns: drained_columns, - schema: self.schema.clone(), - symbol_dicts: drained_dicts, - row_count: self.row_count, - min_ts: self.min_ts, - max_ts: self.max_ts, - max_system_ts, - series_row_counts: std::mem::take(&mut self.series_row_counts), - }; - - self.row_count = 0; - self.memory_bytes = 0; - self.min_ts = i64::MAX; - self.max_ts = i64::MIN; - - result - } - - // -- Mutators -- - - /// Truncate this memtable back to `n` rows. - /// - /// Used during transaction rollback to reverse a `TimeseriesIngest` operation. - /// All column vectors are truncated; aggregate stats are recomputed from the - /// surviving rows. `series_row_counts` is rebuilt from scratch so per-series - /// cardinality remains consistent. - pub fn truncate_to(&mut self, n: u64) { - if n >= self.row_count { - return; - } - let n_usize = n as usize; - let ts_idx = self.schema.timestamp_idx; - for col in &mut self.columns { - match col { - ColumnData::Timestamp(v) | ColumnData::Int64(v) => v.truncate(n_usize), - ColumnData::Float64(v) => v.truncate(n_usize), - ColumnData::Symbol(v) => v.truncate(n_usize), - ColumnData::DictEncoded { ids, valid, .. } => { - ids.truncate(n_usize); - valid.truncate(n_usize); - } - } - } - self.row_count = n; - // Recompute ts range from surviving timestamps. - if n == 0 { - self.min_ts = i64::MAX; - self.max_ts = i64::MIN; - self.series_row_counts.clear(); - } else if let ColumnData::Timestamp(ts) = &self.columns[ts_idx] { - self.min_ts = ts.iter().copied().min().unwrap_or(i64::MAX); - self.max_ts = ts.iter().copied().max().unwrap_or(i64::MIN); - } - // Recompute memory_bytes estimate by re-summing column capacities. - self.memory_bytes = self - .columns - .iter() - .map(|c| match c { - ColumnData::Timestamp(v) | ColumnData::Int64(v) => v.capacity() * 8, - ColumnData::Float64(v) => v.capacity() * 8, - ColumnData::Symbol(v) => v.capacity() * 4, - ColumnData::DictEncoded { - ids, - valid, - dictionary, - .. - } => ids.capacity() * 4 + valid.capacity() + dictionary.len() * 32, - }) - .sum(); - } - - // -- Accessors -- - - pub fn row_count(&self) -> u64 { - self.row_count - } - - /// Approximate memory usage. Uses incremental tracking with periodic - /// recomputation from column capacities for accuracy. - pub fn memory_bytes(&self) -> usize { - let col_bytes: usize = self.columns.iter().map(|c| c.memory_bytes()).sum(); - let dict_bytes: usize = self.symbol_dicts.len() * 256; // rough estimate - self.memory_bytes.max(col_bytes + dict_bytes) - } - - pub fn min_ts(&self) -> i64 { - self.min_ts - } - - pub fn max_ts(&self) -> i64 { - self.max_ts - } - - pub fn series_count(&self) -> usize { - self.series_row_counts.len() - } - - pub fn schema(&self) -> &ColumnarSchema { - &self.schema - } - - /// Return the immutable admission configuration used to construct this - /// memtable. Transaction undo uses this with a snapshot so restoration - /// preserves the original limits even if live operator tuning changed. - pub fn config(&self) -> ColumnarMemtableConfig { - self.config.clone() - } - - /// Restore the pre-transaction resident-byte accounting after replacing - /// the memtable from a logical snapshot. - /// - /// A snapshot preserves values and dictionaries, but rebuilding its vectors - /// intentionally does not preserve their spare capacity. Transaction undo - /// retains the original governor reservation, so it must also reinstate the - /// original reported footprint rather than silently undercounting it. - pub(crate) fn restore_memory_bytes_for_undo(&mut self, memory_bytes: usize) { - self.memory_bytes = memory_bytes; - } - - /// Export a lossless snapshot (carries column types + symbol dicts). INFALLIBLE. - pub fn export_snapshot(&self) -> MemtableSnapshot { - let columns: Vec<_> = self.columns.iter().map(column_to_snapshot).collect(); - - // Stable ordering for deterministic snapshots. - let mut symbol_dicts: Vec<(usize, SymbolDictionary)> = self - .symbol_dicts - .iter() - .map(|(&idx, dict)| (idx, dict.clone())) - .collect(); - symbol_dicts.sort_by_key(|(idx, _)| *idx); - - let mut series_row_counts: Vec<(SeriesId, u64)> = self - .series_row_counts - .iter() - .map(|(&k, &v)| (k, v)) - .collect(); - series_row_counts.sort_by_key(|(id, _)| *id); - - MemtableSnapshot { - schema_columns: self.schema.columns.clone(), - timestamp_idx: self.schema.timestamp_idx, - columns, - symbol_dicts, - series_row_counts, - row_count: self.row_count, - min_ts: self.min_ts, - max_ts: self.max_ts, - } - } - - /// Reconstruct a memtable from a snapshot. Returns a typed error on any - /// schema/row-count mismatch. - /// - /// `config` is not carried in the snapshot — it is operator tuning, not - /// data — so callers pass the live `ColumnarMemtableConfig::from_tuning` - /// value. A memtable keeps its limits for its whole life, so restoring with - /// compiled defaults would silently ignore the operator's configuration. - pub fn from_snapshot( - snap: MemtableSnapshot, - config: ColumnarMemtableConfig, - ) -> crate::Result { - if snap.timestamp_idx >= snap.schema_columns.len() { - return Err(crate::Error::BadRequest { - detail: format!( - "snapshot timestamp_idx {} out of range for {} columns", - snap.timestamp_idx, - snap.schema_columns.len(), - ), - }); - } - - let columns = rebuild_columns(snap.columns, &snap.schema_columns, snap.row_count)?; - let n = snap.schema_columns.len(); - // Codecs are ephemeral — not serialized; rebuild as Auto. - let schema = ColumnarSchema { - columns: snap.schema_columns, - timestamp_idx: snap.timestamp_idx, - codecs: vec![nodedb_codec::ColumnCodec::Auto; n], - }; - - let symbol_dicts: HashMap = - snap.symbol_dicts.into_iter().collect(); - let series_row_counts: HashMap = - snap.series_row_counts.into_iter().collect(); - let memory_bytes: usize = - columns.iter().map(|c| c.memory_bytes()).sum::() + symbol_dicts.len() * 256; - - Ok(Self { - schema, - columns, - series_row_counts, - symbol_dicts, - row_count: snap.row_count, - memory_bytes, - config, - min_ts: snap.min_ts, - max_ts: snap.max_ts, - }) - } - - pub fn column(&self, idx: usize) -> &ColumnData { - &self.columns[idx] - } - - pub fn symbol_dict(&self, col_idx: usize) -> Option<&SymbolDictionary> { - self.symbol_dicts.get(&col_idx) - } - - pub fn is_empty(&self) -> bool { - self.row_count == 0 - } - - /// Add a new column to the memtable schema, backfilling existing rows - /// with NULL-equivalent values. - /// - /// Used for ILP schema evolution: when a new field appears in a later - /// batch, the column is added and old rows get NaN/0/null-symbol. - pub fn add_column(&mut self, name: String, col_type: ColumnType) { - // Don't add duplicates. - if self.schema.columns.iter().any(|(n, _)| n == &name) { - return; - } - let existing_rows = self.row_count as usize; - let col = match col_type { - ColumnType::Float64 => ColumnData::Float64(vec![f64::NAN; existing_rows]), - ColumnType::Int64 => ColumnData::Int64(vec![0; existing_rows]), - ColumnType::Symbol => ColumnData::Symbol(vec![u32::MAX; existing_rows]), - ColumnType::Timestamp => return, // never add a second timestamp - }; - let idx = self.columns.len(); - self.columns.push(col); - self.schema.columns.push((name, col_type)); - self.schema.codecs.push(nodedb_codec::ColumnCodec::Auto); - if col_type == ColumnType::Symbol { - self.symbol_dicts.insert(idx, SymbolDictionary::new()); - } - } -} - -#[cfg(test)] -mod tests { - use nodedb_types::timeseries::MetricSample; - - use super::super::snapshot::ColumnSnapshot; - use super::*; - - fn default_config() -> ColumnarMemtableConfig { - ColumnarMemtableConfig { - max_memory_bytes: 1024 * 1024, - hard_memory_limit: 2 * 1024 * 1024, - max_tag_cardinality: 1000, - } - } - - #[test] - fn empty_memtable() { - let mt = ColumnarMemtable::new_metric(default_config()); - assert_eq!(mt.row_count(), 0); - assert!(mt.is_empty()); - assert_eq!(mt.series_count(), 0); - } - - #[test] - fn ingest_simple_metric() { - let mut mt = ColumnarMemtable::new_metric(default_config()); - let result = mt.ingest_metric( - 1, - MetricSample { - timestamp_ms: 1000, - value: 42.5, - }, - ); - assert_eq!(result, IngestResult::Ok); - assert_eq!(mt.row_count(), 1); - assert_eq!(mt.min_ts(), 1000); - assert_eq!(mt.max_ts(), 1000); - - let ts_col = mt.column(0).as_timestamps(); - assert_eq!(ts_col, &[1000]); - let val_col = mt.column(1).as_f64(); - assert!((val_col[0] - 42.5).abs() < f64::EPSILON); - } - - #[test] - fn ingest_multiple_metrics() { - let mut mt = ColumnarMemtable::new_metric(default_config()); - for i in 0..100 { - mt.ingest_metric( - i % 10, - MetricSample { - timestamp_ms: 1000 + i as i64, - value: i as f64, - }, - ); - } - assert_eq!(mt.row_count(), 100); - assert_eq!(mt.series_count(), 10); - assert_eq!(mt.min_ts(), 1000); - assert_eq!(mt.max_ts(), 1099); - } - - #[test] - fn ingest_row_with_tags() { - let schema = ColumnarSchema { - columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), - ("value".into(), ColumnType::Float64), - ("host".into(), ColumnType::Symbol), - ("dc".into(), ColumnType::Symbol), - ], - timestamp_idx: 0, - codecs: vec![nodedb_codec::ColumnCodec::Auto; 4], - }; - let mut mt = ColumnarMemtable::new(schema, default_config()); - - let result = mt.ingest_row( - 1, - &[ - ColumnValue::Timestamp(5000), - ColumnValue::Float64(99.9), - ColumnValue::Symbol("prod-1".to_string()), - ColumnValue::Symbol("us-east".to_string()), - ], - ); - assert!(result.is_ok()); - assert_eq!(mt.row_count(), 1); - - // Verify symbol dictionaries were populated. - let host_dict = mt.symbol_dict(2).unwrap(); - assert_eq!(host_dict.len(), 1); - assert_eq!(host_dict.get(0), Some("prod-1")); - - let dc_dict = mt.symbol_dict(3).unwrap(); - assert_eq!(dc_dict.get(0), Some("us-east")); - } - - #[test] - fn tag_cardinality_breaker() { - let schema = ColumnarSchema { - columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), - ("value".into(), ColumnType::Float64), - ("tag".into(), ColumnType::Symbol), - ], - timestamp_idx: 0, - codecs: vec![nodedb_codec::ColumnCodec::Auto; 3], - }; - let config = ColumnarMemtableConfig { - max_tag_cardinality: 5, - ..default_config() - }; - let mut mt = ColumnarMemtable::new(schema, config); - - // First 5 unique tags work. - for i in 0..5 { - let tag = format!("val-{i}"); - let r = mt.ingest_row( - i as u64, - &[ - ColumnValue::Timestamp(1000 + i as i64), - ColumnValue::Float64(1.0), - ColumnValue::Symbol(tag.clone()), - ], - ); - assert!(r.is_ok()); - } - assert_eq!(mt.row_count(), 5); - - // 6th unique tag is rejected. - let r = mt.ingest_row( - 99, - &[ - ColumnValue::Timestamp(2000), - ColumnValue::Float64(1.0), - ColumnValue::Symbol("one-too-many".to_string()), - ], - ); - assert!(r.is_err()); - // Row count didn't increase (rolled back). - assert_eq!(mt.row_count(), 5); - } - - #[test] - fn drain_returns_data_and_resets() { - let mut mt = ColumnarMemtable::new_metric(default_config()); - for i in 0..50 { - mt.ingest_metric( - 1, - MetricSample { - timestamp_ms: 1000 + i, - value: i as f64, - }, - ); - } - assert_eq!(mt.row_count(), 50); - - let result = mt.drain(); - assert_eq!(result.row_count, 50); - assert_eq!(result.min_ts, 1000); - assert_eq!(result.max_ts, 1049); - assert_eq!(result.columns.len(), 2); - assert_eq!(result.columns[0].len(), 50); - assert_eq!(result.columns[1].len(), 50); - - // Memtable is reset. - assert_eq!(mt.row_count(), 0); - assert!(mt.is_empty()); - } - - #[test] - fn ingest_metric_accepts_past_hard_limit() { - // A sample reaching the memtable belongs to an already-committed WAL - // record; refusing it would silently drop a durable write on replay. - // So `ingest_metric` accepts every sample regardless of the ceiling — - // it never returns `Rejected`, and the resident footprint overshoots - // the hard limit rather than losing data. - let config = ColumnarMemtableConfig { - max_memory_bytes: 100, - hard_memory_limit: 200, - max_tag_cardinality: 1000, - }; - let mut mt = ColumnarMemtable::new_metric(config); - - for i in 0..1000 { - let r = mt.ingest_metric( - 1, - MetricSample { - timestamp_ms: i, - value: 1.0, - }, - ); - assert_ne!( - r, - IngestResult::Rejected, - "sample {i} must not be rejected — that would drop a durable record on replay" - ); - } - assert_eq!( - mt.row_count(), - 1000, - "every sample past the limit is retained" - ); - assert!( - mt.memory_bytes() >= 200, - "the footprint is allowed to overshoot the hard limit" - ); - } - - #[test] - fn flush_needed_signal() { - let config = ColumnarMemtableConfig { - max_memory_bytes: 100, - hard_memory_limit: 200, - max_tag_cardinality: 1000, - }; - let mut mt = ColumnarMemtable::new_metric(config); - - let mut flush_signaled = false; - for i in 0..100 { - let r = mt.ingest_metric( - 1, - MetricSample { - timestamp_ms: i, - value: 1.0, - }, - ); - if r == IngestResult::FlushNeeded { - flush_signaled = true; - break; - } - } - assert!(flush_signaled); - } - - #[test] - fn type_mismatch_rejected() { - let schema = ColumnarSchema { - columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), - ("value".into(), ColumnType::Float64), - ], - timestamp_idx: 0, - codecs: vec![nodedb_codec::ColumnCodec::Auto; 2], - }; - let mut mt = ColumnarMemtable::new(schema, default_config()); - - let r = mt.ingest_row( - 1, - &[ - ColumnValue::Timestamp(1000), - ColumnValue::Int64(42), // Wrong: schema says Float64 - ], - ); - assert!(r.is_err()); - assert_eq!(mt.row_count(), 0); // Rolled back. - } - - // ----------------------------------------------------------------------- - // Snapshot round-trip tests - // ----------------------------------------------------------------------- - - #[test] - fn snapshot_roundtrip_empty_memtable() { - let mt = ColumnarMemtable::new_metric(default_config()); - let snap = mt.export_snapshot(); - let bytes = zerompk::to_msgpack_vec(&snap).expect("serialize"); - let snap2: MemtableSnapshot = zerompk::from_msgpack(&bytes).expect("deserialize"); - let mt2 = ColumnarMemtable::from_snapshot(snap2, default_config()).expect("from_snapshot"); - assert_eq!(mt2.row_count(), 0); - assert!(mt2.is_empty()); - assert_eq!(mt2.schema().columns.len(), 2); - assert_eq!(mt2.schema().timestamp_idx, 0); - } - - #[test] - fn snapshot_roundtrip_multi_column_with_symbol_tags() { - let schema = ColumnarSchema { - columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), - ("value".into(), ColumnType::Float64), - ("host".into(), ColumnType::Symbol), - ], - timestamp_idx: 0, - codecs: vec![nodedb_codec::ColumnCodec::Auto; 3], - }; - let mut mt = ColumnarMemtable::new(schema, default_config()); - - for i in 0..5u64 { - mt.ingest_row( - i, - &[ - ColumnValue::Timestamp(1000 + i as i64), - ColumnValue::Float64(i as f64 * 1.5), - ColumnValue::Symbol(format!("host-{i}")), - ], - ) - .expect("ingest"); - } - // Also re-use an existing host to verify symbol dict cardinality. - mt.ingest_row( - 99, - &[ - ColumnValue::Timestamp(2000), - ColumnValue::Float64(99.0), - ColumnValue::Symbol("host-0".to_string()), - ], - ) - .expect("ingest existing host"); - - let expected_row_count = mt.row_count(); - let expected_min_ts = mt.min_ts(); - let expected_max_ts = mt.max_ts(); - let expected_schema_cols: Vec<(String, ColumnType)> = mt.schema().columns.clone(); - let expected_ts_idx = mt.schema().timestamp_idx; - let expected_dict_len = mt.symbol_dict(2).map(|d| d.len()).unwrap_or(0); - - let snap = mt.export_snapshot(); - let bytes = zerompk::to_msgpack_vec(&snap).expect("serialize"); - let snap2: MemtableSnapshot = zerompk::from_msgpack(&bytes).expect("deserialize"); - let mt2 = ColumnarMemtable::from_snapshot(snap2, default_config()).expect("from_snapshot"); - - assert_eq!(mt2.row_count(), expected_row_count); - assert_eq!(mt2.min_ts(), expected_min_ts); - assert_eq!(mt2.max_ts(), expected_max_ts); - assert_eq!(mt2.schema().columns, expected_schema_cols); - assert_eq!(mt2.schema().timestamp_idx, expected_ts_idx); - - // Verify symbol dict was faithfully restored. - let dict2 = mt2.symbol_dict(2).expect("host dict present"); - assert_eq!(dict2.len(), expected_dict_len); - assert_eq!(dict2.get(0), Some("host-0")); - - // Verify column data lengths match. - for i in 0..3 { - assert_eq!(mt2.column(i).len(), expected_row_count as usize); - } - } - - #[test] - fn snapshot_roundtrip_dict_encoded_column_rebuilds_reverse() { - // Construct a DictEncoded snapshot directly (bypass ingest path). - let snap = MemtableSnapshot { - schema_columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), - ("tag".into(), ColumnType::Symbol), - ], - timestamp_idx: 0, - columns: vec![ - ColumnSnapshot::Timestamp(vec![1000, 2000, 3000]), - ColumnSnapshot::DictEncoded { - ids: vec![0, 1, 0], - dictionary: vec!["alpha".to_string(), "beta".to_string()], - valid: vec![true, true, true], - }, - ], - symbol_dicts: vec![], - series_row_counts: vec![], - row_count: 3, - min_ts: 1000, - max_ts: 3000, - }; - - let mt = ColumnarMemtable::from_snapshot(snap, default_config()).expect("from_snapshot"); - assert_eq!(mt.row_count(), 3); - // Verify the DictEncoded column has correct data. - match mt.column(1) { - ColumnData::DictEncoded { - ids, - dictionary, - reverse, - valid, - } => { - assert_eq!(ids, &[0u32, 1, 0]); - assert_eq!(dictionary, &["alpha", "beta"]); - assert_eq!(reverse.get("alpha"), Some(&0u32)); - assert_eq!(reverse.get("beta"), Some(&1u32)); - assert_eq!(valid, &[true, true, true]); - } - other => panic!("expected DictEncoded, got {:?}", other), - } - } - - #[test] - fn snapshot_from_invalid_column_lengths_returns_error() { - // row_count says 3 but timestamp column has only 2 rows → mismatch. - let snap = MemtableSnapshot { - schema_columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), - ("value".into(), ColumnType::Float64), - ], - timestamp_idx: 0, - columns: vec![ - ColumnSnapshot::Timestamp(vec![1000, 2000]), // 2 rows - ColumnSnapshot::Float64(vec![1.0, 2.0, 3.0]), // 3 rows — mismatch - ], - symbol_dicts: vec![], - series_row_counts: vec![], - row_count: 3, - min_ts: 1000, - max_ts: 2000, - }; - let result = ColumnarMemtable::from_snapshot(snap, default_config()); - assert!( - matches!(result, Err(crate::Error::BadRequest { .. })), - "expected BadRequest error on length mismatch" - ); - } -} diff --git a/nodedb/src/engine/timeseries/columnar_memtable/memtable/drain.rs b/nodedb/src/engine/timeseries/columnar_memtable/memtable/drain.rs new file mode 100644 index 000000000..129f942a1 --- /dev/null +++ b/nodedb/src/engine/timeseries/columnar_memtable/memtable/drain.rs @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Taking rows out of a [`ColumnarMemtable`]: flush view, drain, truncate. + +use nodedb_types::timeseries::SymbolDictionary; + +use super::super::types::{ + ColumnData, ColumnType, ColumnarDrainResult, ColumnarFlushView, max_system_ts_of, +}; +use super::table::ColumnarMemtable; + +impl ColumnarMemtable { + /// Borrow this memtable's live rows as the payload a flush would write. + /// + /// The read-only half of a flush: a segment can be encoded and landed from + /// this view, and only then does [`Self::drain`] take the rows out. Nothing + /// leaves memory before it is durable, so a failed segment write leaves the + /// memtable exactly as it was. + pub fn flush_view(&self) -> ColumnarFlushView<'_> { + ColumnarFlushView { + columns: &self.columns, + schema: &self.schema, + symbol_dicts: &self.symbol_dicts, + row_count: self.row_count, + min_ts: self.min_ts, + max_ts: self.max_ts, + max_system_ts: max_system_ts_of(&self.schema, &self.columns), + } + } + + /// Drain all data from the memtable, resetting it for reuse. + /// + /// Returns the column data, schema, symbol dicts, and stats. + /// + /// Callers that flush must write the segment from [`Self::flush_view`] FIRST + /// and drain only once that write has committed — the rows here have no + /// other copy but the WAL, and the checkpoint that calls the flush is what + /// authorises deleting it. + pub fn drain(&mut self) -> ColumnarDrainResult { + let mut drained_columns = Vec::with_capacity(self.columns.len()); + for (col, (_, schema_type)) in self.columns.iter_mut().zip(self.schema.columns.iter()) { + // DictEncoded columns are drained by swapping in a fresh Symbol + // placeholder. The flusher converts Symbol → DictEncoded during + // segment encoding once it has enough cardinality data. + let col_type = match col { + ColumnData::Timestamp(_) => *schema_type, + ColumnData::Float64(_) => ColumnType::Float64, + ColumnData::Int64(_) => ColumnType::Int64, + ColumnData::Symbol(_) => ColumnType::Symbol, + ColumnData::DictEncoded { .. } => ColumnType::Symbol, + }; + let mut empty = ColumnData::new(col_type); + std::mem::swap(col, &mut empty); + drained_columns.push(empty); + } + + let drained_dicts = std::mem::take(&mut self.symbol_dicts); + // Reinitialize symbol dicts. + for (i, (_, ty)) in self.schema.columns.iter().enumerate() { + if *ty == ColumnType::Symbol { + self.symbol_dicts.insert(i, SymbolDictionary::new()); + } + } + + // Scan `_ts_system` column (if present) for retention's system-time axis. + let max_system_ts = max_system_ts_of(&self.schema, &drained_columns); + + let result = ColumnarDrainResult { + columns: drained_columns, + schema: self.schema.clone(), + symbol_dicts: drained_dicts, + row_count: self.row_count, + min_ts: self.min_ts, + max_ts: self.max_ts, + max_system_ts, + series_row_counts: std::mem::take(&mut self.series_row_counts), + }; + + self.row_count = 0; + self.memory_bytes = 0; + self.min_ts = i64::MAX; + self.max_ts = i64::MIN; + + result + } + + /// Truncate this memtable back to `n` rows. + /// + /// Used during transaction rollback to reverse a `TimeseriesIngest` operation. + /// All column vectors are truncated; aggregate stats are recomputed from the + /// surviving rows. `series_row_counts` is rebuilt from scratch so per-series + /// cardinality remains consistent. + pub fn truncate_to(&mut self, n: u64) { + if n >= self.row_count { + return; + } + let n_usize = n as usize; + let ts_idx = self.schema.timestamp_idx; + for col in &mut self.columns { + match col { + ColumnData::Timestamp(v) | ColumnData::Int64(v) => v.truncate(n_usize), + ColumnData::Float64(v) => v.truncate(n_usize), + ColumnData::Symbol(v) => v.truncate(n_usize), + ColumnData::DictEncoded { ids, valid, .. } => { + ids.truncate(n_usize); + valid.truncate(n_usize); + } + } + } + self.row_count = n; + // Recompute ts range from surviving timestamps. + if n == 0 { + self.min_ts = i64::MAX; + self.max_ts = i64::MIN; + self.series_row_counts.clear(); + } else if let ColumnData::Timestamp(ts) = &self.columns[ts_idx] { + self.min_ts = ts.iter().copied().min().unwrap_or(i64::MAX); + self.max_ts = ts.iter().copied().max().unwrap_or(i64::MIN); + } + // Recompute memory_bytes estimate by re-summing column capacities. + self.memory_bytes = self + .columns + .iter() + .map(|c| match c { + ColumnData::Timestamp(v) | ColumnData::Int64(v) => v.capacity() * 8, + ColumnData::Float64(v) => v.capacity() * 8, + ColumnData::Symbol(v) => v.capacity() * 4, + ColumnData::DictEncoded { + ids, + valid, + dictionary, + .. + } => ids.capacity() * 4 + valid.capacity() + dictionary.len() * 32, + }) + .sum(); + } +} + +#[cfg(test)] +mod tests { + use nodedb_types::timeseries::MetricSample; + + use super::super::super::types::ColumnarMemtableConfig; + use super::*; + + fn default_config() -> ColumnarMemtableConfig { + ColumnarMemtableConfig { + max_memory_bytes: 1024 * 1024, + hard_memory_limit: 2 * 1024 * 1024, + max_tag_cardinality: 1000, + } + } + + #[test] + fn drain_returns_data_and_resets() { + let mut mt = ColumnarMemtable::new_metric(default_config()); + for i in 0..50 { + mt.ingest_metric( + 1, + MetricSample { + timestamp_ms: 1000 + i, + value: i as f64, + }, + ); + } + assert_eq!(mt.row_count(), 50); + + let result = mt.drain(); + assert_eq!(result.row_count, 50); + assert_eq!(result.min_ts, 1000); + assert_eq!(result.max_ts, 1049); + assert_eq!(result.columns.len(), 2); + assert_eq!(result.columns[0].len(), 50); + assert_eq!(result.columns[1].len(), 50); + + // Memtable is reset. + assert_eq!(mt.row_count(), 0); + assert!(mt.is_empty()); + } +} diff --git a/nodedb/src/engine/timeseries/columnar_memtable/memtable/evolve.rs b/nodedb/src/engine/timeseries/columnar_memtable/memtable/evolve.rs new file mode 100644 index 000000000..e712b46a2 --- /dev/null +++ b/nodedb/src/engine/timeseries/columnar_memtable/memtable/evolve.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Schema evolution on a live [`ColumnarMemtable`]. + +use nodedb_types::timeseries::SymbolDictionary; + +use super::super::types::{ColumnData, ColumnType}; +use super::table::ColumnarMemtable; + +impl ColumnarMemtable { + /// Add a new column to the memtable schema, backfilling existing rows + /// with NULL-equivalent values. + /// + /// Used for ILP schema evolution: when a new field appears in a later + /// batch, the column is added and old rows get NaN/0/null-symbol. + pub fn add_column(&mut self, name: String, col_type: ColumnType) { + // Don't add duplicates. + if self.schema.columns.iter().any(|(n, _)| n == &name) { + return; + } + let existing_rows = self.row_count as usize; + let col = match col_type { + ColumnType::Float64 => ColumnData::Float64(vec![f64::NAN; existing_rows]), + ColumnType::Int64 => ColumnData::Int64(vec![0; existing_rows]), + ColumnType::Symbol => ColumnData::Symbol(vec![u32::MAX; existing_rows]), + ColumnType::Timestamp(_) => return, // never add a second timestamp + }; + let idx = self.columns.len(); + self.columns.push(col); + self.schema.columns.push((name, col_type)); + self.schema.codecs.push(nodedb_codec::ColumnCodec::Auto); + if col_type == ColumnType::Symbol { + self.symbol_dicts.insert(idx, SymbolDictionary::new()); + } + } +} diff --git a/nodedb/src/engine/timeseries/columnar_memtable/memtable/ingest.rs b/nodedb/src/engine/timeseries/columnar_memtable/memtable/ingest.rs new file mode 100644 index 000000000..ab45c7e2d --- /dev/null +++ b/nodedb/src/engine/timeseries/columnar_memtable/memtable/ingest.rs @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Row and sample ingest into a [`ColumnarMemtable`]. + +use nodedb_types::timeseries::{IngestResult, MetricSample, SeriesId}; + +use super::super::types::{ColumnData, ColumnType, ColumnValue}; +use super::table::ColumnarMemtable; + +impl ColumnarMemtable { + /// Ingest a metric sample into the default (timestamp, value) layout. + /// + /// For the simple 2-column schema. For multi-column schemas with tags, + /// use `ingest_row()` instead. + /// + /// Does NOT enforce `hard_memory_limit`, for the same reason `ingest_row` + /// does not (see its doc): the sole production caller is WAL replay + /// (`replay_timeseries_payload`), and a sample reaching here belongs to a + /// record that has ALREADY COMMITTED, so refusing it is not backpressure — + /// it is silent loss of a durable write. Replay must take the record whole; + /// the ceiling lives at the record boundary in the live ingest handler's + /// admission gate, never here. NOTE: if a structured-`TimeseriesWalBatch` + /// ingest producer is ever added, its replay must gain that same + /// record-boundary flush, or a mid-record replay flush would stamp the + /// partition with an LSN covering rows it does not hold. + /// `replay_timeseries_wal` advances `ts_max_ingested_lsn` only AFTER a + /// record is applied, which is what keeps that stamp honest. + pub fn ingest_metric(&mut self, series_id: SeriesId, sample: MetricSample) -> IngestResult { + // Push to timestamp column. + if let ColumnData::Timestamp(ref mut v) = self.columns[self.schema.timestamp_idx] { + v.push(sample.timestamp_ms); + } + + // Push to value column (assume index 1 for default schema). + if self.columns.len() > 1 + && let ColumnData::Float64(ref mut v) = self.columns[1] + { + v.push(sample.value); + } + + self.update_stats(series_id, sample.timestamp_ms, 16); + self.check_flush_state() + } + + /// Ingest a row with explicit column values. + /// + /// `values` must match the schema length. Tag string values are resolved + /// to symbol IDs via the per-column dictionary. + /// + /// ## Why this does NOT enforce `hard_memory_limit` + /// + /// Every row reaching here belongs to a WAL record that has ALREADY + /// COMMITTED, so refusing it is not backpressure — it is silent loss of a + /// durable write. A refusal rescued by flushing and re-ingesting + /// mid-record stamps the flushed partition with the PREVIOUS record's LSN + /// while it holds part of the current one; replay, gated on that stamp, + /// re-appends the whole record on top. + /// + /// The ceiling lives at the record boundary instead (the ingest handler's + /// admission gate flushes BEFORE a record when the memtable is at or over + /// it), so a flush always lands on a whole-record prefix and the + /// partition's stamp is true of every row in it. Errors returned here are + /// therefore genuine per-row data faults — bad arity, type mismatch, + /// exhausted tag dictionary — never "come back later". + pub fn ingest_row( + &mut self, + series_id: SeriesId, + values: &[ColumnValue], + ) -> crate::Result { + let col_types: Vec<(String, ColumnType)> = self.schema.columns.clone(); + + if values.len() != col_types.len() { + return Err(crate::Error::BadRequest { + detail: format!("expected {} columns, got {}", col_types.len(), values.len()), + }); + } + + let mut ts = 0i64; + let mut row_bytes = 0usize; + let max_card = self.config.max_tag_cardinality; + + for (i, (val, (col_name, col_type))) in values.iter().zip(col_types.iter()).enumerate() { + match (val, col_type) { + (ColumnValue::Timestamp(t), ColumnType::Timestamp(_)) => { + if let ColumnData::Timestamp(ref mut v) = self.columns[i] { + v.push(*t); + } + ts = *t; + row_bytes += 8; + } + (ColumnValue::Float64(f), ColumnType::Float64) => { + if let ColumnData::Float64(ref mut v) = self.columns[i] { + v.push(*f); + } + row_bytes += 8; + } + (ColumnValue::Int64(n), ColumnType::Int64) => { + if let ColumnData::Int64(ref mut v) = self.columns[i] { + v.push(*n); + } + row_bytes += 8; + } + (ColumnValue::Symbol(s), ColumnType::Symbol) => { + let dict = + self.symbol_dicts + .get_mut(&i) + .ok_or_else(|| crate::Error::BadRequest { + detail: format!( + "internal error: symbol dict missing for column {i}" + ), + })?; + match dict.resolve(s, max_card) { + Some(sym_id) => { + if let ColumnData::Symbol(ref mut v) = self.columns[i] { + v.push(sym_id); + } + } + None => { + self.rollback_partial_row(i); + return Err(crate::Error::BadRequest { + detail: format!( + "tag cardinality limit ({max_card}) exceeded for column '{col_name}'" + ), + }); + } + } + row_bytes += 4; + } + _ => { + self.rollback_partial_row(i); + return Err(crate::Error::BadRequest { + detail: format!("type mismatch at column {i}: expected {col_type:?}"), + }); + } + } + } + + self.update_stats(series_id, ts, row_bytes); + Ok(self.check_flush_state()) + } + + /// Roll back a partially written row (called on error during `ingest_row`). + fn rollback_partial_row(&mut self, columns_written: usize) { + for col in self.columns.iter_mut().take(columns_written) { + match col { + ColumnData::Timestamp(v) => { + v.pop(); + } + ColumnData::Float64(v) => { + v.pop(); + } + ColumnData::Int64(v) => { + v.pop(); + } + ColumnData::Symbol(v) => { + v.pop(); + } + ColumnData::DictEncoded { ids, valid, .. } => { + ids.pop(); + valid.pop(); + } + } + } + } + + fn update_stats(&mut self, series_id: SeriesId, ts: i64, row_bytes: usize) { + *self.series_row_counts.entry(series_id).or_insert(0) += 1; + self.row_count += 1; + self.memory_bytes += row_bytes; + if ts < self.min_ts { + self.min_ts = ts; + } + if ts > self.max_ts { + self.max_ts = ts; + } + } + + fn check_flush_state(&self) -> IngestResult { + if self.memory_bytes >= self.config.max_memory_bytes { + IngestResult::FlushNeeded + } else { + IngestResult::Ok + } + } +} + +#[cfg(test)] +mod tests { + use super::super::super::types::{ColumnarMemtableConfig, ColumnarSchema, TimeKind}; + use super::*; + + fn default_config() -> ColumnarMemtableConfig { + ColumnarMemtableConfig { + max_memory_bytes: 1024 * 1024, + hard_memory_limit: 2 * 1024 * 1024, + max_tag_cardinality: 1000, + } + } + + #[test] + fn ingest_simple_metric() { + let mut mt = ColumnarMemtable::new_metric(default_config()); + let result = mt.ingest_metric( + 1, + MetricSample { + timestamp_ms: 1000, + value: 42.5, + }, + ); + assert_eq!(result, IngestResult::Ok); + assert_eq!(mt.row_count(), 1); + assert_eq!(mt.min_ts(), 1000); + assert_eq!(mt.max_ts(), 1000); + + let ts_col = mt.column(0).as_timestamps(); + assert_eq!(ts_col, &[1000]); + let val_col = mt.column(1).as_f64(); + assert!((val_col[0] - 42.5).abs() < f64::EPSILON); + } + + #[test] + fn ingest_multiple_metrics() { + let mut mt = ColumnarMemtable::new_metric(default_config()); + for i in 0..100 { + mt.ingest_metric( + i % 10, + MetricSample { + timestamp_ms: 1000 + i as i64, + value: i as f64, + }, + ); + } + assert_eq!(mt.row_count(), 100); + assert_eq!(mt.series_count(), 10); + assert_eq!(mt.min_ts(), 1000); + assert_eq!(mt.max_ts(), 1099); + } + + #[test] + fn ingest_row_with_tags() { + let schema = ColumnarSchema { + columns: vec![ + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), + ("value".into(), ColumnType::Float64), + ("host".into(), ColumnType::Symbol), + ("dc".into(), ColumnType::Symbol), + ], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 4], + }; + let mut mt = ColumnarMemtable::new(schema, default_config()); + + let result = mt.ingest_row( + 1, + &[ + ColumnValue::Timestamp(5000), + ColumnValue::Float64(99.9), + ColumnValue::Symbol("prod-1".to_string()), + ColumnValue::Symbol("us-east".to_string()), + ], + ); + assert!(result.is_ok()); + assert_eq!(mt.row_count(), 1); + + // Verify symbol dictionaries were populated. + let host_dict = mt.symbol_dict(2).unwrap(); + assert_eq!(host_dict.len(), 1); + assert_eq!(host_dict.get(0), Some("prod-1")); + + let dc_dict = mt.symbol_dict(3).unwrap(); + assert_eq!(dc_dict.get(0), Some("us-east")); + } + + #[test] + fn tag_cardinality_breaker() { + let schema = ColumnarSchema { + columns: vec![ + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), + ("value".into(), ColumnType::Float64), + ("tag".into(), ColumnType::Symbol), + ], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 3], + }; + let config = ColumnarMemtableConfig { + max_tag_cardinality: 5, + ..default_config() + }; + let mut mt = ColumnarMemtable::new(schema, config); + + // First 5 unique tags work. + for i in 0..5 { + let tag = format!("val-{i}"); + let r = mt.ingest_row( + i as u64, + &[ + ColumnValue::Timestamp(1000 + i as i64), + ColumnValue::Float64(1.0), + ColumnValue::Symbol(tag.clone()), + ], + ); + assert!(r.is_ok()); + } + assert_eq!(mt.row_count(), 5); + + // 6th unique tag is rejected. + let r = mt.ingest_row( + 99, + &[ + ColumnValue::Timestamp(2000), + ColumnValue::Float64(1.0), + ColumnValue::Symbol("one-too-many".to_string()), + ], + ); + assert!(r.is_err()); + // Row count didn't increase (rolled back). + assert_eq!(mt.row_count(), 5); + } + + #[test] + fn ingest_metric_accepts_past_hard_limit() { + // A sample reaching the memtable belongs to an already-committed WAL + // record; refusing it would silently drop a durable write on replay. + // So `ingest_metric` accepts every sample regardless of the ceiling — + // it never returns `Rejected`, and the resident footprint overshoots + // the hard limit rather than losing data. + let config = ColumnarMemtableConfig { + max_memory_bytes: 100, + hard_memory_limit: 200, + max_tag_cardinality: 1000, + }; + let mut mt = ColumnarMemtable::new_metric(config); + + for i in 0..1000 { + let r = mt.ingest_metric( + 1, + MetricSample { + timestamp_ms: i, + value: 1.0, + }, + ); + assert_ne!( + r, + IngestResult::Rejected, + "sample {i} must not be rejected — that would drop a durable record on replay" + ); + } + assert_eq!( + mt.row_count(), + 1000, + "every sample past the limit is retained" + ); + assert!( + mt.memory_bytes() >= 200, + "the footprint is allowed to overshoot the hard limit" + ); + } + + #[test] + fn flush_needed_signal() { + let config = ColumnarMemtableConfig { + max_memory_bytes: 100, + hard_memory_limit: 200, + max_tag_cardinality: 1000, + }; + let mut mt = ColumnarMemtable::new_metric(config); + + let mut flush_signaled = false; + for i in 0..100 { + let r = mt.ingest_metric( + 1, + MetricSample { + timestamp_ms: i, + value: 1.0, + }, + ); + if r == IngestResult::FlushNeeded { + flush_signaled = true; + break; + } + } + assert!(flush_signaled); + } + + #[test] + fn type_mismatch_rejected() { + let schema = ColumnarSchema { + columns: vec![ + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), + ("value".into(), ColumnType::Float64), + ], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 2], + }; + let mut mt = ColumnarMemtable::new(schema, default_config()); + + let r = mt.ingest_row( + 1, + &[ + ColumnValue::Timestamp(1000), + ColumnValue::Int64(42), // Wrong: schema says Float64 + ], + ); + assert!(r.is_err()); + assert_eq!(mt.row_count(), 0); // Rolled back. + } +} diff --git a/nodedb/src/engine/timeseries/columnar_memtable/memtable/mod.rs b/nodedb/src/engine/timeseries/columnar_memtable/memtable/mod.rs new file mode 100644 index 000000000..462d96b53 --- /dev/null +++ b/nodedb/src/engine/timeseries/columnar_memtable/memtable/mod.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BUSL-1.1 + +mod drain; +mod evolve; +mod ingest; +mod snapshot_io; +mod table; + +pub use table::ColumnarMemtable; diff --git a/nodedb/src/engine/timeseries/columnar_memtable/memtable/snapshot_io.rs b/nodedb/src/engine/timeseries/columnar_memtable/memtable/snapshot_io.rs new file mode 100644 index 000000000..8557b6370 --- /dev/null +++ b/nodedb/src/engine/timeseries/columnar_memtable/memtable/snapshot_io.rs @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Lossless snapshot export and import for a [`ColumnarMemtable`]. + +use std::collections::HashMap; + +use nodedb_types::timeseries::{SeriesId, SymbolDictionary}; + +use super::super::snapshot::{MemtableSnapshot, column_to_snapshot, rebuild_columns}; +use super::super::types::{ColumnarMemtableConfig, ColumnarSchema}; +use super::table::ColumnarMemtable; + +impl ColumnarMemtable { + /// Restore the pre-transaction resident-byte accounting after replacing + /// the memtable from a logical snapshot. + /// + /// A snapshot preserves values and dictionaries, but rebuilding its vectors + /// intentionally does not preserve their spare capacity. Transaction undo + /// retains the original governor reservation, so it must also reinstate the + /// original reported footprint rather than silently undercounting it. + pub(crate) fn restore_memory_bytes_for_undo(&mut self, memory_bytes: usize) { + self.memory_bytes = memory_bytes; + } + + /// Export a lossless snapshot (carries column types + symbol dicts). INFALLIBLE. + pub fn export_snapshot(&self) -> MemtableSnapshot { + let columns: Vec<_> = self.columns.iter().map(column_to_snapshot).collect(); + + // Stable ordering for deterministic snapshots. + let mut symbol_dicts: Vec<(usize, SymbolDictionary)> = self + .symbol_dicts + .iter() + .map(|(&idx, dict)| (idx, dict.clone())) + .collect(); + symbol_dicts.sort_by_key(|(idx, _)| *idx); + + let mut series_row_counts: Vec<(SeriesId, u64)> = self + .series_row_counts + .iter() + .map(|(&k, &v)| (k, v)) + .collect(); + series_row_counts.sort_by_key(|(id, _)| *id); + + MemtableSnapshot { + schema_columns: self.schema.columns.clone(), + timestamp_idx: self.schema.timestamp_idx, + columns, + symbol_dicts, + series_row_counts, + row_count: self.row_count, + min_ts: self.min_ts, + max_ts: self.max_ts, + } + } + + /// Reconstruct a memtable from a snapshot. Returns a typed error on any + /// schema/row-count mismatch. + /// + /// `config` is not carried in the snapshot — it is operator tuning, not + /// data — so callers pass the live `ColumnarMemtableConfig::from_tuning` + /// value. A memtable keeps its limits for its whole life, so restoring with + /// compiled defaults would silently ignore the operator's configuration. + pub fn from_snapshot( + snap: MemtableSnapshot, + config: ColumnarMemtableConfig, + ) -> crate::Result { + if snap.timestamp_idx >= snap.schema_columns.len() { + return Err(crate::Error::BadRequest { + detail: format!( + "snapshot timestamp_idx {} out of range for {} columns", + snap.timestamp_idx, + snap.schema_columns.len(), + ), + }); + } + + let columns = rebuild_columns(snap.columns, &snap.schema_columns, snap.row_count)?; + let n = snap.schema_columns.len(); + // Codecs are ephemeral — not serialized; rebuild as Auto. + let schema = ColumnarSchema { + columns: snap.schema_columns, + timestamp_idx: snap.timestamp_idx, + codecs: vec![nodedb_codec::ColumnCodec::Auto; n], + }; + + let symbol_dicts: HashMap = + snap.symbol_dicts.into_iter().collect(); + let series_row_counts: HashMap = + snap.series_row_counts.into_iter().collect(); + let memory_bytes: usize = + columns.iter().map(|c| c.memory_bytes()).sum::() + symbol_dicts.len() * 256; + + Ok(Self { + schema, + columns, + series_row_counts, + symbol_dicts, + row_count: snap.row_count, + memory_bytes, + config, + min_ts: snap.min_ts, + max_ts: snap.max_ts, + }) + } +} + +#[cfg(test)] +mod tests { + use super::super::super::snapshot::ColumnSnapshot; + use super::super::super::types::{ColumnData, ColumnType, ColumnValue, TimeKind}; + use super::*; + + fn default_config() -> ColumnarMemtableConfig { + ColumnarMemtableConfig { + max_memory_bytes: 1024 * 1024, + hard_memory_limit: 2 * 1024 * 1024, + max_tag_cardinality: 1000, + } + } + + #[test] + fn snapshot_roundtrip_empty_memtable() { + let mt = ColumnarMemtable::new_metric(default_config()); + let snap = mt.export_snapshot(); + let bytes = zerompk::to_msgpack_vec(&snap).expect("serialize"); + let snap2: MemtableSnapshot = zerompk::from_msgpack(&bytes).expect("deserialize"); + let mt2 = ColumnarMemtable::from_snapshot(snap2, default_config()).expect("from_snapshot"); + assert_eq!(mt2.row_count(), 0); + assert!(mt2.is_empty()); + assert_eq!(mt2.schema().columns.len(), 2); + assert_eq!(mt2.schema().timestamp_idx, 0); + } + + #[test] + fn snapshot_roundtrip_multi_column_with_symbol_tags() { + let schema = ColumnarSchema { + columns: vec![ + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), + ("value".into(), ColumnType::Float64), + ("host".into(), ColumnType::Symbol), + ], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 3], + }; + let mut mt = ColumnarMemtable::new(schema, default_config()); + + for i in 0..5u64 { + mt.ingest_row( + i, + &[ + ColumnValue::Timestamp(1000 + i as i64), + ColumnValue::Float64(i as f64 * 1.5), + ColumnValue::Symbol(format!("host-{i}")), + ], + ) + .expect("ingest"); + } + // Also re-use an existing host to verify symbol dict cardinality. + mt.ingest_row( + 99, + &[ + ColumnValue::Timestamp(2000), + ColumnValue::Float64(99.0), + ColumnValue::Symbol("host-0".to_string()), + ], + ) + .expect("ingest existing host"); + + let expected_row_count = mt.row_count(); + let expected_min_ts = mt.min_ts(); + let expected_max_ts = mt.max_ts(); + let expected_schema_cols: Vec<(String, ColumnType)> = mt.schema().columns.clone(); + let expected_ts_idx = mt.schema().timestamp_idx; + let expected_dict_len = mt.symbol_dict(2).map(|d| d.len()).unwrap_or(0); + + let snap = mt.export_snapshot(); + let bytes = zerompk::to_msgpack_vec(&snap).expect("serialize"); + let snap2: MemtableSnapshot = zerompk::from_msgpack(&bytes).expect("deserialize"); + let mt2 = ColumnarMemtable::from_snapshot(snap2, default_config()).expect("from_snapshot"); + + assert_eq!(mt2.row_count(), expected_row_count); + assert_eq!(mt2.min_ts(), expected_min_ts); + assert_eq!(mt2.max_ts(), expected_max_ts); + assert_eq!(mt2.schema().columns, expected_schema_cols); + assert_eq!(mt2.schema().timestamp_idx, expected_ts_idx); + + // Verify symbol dict was faithfully restored. + let dict2 = mt2.symbol_dict(2).expect("host dict present"); + assert_eq!(dict2.len(), expected_dict_len); + assert_eq!(dict2.get(0), Some("host-0")); + + // Verify column data lengths match. + for i in 0..3 { + assert_eq!(mt2.column(i).len(), expected_row_count as usize); + } + } + + /// The time kind survives the snapshot round trip for every kind. + #[test] + fn snapshot_roundtrip_keeps_time_kind() { + for kind in [ + TimeKind::Millis, + TimeKind::Instant(nodedb_types::InstantKind::Naive), + TimeKind::Instant(nodedb_types::InstantKind::Utc), + ] { + let schema = ColumnarSchema { + columns: vec![ + ("ts".into(), ColumnType::Timestamp(kind)), + ("value".into(), ColumnType::Float64), + ], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 2], + }; + let mt = ColumnarMemtable::new(schema, default_config()); + let snap = mt.export_snapshot(); + let bytes = zerompk::to_msgpack_vec(&snap).expect("serialize"); + let snap2: MemtableSnapshot = zerompk::from_msgpack(&bytes).expect("deserialize"); + let mt2 = + ColumnarMemtable::from_snapshot(snap2, default_config()).expect("from_snapshot"); + assert_eq!(mt2.schema().columns[0].1, ColumnType::Timestamp(kind)); + } + } + + #[test] + fn snapshot_roundtrip_dict_encoded_column_rebuilds_reverse() { + // Construct a DictEncoded snapshot directly (bypass ingest path). + let snap = MemtableSnapshot { + schema_columns: vec![ + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), + ("tag".into(), ColumnType::Symbol), + ], + timestamp_idx: 0, + columns: vec![ + ColumnSnapshot::Timestamp(vec![1000, 2000, 3000]), + ColumnSnapshot::DictEncoded { + ids: vec![0, 1, 0], + dictionary: vec!["alpha".to_string(), "beta".to_string()], + valid: vec![true, true, true], + }, + ], + symbol_dicts: vec![], + series_row_counts: vec![], + row_count: 3, + min_ts: 1000, + max_ts: 3000, + }; + + let mt = ColumnarMemtable::from_snapshot(snap, default_config()).expect("from_snapshot"); + assert_eq!(mt.row_count(), 3); + // Verify the DictEncoded column has correct data. + match mt.column(1) { + ColumnData::DictEncoded { + ids, + dictionary, + reverse, + valid, + } => { + assert_eq!(ids, &[0u32, 1, 0]); + assert_eq!(dictionary, &["alpha", "beta"]); + assert_eq!(reverse.get("alpha"), Some(&0u32)); + assert_eq!(reverse.get("beta"), Some(&1u32)); + assert_eq!(valid, &[true, true, true]); + } + other => panic!("expected DictEncoded, got {:?}", other), + } + } + + #[test] + fn snapshot_from_invalid_column_lengths_returns_error() { + // row_count says 3 but timestamp column has only 2 rows → mismatch. + let snap = MemtableSnapshot { + schema_columns: vec![ + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), + ("value".into(), ColumnType::Float64), + ], + timestamp_idx: 0, + columns: vec![ + ColumnSnapshot::Timestamp(vec![1000, 2000]), // 2 rows + ColumnSnapshot::Float64(vec![1.0, 2.0, 3.0]), // 3 rows — mismatch + ], + symbol_dicts: vec![], + series_row_counts: vec![], + row_count: 3, + min_ts: 1000, + max_ts: 2000, + }; + let result = ColumnarMemtable::from_snapshot(snap, default_config()); + assert!( + matches!(result, Err(crate::Error::BadRequest { .. })), + "expected BadRequest error on length mismatch" + ); + } +} diff --git a/nodedb/src/engine/timeseries/columnar_memtable/memtable/table.rs b/nodedb/src/engine/timeseries/columnar_memtable/memtable/table.rs new file mode 100644 index 000000000..7eac30bdb --- /dev/null +++ b/nodedb/src/engine/timeseries/columnar_memtable/memtable/table.rs @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `ColumnarMemtable` — per-column ingest buffer for timeseries data. +//! +//! NOT thread-safe — lives on a single Data Plane core (!Send by design). + +use std::collections::HashMap; + +use nodedb_types::timeseries::{SeriesId, SymbolDictionary}; + +use super::super::types::{ColumnData, ColumnType, ColumnarMemtableConfig, ColumnarSchema}; + +/// Columnar memtable: per-column vectors instead of per-series hash maps. +/// +/// Each row is a flat tuple of (timestamp, value, tag1, tag2, ...). +/// Series identity is derived from the tag columns at query time. +/// This layout is SIMD-friendly: aggregation functions operate on +/// contiguous `&[f64]` or `&[i64]` slices. +pub struct ColumnarMemtable { + pub(super) schema: ColumnarSchema, + pub(super) columns: Vec, + /// Per-series row count for quick cardinality checks. + pub(super) series_row_counts: HashMap, + /// Per-tag-column symbol dictionary. + pub(super) symbol_dicts: HashMap, + pub(super) row_count: u64, + pub(super) memory_bytes: usize, + pub(super) config: ColumnarMemtableConfig, + pub(super) min_ts: i64, + pub(super) max_ts: i64, +} + +impl ColumnarMemtable { + /// Create a new columnar memtable with the given schema. + pub fn new(schema: ColumnarSchema, config: ColumnarMemtableConfig) -> Self { + let columns: Vec = schema + .columns + .iter() + .map(|(_, ty)| ColumnData::new(*ty)) + .collect(); + + // Initialize symbol dicts for tag columns. + let mut symbol_dicts = HashMap::new(); + for (i, (_, ty)) in schema.columns.iter().enumerate() { + if *ty == ColumnType::Symbol { + symbol_dicts.insert(i, SymbolDictionary::new()); + } + } + + Self { + schema, + columns, + series_row_counts: HashMap::new(), + symbol_dicts, + row_count: 0, + memory_bytes: 0, + config, + min_ts: i64::MAX, + max_ts: i64::MIN, + } + } + + /// Create a simple metrics memtable (timestamp + f64 value, no tags). + pub fn new_metric(config: ColumnarMemtableConfig) -> Self { + Self::new(ColumnarSchema::metric_default(), config) + } + + // -- Accessors -- + + pub fn row_count(&self) -> u64 { + self.row_count + } + + /// Approximate memory usage. Uses incremental tracking with periodic + /// recomputation from column capacities for accuracy. + pub fn memory_bytes(&self) -> usize { + let col_bytes: usize = self.columns.iter().map(|c| c.memory_bytes()).sum(); + let dict_bytes: usize = self.symbol_dicts.len() * 256; // rough estimate + self.memory_bytes.max(col_bytes + dict_bytes) + } + + pub fn min_ts(&self) -> i64 { + self.min_ts + } + + pub fn max_ts(&self) -> i64 { + self.max_ts + } + + pub fn series_count(&self) -> usize { + self.series_row_counts.len() + } + + pub fn schema(&self) -> &ColumnarSchema { + &self.schema + } + + /// Return the immutable admission configuration used to construct this + /// memtable. Transaction undo uses this with a snapshot so restoration + /// preserves the original limits even if live operator tuning changed. + pub fn config(&self) -> ColumnarMemtableConfig { + self.config.clone() + } + + pub fn column(&self, idx: usize) -> &ColumnData { + &self.columns[idx] + } + + pub fn symbol_dict(&self, col_idx: usize) -> Option<&SymbolDictionary> { + self.symbol_dicts.get(&col_idx) + } + + pub fn is_empty(&self) -> bool { + self.row_count == 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn default_config() -> ColumnarMemtableConfig { + ColumnarMemtableConfig { + max_memory_bytes: 1024 * 1024, + hard_memory_limit: 2 * 1024 * 1024, + max_tag_cardinality: 1000, + } + } + + #[test] + fn empty_memtable() { + let mt = ColumnarMemtable::new_metric(default_config()); + assert_eq!(mt.row_count(), 0); + assert!(mt.is_empty()); + assert_eq!(mt.series_count(), 0); + } +} diff --git a/nodedb/src/engine/timeseries/columnar_memtable/mod.rs b/nodedb/src/engine/timeseries/columnar_memtable/mod.rs index f7d36f55e..20323f4e6 100644 --- a/nodedb/src/engine/timeseries/columnar_memtable/mod.rs +++ b/nodedb/src/engine/timeseries/columnar_memtable/mod.rs @@ -2,11 +2,12 @@ mod memtable; mod snapshot; +mod time_literal; mod types; pub use memtable::ColumnarMemtable; pub use snapshot::{ColumnSnapshot, MemtableSnapshot}; pub use types::{ ColumnData, ColumnType, ColumnValue, ColumnarDrainResult, ColumnarFlushView, - ColumnarMemtableConfig, ColumnarSchema, + ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; diff --git a/nodedb/src/engine/timeseries/columnar_memtable/snapshot.rs b/nodedb/src/engine/timeseries/columnar_memtable/snapshot.rs index cb2321e67..35cf3c0e2 100644 --- a/nodedb/src/engine/timeseries/columnar_memtable/snapshot.rs +++ b/nodedb/src/engine/timeseries/columnar_memtable/snapshot.rs @@ -66,7 +66,7 @@ pub enum ColumnSnapshot { } // --------------------------------------------------------------------------- -// Column conversion helpers (called from memtable.rs) +// Column conversion helpers (called from `export_snapshot`) // --------------------------------------------------------------------------- /// Convert a single [`ColumnData`] value into its [`ColumnSnapshot`] wire form. @@ -94,7 +94,7 @@ pub(super) fn column_to_snapshot(col: &ColumnData) -> ColumnSnapshot { } // --------------------------------------------------------------------------- -// Column rebuild helper (called from memtable.rs `from_snapshot`) +// Column rebuild helper (called from `from_snapshot`) // --------------------------------------------------------------------------- /// Rebuild a `Vec` from the snapshot columns and validate diff --git a/nodedb/src/engine/timeseries/columnar_memtable/time_literal.rs b/nodedb/src/engine/timeseries/columnar_memtable/time_literal.rs new file mode 100644 index 000000000..50262fc27 --- /dev/null +++ b/nodedb/src/engine/timeseries/columnar_memtable/time_literal.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Lowering between a time column's stored milliseconds and the value its +//! kind denotes, in both directions. +//! +//! A time column stores an `i64` millisecond count whatever its kind. The +//! kind decides what that count means to a reader: an `Instant` column's +//! cell is a typed datetime, a `Millis` column's cell is the integer stored. +//! A predicate literal against the column arrives already typed for that +//! kind (the planner coerces a literal against a declared `TIMESTAMP` / +//! `TIMESTAMPTZ` column to an instant), so lowering it to the stored unit is +//! a kind dispatch with no unit guessing: an instant yields its milliseconds, +//! an integer yields itself, and a value of the wrong kind yields nothing. + +use nodedb_types::datetime::{NdbDateTime, NdbDateTimeError}; +use nodedb_types::value::Value; + +use super::types::TimeKind; + +impl TimeKind { + /// The stored millisecond count a predicate literal (or a cell of the + /// same kind) denotes on a column of this kind, or `None` when the value + /// is not of the kind the column holds. + /// + /// `Instant`: a typed datetime, or text that parses as one (a literal + /// that reached the engine as text). `Millis`: an integer, a finite + /// float truncated toward zero, or text that parses as a datetime — a + /// `BIGINT TIME_KEY` is still a point in time, so a datetime literal + /// names its millisecond count. + pub fn literal_ms(self, value: &Value) -> Option { + match self { + TimeKind::Instant(_) => match value { + Value::DateTime(at) | Value::NaiveDateTime(at) => Some(at.unix_millis()), + Value::String(text) => NdbDateTime::parse(text).map(|at| at.unix_millis()), + Value::Null + | Value::Bool(_) + | Value::Integer(_) + | Value::Float(_) + | Value::Bytes(_) + | Value::Array(_) + | Value::Object(_) + | Value::Uuid(_) + | Value::Ulid(_) + | Value::Duration(_) + | Value::Decimal(_) + | Value::Geometry(_) + | Value::Set(_) + | Value::Regex(_) + | Value::ArrayCell(_) + | Value::Vector(_) => None, + // `Value` is `#[non_exhaustive]`: a kind added later carries + // no instant either. + _ => None, + }, + TimeKind::Millis => match value { + Value::Integer(ms) => Some(*ms), + Value::Float(ms) => ms.is_finite().then(|| ms.trunc() as i64), + Value::String(text) => NdbDateTime::parse(text).map(|at| at.unix_millis()), + Value::Null + | Value::Bool(_) + | Value::Bytes(_) + | Value::Array(_) + | Value::Object(_) + | Value::Uuid(_) + | Value::Ulid(_) + | Value::DateTime(_) + | Value::NaiveDateTime(_) + | Value::Duration(_) + | Value::Decimal(_) + | Value::Geometry(_) + | Value::Set(_) + | Value::Regex(_) + | Value::ArrayCell(_) + | Value::Vector(_) => None, + // `Value` is `#[non_exhaustive]`: a kind added later is not a + // millisecond count either. + _ => None, + }, + } + } + + /// The value a stored millisecond cell denotes on a column of this kind. + /// + /// An `Instant` column yields a typed instant, a `Millis` column the + /// integer stored. `Err` when the milliseconds overflow the microsecond + /// range an instant carries. + pub fn cell_value(self, millis: i64) -> Result { + match self { + TimeKind::Instant(kind) => kind.from_millis(millis), + TimeKind::Millis => Ok(Value::Integer(millis)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_types::InstantKind; + + /// `2020-03-05T10:00:00Z` in epoch milliseconds. + const EARLY_MS: i64 = 1_583_402_400_000; + + const NAIVE: TimeKind = TimeKind::Instant(InstantKind::Naive); + const UTC: TimeKind = TimeKind::Instant(InstantKind::Utc); + + fn early() -> NdbDateTime { + NdbDateTime::from_micros(EARLY_MS * 1_000) + } + + #[test] + fn an_instant_column_lowers_typed_instants_of_either_tag() { + for kind in [NAIVE, UTC] { + assert_eq!( + kind.literal_ms(&Value::NaiveDateTime(early())), + Some(EARLY_MS) + ); + assert_eq!(kind.literal_ms(&Value::DateTime(early())), Some(EARLY_MS)); + } + } + + #[test] + fn an_instant_column_parses_datetime_text() { + assert_eq!( + NAIVE.literal_ms(&Value::String("2020-03-05 10:00:00".into())), + Some(EARLY_MS) + ); + assert_eq!( + UTC.literal_ms(&Value::String("2020-03-05T10:00:00Z".into())), + Some(EARLY_MS) + ); + assert_eq!(NAIVE.literal_ms(&Value::String("not a date".into())), None); + } + + /// A bare number carries no unit, so it is not an instant literal. + #[test] + fn an_instant_column_refuses_numbers_and_other_kinds() { + for value in [ + Value::Integer(EARLY_MS), + Value::Float(EARLY_MS as f64), + Value::String(EARLY_MS.to_string()), + Value::Null, + Value::Bool(true), + Value::Array(vec![Value::Integer(EARLY_MS)]), + ] { + assert_eq!(NAIVE.literal_ms(&value), None, "{value:?}"); + } + } + + #[test] + fn a_millis_column_lowers_integers_and_truncates_finite_floats() { + assert_eq!(TimeKind::Millis.literal_ms(&Value::Integer(42)), Some(42)); + assert_eq!(TimeKind::Millis.literal_ms(&Value::Float(42.9)), Some(42)); + assert_eq!(TimeKind::Millis.literal_ms(&Value::Float(-1.5)), Some(-1)); + assert_eq!(TimeKind::Millis.literal_ms(&Value::Float(f64::NAN)), None); + assert_eq!( + TimeKind::Millis.literal_ms(&Value::Float(f64::INFINITY)), + None + ); + } + + #[test] + fn a_millis_column_refuses_instants_and_other_kinds() { + for value in [ + Value::NaiveDateTime(early()), + Value::DateTime(early()), + Value::String("42".into()), + Value::Null, + Value::Bool(true), + ] { + assert_eq!(TimeKind::Millis.literal_ms(&value), None, "{value:?}"); + } + } + + #[test] + fn a_millis_column_parses_datetime_text_to_its_millisecond_count() { + let value = Value::String("2020-03-05 10:00:00".into()); + assert_eq!( + TimeKind::Millis.literal_ms(&value), + Some(early().unix_millis()) + ); + } + + #[test] + fn a_cell_reads_back_as_its_kind() { + assert_eq!( + NAIVE.cell_value(EARLY_MS).expect("in range"), + Value::NaiveDateTime(early()) + ); + assert_eq!( + UTC.cell_value(EARLY_MS).expect("in range"), + Value::DateTime(early()) + ); + assert_eq!( + TimeKind::Millis.cell_value(EARLY_MS).expect("an integer"), + Value::Integer(EARLY_MS) + ); + assert!(NAIVE.cell_value(i64::MAX).is_err()); + } + + /// A cell written by `cell_value` lowers back to the milliseconds it was + /// written from, on every kind. + #[test] + fn cell_value_and_literal_ms_round_trip() { + for kind in [NAIVE, UTC, TimeKind::Millis] { + let cell = kind.cell_value(EARLY_MS).expect("in range"); + assert_eq!(kind.literal_ms(&cell), Some(EARLY_MS), "{kind:?}"); + } + } +} diff --git a/nodedb/src/engine/timeseries/columnar_memtable/types.rs b/nodedb/src/engine/timeseries/columnar_memtable/types.rs index 3955fda9a..14a74397d 100644 --- a/nodedb/src/engine/timeseries/columnar_memtable/types.rs +++ b/nodedb/src/engine/timeseries/columnar_memtable/types.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; +use nodedb_types::InstantKind; use nodedb_types::columnar::schema::TS_SYSTEM; use nodedb_types::timeseries::{SeriesId, SymbolDictionary}; use serde::{Deserialize, Serialize}; @@ -12,6 +13,25 @@ use serde::{Deserialize, Serialize}; // Schema // --------------------------------------------------------------------------- +/// What a millisecond time column denotes when its cells are read. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum TimeKind { + /// An integer count of milliseconds, read back as the integer stored. + Millis, + /// A declared TIMESTAMP (naive) or TIMESTAMPTZ (utc) instant. + Instant(InstantKind), +} + /// Column data type in a columnar memtable. #[derive( Debug, @@ -25,8 +45,8 @@ use serde::{Deserialize, Serialize}; zerompk::FromMessagePack, )] pub enum ColumnType { - /// Designated timestamp column (i64 millis). - Timestamp, + /// Time column (i64 millis). The kind says how a cell is read back. + Timestamp(TimeKind), /// Floating-point metric value. Float64, /// Integer metric value. @@ -35,6 +55,27 @@ pub enum ColumnType { Symbol, } +impl ColumnType { + /// Whether this is a time column of any kind. + pub fn is_time(&self) -> bool { + matches!(self, ColumnType::Timestamp(_)) + } + + /// The SQL DDL type name a client sees for this column (schema-preview + /// responses, catalog field lists). + pub fn ddl_type_name(&self) -> &'static str { + match self { + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)) => "TIMESTAMPTZ", + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive) | TimeKind::Millis) => { + "TIMESTAMP" + } + ColumnType::Float64 => "FLOAT", + ColumnType::Int64 => "BIGINT", + ColumnType::Symbol => "VARCHAR", + } + } +} + /// Schema for a columnar memtable (column names + types, in order). #[derive(Debug, Clone)] pub struct ColumnarSchema { @@ -51,7 +92,7 @@ impl ColumnarSchema { pub fn metric_default() -> Self { Self { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), ("value".into(), ColumnType::Float64), ], timestamp_idx: 0, @@ -106,7 +147,7 @@ pub enum ColumnData { impl ColumnData { pub(super) fn new(ty: ColumnType) -> Self { match ty { - ColumnType::Timestamp => Self::Timestamp(Vec::with_capacity(4096)), + ColumnType::Timestamp(_) => Self::Timestamp(Vec::with_capacity(4096)), ColumnType::Float64 => Self::Float64(Vec::with_capacity(4096)), ColumnType::Int64 => Self::Int64(Vec::with_capacity(4096)), ColumnType::Symbol => Self::Symbol(Vec::with_capacity(4096)), diff --git a/nodedb/src/engine/timeseries/columnar_segment/codec.rs b/nodedb/src/engine/timeseries/columnar_segment/codec.rs index b4fc0d936..d942856bc 100644 --- a/nodedb/src/engine/timeseries/columnar_segment/codec.rs +++ b/nodedb/src/engine/timeseries/columnar_segment/codec.rs @@ -10,7 +10,7 @@ use super::error::SegmentError; /// Legacy default codecs for partitions written before V2 codec metadata. pub(super) fn legacy_default_codec(col_type: ColumnType) -> ResolvedColumnCodec { match col_type { - ColumnType::Timestamp => ResolvedColumnCodec::Gorilla, + ColumnType::Timestamp(_) => ResolvedColumnCodec::Gorilla, ColumnType::Float64 => ResolvedColumnCodec::Gorilla, ColumnType::Int64 => ResolvedColumnCodec::Raw, ColumnType::Symbol => ResolvedColumnCodec::Raw, @@ -28,7 +28,7 @@ pub(super) fn encode_column( requested_codec: ColumnCodec, ) -> Result<(Vec, ResolvedColumnCodec, ColumnStatistics), SegmentError> { match col_type { - ColumnType::Timestamp => { + ColumnType::Timestamp(_) => { let values = col_data.as_timestamps(); let codec = if requested_codec == ColumnCodec::Auto { nodedb_codec::detect::detect_i64_codec(values) @@ -99,7 +99,7 @@ pub(super) fn decode_column( let map_err = |e: nodedb_codec::CodecError| SegmentError::Corrupt(format!("{codec}: {e}")); match col_type { - ColumnType::Timestamp => { + ColumnType::Timestamp(_) => { let values = nodedb_codec::decode_i64_pipeline(data, codec.into_column_codec()) .map_err(map_err)?; Ok(ColumnData::Timestamp(values)) diff --git a/nodedb/src/engine/timeseries/columnar_segment/reader.rs b/nodedb/src/engine/timeseries/columnar_segment/reader.rs index 36f0ecd04..e200c59cb 100644 --- a/nodedb/src/engine/timeseries/columnar_segment/reader.rs +++ b/nodedb/src/engine/timeseries/columnar_segment/reader.rs @@ -229,7 +229,7 @@ impl ColumnarSegmentReader { } let data = match col_type { - ColumnType::Timestamp => ColumnData::Timestamp(all_values), + ColumnType::Timestamp(_) => ColumnData::Timestamp(all_values), ColumnType::Int64 => ColumnData::Int64(all_values), ColumnType::Float64 => { let f64_vals: Vec = all_values @@ -417,11 +417,13 @@ mod tests { use tempfile::TempDir; use super::super::super::columnar_memtable::{ - ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, + ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, TimeKind, }; use super::super::writer::ColumnarSegmentWriter; use super::*; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + fn test_config() -> ColumnarMemtableConfig { ColumnarMemtableConfig { max_memory_bytes: 10 * 1024 * 1024, @@ -459,7 +461,7 @@ mod tests { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("value".into(), ColumnType::Float64), ("extra".into(), ColumnType::Int64), ], @@ -492,7 +494,7 @@ mod tests { let projected = ColumnarSegmentReader::read_columns( &part_dir, &[ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("value".into(), ColumnType::Float64), ], None, @@ -544,7 +546,7 @@ mod tests { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ], timestamp_idx: 0, @@ -720,13 +722,8 @@ mod tests { bytes[nodedb_wal::crypto::SEGMENT_ENVELOPE_PREAMBLE_SIZE + 2] ^= 0xFF; std::fs::write(&col_path, &bytes).unwrap(); - let err = ColumnarSegmentReader::read_column( - &part_dir, - "timestamp", - ColumnType::Timestamp, - Some(&kek), - ) - .unwrap_err(); + let err = ColumnarSegmentReader::read_column(&part_dir, "timestamp", MILLIS, Some(&kek)) + .unwrap_err(); assert!( matches!(err, SegmentError::DecryptionFailed(_)), "expected DecryptionFailed, got {err:?}" diff --git a/nodedb/src/engine/timeseries/columnar_segment/schema.rs b/nodedb/src/engine/timeseries/columnar_segment/schema.rs index d8324c8e5..d629b41eb 100644 --- a/nodedb/src/engine/timeseries/columnar_segment/schema.rs +++ b/nodedb/src/engine/timeseries/columnar_segment/schema.rs @@ -3,10 +3,18 @@ //! Schema serialization (V2: includes codec per column). use nodedb_codec::ColumnCodec; +use nodedb_types::InstantKind; -use super::super::columnar_memtable::{ColumnType, ColumnarSchema}; +use super::super::columnar_memtable::{ColumnType, ColumnarSchema, TimeKind}; use super::error::SegmentError; +/// On-disk type name of a millisecond time column read back as an integer. +const TYPE_TIMESTAMP: &str = "timestamp"; +/// On-disk type name of a time column read back as a naive instant. +const TYPE_INSTANT_NAIVE: &str = "instant_naive"; +/// On-disk type name of a time column read back as a UTC instant. +const TYPE_INSTANT_UTC: &str = "instant_utc"; + /// Schema entry for JSON serialization. #[derive(serde::Serialize, serde::Deserialize)] pub(super) struct SchemaEntry { @@ -46,7 +54,9 @@ pub(super) fn schema_to_json(schema: &ColumnarSchema) -> Vec { .enumerate() .map(|(i, (name, ty))| { let ty_str = match ty { - ColumnType::Timestamp => "timestamp", + ColumnType::Timestamp(TimeKind::Millis) => TYPE_TIMESTAMP, + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive)) => TYPE_INSTANT_NAIVE, + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)) => TYPE_INSTANT_UTC, ColumnType::Float64 => "float64", ColumnType::Int64 => "int64", ColumnType::Symbol => "symbol", @@ -66,10 +76,7 @@ pub(super) fn schema_to_json(schema: &ColumnarSchema) -> Vec { /// (pre-marker) schema carries implicitly, since those were written by /// inference, which always emits the designated column first. fn first_timestamp_idx(columns: &[(String, ColumnType)]) -> usize { - columns - .iter() - .position(|(_, ty)| *ty == ColumnType::Timestamp) - .unwrap_or(0) + columns.iter().position(|(_, ty)| ty.is_time()).unwrap_or(0) } pub(super) fn schema_from_parsed(json: &SchemaJson) -> Result { @@ -81,7 +88,7 @@ pub(super) fn schema_from_parsed(json: &SchemaJson) -> Result Result Result { match ty_str { - "timestamp" => Ok(ColumnType::Timestamp), + TYPE_TIMESTAMP => Ok(ColumnType::Timestamp(TimeKind::Millis)), + TYPE_INSTANT_NAIVE => Ok(ColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive))), + TYPE_INSTANT_UTC => Ok(ColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc))), "float64" => Ok(ColumnType::Float64), "int64" => Ok(ColumnType::Int64), "symbol" => Ok(ColumnType::Symbol), @@ -133,7 +142,7 @@ mod tests { fn schema_v2_roundtrip() { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), ("cpu".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], @@ -152,4 +161,48 @@ mod tests { assert_eq!(recovered.timestamp_idx, 0); assert_eq!(recovered.codecs, schema.codecs); } + + /// Every time kind has its own on-disk type name and reads back as itself. + #[test] + fn schema_keeps_the_time_kind_of_every_time_column() { + let schema = ColumnarSchema { + columns: vec![ + ( + "ts".into(), + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive)), + ), + ( + "seen_at".into(), + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)), + ), + ("_ts_system".into(), ColumnType::Timestamp(TimeKind::Millis)), + ], + timestamp_idx: 0, + codecs: vec![ColumnCodec::Auto; 3], + }; + let entries = schema_to_json(&schema); + assert_eq!(entries[0].col_type, TYPE_INSTANT_NAIVE); + assert_eq!(entries[1].col_type, TYPE_INSTANT_UTC); + assert_eq!(entries[2].col_type, TYPE_TIMESTAMP); + + let json = sonic_rs::to_vec(&entries).unwrap(); + let parsed: SchemaJson = sonic_rs::from_slice(&json).unwrap(); + let recovered = schema_from_parsed(&parsed).unwrap(); + assert_eq!(recovered.columns, schema.columns); + assert_eq!(recovered.timestamp_idx, 0); + } + + /// A V1 tuple schema names its time column `timestamp` and reads back as + /// `Millis`. + #[test] + fn v1_schema_time_column_is_millis() { + let json = br#"[["timestamp","timestamp"],["value","float64"]]"#; + let parsed: SchemaJson = sonic_rs::from_slice(json).unwrap(); + let recovered = schema_from_parsed(&parsed).unwrap(); + assert_eq!( + recovered.columns[0].1, + ColumnType::Timestamp(TimeKind::Millis) + ); + assert_eq!(recovered.timestamp_idx, 0); + } } diff --git a/nodedb/src/engine/timeseries/columnar_segment/writer.rs b/nodedb/src/engine/timeseries/columnar_segment/writer.rs index e93961fb2..83533ae8c 100644 --- a/nodedb/src/engine/timeseries/columnar_segment/writer.rs +++ b/nodedb/src/engine/timeseries/columnar_segment/writer.rs @@ -74,10 +74,10 @@ impl ColumnarSegmentWriter { // declared collection to reach the inference fallback and flush a // partition under inferred names — `timestamp.col` instead of the // declared TIME_KEY — the seed would have to arrive empty, which needs - // one of: the catalog unreadable at boot (now an error rather than a - // silent empty seed, see `CatalogForRead::open`), a core spawned - // without `doc_config_seed`, or the collection missing from the - // catalog. All three are boot-integrity failures, not steady state. + // one of: the catalog unreadable at boot (an error, see + // `CatalogForRead::open`), a core spawned without `doc_config_seed`, + // or the collection missing from the catalog. All three are + // boot-integrity failures, not steady state. // // If that ever regresses, the damage is durable and silent: those // partitions keep projecting under the inferred name after the @@ -198,11 +198,13 @@ mod tests { use tempfile::TempDir; use super::super::super::columnar_memtable::{ - ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, + ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, TimeKind, }; use super::super::reader::ColumnarSegmentReader; use super::*; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + fn test_config() -> ColumnarMemtableConfig { ColumnarMemtableConfig { max_memory_bytes: 10 * 1024 * 1024, @@ -281,7 +283,7 @@ mod tests { let ts_col = ColumnarSegmentReader::read_column_with_codec( &part_dir, "timestamp", - ColumnType::Timestamp, + MILLIS, Some(ResolvedColumnCodec::DoubleDelta), None, ) @@ -312,7 +314,7 @@ mod tests { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], @@ -367,7 +369,7 @@ mod tests { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("value".into(), ColumnType::Float64), ], timestamp_idx: 0, @@ -401,7 +403,7 @@ mod tests { let ts_col = ColumnarSegmentReader::read_column_with_codec( &part_dir, "timestamp", - ColumnType::Timestamp, + MILLIS, Some(ResolvedColumnCodec::Gorilla), None, ) @@ -507,13 +509,8 @@ mod tests { let meta = ColumnarSegmentReader::read_meta(&part_dir, Some(&kek)).unwrap(); assert_eq!(meta.row_count, 100); - let ts_col = ColumnarSegmentReader::read_column( - &part_dir, - "timestamp", - ColumnType::Timestamp, - Some(&kek), - ) - .unwrap(); + let ts_col = + ColumnarSegmentReader::read_column(&part_dir, "timestamp", MILLIS, Some(&kek)).unwrap(); assert_eq!(ts_col.as_timestamps().len(), 100); let sparse = ColumnarSegmentReader::read_sparse_index(&part_dir, Some(&kek)) diff --git a/nodedb/src/engine/timeseries/continuous_agg/manager.rs b/nodedb/src/engine/timeseries/continuous_agg/manager.rs index ac8eea5eb..f89ad2911 100644 --- a/nodedb/src/engine/timeseries/continuous_agg/manager.rs +++ b/nodedb/src/engine/timeseries/continuous_agg/manager.rs @@ -331,7 +331,7 @@ pub struct AggregateInfo { mod tests { use super::*; use crate::engine::timeseries::columnar_memtable::{ - ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, + ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; use crate::engine::timeseries::continuous_agg::definition::{ AggFunction, AggregateExpr, RefreshPolicy, @@ -451,7 +451,7 @@ mod tests { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), ("value".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], diff --git a/nodedb/src/engine/timeseries/grouped_filter.rs b/nodedb/src/engine/timeseries/grouped_filter.rs index 04f17dbab..de8528403 100644 --- a/nodedb/src/engine/timeseries/grouped_filter.rs +++ b/nodedb/src/engine/timeseries/grouped_filter.rs @@ -6,107 +6,143 @@ //! returning packed `Vec` bitmasks. Uses SIMD kernels from //! `nodedb_query::simd_filter` for numeric and symbol comparisons. +use nodedb_query::scan_filter::FilterOp; use nodedb_query::simd_filter; use super::columnar_memtable::{ColumnData, ColumnType}; use crate::bridge::scan_filter::ScanFilter; +/// A predicate the grouped scan cannot lower onto typed column vectors. +/// +/// The grouped scan evaluates predicates only as SIMD bitmasks, so a shape +/// it cannot lower fails the aggregate instead of aggregating rows the +/// predicate never excluded. The message names the predicate and why. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("predicate `{field} {op}` cannot be evaluated by the timeseries grouped scan: {reason}")] +pub struct UnsupportedPredicate { + pub field: String, + pub op: &'static str, + pub reason: &'static str, +} + +impl UnsupportedPredicate { + fn new(f: &ScanFilter, reason: &'static str) -> Self { + Self { + field: f.field.clone(), + op: f.op.as_str(), + reason, + } + } +} + /// Evaluate ScanFilter predicates on columnar data, returning a bitmask. /// -/// Bit *i* is set iff row *i* passes ALL filters. -/// Returns `None` for unsupported patterns (OR clauses, contains, etc.). +/// Bit *i* is set iff row *i* passes ALL filters. Supported shapes: a +/// single `(column, op, literal)` comparison with `eq`/`ne` on symbol +/// columns and `eq`/`ne`/`gt`/`gte`/`lt`/`lte` on numeric and time +/// columns. Every other shape (OR clauses, expressions, `in`, `like`, an +/// unknown column, a literal of the wrong type) is an +/// [`UnsupportedPredicate`] error. pub fn eval_filters_to_bitmask<'a>( filters: &[ScanFilter], schema: &[(String, ColumnType)], columns: &[Option<&'a ColumnData>], sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, row_count: usize, -) -> Option> { +) -> Result, UnsupportedPredicate> { let rt = simd_filter::filter_runtime(); let mut mask = simd_filter::bitmask_all(row_count); for f in filters { - if f.op == nodedb_query::scan_filter::FilterOp::MatchAll { + if f.op == FilterOp::MatchAll { continue; } if !f.clauses.is_empty() { - return None; + return Err(UnsupportedPredicate::new(f, "OR clauses")); + } + if f.expr.is_some() { + return Err(UnsupportedPredicate::new(f, "expression predicate")); } - let col_pos = schema.iter().position(|(n, _)| n == &f.field)?; + let col_pos = schema + .iter() + .position(|(n, _)| n == &f.field) + .ok_or(UnsupportedPredicate::new(f, "column not in the schema"))?; let (_, col_type) = &schema[col_pos]; - let col_data = columns[col_pos]?; + let col_data = + columns[col_pos].ok_or(UnsupportedPredicate::new(f, "column data not loaded"))?; let filter_mask = match col_type { ColumnType::Float64 => { - let fv = f.value.as_f64()?; + let fv = f + .value + .as_f64() + .ok_or(UnsupportedPredicate::new(f, "literal is not a float"))?; let vals = col_data.as_f64(); let slice = &vals[..row_count.min(vals.len())]; - match f.op.as_str() { - "gt" => (rt.gt_f64)(slice, fv), - "gte" => (rt.gte_f64)(slice, fv), - "lt" => (rt.lt_f64)(slice, fv), - "lte" => (rt.lte_f64)(slice, fv), - "eq" => { + match f.op { + FilterOp::Gt => (rt.gt_f64)(slice, fv), + FilterOp::Gte => (rt.gte_f64)(slice, fv), + FilterOp::Lt => (rt.lt_f64)(slice, fv), + FilterOp::Lte => (rt.lte_f64)(slice, fv), + FilterOp::Eq => { let a = (rt.gte_f64)(slice, fv - f64::EPSILON); let b = (rt.lte_f64)(slice, fv + f64::EPSILON); simd_filter::bitmask_and(&a, &b) } - "ne" => { + FilterOp::Ne => { let a = (rt.gte_f64)(slice, fv - f64::EPSILON); let b = (rt.lte_f64)(slice, fv + f64::EPSILON); simd_filter::bitmask_not(&simd_filter::bitmask_and(&a, &b), row_count) } - _ => return None, - } - } - ColumnType::Int64 | ColumnType::Timestamp => { - let fv = f.value.as_i64()?; - let vals = if *col_type == ColumnType::Int64 { - col_data.as_i64() - } else { - col_data.as_timestamps() - }; - let slice = &vals[..row_count.min(vals.len())]; - match f.op.as_str() { - "gt" => (rt.gt_i64)(slice, fv), - "gte" => (rt.gte_i64)(slice, fv), - "lt" => (rt.lt_i64)(slice, fv), - "lte" => (rt.lte_i64)(slice, fv), - "eq" => { - let a = (rt.gte_i64)(slice, fv); - let b = (rt.lte_i64)(slice, fv); - simd_filter::bitmask_and(&a, &b) + _ => { + return Err(UnsupportedPredicate::new(f, "operator on a float column")); } - "ne" => { - let a = (rt.gte_i64)(slice, fv); - let b = (rt.lte_i64)(slice, fv); - simd_filter::bitmask_not(&simd_filter::bitmask_and(&a, &b), row_count) - } - _ => return None, } } + ColumnType::Int64 => { + let fv = f + .value + .as_i64() + .ok_or(UnsupportedPredicate::new(f, "literal is not an integer"))?; + i64_column_mask(rt, f, col_data.as_i64(), fv, row_count)? + } + ColumnType::Timestamp(kind) => { + // A time column stores epoch milliseconds; its literal lowers + // to that unit by the column's kind (an instant for a declared + // TIMESTAMP, an integer for a millisecond column). + let fv = kind + .literal_ms(&f.value) + .ok_or(UnsupportedPredicate::new(f, "literal is not an instant"))?; + i64_column_mask(rt, f, col_data.as_timestamps(), fv, row_count)? + } ColumnType::Symbol => { - let filter_str = f.value.as_str()?; - let dict = sym_lookup(col_pos)?; + let filter_str = f + .value + .as_str() + .ok_or(UnsupportedPredicate::new(f, "literal is not a string"))?; + let dict = sym_lookup(col_pos) + .ok_or(UnsupportedPredicate::new(f, "symbol dictionary not loaded"))?; let sym_ids = col_data.as_symbols(); let slice = &sym_ids[..row_count.min(sym_ids.len())]; - match f.op.as_str() { - "eq" => { + match f.op { + FilterOp::Eq => { if let Some(target_id) = dict.get_id(filter_str) { (rt.eq_u32)(slice, target_id) } else { vec![0u64; simd_filter::words_for(row_count)] } } - "ne" => { + FilterOp::Ne => { if let Some(target_id) = dict.get_id(filter_str) { (rt.ne_u32)(slice, target_id) } else { simd_filter::bitmask_all(row_count) } } - _ => return None, + _ => { + return Err(UnsupportedPredicate::new(f, "operator on a symbol column")); + } } } }; @@ -114,7 +150,7 @@ pub fn eval_filters_to_bitmask<'a>( mask = simd_filter::bitmask_and(&mask, &filter_mask); } - Some(mask) + Ok(mask) } /// Apply sparse index block-level skip to a bitmask. @@ -156,3 +192,37 @@ pub fn apply_sparse_skip( } } } + +/// The bitmask of rows in an `i64` column (integer or time) that satisfy +/// `f.op` against the lowered literal `fv`. +fn i64_column_mask( + rt: &simd_filter::FilterSimdRuntime, + f: &ScanFilter, + vals: &[i64], + fv: i64, + row_count: usize, +) -> Result, UnsupportedPredicate> { + let slice = &vals[..row_count.min(vals.len())]; + Ok(match f.op { + FilterOp::Gt => (rt.gt_i64)(slice, fv), + FilterOp::Gte => (rt.gte_i64)(slice, fv), + FilterOp::Lt => (rt.lt_i64)(slice, fv), + FilterOp::Lte => (rt.lte_i64)(slice, fv), + FilterOp::Eq => { + let a = (rt.gte_i64)(slice, fv); + let b = (rt.lte_i64)(slice, fv); + simd_filter::bitmask_and(&a, &b) + } + FilterOp::Ne => { + let a = (rt.gte_i64)(slice, fv); + let b = (rt.lte_i64)(slice, fv); + simd_filter::bitmask_not(&simd_filter::bitmask_and(&a, &b), row_count) + } + _ => { + return Err(UnsupportedPredicate::new( + f, + "operator on an integer or time column", + )); + } + }) +} diff --git a/nodedb/src/engine/timeseries/grouped_scan/partition.rs b/nodedb/src/engine/timeseries/grouped_scan/partition.rs index fdcd7e311..670afab00 100644 --- a/nodedb/src/engine/timeseries/grouped_scan/partition.rs +++ b/nodedb/src/engine/timeseries/grouped_scan/partition.rs @@ -12,7 +12,7 @@ use nodedb_query::simd_filter; use super::super::columnar_memtable::{ColumnData, ColumnType, ColumnarMemtable}; use super::super::columnar_segment::ColumnarSegmentReader; -use super::super::grouped_filter; +use super::super::grouped_filter::{self, UnsupportedPredicate}; use super::strategies::dispatch_grouping; use super::types::{GroupedAggResult, resolve_schema}; use crate::bridge::envelope::Priority; @@ -20,6 +20,11 @@ use crate::bridge::scan_filter::ScanFilter; use crate::data::io::IoMetrics; /// Aggregate from a columnar memtable with GROUP BY + optional time_bucket. +/// +/// `Ok(None)` when a GROUP BY or aggregate column is not in the memtable's +/// schema. `Err` when a predicate in `filters` cannot be lowered onto the +/// typed columns: the caller fails the statement rather than aggregating +/// rows the predicate never excluded. pub fn aggregate_memtable( mt: &ColumnarMemtable, group_by: &[String], @@ -27,15 +32,19 @@ pub fn aggregate_memtable( filters: &[ScanFilter], time_range: (i64, i64), bucket_interval_ms: i64, -) -> Option { +) -> Result, UnsupportedPredicate> { let schema = mt.schema(); let num_aggs = aggregates.len(); let row_count = mt.row_count() as usize; if row_count == 0 { - return Some(GroupedAggResult::new(num_aggs)); + return Ok(Some(GroupedAggResult::new(num_aggs))); } - let resolved = resolve_schema(&schema.columns, schema.timestamp_idx, group_by, aggregates)?; + let Some(resolved) = + resolve_schema(&schema.columns, schema.timestamp_idx, group_by, aggregates) + else { + return Ok(None); + }; let col_refs: Vec> = (0..schema.columns.len()) .map(|i| Some(mt.column(i))) @@ -63,7 +72,7 @@ pub fn aggregate_memtable( } if simd_filter::popcount(&mask) == 0 { - return Some(GroupedAggResult::new(num_aggs)); + return Ok(Some(GroupedAggResult::new(num_aggs))); } let timestamps = if bucket_interval_ms > 0 { @@ -72,7 +81,7 @@ pub fn aggregate_memtable( None }; - Some(dispatch_grouping( + Ok(Some(dispatch_grouping( super::strategies::GroupedScanInputs { resolved: &resolved, columns: &col_refs, @@ -84,7 +93,7 @@ pub fn aggregate_memtable( &sym_lookup, timestamps, bucket_interval_ms, - )) + ))) } /// Parameters for partition-level grouped aggregation. @@ -109,22 +118,36 @@ pub struct PartitionAggParams<'a> { /// /// When `uring_reader` is `Some`, column files are batch-read via io_uring /// (parallel kernel I/O). When `None`, falls back to fadvise + std::fs::read. -pub fn aggregate_partition(p: PartitionAggParams<'_>) -> Option { +/// +/// `Ok(None)` when the partition's schema or metadata cannot be read or does +/// not carry a GROUP BY / aggregate column. `Err` when a predicate in +/// `filters` cannot be lowered onto the partition's typed columns: the +/// caller fails the statement rather than aggregating rows the predicate +/// never excluded. +pub fn aggregate_partition( + p: PartitionAggParams<'_>, +) -> Result, UnsupportedPredicate> { let num_aggs = p.aggregates.len(); - let schema = ColumnarSegmentReader::read_schema(p.partition_dir, None).ok()?; - let meta = ColumnarSegmentReader::read_meta(p.partition_dir, None).ok()?; + let Ok(schema) = ColumnarSegmentReader::read_schema(p.partition_dir, None) else { + return Ok(None); + }; + let Ok(meta) = ColumnarSegmentReader::read_meta(p.partition_dir, None) else { + return Ok(None); + }; let row_count = meta.row_count as usize; if row_count == 0 { - return Some(GroupedAggResult::new(num_aggs)); + return Ok(Some(GroupedAggResult::new(num_aggs))); } - let resolved = resolve_schema( + let Some(resolved) = resolve_schema( &schema.columns, schema.timestamp_idx, p.group_by, p.aggregates, - )?; + ) else { + return Ok(None); + }; // Load sparse index for block-level skip. let sparse_idx = ColumnarSegmentReader::read_sparse_index(p.partition_dir, None) @@ -203,7 +226,9 @@ pub fn aggregate_partition(p: PartitionAggParams<'_>) -> Option) -> Option) -> Option> = col_data.iter().map(|c| c.as_ref()).collect(); let sym_lookup_tmp = |col_idx: usize| -> Option<&nodedb_types::timeseries::SymbolDictionary> { sym_dicts.get(&col_idx) }; - if let Some(filter_mask) = grouped_filter::eval_filters_to_bitmask( + let filter_mask = grouped_filter::eval_filters_to_bitmask( p.filters, &schema.columns, &col_refs_tmp, &sym_lookup_tmp, effective_row_count, - ) { - mask = simd_filter::bitmask_and(&mask, &filter_mask); - } + )?; + mask = simd_filter::bitmask_and(&mask, &filter_mask); } if simd_filter::popcount(&mask) == 0 { - return Some(GroupedAggResult::new(num_aggs)); + return Ok(Some(GroupedAggResult::new(num_aggs))); } let col_refs: Vec> = col_data.iter().map(|c| c.as_ref()).collect(); @@ -285,7 +312,7 @@ pub fn aggregate_partition(p: PartitionAggParams<'_>) -> Option { diff --git a/nodedb/src/engine/timeseries/grouped_scan/strategies.rs b/nodedb/src/engine/timeseries/grouped_scan/strategies.rs deleted file mode 100644 index 0389af52a..000000000 --- a/nodedb/src/engine/timeseries/grouped_scan/strategies.rs +++ /dev/null @@ -1,537 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Tiered grouping strategies: dispatch, direct-index, FxHash, generic, -//! time-bucket, and integer-to-string key resolution. - -use rustc_hash::FxHashMap; - -use super::super::columnar_agg::AggAccum; -use super::super::columnar_memtable::{ColumnData, ColumnType}; -use super::types::{GroupedAggResult, ResolvedSchema, accumulate_row, for_each_set_bit}; - -const DIRECT_INDEX_MAX_CARDINALITY: u32 = 65536; - -/// Integer group key for local (per-source) grouping. -#[derive(Clone, PartialEq, Eq, Hash)] -pub(super) enum IntGroupKey { - None, - SingleU32(u32), - Multi(Vec), -} - -/// Row-level scan inputs shared by every grouping strategy: the resolved -/// schema, per-column data, the row-selection bitmask, row count, and -/// aggregate count. Bundled so the top-level dispatch/bucket entry points -/// stay within clippy's argument budget. -#[derive(Clone, Copy)] -pub(super) struct GroupedScanInputs<'a> { - pub resolved: &'a ResolvedSchema, - pub columns: &'a [Option<&'a ColumnData>], - pub mask: &'a [u64], - pub row_count: usize, - pub num_aggs: usize, -} - -pub(super) fn dispatch_grouping<'a>( - inputs: GroupedScanInputs<'a>, - group_by: &[String], - sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, - timestamps: Option<&[i64]>, - bucket_interval_ms: i64, -) -> GroupedAggResult { - let GroupedScanInputs { - resolved, - columns, - mask, - row_count, - num_aggs, - } = inputs; - let has_bucket = bucket_interval_ms > 0 && timestamps.is_some(); - - if has_bucket && let Some(ts) = timestamps { - return aggregate_with_bucket(inputs, ts, bucket_interval_ms, sym_lookup); - } - - let local_groups = if group_by.is_empty() { - aggregate_no_group(resolved, columns, mask, row_count, num_aggs) - } else if group_by.len() == 1 && resolved.group_cols[0].1 == ColumnType::Symbol { - let (col_idx, _) = resolved.group_cols[0]; - if let Some(data) = columns[col_idx] { - let sym_ids = data.as_symbols(); - let cardinality = sym_lookup(col_idx).map(|d| d.len() as u32).unwrap_or(0); - if cardinality <= DIRECT_INDEX_MAX_CARDINALITY && cardinality > 0 { - aggregate_direct_index( - resolved, - columns, - mask, - row_count, - num_aggs, - sym_ids, - cardinality, - ) - } else { - aggregate_hash_u32(resolved, columns, mask, row_count, num_aggs, sym_ids) - } - } else { - aggregate_hash_generic(resolved, columns, mask, row_count, num_aggs) - } - } else if row_count > 100_000 { - aggregate_two_level(resolved, columns, mask, row_count, num_aggs) - } else { - aggregate_hash_generic(resolved, columns, mask, row_count, num_aggs) - }; - - // Resolve integer keys to strings. - let mut result = GroupedAggResult::new(num_aggs); - for (int_key, accums) in local_groups { - let str_key = resolve_group_key(&int_key, resolved, sym_lookup); - let entry = result - .groups - .entry(str_key) - .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); - for (i, a) in accums.iter().enumerate() { - entry[i].merge(a); - } - } - result -} - -// --------------------------------------------------------------------------- -// Grouping strategies -// --------------------------------------------------------------------------- - -fn aggregate_no_group( - resolved: &ResolvedSchema, - columns: &[Option<&ColumnData>], - mask: &[u64], - row_count: usize, - num_aggs: usize, -) -> Vec<(IntGroupKey, Vec)> { - let mut accums: Vec = (0..num_aggs).map(|_| AggAccum::default()).collect(); - for_each_set_bit(mask, row_count, |row_idx| { - accumulate_row(&mut accums, resolved, columns, row_idx); - }); - vec![(IntGroupKey::None, accums)] -} - -fn aggregate_direct_index( - resolved: &ResolvedSchema, - columns: &[Option<&ColumnData>], - mask: &[u64], - row_count: usize, - num_aggs: usize, - sym_ids: &[u32], - cardinality: u32, -) -> Vec<(IntGroupKey, Vec)> { - let card = cardinality as usize; - let mut table: Vec> = (0..card) - .map(|_| (0..num_aggs).map(|_| AggAccum::default()).collect()) - .collect(); - - for_each_set_bit(mask, row_count, |row_idx| { - let id = sym_ids[row_idx] as usize; - if id < card { - accumulate_row(&mut table[id], resolved, columns, row_idx); - } - }); - - table - .into_iter() - .enumerate() - .filter(|(_, accums)| accums.iter().any(|a| a.count > 0)) - .map(|(id, accums)| (IntGroupKey::SingleU32(id as u32), accums)) - .collect() -} - -fn aggregate_hash_u32( - resolved: &ResolvedSchema, - columns: &[Option<&ColumnData>], - mask: &[u64], - row_count: usize, - num_aggs: usize, - sym_ids: &[u32], -) -> Vec<(IntGroupKey, Vec)> { - let mut groups: FxHashMap> = FxHashMap::default(); - - for_each_set_bit(mask, row_count, |row_idx| { - let id = sym_ids[row_idx]; - let accums = groups - .entry(id) - .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); - accumulate_row(accums, resolved, columns, row_idx); - }); - - groups - .into_iter() - .map(|(id, accums)| (IntGroupKey::SingleU32(id), accums)) - .collect() -} - -fn aggregate_hash_generic( - resolved: &ResolvedSchema, - columns: &[Option<&ColumnData>], - mask: &[u64], - row_count: usize, - num_aggs: usize, -) -> Vec<(IntGroupKey, Vec)> { - let mut groups: FxHashMap, Vec> = FxHashMap::default(); - - for_each_set_bit(mask, row_count, |row_idx| { - let key = build_generic_key(resolved, columns, row_idx); - let accums = groups - .entry(key) - .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); - accumulate_row(accums, resolved, columns, row_idx); - }); - - groups - .into_iter() - .map(|(key, accums)| (IntGroupKey::Multi(key), accums)) - .collect() -} - -/// Two-level aggregation for high-cardinality GROUP BY (2M+ keys). -/// -/// Phase 1: Partition rows into buckets by hash prefix (top 8 bits → 256 buckets). -/// Phase 2: Aggregate each bucket independently (small HashMap, cache-friendly). -/// Phase 3: Flatten all buckets into the final result. -fn aggregate_two_level( - resolved: &ResolvedSchema, - columns: &[Option<&ColumnData>], - mask: &[u64], - row_count: usize, - num_aggs: usize, -) -> Vec<(IntGroupKey, Vec)> { - const NUM_BUCKETS: usize = 256; - - // Phase 1: Build key for each row, hash it, and partition into buckets by top 8 bits. - // The key is stored alongside the row index so Phase 2 does not recompute it. - let mut buckets: Vec)>> = (0..NUM_BUCKETS).map(|_| Vec::new()).collect(); - for_each_set_bit(mask, row_count, |row_idx| { - let key = build_generic_key(resolved, columns, row_idx); - let hash = fx_hash_key(&key); - let bucket = (hash >> 56) as usize; - buckets[bucket].push((row_idx, key)); - }); - - // Phase 2 + 3: Aggregate each bucket independently and flatten. - let mut all_results: Vec<(IntGroupKey, Vec)> = Vec::new(); - for bucket_rows in buckets { - if bucket_rows.is_empty() { - continue; - } - let mut groups: FxHashMap, Vec> = FxHashMap::default(); - for (row_idx, key) in bucket_rows { - let accums = groups - .entry(key) - .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); - accumulate_row(accums, resolved, columns, row_idx); - } - all_results.extend(groups.into_iter().map(|(k, a)| (IntGroupKey::Multi(k), a))); - } - - all_results -} - -/// Time-bucket aggregation with integer keys. -/// -/// Packs (bucket_ts, group_key_parts...) into a `Vec`: -/// - `parts[0]` = bucket_ts as u64 -/// - `parts[1..]` = group column values (symbol IDs, i64, f64 bits) -/// -/// Resolves to string keys with "bucket_ts\0group1\0group2" format -/// so `emit_grouped_results` can parse them. -fn aggregate_with_bucket<'a>( - inputs: GroupedScanInputs<'a>, - timestamps: &[i64], - bucket_interval_ms: i64, - sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, -) -> GroupedAggResult { - let GroupedScanInputs { - resolved, - columns, - mask, - row_count, - num_aggs, - } = inputs; - let key_len = 1 + resolved.group_cols.len(); // bucket + group columns - - let mut groups: FxHashMap, Vec> = FxHashMap::default(); - - for_each_set_bit(mask, row_count, |row_idx| { - let bucket = - super::super::time_bucket::time_bucket(bucket_interval_ms, timestamps[row_idx]); - - let mut key = Vec::with_capacity(key_len); - key.push(bucket as u64); - - // Pack group-by columns as integers. - for &(col_idx, ty) in &resolved.group_cols { - let part = columns[col_idx] - .map(|data| match ty { - ColumnType::Symbol => { - if let ColumnData::Symbol(ids) = data { - ids[row_idx] as u64 - } else { - u64::MAX - } - } - ColumnType::Int64 => { - if let ColumnData::Int64(v) = data { - v[row_idx] as u64 - } else { - u64::MAX - } - } - ColumnType::Float64 => { - if let ColumnData::Float64(v) = data { - v[row_idx].to_bits() - } else { - u64::MAX - } - } - ColumnType::Timestamp => { - if let ColumnData::Timestamp(v) = data { - v[row_idx] as u64 - } else { - u64::MAX - } - } - }) - .unwrap_or(u64::MAX); - key.push(part); - } - - let accums = groups - .entry(key) - .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); - accumulate_row(accums, resolved, columns, row_idx); - }); - - // Resolve integer keys to string keys: "bucket_ts\0group1\0group2" - let mut result = GroupedAggResult::new(num_aggs); - for (key_parts, accums) in groups { - let bucket_ts = key_parts[0] as i64; - let mut str_key = bucket_ts.to_string(); - - for (i, &part) in key_parts[1..].iter().enumerate() { - str_key.push('\0'); - if i < resolved.group_cols.len() { - let (col_idx, ty) = resolved.group_cols[i]; - match ty { - ColumnType::Symbol => { - if let Some(dict) = sym_lookup(col_idx) - && let Some(name) = dict.get(part as u32) - { - str_key.push_str(name); - } - } - ColumnType::Int64 | ColumnType::Timestamp => { - use std::fmt::Write; - let _ = write!(str_key, "{}", part as i64); - } - ColumnType::Float64 => { - use std::fmt::Write; - let _ = write!(str_key, "{}", f64::from_bits(part)); - } - } - } - } - - let entry = result - .groups - .entry(str_key) - .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); - for (i, a) in accums.iter().enumerate() { - entry[i].merge(a); - } - } - - result -} - -// --------------------------------------------------------------------------- -// Key helpers -// --------------------------------------------------------------------------- - -/// FxHash-style multiplicative hash over a slice of u64 values. -/// -/// Used by `aggregate_two_level` to assign rows to buckets in Phase 1. -#[inline] -fn fx_hash_key(key: &[u64]) -> u64 { - let mut hash: u64 = 0; - for &v in key { - hash = hash.wrapping_mul(0x517cc1b727220a95).wrapping_add(v); - } - hash -} - -fn build_generic_key( - resolved: &ResolvedSchema, - columns: &[Option<&ColumnData>], - row_idx: usize, -) -> Vec { - resolved - .group_cols - .iter() - .map(|&(idx, ty)| { - columns[idx] - .map(|data| match ty { - ColumnType::Symbol => { - if let ColumnData::Symbol(ids) = data { - ids[row_idx] as u64 - } else { - u64::MAX - } - } - ColumnType::Int64 => { - if let ColumnData::Int64(vals) = data { - vals[row_idx] as u64 - } else { - u64::MAX - } - } - ColumnType::Float64 => { - if let ColumnData::Float64(vals) = data { - vals[row_idx].to_bits() - } else { - u64::MAX - } - } - ColumnType::Timestamp => { - if let ColumnData::Timestamp(vals) = data { - vals[row_idx] as u64 - } else { - u64::MAX - } - } - }) - .unwrap_or(u64::MAX) - }) - .collect() -} - -fn resolve_group_key<'a>( - key: &IntGroupKey, - resolved: &ResolvedSchema, - sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, -) -> String { - match key { - IntGroupKey::None => String::new(), - IntGroupKey::SingleU32(id) => { - let col_idx = resolved.group_cols[0].0; - if resolved.group_cols[0].1 == ColumnType::Symbol { - sym_lookup(col_idx) - .and_then(|d: &nodedb_types::timeseries::SymbolDictionary| d.get(*id)) - .unwrap_or("") - .to_string() - } else { - id.to_string() - } - } - IntGroupKey::Multi(parts) => { - let mut s = String::with_capacity(parts.len() * 16); - for (i, &part) in parts.iter().enumerate() { - if i > 0 { - s.push('\0'); - } - let (col_idx, ty) = resolved.group_cols[i]; - match ty { - ColumnType::Symbol => { - if let Some(dict) = sym_lookup(col_idx) - && let Some(name) = dict.get(part as u32) - { - s.push_str(name); - } - } - ColumnType::Int64 | ColumnType::Timestamp => { - use std::fmt::Write; - let _ = write!(s, "{}", part as i64); - } - ColumnType::Float64 => { - use std::fmt::Write; - let _ = write!(s, "{}", f64::from_bits(part)); - } - } - } - s - } - } -} - -#[cfg(test)] -mod tests { - use super::super::super::columnar_memtable::ColumnType; - use super::super::types::AggColInfo; - use super::super::types::ResolvedSchema; - use super::*; - - fn make_resolved_count(col_types: &[(usize, ColumnType)]) -> ResolvedSchema { - ResolvedSchema { - group_cols: col_types.to_vec(), - agg_cols: vec![AggColInfo::CountStar], - ts_idx: 0, - } - } - - /// Build a full-set mask for `row_count` rows. - fn full_mask(row_count: usize) -> Vec { - let words = row_count.div_ceil(64); - let mut mask = vec![u64::MAX; words]; - let rem = row_count % 64; - if rem != 0 { - *mask.last_mut().unwrap() = (1u64 << rem) - 1; - } - mask - } - - /// Collect aggregation results into a sorted Vec of (key, count) for comparison. - fn collect_counts(results: Vec<(IntGroupKey, Vec)>) -> Vec<(Vec, u64)> { - let mut out: Vec<(Vec, u64)> = results - .into_iter() - .map(|(key, accums)| { - let k = match key { - IntGroupKey::Multi(v) => v, - IntGroupKey::SingleU32(v) => vec![v as u64], - IntGroupKey::None => vec![], - }; - let count = accums.first().map(|a| a.count).unwrap_or(0); - (k, count) - }) - .collect(); - out.sort_by(|a, b| a.0.cmp(&b.0)); - out - } - - #[test] - fn two_level_matches_generic_multi_column() { - // Build two Int64 columns with 8 distinct combinations repeated many times. - let n = 200_000usize; - let col_a: Vec = (0..n).map(|i| (i % 4) as i64).collect(); - let col_b: Vec = (0..n).map(|i| (i % 2) as i64).collect(); - - let data_a = ColumnData::Int64(col_a); - let data_b = ColumnData::Int64(col_b); - let columns: Vec> = vec![Some(&data_a), Some(&data_b)]; - - let resolved = make_resolved_count(&[(0, ColumnType::Int64), (1, ColumnType::Int64)]); - let mask = full_mask(n); - - // num_aggs = 1 so AggAccum::count tracks row membership. - let generic = aggregate_hash_generic(&resolved, &columns, &mask, n, 1); - let two_level = aggregate_two_level(&resolved, &columns, &mask, n, 1); - - assert_eq!( - collect_counts(generic), - collect_counts(two_level), - "two-level and generic must produce identical group counts" - ); - } - - #[test] - fn fx_hash_key_deterministic() { - let key = vec![1u64, 2, 3, 4]; - assert_eq!(fx_hash_key(&key), fx_hash_key(&key)); - assert_ne!(fx_hash_key(&[1, 2]), fx_hash_key(&[2, 1])); - } -} diff --git a/nodedb/src/engine/timeseries/grouped_scan/strategies/bucket.rs b/nodedb/src/engine/timeseries/grouped_scan/strategies/bucket.rs new file mode 100644 index 000000000..0afa20f65 --- /dev/null +++ b/nodedb/src/engine/timeseries/grouped_scan/strategies/bucket.rs @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Time-bucket grouping: rows keyed by `time_bucket(ts)` plus the group +//! columns, resolved to `"bucket_ts\0group1\0group2"` string keys. + +use rustc_hash::FxHashMap; + +use super::super::super::columnar_agg::AggAccum; +use super::super::types::{GroupedAggResult, accumulate_row, for_each_set_bit}; +use super::dispatch::GroupedScanInputs; +use super::keys::{key_part, push_key_part}; + +/// Time-bucket aggregation with integer keys. +/// +/// Packs (bucket_ts, group_key_parts...) into a `Vec`: +/// - `parts[0]` = bucket_ts as u64 +/// - `parts[1..]` = group column values (symbol IDs, i64, f64 bits) +/// +/// Resolves to string keys with "bucket_ts\0group1\0group2" format +/// so `emit_grouped_results` can parse them. +pub(super) fn aggregate_with_bucket<'a>( + inputs: GroupedScanInputs<'a>, + timestamps: &[i64], + bucket_interval_ms: i64, + sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, +) -> GroupedAggResult { + let GroupedScanInputs { + resolved, + columns, + mask, + row_count, + num_aggs, + } = inputs; + let key_len = 1 + resolved.group_cols.len(); // bucket + group columns + + let mut groups: FxHashMap, Vec> = FxHashMap::default(); + + for_each_set_bit(mask, row_count, |row_idx| { + let bucket = + super::super::super::time_bucket::time_bucket(bucket_interval_ms, timestamps[row_idx]); + + let mut key = Vec::with_capacity(key_len); + key.push(bucket as u64); + + // Pack group-by columns as integers. + for &(col_idx, ty) in &resolved.group_cols { + key.push(key_part(ty, columns[col_idx], row_idx)); + } + + let accums = groups + .entry(key) + .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); + accumulate_row(accums, resolved, columns, row_idx); + }); + + // Resolve integer keys to string keys: "bucket_ts\0group1\0group2" + let mut result = GroupedAggResult::new(num_aggs); + for (key_parts, accums) in groups { + let bucket_ts = key_parts[0] as i64; + let mut str_key = bucket_ts.to_string(); + + for (i, &part) in key_parts[1..].iter().enumerate() { + str_key.push('\0'); + if i < resolved.group_cols.len() { + let (col_idx, ty) = resolved.group_cols[i]; + push_key_part(&mut str_key, col_idx, ty, part, sym_lookup); + } + } + + result.merge_group(str_key, &accums); + } + + result +} diff --git a/nodedb/src/engine/timeseries/grouped_scan/strategies/dispatch.rs b/nodedb/src/engine/timeseries/grouped_scan/strategies/dispatch.rs new file mode 100644 index 000000000..7ee21c3c2 --- /dev/null +++ b/nodedb/src/engine/timeseries/grouped_scan/strategies/dispatch.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Strategy selection: picks the grouping strategy for a scan from its +//! GROUP BY shape, symbol cardinality, and row count, then resolves the +//! integer keys to strings. + +use super::super::super::columnar_memtable::{ColumnData, ColumnType}; +use super::super::types::{GroupedAggResult, ResolvedSchema}; +use super::bucket::aggregate_with_bucket; +use super::hashed::{ + aggregate_direct_index, aggregate_hash_generic, aggregate_hash_u32, aggregate_no_group, + aggregate_two_level, +}; +use super::keys::resolve_group_key; + +const DIRECT_INDEX_MAX_CARDINALITY: u32 = 65536; + +/// Row-level scan inputs shared by every grouping strategy: the resolved +/// schema, per-column data, the row-selection bitmask, row count, and +/// aggregate count. Bundled so the top-level dispatch/bucket entry points +/// stay within clippy's argument budget. +#[derive(Clone, Copy)] +pub(in crate::engine::timeseries::grouped_scan) struct GroupedScanInputs<'a> { + pub resolved: &'a ResolvedSchema, + pub columns: &'a [Option<&'a ColumnData>], + pub mask: &'a [u64], + pub row_count: usize, + pub num_aggs: usize, +} + +pub(in crate::engine::timeseries::grouped_scan) fn dispatch_grouping<'a>( + inputs: GroupedScanInputs<'a>, + group_by: &[String], + sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, + timestamps: Option<&[i64]>, + bucket_interval_ms: i64, +) -> GroupedAggResult { + let GroupedScanInputs { + resolved, + columns, + mask, + row_count, + num_aggs, + } = inputs; + let has_bucket = bucket_interval_ms > 0 && timestamps.is_some(); + + if has_bucket && let Some(ts) = timestamps { + return aggregate_with_bucket(inputs, ts, bucket_interval_ms, sym_lookup); + } + + let local_groups = if group_by.is_empty() { + aggregate_no_group(resolved, columns, mask, row_count, num_aggs) + } else if group_by.len() == 1 && resolved.group_cols[0].1 == ColumnType::Symbol { + let (col_idx, _) = resolved.group_cols[0]; + if let Some(data) = columns[col_idx] { + let sym_ids = data.as_symbols(); + let cardinality = sym_lookup(col_idx).map(|d| d.len() as u32).unwrap_or(0); + if cardinality <= DIRECT_INDEX_MAX_CARDINALITY && cardinality > 0 { + aggregate_direct_index( + resolved, + columns, + mask, + row_count, + num_aggs, + sym_ids, + cardinality, + ) + } else { + aggregate_hash_u32(resolved, columns, mask, row_count, num_aggs, sym_ids) + } + } else { + aggregate_hash_generic(resolved, columns, mask, row_count, num_aggs) + } + } else if row_count > 100_000 { + aggregate_two_level(resolved, columns, mask, row_count, num_aggs) + } else { + aggregate_hash_generic(resolved, columns, mask, row_count, num_aggs) + }; + + // Resolve integer keys to strings. + let mut result = GroupedAggResult::new(num_aggs); + for (int_key, accums) in local_groups { + let str_key = resolve_group_key(&int_key, resolved, sym_lookup); + result.merge_group(str_key, &accums); + } + result +} diff --git a/nodedb/src/engine/timeseries/grouped_scan/strategies/hashed.rs b/nodedb/src/engine/timeseries/grouped_scan/strategies/hashed.rs new file mode 100644 index 000000000..429153ac8 --- /dev/null +++ b/nodedb/src/engine/timeseries/grouped_scan/strategies/hashed.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Grouping strategies over integer keys: no-group, direct-index, FxHash +//! on a single symbol column, generic multi-column hash, and two-level +//! bucketed hash for high cardinality. + +use rustc_hash::FxHashMap; + +use super::super::super::columnar_agg::AggAccum; +use super::super::super::columnar_memtable::ColumnData; +use super::super::types::{ResolvedSchema, accumulate_row, for_each_set_bit}; +use super::keys::{IntGroupKey, build_generic_key, fx_hash_key}; + +pub(super) fn aggregate_no_group( + resolved: &ResolvedSchema, + columns: &[Option<&ColumnData>], + mask: &[u64], + row_count: usize, + num_aggs: usize, +) -> Vec<(IntGroupKey, Vec)> { + let mut accums: Vec = (0..num_aggs).map(|_| AggAccum::default()).collect(); + for_each_set_bit(mask, row_count, |row_idx| { + accumulate_row(&mut accums, resolved, columns, row_idx); + }); + vec![(IntGroupKey::None, accums)] +} + +pub(super) fn aggregate_direct_index( + resolved: &ResolvedSchema, + columns: &[Option<&ColumnData>], + mask: &[u64], + row_count: usize, + num_aggs: usize, + sym_ids: &[u32], + cardinality: u32, +) -> Vec<(IntGroupKey, Vec)> { + let card = cardinality as usize; + let mut table: Vec> = (0..card) + .map(|_| (0..num_aggs).map(|_| AggAccum::default()).collect()) + .collect(); + + for_each_set_bit(mask, row_count, |row_idx| { + let id = sym_ids[row_idx] as usize; + if id < card { + accumulate_row(&mut table[id], resolved, columns, row_idx); + } + }); + + table + .into_iter() + .enumerate() + .filter(|(_, accums)| accums.iter().any(|a| a.count > 0)) + .map(|(id, accums)| (IntGroupKey::SingleU32(id as u32), accums)) + .collect() +} + +pub(super) fn aggregate_hash_u32( + resolved: &ResolvedSchema, + columns: &[Option<&ColumnData>], + mask: &[u64], + row_count: usize, + num_aggs: usize, + sym_ids: &[u32], +) -> Vec<(IntGroupKey, Vec)> { + let mut groups: FxHashMap> = FxHashMap::default(); + + for_each_set_bit(mask, row_count, |row_idx| { + let id = sym_ids[row_idx]; + let accums = groups + .entry(id) + .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); + accumulate_row(accums, resolved, columns, row_idx); + }); + + groups + .into_iter() + .map(|(id, accums)| (IntGroupKey::SingleU32(id), accums)) + .collect() +} + +pub(super) fn aggregate_hash_generic( + resolved: &ResolvedSchema, + columns: &[Option<&ColumnData>], + mask: &[u64], + row_count: usize, + num_aggs: usize, +) -> Vec<(IntGroupKey, Vec)> { + let mut groups: FxHashMap, Vec> = FxHashMap::default(); + + for_each_set_bit(mask, row_count, |row_idx| { + let key = build_generic_key(resolved, columns, row_idx); + let accums = groups + .entry(key) + .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); + accumulate_row(accums, resolved, columns, row_idx); + }); + + groups + .into_iter() + .map(|(key, accums)| (IntGroupKey::Multi(key), accums)) + .collect() +} + +/// Two-level aggregation for high-cardinality GROUP BY (2M+ keys). +/// +/// Phase 1: Partition rows into buckets by hash prefix (top 8 bits → 256 buckets). +/// Phase 2: Aggregate each bucket independently (small HashMap, cache-friendly). +/// Phase 3: Flatten all buckets into the final result. +pub(super) fn aggregate_two_level( + resolved: &ResolvedSchema, + columns: &[Option<&ColumnData>], + mask: &[u64], + row_count: usize, + num_aggs: usize, +) -> Vec<(IntGroupKey, Vec)> { + const NUM_BUCKETS: usize = 256; + + // Phase 1: Build key for each row, hash it, and partition into buckets by top 8 bits. + // The key is stored alongside the row index so Phase 2 does not recompute it. + let mut buckets: Vec)>> = (0..NUM_BUCKETS).map(|_| Vec::new()).collect(); + for_each_set_bit(mask, row_count, |row_idx| { + let key = build_generic_key(resolved, columns, row_idx); + let hash = fx_hash_key(&key); + let bucket = (hash >> 56) as usize; + buckets[bucket].push((row_idx, key)); + }); + + // Phase 2 + 3: Aggregate each bucket independently and flatten. + let mut all_results: Vec<(IntGroupKey, Vec)> = Vec::new(); + for bucket_rows in buckets { + if bucket_rows.is_empty() { + continue; + } + let mut groups: FxHashMap, Vec> = FxHashMap::default(); + for (row_idx, key) in bucket_rows { + let accums = groups + .entry(key) + .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); + accumulate_row(accums, resolved, columns, row_idx); + } + all_results.extend(groups.into_iter().map(|(k, a)| (IntGroupKey::Multi(k), a))); + } + + all_results +} + +#[cfg(test)] +mod tests { + use super::super::super::super::columnar_memtable::ColumnType; + use super::super::super::types::AggColInfo; + use super::*; + + fn make_resolved_count(col_types: &[(usize, ColumnType)]) -> ResolvedSchema { + ResolvedSchema { + group_cols: col_types.to_vec(), + agg_cols: vec![AggColInfo::CountStar], + ts_idx: 0, + } + } + + /// Build a full-set mask for `row_count` rows. + fn full_mask(row_count: usize) -> Vec { + let words = row_count.div_ceil(64); + let mut mask = vec![u64::MAX; words]; + let rem = row_count % 64; + if rem != 0 { + *mask.last_mut().unwrap() = (1u64 << rem) - 1; + } + mask + } + + /// Collect aggregation results into a sorted Vec of (key, count) for comparison. + fn collect_counts(results: Vec<(IntGroupKey, Vec)>) -> Vec<(Vec, u64)> { + let mut out: Vec<(Vec, u64)> = results + .into_iter() + .map(|(key, accums)| { + let k = match key { + IntGroupKey::Multi(v) => v, + IntGroupKey::SingleU32(v) => vec![v as u64], + IntGroupKey::None => vec![], + }; + let count = accums.first().map(|a| a.count).unwrap_or(0); + (k, count) + }) + .collect(); + out.sort_by(|a, b| a.0.cmp(&b.0)); + out + } + + #[test] + fn two_level_matches_generic_multi_column() { + // Build two Int64 columns with 8 distinct combinations repeated many times. + let n = 200_000usize; + let col_a: Vec = (0..n).map(|i| (i % 4) as i64).collect(); + let col_b: Vec = (0..n).map(|i| (i % 2) as i64).collect(); + + let data_a = ColumnData::Int64(col_a); + let data_b = ColumnData::Int64(col_b); + let columns: Vec> = vec![Some(&data_a), Some(&data_b)]; + + let resolved = make_resolved_count(&[(0, ColumnType::Int64), (1, ColumnType::Int64)]); + let mask = full_mask(n); + + // num_aggs = 1 so AggAccum::count tracks row membership. + let generic = aggregate_hash_generic(&resolved, &columns, &mask, n, 1); + let two_level = aggregate_two_level(&resolved, &columns, &mask, n, 1); + + assert_eq!( + collect_counts(generic), + collect_counts(two_level), + "two-level and generic must produce identical group counts" + ); + } +} diff --git a/nodedb/src/engine/timeseries/grouped_scan/strategies/keys.rs b/nodedb/src/engine/timeseries/grouped_scan/strategies/keys.rs new file mode 100644 index 000000000..c7b86b21d --- /dev/null +++ b/nodedb/src/engine/timeseries/grouped_scan/strategies/keys.rs @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Integer group keys: packing a row's group columns into `u64` parts, +//! hashing them, and resolving them back to the string key emission reads. + +use super::super::super::columnar_memtable::{ColumnData, ColumnType}; +use super::super::types::ResolvedSchema; + +/// Integer group key for local (per-source) grouping. +#[derive(Clone, PartialEq, Eq, Hash)] +pub(super) enum IntGroupKey { + None, + SingleU32(u32), + Multi(Vec), +} + +/// FxHash-style multiplicative hash over a slice of u64 values. +/// +/// Used by `aggregate_two_level` to assign rows to buckets in Phase 1. +#[inline] +pub(super) fn fx_hash_key(key: &[u64]) -> u64 { + let mut hash: u64 = 0; + for &v in key { + hash = hash.wrapping_mul(0x517cc1b727220a95).wrapping_add(v); + } + hash +} + +/// Pack one group column's cell at `row_idx` into a `u64` key part. +/// +/// Symbols and integers pack as their value, floats as their bit pattern, +/// and a time column as its millisecond count. A column whose data is absent +/// or of the wrong shape packs as `u64::MAX`. +#[inline] +pub(super) fn key_part(ty: ColumnType, data: Option<&ColumnData>, row_idx: usize) -> u64 { + data.map(|data| match ty { + ColumnType::Symbol => { + if let ColumnData::Symbol(ids) = data { + ids[row_idx] as u64 + } else { + u64::MAX + } + } + ColumnType::Int64 => { + if let ColumnData::Int64(vals) = data { + vals[row_idx] as u64 + } else { + u64::MAX + } + } + ColumnType::Float64 => { + if let ColumnData::Float64(vals) = data { + vals[row_idx].to_bits() + } else { + u64::MAX + } + } + ColumnType::Timestamp(_) => { + if let ColumnData::Timestamp(vals) = data { + vals[row_idx] as u64 + } else { + u64::MAX + } + } + }) + .unwrap_or(u64::MAX) +} + +pub(super) fn build_generic_key( + resolved: &ResolvedSchema, + columns: &[Option<&ColumnData>], + row_idx: usize, +) -> Vec { + resolved + .group_cols + .iter() + .map(|&(idx, ty)| key_part(ty, columns[idx], row_idx)) + .collect() +} + +/// Append the string form of one packed key part for a column of type `ty`. +/// +/// A symbol resolves through its dictionary and appends nothing when the id +/// is unknown. Integers and time columns render as the signed integer packed, +/// floats from their bit pattern. +pub(super) fn push_key_part<'a>( + out: &mut String, + col_idx: usize, + ty: ColumnType, + part: u64, + sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, +) { + use std::fmt::Write; + match ty { + ColumnType::Symbol => { + if let Some(dict) = sym_lookup(col_idx) + && let Some(name) = dict.get(part as u32) + { + out.push_str(name); + } + } + ColumnType::Int64 | ColumnType::Timestamp(_) => { + let _ = write!(out, "{}", part as i64); + } + ColumnType::Float64 => { + let _ = write!(out, "{}", f64::from_bits(part)); + } + } +} + +pub(super) fn resolve_group_key<'a>( + key: &IntGroupKey, + resolved: &ResolvedSchema, + sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, +) -> String { + match key { + IntGroupKey::None => String::new(), + IntGroupKey::SingleU32(id) => { + let col_idx = resolved.group_cols[0].0; + if resolved.group_cols[0].1 == ColumnType::Symbol { + sym_lookup(col_idx) + .and_then(|d: &nodedb_types::timeseries::SymbolDictionary| d.get(*id)) + .unwrap_or("") + .to_string() + } else { + id.to_string() + } + } + IntGroupKey::Multi(parts) => { + let mut s = String::with_capacity(parts.len() * 16); + for (i, &part) in parts.iter().enumerate() { + if i > 0 { + s.push('\0'); + } + let (col_idx, ty) = resolved.group_cols[i]; + push_key_part(&mut s, col_idx, ty, part, sym_lookup); + } + s + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fx_hash_key_deterministic() { + let key = vec![1u64, 2, 3, 4]; + assert_eq!(fx_hash_key(&key), fx_hash_key(&key)); + assert_ne!(fx_hash_key(&[1, 2]), fx_hash_key(&[2, 1])); + } +} diff --git a/nodedb/src/engine/timeseries/grouped_scan/strategies/mod.rs b/nodedb/src/engine/timeseries/grouped_scan/strategies/mod.rs new file mode 100644 index 000000000..19512c80a --- /dev/null +++ b/nodedb/src/engine/timeseries/grouped_scan/strategies/mod.rs @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: BUSL-1.1 + +mod bucket; +mod dispatch; +mod hashed; +mod keys; + +pub(super) use dispatch::{GroupedScanInputs, dispatch_grouping}; diff --git a/nodedb/src/engine/timeseries/grouped_scan/types.rs b/nodedb/src/engine/timeseries/grouped_scan/types.rs index 461c2af1f..a033ae539 100644 --- a/nodedb/src/engine/timeseries/grouped_scan/types.rs +++ b/nodedb/src/engine/timeseries/grouped_scan/types.rs @@ -25,14 +25,20 @@ impl GroupedAggResult { pub fn merge(&mut self, other: &GroupedAggResult) { for (key, other_accums) in &other.groups { - let accums = self - .groups - .entry(key.clone()) - .or_insert_with(|| (0..self.num_aggs).map(|_| AggAccum::default()).collect()); - for (i, a) in other_accums.iter().enumerate() { - if i < accums.len() { - accums[i].merge(a); - } + self.merge_group(key.clone(), other_accums); + } + } + + /// Merge one group's accumulators into this result, creating the group's + /// entry if it is not already present. + pub(super) fn merge_group(&mut self, key: String, accums: &[AggAccum]) { + let entry = self + .groups + .entry(key) + .or_insert_with(|| (0..self.num_aggs).map(|_| AggAccum::default()).collect()); + for (i, a) in accums.iter().enumerate() { + if i < entry.len() { + entry[i].merge(a); } } } @@ -91,10 +97,10 @@ pub(super) fn resolve_schema( AggColInfo::CountField } else { match ty { - ColumnType::Float64 | ColumnType::Int64 | ColumnType::Timestamp => { + ColumnType::Float64 | ColumnType::Int64 | ColumnType::Timestamp(_) => { AggColInfo::Numeric(idx) } - _ => AggColInfo::Skip, + ColumnType::Symbol => AggColInfo::Skip, } } } else if op == "count" { diff --git a/nodedb/src/engine/timeseries/ilp_ingest.rs b/nodedb/src/engine/timeseries/ilp_ingest.rs index e7b932423..106983d0d 100644 --- a/nodedb/src/engine/timeseries/ilp_ingest.rs +++ b/nodedb/src/engine/timeseries/ilp_ingest.rs @@ -140,10 +140,10 @@ pub fn ingest_batch_with_lvc(args: IngestBatchArgs<'_, '_>) -> IngestBatchOutcom // Only the designated time column takes the line's timestamp. // Any other timestamp column is an ordinary column and carries // whatever the row itself supplied. - ColumnType::Timestamp if col_idx == schema.timestamp_idx => { + ColumnType::Timestamp(_) if col_idx == schema.timestamp_idx => { values.push(ColumnValue::Timestamp(ts_ms)); } - ColumnType::Timestamp => { + ColumnType::Timestamp(_) => { values.push(ColumnValue::Timestamp(find_field_timestamp_ms( &line.fields, col_name, @@ -325,7 +325,9 @@ fn find_field_i64_opt<'a>(fields: &[(Cow<'a, str>, FieldValue<'a>)], name: &str) #[cfg(test)] mod tests { use super::*; - use crate::engine::timeseries::columnar_memtable::{ColumnData, ColumnarMemtableConfig}; + use crate::engine::timeseries::columnar_memtable::{ + ColumnData, ColumnarMemtableConfig, TimeKind, + }; use crate::engine::timeseries::ilp::parse_batch; fn default_config() -> ColumnarMemtableConfig { @@ -347,7 +349,7 @@ mod tests { assert_eq!(schema.columns.len(), 5); assert_eq!( schema.columns[0], - ("timestamp".into(), ColumnType::Timestamp) + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)) ); assert_eq!(schema.columns[1].1, ColumnType::Symbol); // host assert_eq!(schema.columns[2].1, ColumnType::Symbol); // dc diff --git a/nodedb/src/engine/timeseries/ilp_schema.rs b/nodedb/src/engine/timeseries/ilp_schema.rs index 0efe4a229..94d025f1e 100644 --- a/nodedb/src/engine/timeseries/ilp_schema.rs +++ b/nodedb/src/engine/timeseries/ilp_schema.rs @@ -13,13 +13,15 @@ use nodedb_types::columnar::schema::{TS_SYSTEM, TS_VALID_FROM, TS_VALID_UNTIL}; -use super::columnar_memtable::{ColumnType, ColumnarMemtable, ColumnarSchema}; +use super::columnar_memtable::{ColumnType, ColumnarMemtable, ColumnarSchema, TimeKind}; use super::ilp::{FieldValue, IlpLine}; /// Infers a columnar schema from a batch of ILP lines. /// /// Scans all lines to discover tag keys and field keys, then builds /// a schema: timestamp + tag columns (Symbol) + field columns (typed). +/// The inferred time column is `Millis`: no declaration names it an +/// instant, so it reads back as the integer the line carried. pub fn infer_schema(lines: &[IlpLine<'_>]) -> ColumnarSchema { let mut tag_keys: Vec = Vec::new(); let mut field_keys: Vec<(String, ColumnType)> = Vec::new(); @@ -46,7 +48,10 @@ pub fn infer_schema(lines: &[IlpLine<'_>]) -> ColumnarSchema { } let mut columns = Vec::with_capacity(1 + tag_keys.len() + field_keys.len()); - columns.push(("timestamp".to_string(), ColumnType::Timestamp)); + columns.push(( + "timestamp".to_string(), + ColumnType::Timestamp(TimeKind::Millis), + )); for tag in &tag_keys { columns.push((tag.clone(), ColumnType::Symbol)); } diff --git a/nodedb/src/engine/timeseries/merge/o3.rs b/nodedb/src/engine/timeseries/merge/o3.rs index fec4d97f9..98eab09c3 100644 --- a/nodedb/src/engine/timeseries/merge/o3.rs +++ b/nodedb/src/engine/timeseries/merge/o3.rs @@ -38,13 +38,17 @@ pub fn merge_o3_into_partition( // Read existing partition data. let existing_meta = ColumnarSegmentReader::read_meta(&partition_dir, None)?; let existing_schema = ColumnarSegmentReader::read_schema(&partition_dir, None)?; - - let ts_col = ColumnarSegmentReader::read_column( - &partition_dir, - "timestamp", - ColumnType::Timestamp, - None, - )?; + let ts_type = existing_schema + .columns + .get(existing_schema.timestamp_idx) + .map(|(_, ty)| *ty) + .ok_or_else(|| { + SegmentError::Corrupt(format!( + "partition {partition_dir_name} schema has no designated time column" + )) + })?; + + let ts_col = ColumnarSegmentReader::read_column(&partition_dir, "timestamp", ts_type, None)?; let val_col = ColumnarSegmentReader::read_column(&partition_dir, "value", ColumnType::Float64, None)?; diff --git a/nodedb/src/engine/timeseries/merge/partitions.rs b/nodedb/src/engine/timeseries/merge/partitions.rs index 0e08ec248..d095bd58e 100644 --- a/nodedb/src/engine/timeseries/merge/partitions.rs +++ b/nodedb/src/engine/timeseries/merge/partitions.rs @@ -194,7 +194,7 @@ pub struct MergeResult { impl ColumnData { pub(super) fn new_empty(ty: ColumnType) -> Self { match ty { - ColumnType::Timestamp => Self::Timestamp(Vec::new()), + ColumnType::Timestamp(_) => Self::Timestamp(Vec::new()), ColumnType::Float64 => Self::Float64(Vec::new()), ColumnType::Int64 => Self::Int64(Vec::new()), ColumnType::Symbol => Self::Symbol(Vec::new()), @@ -258,11 +258,13 @@ mod tests { use tempfile::TempDir; use crate::engine::timeseries::columnar_memtable::{ - ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, + ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; use super::*; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + fn test_config() -> ColumnarMemtableConfig { ColumnarMemtableConfig { max_memory_bytes: 10 * 1024 * 1024, @@ -305,13 +307,8 @@ mod tests { // Read back merged data. let merged_dir = tmp.path().join("ts-merged"); - let ts_col = ColumnarSegmentReader::read_column( - &merged_dir, - "timestamp", - ColumnType::Timestamp, - None, - ) - .unwrap(); + let ts_col = + ColumnarSegmentReader::read_column(&merged_dir, "timestamp", MILLIS, None).unwrap(); let timestamps = ts_col.as_timestamps(); assert_eq!(timestamps.len(), 100); // Should be sorted. @@ -325,7 +322,7 @@ mod tests { let tmp = TempDir::new().unwrap(); let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("value".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], @@ -384,13 +381,8 @@ mod tests { ); let merged_dir = tmp.path().join("ts-sorted"); - let ts_col = ColumnarSegmentReader::read_column( - &merged_dir, - "timestamp", - ColumnType::Timestamp, - None, - ) - .unwrap(); + let ts_col = + ColumnarSegmentReader::read_column(&merged_dir, "timestamp", MILLIS, None).unwrap(); let timestamps = ts_col.as_timestamps(); // All timestamps from partition 2 (1000-1019) should come before partition 1 (5000-5019). assert_eq!(timestamps[0], 1000); diff --git a/nodedb/src/engine/timeseries/projection.rs b/nodedb/src/engine/timeseries/projection.rs index 20fd9e4f3..1b854fa30 100644 --- a/nodedb/src/engine/timeseries/projection.rs +++ b/nodedb/src/engine/timeseries/projection.rs @@ -104,9 +104,11 @@ pub fn should_use_tag_projection( mod tests { use super::*; use crate::engine::timeseries::columnar_memtable::{ - ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, + ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, TimeKind, }; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + fn test_config() -> ColumnarMemtableConfig { ColumnarMemtableConfig { max_memory_bytes: 10 * 1024 * 1024, @@ -119,7 +121,7 @@ mod tests { fn sort_by_tag_then_timestamp() { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("value".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], @@ -158,7 +160,7 @@ mod tests { fn identity_permutation_for_sorted_data() { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("value".into(), ColumnType::Float64), ], timestamp_idx: 0, @@ -182,7 +184,7 @@ mod tests { fn tag_projection_heuristic() { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("host".into(), ColumnType::Symbol), ], timestamp_idx: 0, diff --git a/nodedb/src/engine/timeseries/schema_evolution.rs b/nodedb/src/engine/timeseries/schema_evolution.rs index aa7251e98..ce222b159 100644 --- a/nodedb/src/engine/timeseries/schema_evolution.rs +++ b/nodedb/src/engine/timeseries/schema_evolution.rs @@ -46,7 +46,9 @@ pub fn build_column_mappings( { Some(idx) => { let (_, p_type) = &partition_schema.columns[idx]; - if p_type == q_type { + // Every time kind shares one millisecond storage, so a + // time column is present whatever kind either side names. + if p_type == q_type || (p_type.is_time() && q_type.is_time()) { ColumnMapping::Present(idx) } else if can_widen(*p_type, *q_type) { ColumnMapping::Widen { @@ -98,7 +100,7 @@ pub fn apply_mappings( /// Create a NULL-equivalent column of the given type and length. fn null_column(ty: ColumnType, rows: usize) -> ColumnData { match ty { - ColumnType::Timestamp => ColumnData::Timestamp(vec![0; rows]), + ColumnType::Timestamp(_) => ColumnData::Timestamp(vec![0; rows]), ColumnType::Float64 => ColumnData::Float64(vec![f64::NAN; rows]), ColumnType::Int64 => ColumnData::Int64(vec![0; rows]), ColumnType::Symbol => ColumnData::Symbol(vec![u32::MAX; rows]), // sentinel @@ -188,12 +190,15 @@ pub fn apply_schema_changes( #[cfg(test)] mod tests { + use super::super::columnar_memtable::TimeKind; use super::*; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + fn schema_v1() -> ColumnarSchema { ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], @@ -205,7 +210,7 @@ mod tests { fn schema_v2_added_column() -> ColumnarSchema { ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ("mem".into(), ColumnType::Float64), @@ -258,7 +263,7 @@ mod tests { fn widen_int64_to_float64() { let query = ColumnarSchema { columns: vec![ - ("ts".into(), ColumnType::Timestamp), + ("ts".into(), MILLIS), ("val".into(), ColumnType::Float64), // query expects f64 ], timestamp_idx: 0, @@ -266,7 +271,7 @@ mod tests { }; let partition = ColumnarSchema { columns: vec![ - ("ts".into(), ColumnType::Timestamp), + ("ts".into(), MILLIS), ("val".into(), ColumnType::Int64), // partition has i64 ], timestamp_idx: 0, @@ -284,6 +289,27 @@ mod tests { assert!((val[0] - 42.0).abs() < f64::EPSILON); } + /// A partition whose time column carries another kind still maps as + /// present: the kind is how a cell is read, not what is stored. + #[test] + fn time_column_maps_present_across_kinds() { + let query = ColumnarSchema { + columns: vec![( + "ts".into(), + ColumnType::Timestamp(TimeKind::Instant(nodedb_types::InstantKind::Naive)), + )], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 1], + }; + let partition = ColumnarSchema { + columns: vec![("ts".into(), MILLIS)], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 1], + }; + let mappings = build_column_mappings(&query, &partition); + assert!(matches!(mappings[0], ColumnMapping::Present(0))); + } + #[test] fn add_column_schema_change() { let s = schema_v1(); diff --git a/nodedb/src/engine/timeseries/sparse_index/index.rs b/nodedb/src/engine/timeseries/sparse_index/index.rs index 0b2e344d5..b1abe762b 100644 --- a/nodedb/src/engine/timeseries/sparse_index/index.rs +++ b/nodedb/src/engine/timeseries/sparse_index/index.rs @@ -356,7 +356,7 @@ fn compute_block_stats( row_end: usize, ) -> BlockColumnStats { match (col, col_type) { - (ColumnData::Timestamp(v), ColumnType::Timestamp) => { + (ColumnData::Timestamp(v), ColumnType::Timestamp(_)) => { let slice = &v[row_start..row_end]; if slice.is_empty() { return BlockColumnStats::none(); @@ -424,9 +424,13 @@ fn compute_block_stats( #[cfg(test)] mod tests { - use super::super::super::columnar_memtable::{ColumnData, ColumnType, ColumnarSchema}; + use super::super::super::columnar_memtable::{ + ColumnData, ColumnType, ColumnarSchema, TimeKind, + }; use super::*; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + fn make_test_columns(row_count: usize) -> (Vec, ColumnarSchema) { let timestamps: Vec = (0..row_count as i64) .map(|i| 1_700_000_000_000 + i * 10_000) @@ -439,7 +443,7 @@ mod tests { ]; let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ], timestamp_idx: 0, @@ -475,7 +479,7 @@ mod tests { let columns = vec![ColumnData::Timestamp(vec![]), ColumnData::Float64(vec![])]; let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ], timestamp_idx: 0, @@ -570,7 +574,7 @@ mod tests { ]; let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ], timestamp_idx: 0, diff --git a/nodedb/src/engine/timeseries/tag_autocomplete.rs b/nodedb/src/engine/timeseries/tag_autocomplete.rs index e0d4bf3d9..7bd35d080 100644 --- a/nodedb/src/engine/timeseries/tag_autocomplete.rs +++ b/nodedb/src/engine/timeseries/tag_autocomplete.rs @@ -109,7 +109,7 @@ pub enum TagAutoError { mod tests { use super::*; use crate::engine::timeseries::columnar_memtable::{ - ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, + ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; use crate::engine::timeseries::columnar_segment::ColumnarSegmentWriter; use tempfile::TempDir; @@ -125,7 +125,7 @@ mod tests { fn make_tagged_memtable() -> ColumnarMemtable { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), ("value".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ("region".into(), ColumnType::Symbol), diff --git a/nodedb/src/engine/timeseries/ts_detect.rs b/nodedb/src/engine/timeseries/ts_detect.rs index 824422b3c..a624ffa91 100644 --- a/nodedb/src/engine/timeseries/ts_detect.rs +++ b/nodedb/src/engine/timeseries/ts_detect.rs @@ -70,7 +70,7 @@ pub fn detect_timestamp( ) -> TsDetection { // Tier 1: Column type is Timestamp. for (i, (name, ty)) in columns.iter().enumerate() { - if *ty == ColumnType::Timestamp { + if ty.is_time() { return TsDetection::ByType { column_index: i, column_name: name.clone(), @@ -133,13 +133,16 @@ fn looks_epoch_like(values: &[i64]) -> bool { #[cfg(test)] mod tests { + use super::super::columnar_memtable::TimeKind; use super::*; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + #[test] fn detect_by_type() { let cols = vec![ ("id".into(), ColumnType::Int64), - ("ts".into(), ColumnType::Timestamp), + ("ts".into(), MILLIS), ("value".into(), ColumnType::Float64), ]; let result = detect_timestamp(&cols, None); @@ -238,7 +241,7 @@ mod tests { fn type_takes_priority_over_name() { let cols = vec![ ("timestamp".into(), ColumnType::Int64), // name match - ("t".into(), ColumnType::Timestamp), // type match + ("t".into(), MILLIS), // type match ]; let result = detect_timestamp(&cols, None); // Type should win. diff --git a/nodedb/src/engine/timeseries/verification.rs b/nodedb/src/engine/timeseries/verification.rs index a41a00207..a6780cff9 100644 --- a/nodedb/src/engine/timeseries/verification.rs +++ b/nodedb/src/engine/timeseries/verification.rs @@ -15,6 +15,8 @@ mod tests { use nodedb_types::timeseries::*; use tempfile::TempDir; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + fn test_memtable_config() -> ColumnarMemtableConfig { ColumnarMemtableConfig { max_memory_bytes: 10 * 1024 * 1024, @@ -188,13 +190,8 @@ mod tests { let mut total_rows = 0; for entry in &matching { let part_dir = tmp.path().join(&entry.dir_name); - let ts_col = ColumnarSegmentReader::read_column( - &part_dir, - "timestamp", - ColumnType::Timestamp, - None, - ) - .unwrap(); + let ts_col = + ColumnarSegmentReader::read_column(&part_dir, "timestamp", MILLIS, None).unwrap(); total_rows += ts_col.len(); } assert_eq!(total_rows, 300, "all 300 rows should be readable"); @@ -240,7 +237,7 @@ mod tests { // V1 schema: timestamp + cpu let schema_v1 = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ], timestamp_idx: 0, @@ -262,7 +259,7 @@ mod tests { // V2 schema: timestamp + cpu + mem (added) let schema_v2 = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ("mem".into(), ColumnType::Float64), ], @@ -295,7 +292,7 @@ mod tests { ColumnarSegmentReader::read_column( &tmp.path().join("ts-v1"), "timestamp", - ColumnType::Timestamp, + MILLIS, None, ) .unwrap(), @@ -332,7 +329,7 @@ mod tests { fn symbol_cardinality_breaker_rejects_with_message() { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("value".into(), ColumnType::Float64), ("uuid_tag".into(), ColumnType::Symbol), ], diff --git a/nodedb/src/error_from.rs b/nodedb/src/error_from.rs index 7a5fd470e..67eaf8316 100644 --- a/nodedb/src/error_from.rs +++ b/nodedb/src/error_from.rs @@ -45,6 +45,14 @@ impl From for Error { } } +impl From for Error { + fn from(e: crate::engine::timeseries::grouped_filter::UnsupportedPredicate) -> Self { + Self::FeatureNotSupported { + detail: e.to_string(), + } + } +} + impl From for Error { fn from(e: crate::engine::timeseries::query::QueryError) -> Self { Self::Storage { diff --git a/nodedb/src/storage/cold_filter.rs b/nodedb/src/storage/cold_filter.rs index 64bcdda69..42bd5d20d 100644 --- a/nodedb/src/storage/cold_filter.rs +++ b/nodedb/src/storage/cold_filter.rs @@ -22,13 +22,14 @@ use std::sync::{Arc, Mutex}; use arrow::array::{Array, Float64Array, Int64Array, RecordBatch, StringArray}; use arrow::datatypes::DataType; use bytes::Bytes; +use nodedb_types::Value; use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::{ArrowPredicateFn, ParquetRecordBatchReaderBuilder, RowFilter}; use parquet::file::metadata::RowGroupMetaData; use parquet::file::statistics::Statistics; use tracing::debug; -use crate::bridge::scan_filter::ScanFilter; +use crate::bridge::scan_filter::{FilterOp, ScanFilter}; /// Read a Parquet file with both row-group pruning and row-level filtering. /// @@ -221,19 +222,41 @@ fn row_group_might_match(rg: &RowGroupMetaData, filter: &ScanFilter) -> bool { return true; // No statistics — can't prune. }; - // Use min/max statistics to prune. - use nodedb_query::scan_filter::FilterOp; - let json_val: serde_json::Value = filter.value.clone().into(); - match &filter.op { - FilterOp::Eq => stat_might_contain(stats, &json_val), - FilterOp::Gt | FilterOp::Gte => stat_max_gte(stats, &json_val), - FilterOp::Lt | FilterOp::Lte => stat_min_lte(stats, &json_val), - _ => true, // Complex operators (contains, like, etc.) — can't prune. + // Use min/max statistics to prune. Only a scalar range predicate has a + // min/max form; every other operator keeps the row group. + match filter.op { + FilterOp::Eq => stat_might_contain(stats, &filter.value), + FilterOp::Gt | FilterOp::Gte => stat_max_gte(stats, &filter.value), + FilterOp::Lt | FilterOp::Lte => stat_min_lte(stats, &filter.value), + FilterOp::Ne + | FilterOp::Contains + | FilterOp::Like + | FilterOp::NotLike + | FilterOp::Ilike + | FilterOp::NotIlike + | FilterOp::In + | FilterOp::NotIn + | FilterOp::IsNull + | FilterOp::IsNotNull + | FilterOp::ArrayContains + | FilterOp::ArrayContainsAll + | FilterOp::ArrayOverlap + | FilterOp::MatchAll + | FilterOp::Exists + | FilterOp::NotExists + | FilterOp::Or + | FilterOp::Expr + | FilterOp::GtColumn + | FilterOp::GteColumn + | FilterOp::LtColumn + | FilterOp::LteColumn + | FilterOp::EqColumn + | FilterOp::NeColumn => true, } } /// Check if column statistics allow an equality match. -fn stat_might_contain(stats: &Statistics, value: &serde_json::Value) -> bool { +fn stat_might_contain(stats: &Statistics, value: &Value) -> bool { match stats { Statistics::Int64(s) => { let (Some(min), Some(max)) = (s.min_opt(), s.max_opt()) else { @@ -272,7 +295,7 @@ fn stat_might_contain(stats: &Statistics, value: &serde_json::Value) -> bool { } /// Check if the column max >= value (for gt/gte predicates). -fn stat_max_gte(stats: &Statistics, value: &serde_json::Value) -> bool { +fn stat_max_gte(stats: &Statistics, value: &Value) -> bool { match stats { Statistics::Int64(s) => { let Some(max) = s.max_opt() else { return true }; @@ -287,7 +310,7 @@ fn stat_max_gte(stats: &Statistics, value: &serde_json::Value) -> bool { } /// Check if the column min <= value (for lt/lte predicates). -fn stat_min_lte(stats: &Statistics, value: &serde_json::Value) -> bool { +fn stat_min_lte(stats: &Statistics, value: &Value) -> bool { match stats { Statistics::Int64(s) => { let Some(min) = s.min_opt() else { return true }; @@ -481,7 +504,7 @@ mod tests { let filters = vec![ScanFilter { field: "age".into(), - op: "gt".into(), + op: crate::bridge::scan_filter::FilterOp::Gt, value: nodedb_types::Value::Integer(25), clauses: vec![], expr: None, @@ -509,7 +532,7 @@ mod tests { let filters = vec![ScanFilter { field: String::new(), - op: "expr".into(), + op: crate::bridge::scan_filter::FilterOp::Expr, value: nodedb_types::Value::Null, clauses: vec![], expr: Some(SqlExpr::BinaryOp { @@ -538,7 +561,7 @@ mod tests { // `10 / age > 1` is true only for d1 (10/2 = 5 > 1); d2 (10/30 = 0) fails. let filters = vec![ScanFilter { field: String::new(), - op: "expr".into(), + op: crate::bridge::scan_filter::FilterOp::Expr, value: nodedb_types::Value::Null, clauses: vec![], expr: Some(SqlExpr::BinaryOp { diff --git a/nodedb/src/util.rs b/nodedb/src/util.rs index da33524e5..909036999 100644 --- a/nodedb/src/util.rs +++ b/nodedb/src/util.rs @@ -4,6 +4,8 @@ pub mod bounded_json; pub mod bounded_msgpack; +pub mod rmpv_value; +pub mod wire_json; /// FNV-1a 64-bit hash of a byte slice. /// diff --git a/nodedb/src/util/rmpv_value.rs b/nodedb/src/util/rmpv_value.rs new file mode 100644 index 000000000..f79e5ca26 --- /dev/null +++ b/nodedb/src/util/rmpv_value.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The one converter pair between `rmpv::Value` and `nodedb_types::Value`. +//! +//! Timeseries rows, trigger batches and `RETURNING` projections all move +//! between the two dynamic forms. Both directions live here so an instant, +//! a binary blob or a nested map renders the same way on every path. +//! +//! Instants cross as the msgpack ext `nodedb_types::write_instant` produces: +//! `rmpv::Value::Ext(1 | 2, )`. Every other ext type reads as +//! `Value::Null`, the rule `nodedb_types::value_from_msgpack` applies. + +use std::collections::HashMap; + +use nodedb_types::json_msgpack::instant_from_ext; +use nodedb_types::{InstantKind, Value}; + +/// Convert an `rmpv::Value` to a `nodedb_types::Value`. +/// +/// - `Integer` outside `i64` reads as its `u64` bits cast to `i64`. +/// - A `String` that is not UTF-8 reads as `Value::Bytes` of its raw bytes. +/// - A map key that is not a string renders through rmpv's `Display`. +/// - `Ext` reads as an instant for ext types 1 and 2, else `Value::Null`. +pub fn rmpv_to_value(v: &rmpv::Value) -> Value { + match v { + rmpv::Value::Nil => Value::Null, + rmpv::Value::Boolean(b) => Value::Bool(*b), + rmpv::Value::Integer(n) => match (n.as_i64(), n.as_u64()) { + (Some(i), _) => Value::Integer(i), + (None, Some(u)) => Value::Integer(u as i64), + (None, None) => Value::Null, + }, + rmpv::Value::F32(f) => Value::Float(f64::from(*f)), + rmpv::Value::F64(f) => Value::Float(*f), + rmpv::Value::String(s) => match s.as_str() { + Some(text) => Value::String(text.to_string()), + None => Value::Bytes(s.as_bytes().to_vec()), + }, + rmpv::Value::Binary(b) => Value::Bytes(b.clone()), + rmpv::Value::Array(items) => Value::Array(items.iter().map(rmpv_to_value).collect()), + rmpv::Value::Map(entries) => { + let mut map = HashMap::with_capacity(entries.len()); + for (key, value) in entries { + let name = match key { + rmpv::Value::String(s) => match s.as_str() { + Some(text) => text.to_string(), + None => key.to_string(), + }, + other => other.to_string(), + }; + map.insert(name, rmpv_to_value(value)); + } + Value::Object(map) + } + rmpv::Value::Ext(ext_type, payload) => match instant_from_ext(*ext_type, payload) { + Some((kind, micros)) => kind.from_micros(micros), + None => Value::Null, + }, + } +} + +/// Convert a `nodedb_types::Value` to an `rmpv::Value`. +/// +/// Mirrors `nodedb_types::value_to_msgpack` variant for variant, so an rmpv +/// row encodes to the same bytes a `Value` row does: +/// +/// - `Uuid`, `Ulid`, `Regex`, `Duration`, `Decimal` and `Geometry` are strings. +/// - `Set` is an array. `Vector` is an array of `F64`. +/// - `ArrayCell` is a `{coords, attrs}` map, plus `_ts_system` when versioned. +/// - `Range` and `Record` have no rmpv form and become `Nil`. +pub fn value_to_rmpv(v: &Value) -> rmpv::Value { + match v { + Value::Null => rmpv::Value::Nil, + Value::Bool(b) => rmpv::Value::Boolean(*b), + Value::Integer(n) => rmpv::Value::Integer((*n).into()), + Value::Float(f) => rmpv::Value::F64(*f), + Value::String(s) | Value::Uuid(s) | Value::Ulid(s) | Value::Regex(s) => { + rmpv::Value::String(s.as_str().into()) + } + Value::Bytes(b) => rmpv::Value::Binary(b.clone()), + Value::Array(items) | Value::Set(items) => { + rmpv::Value::Array(items.iter().map(value_to_rmpv).collect()) + } + Value::Object(map) => rmpv::Value::Map( + map.iter() + .map(|(k, v)| (rmpv::Value::String(k.as_str().into()), value_to_rmpv(v))) + .collect(), + ), + Value::DateTime(dt) => instant_ext(InstantKind::Utc, dt.micros), + Value::NaiveDateTime(dt) => instant_ext(InstantKind::Naive, dt.micros), + Value::Duration(d) => rmpv::Value::String(d.to_string().into()), + Value::Decimal(d) => rmpv::Value::String(d.to_string().into()), + Value::Geometry(g) => match sonic_rs::to_string(g) { + Ok(text) => rmpv::Value::String(text.into()), + Err(_) => rmpv::Value::Nil, + }, + Value::Range { .. } | Value::Record { .. } => rmpv::Value::Nil, + Value::Vector(floats) => rmpv::Value::Array( + floats + .iter() + .map(|f| rmpv::Value::F64(f64::from(*f))) + .collect(), + ), + Value::ArrayCell(cell) => { + let mut fields = vec![ + ( + rmpv::Value::String("coords".into()), + rmpv::Value::Array(cell.coords.iter().map(value_to_rmpv).collect()), + ), + ( + rmpv::Value::String("attrs".into()), + rmpv::Value::Array(cell.attrs.iter().map(value_to_rmpv).collect()), + ), + ]; + if let Some(ts) = cell.system_time { + fields.push(( + rmpv::Value::String("_ts_system".into()), + rmpv::Value::Integer(ts.into()), + )); + } + rmpv::Value::Map(fields) + } + // `Value` is `#[non_exhaustive]`: a variant this crate cannot name + // has no rmpv form. + _ => rmpv::Value::Nil, + } +} + +fn instant_ext(kind: InstantKind, micros: i64) -> rmpv::Value { + rmpv::Value::Ext(kind.ext_type(), micros.to_be_bytes().to_vec()) +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_types::NdbDateTime; + + #[test] + fn instants_round_trip_both_kinds() { + for v in [ + Value::DateTime(NdbDateTime::from_micros(1_583_402_400_000_000)), + Value::NaiveDateTime(NdbDateTime::from_micros(1_583_402_400_000_000)), + ] { + let ext = value_to_rmpv(&v); + assert!(matches!(ext, rmpv::Value::Ext(1 | 2, ref b) if b.len() == 8)); + assert_eq!(rmpv_to_value(&ext), v); + } + } + + #[test] + fn negative_micros_round_trip() { + let v = Value::NaiveDateTime(NdbDateTime::from_micros(-86_400_000_000)); + assert_eq!(rmpv_to_value(&value_to_rmpv(&v)), v); + } + + #[test] + fn instant_ext_matches_the_native_encoding() { + let v = Value::DateTime(NdbDateTime::from_micros(42)); + let mut via_rmpv = Vec::new(); + rmpv::encode::write_value(&mut via_rmpv, &value_to_rmpv(&v)).expect("encode"); + let native = nodedb_types::value_to_msgpack(&v).expect("native"); + assert_eq!(via_rmpv, native); + } + + #[test] + fn nested_object_and_array_round_trip() { + let mut inner = HashMap::new(); + inner.insert("n".to_string(), Value::Integer(-7)); + inner.insert("b".to_string(), Value::Bytes(vec![9, 8])); + let mut outer = HashMap::new(); + outer.insert( + "arr".to_string(), + Value::Array(vec![ + Value::Null, + Value::Bool(true), + Value::Float(1.5), + Value::String("x".into()), + Value::Object(inner), + ]), + ); + let v = Value::Object(outer); + assert_eq!(rmpv_to_value(&value_to_rmpv(&v)), v); + } + + #[test] + fn unknown_ext_reads_as_null() { + assert_eq!(rmpv_to_value(&rmpv::Value::Ext(7, vec![0; 8])), Value::Null); + } + + #[test] + fn u64_beyond_i64_reads_as_its_bits() { + let v = rmpv::Value::Integer(rmpv::Integer::from(u64::MAX)); + assert_eq!(rmpv_to_value(&v), Value::Integer(-1)); + } +} diff --git a/nodedb/src/util/wire_json.rs b/nodedb/src/util/wire_json.rs new file mode 100644 index 000000000..989cf247f --- /dev/null +++ b/nodedb/src/util/wire_json.rs @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Edge conversion of a typed cell to the JSON a text protocol emits. +//! +//! HTTP renders JSON text, pgwire renders a composite cell's JSON text +//! (`response_shape::cell::cell_text`), and `control::security` redacts a +//! typed cell before rendering it into a redaction preview; each converts +//! through [`value_to_wire_json`], and nowhere else: the one place a byte +//! cell picks its text form is here. This lives outside +//! `control::server::response_shape` so `control::security` — which +//! `response_shape` itself depends on for redaction — never depends back +//! on `control::server`. +//! +//! A `Value::Bytes` cell renders as unpadded standard base64, the text the +//! msgpack → JSON transcoder (`nodedb_types::msgpack_to_json_string`) emits +//! for a msgpack `bin` — not the hex `serde_json::Value::from(Value)` +//! produces. Every other variant renders exactly as that `From` does. + +use base64::Engine; +use nodedb_types::Value; + +/// Render one typed cell as the JSON value a text protocol emits. +pub fn value_to_wire_json(cell: &Value) -> serde_json::Value { + match cell { + Value::Bytes(bytes) => serde_json::Value::String( + base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes), + ), + Value::Array(items) | Value::Set(items) => { + serde_json::Value::Array(items.iter().map(value_to_wire_json).collect()) + } + Value::Object(map) => serde_json::Value::Object( + map.iter() + .map(|(k, v)| (k.clone(), value_to_wire_json(v))) + .collect(), + ), + Value::Null + | Value::Bool(_) + | Value::Integer(_) + | Value::Float(_) + | Value::String(_) + | Value::Uuid(_) + | Value::Ulid(_) + | Value::DateTime(_) + | Value::NaiveDateTime(_) + | Value::Duration(_) + | Value::Decimal(_) + | Value::Geometry(_) + | Value::Regex(_) + | Value::Range { .. } + | Value::Record { .. } + | Value::ArrayCell(_) + | Value::Vector(_) => serde_json::Value::from(cell.clone()), + // `Value` is `#[non_exhaustive]`: a variant this crate cannot name + // renders through the shared `From`, like the scalar arm above. + _ => serde_json::Value::from(cell.clone()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_types::NdbDateTime; + + /// A byte cell renders as the same unpadded base64 the msgpack → JSON + /// transcoder emits for a `bin`, nested or not. + #[test] + fn bytes_render_as_unpadded_base64_like_the_transcoder() { + let bytes = vec![0u8, 255, 7, 1]; + let transcoded = nodedb_types::msgpack_to_json_string( + &nodedb_types::value_to_msgpack(&Value::Bytes(bytes.clone())).expect("encode"), + ) + .expect("transcode"); + let expected: serde_json::Value = serde_json::from_str(&transcoded).expect("json"); + + assert_eq!(value_to_wire_json(&Value::Bytes(bytes.clone())), expected); + assert_eq!( + value_to_wire_json(&Value::Array(vec![Value::Bytes(bytes)])), + serde_json::Value::Array(vec![expected]) + ); + } + + /// An instant renders as ISO-8601 text, a number as itself, NULL as null. + #[test] + fn scalars_render_through_the_shared_from() { + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + assert_eq!( + value_to_wire_json(&Value::NaiveDateTime(at)), + serde_json::Value::String("2020-03-05T10:00:00.000000Z".into()) + ); + assert_eq!( + value_to_wire_json(&Value::Integer(7)), + serde_json::Value::from(7i64) + ); + assert_eq!(value_to_wire_json(&Value::Null), serde_json::Value::Null); + } +} diff --git a/nodedb/tests/inproc/cases/catalog_recovery_check.rs b/nodedb/tests/inproc/cases/catalog_recovery_check.rs index a8e07476e..97bfdf70d 100644 --- a/nodedb/tests/inproc/cases/catalog_recovery_check.rs +++ b/nodedb/tests/inproc/cases/catalog_recovery_check.rs @@ -207,11 +207,12 @@ async fn rls_policy_orphan_refuses_startup() { let stored = nodedb::control::security::catalog::rls::StoredRlsPolicy { tenant_id: 1, + database_id: 0, collection: "orders".to_string(), display_collection: "orders".to_string(), name: "only_own_orders".to_string(), policy_type_tag: 0, - compiled_predicate_json: String::new(), + predicate_text: String::new(), mode_tag: 0, on_deny_json: r#""Silent""#.to_string(), enabled: true, @@ -517,11 +518,12 @@ async fn rls_policy_value_mismatch_detected() { let stored = nodedb::control::security::catalog::rls::StoredRlsPolicy { tenant_id: 1, + database_id: 0, collection: "docs".to_string(), display_collection: "docs".to_string(), name: "read_own".to_string(), policy_type_tag: 0, - compiled_predicate_json: String::new(), + predicate_text: String::new(), mode_tag: 0, on_deny_json: r#""Silent""#.to_string(), enabled: true, @@ -531,7 +533,7 @@ async fn rls_policy_value_mismatch_detected() { catalog.put_rls_policy(&stored).unwrap(); // Insert into memory with enabled=false — value mismatch. - let mut policy = stored.to_runtime().unwrap(); + let mut policy = stored.to_runtime(catalog).unwrap(); policy.enabled = false; shared.rls.install_replicated_policy(policy); diff --git a/nodedb/tests/inproc/cases/collection_cascade_enumeration.rs b/nodedb/tests/inproc/cases/collection_cascade_enumeration.rs index b5df0bfc2..bb7cc4bff 100644 --- a/nodedb/tests/inproc/cases/collection_cascade_enumeration.rs +++ b/nodedb/tests/inproc/cases/collection_cascade_enumeration.rs @@ -24,11 +24,12 @@ use super::catalog_integrity_helpers::{ fn put_rls(catalog: &nodedb::control::security::catalog::SystemCatalog, name: &str, coll: &str) { let p = StoredRlsPolicy { tenant_id: TENANT, + database_id: 0, collection: coll.into(), display_collection: coll.into(), name: name.into(), policy_type_tag: 0, - compiled_predicate_json: String::new(), + predicate_text: String::new(), mode_tag: 0, on_deny_json: String::new(), enabled: true, diff --git a/nodedb/tests/inproc/cases/executor_tests/test_array_ops.rs b/nodedb/tests/inproc/cases/executor_tests/test_array_ops.rs index 66cf51961..76480c495 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_array_ops.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_array_ops.rs @@ -2,7 +2,7 @@ //! Integration tests for array operators and array aggregate functions. -use nodedb::bridge::scan_filter::ScanFilter; +use nodedb::bridge::scan_filter::{FilterOp, ScanFilter}; use nodedb_physical::physical_plan::{ AggregateSpec, DocumentOp, GroupKeySpec, PhysicalPlan, QueryOp, }; @@ -45,10 +45,10 @@ fn insert_product( ); } -fn filter(field: &str, op: &str, value: nodedb_types::Value) -> ScanFilter { +fn filter(field: &str, op: FilterOp, value: nodedb_types::Value) -> ScanFilter { ScanFilter { field: field.into(), - op: op.into(), + op, value, clauses: Vec::new(), expr: None, @@ -106,7 +106,7 @@ fn array_contains_filter() { // Products where tags contains "sale". let filters = vec![filter( "tags", - "array_contains", + FilterOp::ArrayContains, nodedb_types::Value::String("sale".into()), )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); @@ -145,7 +145,7 @@ fn array_contains_all_filter() { // Products where sizes contains ALL of ["S", "M"]. let filters = vec![filter( "sizes", - "array_contains_all", + FilterOp::ArrayContainsAll, nodedb_types::Value::Array(vec![ nodedb_types::Value::String("S".into()), nodedb_types::Value::String("M".into()), @@ -187,7 +187,7 @@ fn array_overlap_filter() { // Products where tags overlaps ["sale", "premium"]. let filters = vec![filter( "tags", - "array_overlap", + FilterOp::ArrayOverlap, nodedb_types::Value::Array(vec![ nodedb_types::Value::String("sale".into()), nodedb_types::Value::String("premium".into()), @@ -322,7 +322,7 @@ fn no_match_returns_zero() { // No product has tag "nonexistent". let filters = vec![filter( "tags", - "array_contains", + FilterOp::ArrayContains, nodedb_types::Value::String("nonexistent".into()), )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); diff --git a/nodedb/tests/inproc/cases/executor_tests/test_conditional_update.rs b/nodedb/tests/inproc/cases/executor_tests/test_conditional_update.rs index ed824abee..bc81c1b8c 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_conditional_update.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_conditional_update.rs @@ -10,15 +10,15 @@ //! - PointUpdate returns affected count use nodedb::bridge::envelope::Status; -use nodedb::bridge::scan_filter::ScanFilter; +use nodedb::bridge::scan_filter::{FilterOp, ScanFilter}; use nodedb_physical::physical_plan::{DocumentOp, MetaOp, PhysicalPlan, UpdateValue}; use super::helpers::*; -fn filter(field: &str, op: &str, value: nodedb_types::Value) -> ScanFilter { +fn filter(field: &str, op: FilterOp, value: nodedb_types::Value) -> ScanFilter { ScanFilter { field: field.into(), - op: op.into(), + op, value, clauses: Vec::new(), expr: None, @@ -107,7 +107,11 @@ fn bulk_update_returns_affected_count() { insert_product(&mut core, &mut tx, &mut rx, "p3", 0); // Bulk update: SET stock = 99 WHERE stock > 0 (should match p1 and p2). - let filters = vec![filter("stock", "gt", nodedb_types::Value::Integer(0))]; + let filters = vec![filter( + "stock", + FilterOp::Gt, + nodedb_types::Value::Integer(0), + )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); let updates = vec![( "stock".to_string(), @@ -153,7 +157,11 @@ fn conditional_decrement_stops_at_zero() { for i in 0..10 { let current_stock = get_stock(&mut core, &mut tx, &mut rx, "flash-deal"); - let filters = vec![filter("stock", "gte", nodedb_types::Value::Integer(1))]; + let filters = vec![filter( + "stock", + FilterOp::Gte, + nodedb_types::Value::Integer(1), + )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); let new_stock = current_stock.saturating_sub(1); @@ -209,7 +217,11 @@ fn bulk_update_zero_match_returns_zero_affected() { insert_product(&mut core, &mut tx, &mut rx, "p1", 0); - let filters = vec![filter("stock", "gte", nodedb_types::Value::Integer(100))]; + let filters = vec![filter( + "stock", + FilterOp::Gte, + nodedb_types::Value::Integer(100), + )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); let updates = vec![( "stock".to_string(), @@ -248,7 +260,11 @@ fn bulk_update_returning_returns_updated_documents() { insert_product(&mut core, &mut tx, &mut rx, "r1", 10); insert_product(&mut core, &mut tx, &mut rx, "r2", 20); - let filters = vec![filter("stock", "gt", nodedb_types::Value::Integer(0))]; + let filters = vec![filter( + "stock", + FilterOp::Gt, + nodedb_types::Value::Integer(0), + )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); let updates = vec![( "stock".to_string(), @@ -286,7 +302,11 @@ fn bulk_update_returning_zero_match_returns_affected_zero() { insert_product(&mut core, &mut tx, &mut rx, "p1", 0); - let filters = vec![filter("stock", "gte", nodedb_types::Value::Integer(100))]; + let filters = vec![filter( + "stock", + FilterOp::Gte, + nodedb_types::Value::Integer(100), + )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); let updates = vec![( "stock".to_string(), @@ -402,14 +422,14 @@ fn transaction_batch_does_not_abort_on_zero_row_update() { // Batch should NOT auto-abort on 0-row update. let filters_match = zerompk::to_msgpack_vec(&vec![filter( "stock", - "gte", + FilterOp::Gte, nodedb_types::Value::Integer(1), )]) .unwrap(); let filters_nomatch = zerompk::to_msgpack_vec(&vec![filter( "stock", - "gte", + FilterOp::Gte, nodedb_types::Value::Integer(100), )]) .unwrap(); diff --git a/nodedb/tests/inproc/cases/executor_tests/test_cross_engine_validation.rs b/nodedb/tests/inproc/cases/executor_tests/test_cross_engine_validation.rs index e6330672c..40ba6d91a 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_cross_engine_validation.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_cross_engine_validation.rs @@ -150,7 +150,7 @@ fn cross_model_query_vector_graph_relational() { // 6. Relational filter: scan papers with year >= 2023. let filter = vec![nodedb::bridge::scan_filter::ScanFilter { field: "year".into(), - op: "gte".into(), + op: nodedb::bridge::scan_filter::FilterOp::Gte, value: nodedb_types::Value::Integer(2023), clauses: Vec::new(), expr: None, diff --git a/nodedb/tests/inproc/cases/executor_tests/test_facet.rs b/nodedb/tests/inproc/cases/executor_tests/test_facet.rs index e072fca0c..17b8271ee 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_facet.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_facet.rs @@ -9,7 +9,7 @@ //! - Zero-match filter returns empty facets //! - Limit per facet (top-N truncation) -use nodedb::bridge::scan_filter::ScanFilter; +use nodedb::bridge::scan_filter::{FilterOp, ScanFilter}; use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan, QueryOp}; use super::helpers::*; @@ -133,10 +133,10 @@ fn seed_products( ); } -fn filter(field: &str, op: &str, value: nodedb_types::Value) -> ScanFilter { +fn filter(field: &str, op: FilterOp, value: nodedb_types::Value) -> ScanFilter { ScanFilter { field: field.into(), - op: op.into(), + op, value, clauses: Vec::new(), expr: None, @@ -193,7 +193,7 @@ fn filtered_facet_counts() { // Filter: brand = 'Nike' — should only count Nike products. let filters = vec![filter( "brand", - "eq", + FilterOp::Eq, nodedb_types::Value::String("Nike".into()), )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); @@ -236,7 +236,7 @@ fn zero_match_filter_returns_empty_facets() { // Filter: brand = 'NonExistent'. let filters = vec![filter( "brand", - "eq", + FilterOp::Eq, nodedb_types::Value::String("NonExistent".into()), )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); diff --git a/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs b/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs index d96111c4a..3b1cab4c5 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs @@ -86,7 +86,6 @@ fn sql_to_physical(sql: &str) -> PhysicalPlan { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, }; let tenant_id = nodedb::types::TenantId::new(1); let tasks = convert(&plans, tenant_id, &ctx).unwrap(); diff --git a/nodedb/tests/inproc/cases/executor_tests/test_ollp_verification.rs b/nodedb/tests/inproc/cases/executor_tests/test_ollp_verification.rs index 9948252af..ac6fc2d29 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_ollp_verification.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_ollp_verification.rs @@ -40,7 +40,7 @@ const COLLECTION: &str = "ollp_items"; fn filter_active() -> Vec { let f = ScanFilter { field: "active".into(), - op: "eq".into(), + op: nodedb::bridge::scan_filter::FilterOp::Eq, value: nodedb_types::Value::Bool(true), clauses: Vec::new(), expr: None, diff --git a/nodedb/tests/inproc/cases/executor_tests/test_timeseries.rs b/nodedb/tests/inproc/cases/executor_tests/test_timeseries.rs index bc3e9945f..bd266b0e4 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_timeseries.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_timeseries.rs @@ -317,7 +317,7 @@ fn where_predicate_filters_count() { 0, vec![nodedb::bridge::scan_filter::ScanFilter { field: "qtype".into(), - op: "eq".into(), + op: nodedb::bridge::scan_filter::FilterOp::Eq, value: nodedb_types::Value::String("A".into()), clauses: vec![], expr: None, diff --git a/nodedb/tests/inproc/cases/oidc_provider_ddl.rs b/nodedb/tests/inproc/cases/oidc_provider_ddl.rs index 12be7b18a..afbd0db81 100644 --- a/nodedb/tests/inproc/cases/oidc_provider_ddl.rs +++ b/nodedb/tests/inproc/cases/oidc_provider_ddl.rs @@ -224,7 +224,7 @@ async fn show_oidc_providers_exposes_tenant_binding() { assert_eq!( rows.rows[0] .get("tenant_id") - .and_then(serde_json::Value::as_str), + .and_then(nodedb_types::Value::as_str), Some("42") ); } diff --git a/nodedb/tests/inproc/cases/startup_failure.rs b/nodedb/tests/inproc/cases/startup_failure.rs index aea3e9fc5..42ddb059e 100644 --- a/nodedb/tests/inproc/cases/startup_failure.rs +++ b/nodedb/tests/inproc/cases/startup_failure.rs @@ -103,11 +103,12 @@ fn nodedb_exits_nonzero_on_catalog_integrity_violation() { catalog .put_rls_policy(&StoredRlsPolicy { tenant_id: 1, + database_id: 0, collection: "collection_that_was_never_created".to_string(), display_collection: "collection_that_was_never_created".to_string(), name: "dangling_policy".to_string(), policy_type_tag: 0, - compiled_predicate_json: String::new(), + predicate_text: String::new(), mode_tag: 0, on_deny_json: r#""Silent""#.to_string(), enabled: true, diff --git a/nodedb/tests/wire/cases/columnar_read_row_level_security.rs b/nodedb/tests/wire/cases/columnar_read_row_level_security.rs new file mode 100644 index 000000000..b790184c0 --- /dev/null +++ b/nodedb/tests/wire/cases/columnar_read_row_level_security.rs @@ -0,0 +1,364 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Row-level security on columnar reads. +//! +//! A `FOR READ` policy on a columnar collection governs every row a `SELECT` +//! returns, from the live memtable and from flushed segments alike. The scan +//! applies the policy after block pruning and the query's own WHERE, before +//! projection, sort, and limit, so a row the policy excludes never reaches +//! the client, never consumes a `LIMIT` slot, and never joins a partner. +//! A spatial collection runs the same scan and is governed the same way. + +use crate::harness::TestServer; + +const PASSWORD: &str = "probe-secret-99"; + +/// Create a columnar `collection` holding one row owned by `user` and two +/// owned by someone else, plus `user` with the readwrite role. +async fn seed(server: &TestServer, collection: &str, user: &str) { + server + .exec(&format!( + "CREATE COLLECTION {collection} \ + (id TEXT PRIMARY KEY, owner TEXT, note TEXT) \ + WITH (engine='columnar')" + )) + .await + .unwrap_or_else(|e| panic!("create {collection}: {e}")); + server + .exec(&format!( + "INSERT INTO {collection} (id, owner, note) VALUES \ + ('r_mine', '{user}', 'mine'), \ + ('r_a', 'someone_else', 'theirs'), \ + ('r_b', 'someone_else', 'theirs too')" + )) + .await + .unwrap_or_else(|e| panic!("seed {collection}: {e}")); + server + .exec(&format!("CREATE USER {user} PASSWORD '{PASSWORD}'")) + .await + .unwrap_or_else(|e| panic!("create user {user}: {e}")); + server + .exec(&format!("GRANT ROLE readwrite TO {user}")) + .await + .unwrap_or_else(|e| panic!("grant readwrite to {user}: {e}")); + server + .exec(&format!( + "CREATE RLS POLICY {collection}_owner ON {collection} FOR READ \ + USING (owner = $auth.username)" + )) + .await + .unwrap_or_else(|e| panic!("create read policy on {collection}: {e}")); +} + +/// Run `sql` as `user` and return each row's cells joined by `|`. +async fn rows_as(server: &TestServer, user: &str, sql: &str) -> Vec { + let (client, handle) = server + .connect_as(user, PASSWORD) + .await + .unwrap_or_else(|e| panic!("connect as {user}: {e}")); + let messages = client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("{user} runs {sql}: {e}")); + let mut out = Vec::new(); + for message in messages { + if let tokio_postgres::SimpleQueryMessage::Row(row) = message { + let mut cells = Vec::new(); + for i in 0..row.len() { + cells.push(row.get(i).unwrap_or("").to_string()); + } + out.push(cells.join("|")); + } + } + drop(client); + handle.abort(); + out +} + +/// A `SELECT` over the live memtable returns only the rows the read policy +/// admits for the caller. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_columnar_select_returns_only_policy_admitted_rows() { + let server = TestServer::start().await; + seed(&server, "col_rls_read", "col_rls_reader").await; + + let rows = rows_as( + &server, + "col_rls_reader", + "SELECT id, note FROM col_rls_read ORDER BY id", + ) + .await; + assert_eq!( + rows, + vec!["r_mine|mine".to_string()], + "the read policy admits one row for this caller: {rows:?}" + ); +} + +/// A `SELECT` over flushed segments returns only the rows the read policy +/// admits for the caller. The policy applies after block pruning, so a +/// flushed row is governed the same as a live one. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_columnar_select_over_flushed_segments_returns_only_policy_admitted_rows() { + let server = TestServer::start_with_columnar_flush_threshold(2).await; + seed(&server, "col_rls_read_flushed", "col_rls_flushed_reader").await; + + let rows = rows_as( + &server, + "col_rls_flushed_reader", + "SELECT id, note FROM col_rls_read_flushed ORDER BY id", + ) + .await; + assert_eq!( + rows, + vec!["r_mine|mine".to_string()], + "the read policy admits one row for this caller: {rows:?}" + ); +} + +/// A `COUNT(*)` over a governed columnar collection counts only the rows the +/// read policy admits. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_columnar_count_counts_only_policy_admitted_rows() { + let server = TestServer::start().await; + seed(&server, "col_rls_count", "col_rls_counter").await; + + let rows = rows_as( + &server, + "col_rls_counter", + "SELECT COUNT(*) FROM col_rls_count", + ) + .await; + assert_eq!( + rows, + vec!["1".to_string()], + "the read policy admits one row for this caller: {rows:?}" + ); +} + +/// A `LIMIT` counts admitted rows only: with one admitted row and two +/// excluded rows seeded, `LIMIT 1` returns the admitted row, never an +/// excluded row that happened to be scanned first. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_columnar_select_with_a_limit_counts_only_admitted_rows() { + let server = TestServer::start().await; + seed(&server, "col_rls_limit", "col_rls_limiter").await; + + let rows = rows_as( + &server, + "col_rls_limiter", + "SELECT id FROM col_rls_limit ORDER BY id LIMIT 1", + ) + .await; + assert_eq!( + rows, + vec!["r_mine".to_string()], + "the limit applies to admitted rows only: {rows:?}" + ); +} + +/// A join reads the governed columnar side on the caller's behalf, so its +/// policy applies to that side before the join: excluded rows neither match +/// a partner nor reach the client. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_join_over_a_governed_columnar_collection_excludes_policy_filtered_rows() { + let server = TestServer::start().await; + seed(&server, "col_rls_join_c", "col_rls_joiner").await; + server + .exec( + "CREATE COLLECTION col_rls_join_d (id TEXT PRIMARY KEY, tag TEXT) \ + WITH (engine='document_strict')", + ) + .await + .expect("create document side"); + server + .exec( + "INSERT INTO col_rls_join_d (id, tag) VALUES \ + ('r_mine', 't_mine'), ('r_a', 't_a'), ('r_b', 't_b')", + ) + .await + .expect("seed document side"); + + let rows = rows_as( + &server, + "col_rls_joiner", + "SELECT c.id, d.tag FROM col_rls_join_c c \ + JOIN col_rls_join_d d ON c.id = d.id ORDER BY c.id", + ) + .await; + assert_eq!( + rows, + vec!["r_mine|t_mine".to_string()], + "the join surfaced columnar rows the read policy excludes: {rows:?}" + ); +} + +/// A plain `SELECT` over a spatial collection runs the same columnar scan, +/// so the read policy governs it the same way. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_spatial_select_returns_only_policy_admitted_rows() { + let server = TestServer::start().await; + let (collection, user) = ("sp_rls_read", "sp_rls_reader"); + server + .exec(&format!( + "CREATE COLLECTION {collection} \ + COLUMNS (id TEXT, owner TEXT, loc GEOMETRY) \ + WITH (engine='spatial')" + )) + .await + .unwrap_or_else(|e| panic!("create {collection}: {e}")); + for (id, owner, wkt) in [ + ("r_mine", user, "POINT(1 1)"), + ("r_a", "someone_else", "POINT(2 2)"), + ("r_b", "someone_else", "POINT(3 3)"), + ] { + server + .exec(&format!( + "INSERT INTO {collection} (id, owner, loc) \ + VALUES ('{id}', '{owner}', ST_GeomFromText('{wkt}'))" + )) + .await + .unwrap_or_else(|e| panic!("seed {collection} row {id}: {e}")); + } + server + .exec(&format!("CREATE USER {user} PASSWORD '{PASSWORD}'")) + .await + .unwrap_or_else(|e| panic!("create user {user}: {e}")); + server + .exec(&format!("GRANT ROLE readwrite TO {user}")) + .await + .unwrap_or_else(|e| panic!("grant readwrite to {user}: {e}")); + server + .exec(&format!( + "CREATE RLS POLICY {collection}_owner ON {collection} FOR READ \ + USING (owner = $auth.username)" + )) + .await + .unwrap_or_else(|e| panic!("create read policy on {collection}: {e}")); + + let rows = rows_as( + &server, + user, + &format!("SELECT id FROM {collection} ORDER BY id"), + ) + .await; + assert_eq!( + rows, + vec!["r_mine".to_string()], + "the read policy admits one row for this caller: {rows:?}" + ); +} + +/// `2020-03-05T10:00:00Z` as epoch milliseconds: the instant the policies +/// below compare against. +const CUTOFF_MS: i64 = 1_583_402_400_000; +const CUTOFF_TEXT: &str = "2020-03-05 10:00:00"; + +/// Create a columnar `collection` with a declared `TIMESTAMP` column holding +/// one row before, one at, and one after the cutoff, plus `user` with the +/// readwrite role. No policy: each test creates its own. +async fn seed_instant(server: &TestServer, collection: &str, user: &str) { + server + .exec(&format!( + "CREATE COLLECTION {collection} \ + (id TEXT PRIMARY KEY, owner TEXT, captured_at TIMESTAMP) \ + WITH (engine='columnar')" + )) + .await + .unwrap_or_else(|e| panic!("create {collection}: {e}")); + server + .exec(&format!( + "INSERT INTO {collection} (id, owner, captured_at) VALUES \ + ('before', '{user}', '2020-03-05 09:00:00'), \ + ('at', '{user}', '{CUTOFF_TEXT}'), \ + ('after', '{user}', '2020-03-05 11:00:00')" + )) + .await + .unwrap_or_else(|e| panic!("seed {collection}: {e}")); + server + .exec(&format!("CREATE USER {user} PASSWORD '{PASSWORD}'")) + .await + .unwrap_or_else(|e| panic!("create user {user}: {e}")); + server + .exec(&format!("GRANT ROLE readwrite TO {user}")) + .await + .unwrap_or_else(|e| panic!("grant readwrite to {user}: {e}")); +} + +/// A numeric policy literal compared against a declared `TIMESTAMP` column +/// is epoch milliseconds, typed at `CREATE RLS POLICY` by the rule a query +/// predicate follows: the governed user sees the rows at and after the +/// cutoff, and the row before it is excluded. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_numeric_policy_literal_on_a_timestamp_column_is_an_instant() { + let server = TestServer::start().await; + let user = "col_rls_ms_reader"; + seed_instant(&server, "col_rls_ms", user).await; + server + .exec(&format!( + "CREATE RLS POLICY col_rls_ms_recent ON col_rls_ms FOR READ \ + USING (captured_at >= {CUTOFF_MS})" + )) + .await + .expect("a numeric literal on a TIMESTAMP column is epoch milliseconds"); + + let rows = rows_as( + &server, + user, + "SELECT id FROM col_rls_ms ORDER BY captured_at", + ) + .await; + assert_eq!( + rows, + vec!["at".to_string(), "after".to_string()], + "the policy admits the rows at and after the cutoff: {rows:?}" + ); +} + +/// A text policy literal compared against a declared `TIMESTAMP` column is +/// parsed as ISO-8601 and enforced as the same instant. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_text_policy_literal_on_a_timestamp_column_is_an_instant() { + let server = TestServer::start().await; + let user = "col_rls_text_reader"; + seed_instant(&server, "col_rls_text", user).await; + server + .exec(&format!( + "CREATE RLS POLICY col_rls_text_recent ON col_rls_text FOR READ \ + USING (captured_at >= '{CUTOFF_TEXT}')" + )) + .await + .expect("a text literal on a TIMESTAMP column is parsed as an instant"); + + let rows = rows_as( + &server, + user, + "SELECT id FROM col_rls_text ORDER BY captured_at", + ) + .await; + assert_eq!( + rows, + vec!["at".to_string(), "after".to_string()], + "the policy admits the rows at and after the cutoff: {rows:?}" + ); +} + +/// A policy literal no instant can be read from is refused at +/// `CREATE RLS POLICY`, naming the column, so an unenforceable policy is +/// never stored. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_non_instant_policy_literal_on_a_timestamp_column_is_refused() { + let server = TestServer::start().await; + seed_instant(&server, "col_rls_bool", "col_rls_bool_reader").await; + let error = server + .exec( + "CREATE RLS POLICY col_rls_bool_bad ON col_rls_bool FOR READ \ + USING (captured_at >= true)", + ) + .await + .expect_err("a boolean is not an instant"); + assert!( + error.contains("captured_at"), + "the refusal must name the column: {error}" + ); +} diff --git a/nodedb/tests/wire/cases/kv_column_defaults.rs b/nodedb/tests/wire/cases/kv_column_defaults.rs index e0bd55a92..d0d4b056b 100644 --- a/nodedb/tests/wire/cases/kv_column_defaults.rs +++ b/nodedb/tests/wire/cases/kv_column_defaults.rs @@ -177,29 +177,17 @@ async fn upsert_materializes_an_omitted_columns_default() { ); } -/// A materialized default is validated exactly like a supplied literal: it is -/// appended to the row BEFORE the declared-type coercion and range checks run. -/// Were it filled in afterwards, a DEFAULT would be a way to store a value the -/// same literal is rejected for. +/// A `DEFAULT` beyond the declared width is refused where it is declared, +/// so no insert can reach it: the `CREATE` fails with the same range error a +/// supplied literal gets, naming the column. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_default_beyond_the_declared_width_is_rejected_like_a_supplied_literal() { let server = TestServer::start().await; - create_kv( - &server, - "kv_def_range", - "key TEXT PRIMARY KEY, n SMALLINT DEFAULT 999999", - ) - .await; - - // The same literal supplied directly is rejected... server .expect_error( - "INSERT INTO kv_def_range (key, n) VALUES ('k1', 999999)", + "CREATE COLLECTION kv_def_range (key TEXT PRIMARY KEY, n SMALLINT DEFAULT 999999) \ + WITH (engine='kv')", "range", ) .await; - // ...so arriving via the DEFAULT must not be a way around it. - server - .expect_error("INSERT INTO kv_def_range (key) VALUES ('k2')", "range") - .await; } diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index e47f21052..257088f8d 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -37,6 +37,7 @@ mod clone_tombstone_hides_source_row; mod clone_write_isolation; mod clone_write_isolation_fuzz; mod clone_write_suppresses_source_row; +mod columnar_read_row_level_security; mod columnar_write_row_level_security; mod command_complete_tag_conformance; mod crdt_write_rls_database_scope; @@ -153,6 +154,7 @@ mod sql_alter_after_drop; mod sql_arithmetic_overflow; mod sql_backup_restore_columnar; mod sql_backup_restore_columnar_restart; +mod sql_backup_restore_timeseries; mod sql_backup_restore_vector_params; mod sql_backup_restore_vector_restart; mod sql_backup_restore_wire; @@ -165,6 +167,7 @@ mod sql_copy_from; mod sql_copy_to; mod sql_cursors; mod sql_declared_column_types; +mod sql_default_declared_types; mod sql_default_expressions; mod sql_default_vector_primary; mod sql_default_volatility; @@ -194,6 +197,7 @@ mod sql_limit_offset_bounds; mod sql_maintenance; mod sql_materialized_view_refresh; mod sql_merge; +mod sql_merge_declared_types; mod sql_multi_statement_batch; mod sql_null_predicate_parity; mod sql_object_literal_insert; @@ -260,6 +264,7 @@ mod sql_update_from; mod sql_utf8_expressions; mod sql_vector_index_ddl; mod sql_where_expressions; +mod sql_where_instant_literals; mod sql_where_vector_search; mod sql_window_frames; mod sql_window_functions; @@ -267,8 +272,10 @@ mod streaming_select; mod strict_bitemporal_audit_query; mod strict_bitemporal_select_star; mod strict_schema_restart; +mod strict_typed_column_rendering; mod timeseries_declared_time_key; mod timeseries_join_time_rendering; +mod timeseries_read_row_level_security; mod timeseries_write_row_level_security; mod transactional_ddl_atomicity; mod transactional_ddl_compensation; diff --git a/nodedb/tests/wire/cases/pgwire_extended_query.rs b/nodedb/tests/wire/cases/pgwire_extended_query.rs index e1080ac1f..612ee388c 100644 --- a/nodedb/tests/wire/cases/pgwire_extended_query.rs +++ b/nodedb/tests/wire/cases/pgwire_extended_query.rs @@ -555,12 +555,13 @@ async fn extended_query_binary_typed_columns_decode() { assert_eq!(name, "hello"); } -/// A `TIMESTAMP` column is feature-blocked for binary encoding, so it stays -/// text even when the client requests binary. The extended query still -/// succeeds and its binary-capable sibling column decodes; the same timestamp -/// is retrievable as text over the simple-query path. +/// A `TIMESTAMP` column honours a binary result request: the cell arrives as +/// PostgreSQL binary `timestamp` (microseconds since 2000-01-01) and decodes +/// through the driver's `SystemTime` reader to the stored instant, next to +/// its integer sibling. The same timestamp renders as ISO-8601 text over the +/// simple-query path. #[tokio::test] -async fn extended_query_timestamp_text_fallback_with_binary_sibling() { +async fn extended_query_timestamp_decodes_from_binary_with_sibling() { let server = TestServer::start().await; server .exec( @@ -574,9 +575,9 @@ async fn extended_query_timestamp_text_fallback_with_binary_sibling() { .await .unwrap(); - // Extended path: the query carrying a timestamp column must succeed, and - // the integer sibling decodes from binary. `n` is declared INT, so it - // advertises OID 23 and decodes as i32. + // Extended path: `tokio-postgres` requests binary for every result + // column. `n` is declared INT, so it advertises OID 23 and decodes as + // i32; `ts` decodes as the stored instant, 2024-01-01T00:00:00Z. let rows = server .client .query("SELECT n, ts FROM ev WHERE id = $1", &[&"a"]) @@ -585,18 +586,19 @@ async fn extended_query_timestamp_text_fallback_with_binary_sibling() { assert_eq!(rows.len(), 1); let n: i32 = rows[0].get("n"); assert_eq!(n, 7); + let ts: std::time::SystemTime = rows[0].get("ts"); + assert_eq!( + ts, + std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_704_067_200), + "binary timestamp must decode to the stored instant" + ); - // Simple-query path returns every column as text, including the timestamp. + // Simple-query path renders the timestamp as ISO-8601 text. let text_rows = server .query_text("SELECT ts FROM ev WHERE id = 'a'") .await .expect("simple-query text select should succeed"); - assert_eq!(text_rows.len(), 1); - assert!( - !text_rows[0].is_empty(), - "timestamp must be present as text on the simple-query path, got {:?}", - text_rows[0] - ); + assert_eq!(text_rows, vec!["2024-01-01T00:00:00.000000Z".to_string()]); } /// Regression lock: when the client declares a parameter's type (via diff --git a/nodedb/tests/wire/cases/sql_backup_restore_timeseries.rs b/nodedb/tests/wire/cases/sql_backup_restore_timeseries.rs new file mode 100644 index 000000000..a113e11e9 --- /dev/null +++ b/nodedb/tests/wire/cases/sql_backup_restore_timeseries.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! BACKUP / RESTORE of a timeseries collection with a declared time key. +//! +//! A row restored into a timeseries collection reads back exactly as the row +//! that was inserted: a declared `TIMESTAMP` / `TIMESTAMPTZ` time key is the +//! same instant, a `BIGINT` time key the same integer, and the other columns +//! are intact. This holds whether the rows sat in the memtable or had been +//! flushed to a partition at backup time, and it survives a restart of the +//! restored server. +//! +//! The restore target is a genuinely clean one: the collection is hard-purged +//! before the restore (Data Plane registration, memtable and partitions all +//! gone), or the restore lands on a fresh server. A restore that inferred the +//! collection shape from the reissued rows, instead of registering the +//! declared one first, would stamp the rows with the restore-time clock and +//! leave the time key as an integer column. + +use crate::harness::TestServer; + +use bytes::Bytes; +use futures::SinkExt; +use futures::StreamExt; + +const TENANT: u64 = 1; + +/// Event time, years in the past, so a restore-time "now" stamp is trivially +/// separable from the inserted value. +const EARLY: &str = "2020-03-05 10:00:00"; +/// `EARLY` as a declared instant column renders it over pgwire. +const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; +/// A `BIGINT` time key value in epoch milliseconds. +const EARLY_MS: &str = "1700000000000"; + +async fn drain_backup(server: &TestServer, tenant: u64) -> Vec { + let stream = server + .client + .copy_out(&format!("COPY (BACKUP TENANT {tenant}) TO STDOUT")) + .await + .expect("copy_out: BACKUP TENANT"); + let mut bytes = Vec::new(); + let mut s = Box::pin(stream); + while let Some(chunk) = s.next().await { + bytes.extend_from_slice(&chunk.expect("copy_out chunk")); + } + bytes +} + +async fn push_restore(server: &TestServer, tenant: u64, bytes: Vec) { + let sink = server + .client + .copy_in::<_, Bytes>(&format!("COPY tenant_restore({tenant}) FROM STDIN")) + .await + .expect("copy_in: RESTORE TENANT"); + let mut sink = Box::pin(sink); + sink.as_mut() + .send(Bytes::from(bytes)) + .await + .expect("send backup bytes"); + sink.as_mut() + .finish() + .await + .expect("finish copy_in: RESTORE TENANT"); +} + +/// Create the collection, insert one row, and return what the time key reads +/// back as before the backup. +async fn create_and_insert( + srv: &TestServer, + name: &str, + time_key_type: &str, + time: &str, +) -> String { + srv.exec(&format!( + "CREATE COLLECTION {name} \ + (captured_at {time_key_type} TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .unwrap_or_else(|e| panic!("CREATE COLLECTION {name}: {e}")); + srv.exec(&format!( + "INSERT INTO {name} (captured_at, host, v) VALUES ('{time}', 'h1', 1.5)" + )) + .await + .unwrap_or_else(|e| panic!("INSERT INTO {name}: {e}")); + let rows = srv + .query_text(&format!("SELECT captured_at FROM {name}")) + .await + .unwrap_or_else(|e| panic!("SELECT captured_at FROM {name}: {e}")); + assert_eq!(rows.len(), 1, "one inserted row must read back: {rows:?}"); + rows[0].clone() +} + +/// The restored row carries the same time key and the same other columns. +async fn assert_restored_row(srv: &TestServer, name: &str, expected_time: &str) { + let times = srv + .query_text(&format!("SELECT captured_at FROM {name}")) + .await + .unwrap_or_else(|e| panic!("post-restore SELECT captured_at FROM {name}: {e}")); + assert_eq!( + times, + vec![expected_time.to_string()], + "the restored time key must read back as the inserted value" + ); + let hosts = srv + .query_text(&format!("SELECT host FROM {name}")) + .await + .unwrap_or_else(|e| panic!("post-restore SELECT host FROM {name}: {e}")); + assert_eq!( + hosts, + vec!["h1".to_string()], + "the restored row must carry its other columns intact" + ); +} + +/// Backup, hard-purge the collection on the same server, restore, and check +/// the row. The purge removes the catalog row, the Data Plane registration, +/// the memtable and every partition, so the restore lands on a clean target. +async fn backup_purge_restore(srv: &TestServer, name: &str, expected_time: &str) { + let backup_bytes = drain_backup(srv, TENANT).await; + assert!( + !backup_bytes.is_empty(), + "backup envelope must not be empty" + ); + + srv.exec(&format!("DROP COLLECTION {name} PURGE")) + .await + .unwrap_or_else(|e| panic!("DROP COLLECTION {name} PURGE: {e}")); + + push_restore(srv, TENANT, backup_bytes).await; + assert_restored_row(srv, name, expected_time).await; +} + +#[tokio::test] +async fn a_declared_time_key_survives_backup_and_restore() { + let srv = TestServer::start().await; + let before = create_and_insert(&srv, "ts_bk_naive", "TIMESTAMP", EARLY).await; + assert_eq!( + before, EARLY_ISO, + "the inserted instant reads back before the backup" + ); + backup_purge_restore(&srv, "ts_bk_naive", &before).await; +} + +/// The same round trip once every row has been flushed to a partition: the +/// partition schema carries the time kind, and the restore reads it from +/// there rather than from the memtable snapshot. +#[tokio::test] +async fn a_declared_time_key_survives_backup_and_restore_from_a_flushed_partition() { + let srv = TestServer::start_with_timeseries_memtable_budget(1).await; + let before = create_and_insert(&srv, "ts_bk_flushed", "TIMESTAMP", EARLY).await; + assert_eq!( + before, EARLY_ISO, + "the inserted instant reads back before the backup" + ); + backup_purge_restore(&srv, "ts_bk_flushed", &before).await; +} + +#[tokio::test] +async fn a_declared_timestamptz_time_key_survives_backup_and_restore() { + let srv = TestServer::start().await; + let before = create_and_insert(&srv, "ts_bk_utc", "TIMESTAMPTZ", EARLY).await; + assert_eq!( + before, EARLY_ISO, + "the inserted instant reads back before the backup" + ); + backup_purge_restore(&srv, "ts_bk_utc", &before).await; +} + +/// A `BIGINT` time key is not an instant: it restores as the integer that +/// was inserted. +#[tokio::test] +async fn a_bigint_time_key_stays_an_integer_after_restore() { + let srv = TestServer::start().await; + let before = create_and_insert(&srv, "ts_bk_bigint", "BIGINT", EARLY_MS).await; + assert_eq!( + before, EARLY_MS, + "the inserted integer reads back before the backup" + ); + backup_purge_restore(&srv, "ts_bk_bigint", &before).await; +} + +/// Restore into a fresh server, then restart it on the same data directory: +/// the reissued row is WAL-durable and the boot-time registration of the +/// restored collection types it the same way the restore-time one did. +#[tokio::test] +async fn a_declared_time_key_survives_restore_and_restart() { + let srv_a = TestServer::start().await; + let before = create_and_insert(&srv_a, "ts_bk_restart", "TIMESTAMP", EARLY).await; + assert_eq!( + before, EARLY_ISO, + "the inserted instant reads back before the backup" + ); + let backup_bytes = drain_backup(&srv_a, TENANT).await; + assert!( + !backup_bytes.is_empty(), + "backup envelope must not be empty" + ); + drop(srv_a); + + let srv_b = TestServer::start().await; + push_restore(&srv_b, TENANT, backup_bytes).await; + assert_restored_row(&srv_b, "ts_bk_restart", &before).await; + + let (srv_b, dir) = srv_b.take_dir(); + srv_b.graceful_shutdown().await; + let (srv_c, _dir) = TestServer::open_on_path(dir).await; + assert_restored_row(&srv_c, "ts_bk_restart", &before).await; +} diff --git a/nodedb/tests/wire/cases/sql_default_declared_types.rs b/nodedb/tests/wire/cases/sql_default_declared_types.rs new file mode 100644 index 000000000..490426e1e --- /dev/null +++ b/nodedb/tests/wire/cases/sql_default_declared_types.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Declared-type coercion for DEFAULT literals. +//! +//! A DEFAULT literal materialized at insert passes the same declared-type +//! coercion an explicit `VALUES` literal gets: a numeric literal defaulted +//! into a `TIMESTAMP` column is epoch milliseconds, a text literal is parsed, +//! and a literal the column cannot hold is refused naming the column. + +use crate::harness::TestServer; + +/// A numeric `DEFAULT` on a `TIMESTAMP` column is epoch milliseconds, the +/// same unit an explicit `VALUES` literal takes — on `document_strict` and +/// `columnar`. +async fn a_numeric_default_on_a_timestamp_column_is_epoch_milliseconds_on(engine: &str) { + let server = TestServer::start().await; + let name = format!("def_ts_epoch_{engine}"); + + server + .exec(&format!( + "CREATE COLLECTION {name} (\ + id TEXT PRIMARY KEY, \ + at TIMESTAMP DEFAULT 1583402400000) WITH (engine='{engine}')" + )) + .await + .unwrap_or_else(|e| panic!("create {name}: {e}")); + + server + .exec(&format!("INSERT INTO {name} (id) VALUES ('r1')")) + .await + .unwrap_or_else(|e| panic!("insert into {name}: {e}")); + + let rows = server + .query_text(&format!("SELECT at FROM {name} WHERE id = 'r1'")) + .await + .unwrap_or_else(|e| panic!("select from {name}: {e}")); + assert_eq!(rows.len(), 1, "row should exist"); + assert_eq!( + rows[0], "2020-03-05T10:00:00.000000Z", + "DEFAULT 1583402400000 on a TIMESTAMP column must render as epoch milliseconds: {:?}", + rows[0] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_numeric_default_on_a_timestamp_column_is_epoch_milliseconds_document_strict() { + a_numeric_default_on_a_timestamp_column_is_epoch_milliseconds_on("document_strict").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_numeric_default_on_a_timestamp_column_is_epoch_milliseconds_columnar() { + a_numeric_default_on_a_timestamp_column_is_epoch_milliseconds_on("columnar").await; +} + +/// The timeseries engine has no `id TEXT PRIMARY KEY`; its row identity is +/// the `TIME_KEY` column, so the numeric `DEFAULT` sits on it. A row that +/// omits the time key takes the declared default, not the ingest clock. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_numeric_default_on_a_timestamp_column_is_epoch_milliseconds_timeseries() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION def_ts_epoch_timeseries \ + COLUMNS (at TIMESTAMP TIME_KEY DEFAULT 1583402400000, host TEXT, v FLOAT) \ + WITH (engine='timeseries')", + ) + .await + .expect("create def_ts_epoch_timeseries"); + + server + .exec("INSERT INTO def_ts_epoch_timeseries (host, v) VALUES ('h0', 1.0)") + .await + .expect("insert into def_ts_epoch_timeseries"); + + let rows = server + .query_text("SELECT at FROM def_ts_epoch_timeseries") + .await + .expect("select from def_ts_epoch_timeseries"); + assert_eq!( + rows, + vec!["2020-03-05T10:00:00.000000Z".to_string()], + "DEFAULT 1583402400000 on TIME_KEY is the row's time key, in epoch milliseconds" + ); +} + +/// A text `DEFAULT` on a `TIMESTAMP` column is parsed the same way an +/// explicit text literal is. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_text_default_on_a_timestamp_column_is_parsed() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION def_ts_text (\ + id TEXT PRIMARY KEY, \ + at TIMESTAMP DEFAULT '2020-03-05 10:00:00') WITH (engine='document_strict')", + ) + .await + .expect("create def_ts_text"); + + server + .exec("INSERT INTO def_ts_text (id) VALUES ('r1')") + .await + .expect("insert into def_ts_text"); + + let rows = server + .query_text("SELECT at FROM def_ts_text WHERE id = 'r1'") + .await + .expect("select from def_ts_text"); + assert_eq!(rows.len(), 1, "row should exist"); + assert_eq!( + rows[0], "2020-03-05T10:00:00.000000Z", + "DEFAULT '2020-03-05 10:00:00' must parse to the same instant: {:?}", + rows[0] + ); +} + +/// A `DEFAULT` literal the declared column type cannot hold is refused at +/// `CREATE`, naming the column. A default is checked where it is declared, +/// so no insert can ever materialize a value the column cannot hold. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_default_the_column_cannot_hold_is_refused() { + let server = TestServer::start().await; + + server + .expect_error( + "CREATE COLLECTION def_bad_ts (\ + id TEXT PRIMARY KEY, \ + at TIMESTAMP DEFAULT 'not a date') WITH (engine='document_strict')", + "'at'", + ) + .await; + + server + .expect_error( + "CREATE COLLECTION def_bad_smallint (\ + id TEXT PRIMARY KEY, \ + s SMALLINT DEFAULT 999999) WITH (engine='document_strict')", + "'s'", + ) + .await; +} + +/// A `DEFAULT` on a timeseries collection's non-key column lands, the same +/// way it lands on every other engine. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_default_on_a_timeseries_collection_lands() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION def_ts_host_default \ + COLUMNS (ts BIGINT TIME_KEY, host TEXT DEFAULT 'h0', v FLOAT) \ + WITH (engine='timeseries')", + ) + .await + .expect("create def_ts_host_default"); + + server + .exec("INSERT INTO def_ts_host_default (ts, v) VALUES (1700000000000, 1.0)") + .await + .expect("insert into def_ts_host_default"); + + let rows = server + .query_text("SELECT host FROM def_ts_host_default") + .await + .expect("select from def_ts_host_default"); + assert_eq!(rows.len(), 1, "row should exist"); + assert_eq!( + rows[0], "h0", + "DEFAULT 'h0' must land on host: {:?}", + rows[0] + ); +} diff --git a/nodedb/tests/wire/cases/sql_division_by_zero.rs b/nodedb/tests/wire/cases/sql_division_by_zero.rs index 52db70018..2ac7c0a2d 100644 --- a/nodedb/tests/wire/cases/sql_division_by_zero.rs +++ b/nodedb/tests/wire/cases/sql_division_by_zero.rs @@ -312,11 +312,11 @@ async fn columnar_where_clause_division_by_zero_errors_22012() { } /// Columnar engine, computed SELECT column division by zero: -/// `row_to_projected_json`'s `Err` arm in `execute_columnar_scan`'s +/// `row_to_projected_value`'s `Err` arm in `execute_columnar_scan`'s /// live-memtable phase (a real, non-empty `computed_cols`). /// /// `denom` must be listed explicitly alongside the computed `ratio` column -/// (not just referenced inside the expression): `row_to_projected_json` +/// (not just referenced inside the expression): `row_to_projected_value` /// only includes a stored column in the row object it hands to the computed /// expression evaluator when that column is itself in the projection list /// (or force-included) — an explicit, non-computed `SELECT` entry for every diff --git a/nodedb/tests/wire/cases/sql_merge_declared_types.rs b/nodedb/tests/wire/cases/sql_merge_declared_types.rs new file mode 100644 index 000000000..714a71b64 --- /dev/null +++ b/nodedb/tests/wire/cases/sql_merge_declared_types.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Declared-type coercion for MERGE literals. +//! +//! A literal reaching storage through `INSERT VALUES` or `UPDATE SET` passes +//! the declared-type coercion the planner applies once, in +//! `declared_type_coerce`. A MERGE `WHEN NOT MATCHED THEN INSERT` arm and a +//! `WHEN MATCHED THEN UPDATE SET` arm carry the same literals through the same +//! planner and must be coerced the same way, on every engine that declares +//! columns. + +use crate::harness::TestServer; + +async fn create_typed_target(server: &TestServer, name: &str, engine: &str, id_type: &str) { + server + .exec(&format!( + "CREATE COLLECTION {name} (id {id_type} PRIMARY KEY, n INT, at TIMESTAMP) \ + WITH (engine='{engine}')" + )) + .await + .unwrap_or_else(|e| panic!("create {name} ({engine}): {e}")); +} + +async fn create_typed_source(server: &TestServer, name: &str, engine: &str, id_type: &str) { + server + .exec(&format!( + "CREATE COLLECTION {name} (id {id_type} PRIMARY KEY) WITH (engine='{engine}')" + )) + .await + .unwrap_or_else(|e| panic!("create {name} ({engine}): {e}")); +} + +/// `MERGE ... WHEN NOT MATCHED THEN INSERT` runs its literals through the same +/// declared-type coercion an ordinary `INSERT VALUES` gets: a text literal +/// into `n INT` stores an integer, and an epoch-millisecond numeric literal +/// into `at TIMESTAMP` stores that instant, on every engine whose rule +/// plans MERGE (the key-value and columnar rules refuse MERGE outright). +async fn merge_insert_literals_take_the_declared_column_types_on(engine: &str, id_type: &str) { + let server = TestServer::start().await; + create_typed_target(&server, "merge_typed_target", engine, id_type).await; + create_typed_source(&server, "merge_typed_source", engine, id_type).await; + + let source_id = if id_type == "TEXT" { "'1'" } else { "1" }; + server + .exec(&format!( + "INSERT INTO merge_typed_source (id) VALUES ({source_id})" + )) + .await + .expect("seed source"); + + server + .exec( + "MERGE INTO merge_typed_target t \ + USING merge_typed_source s ON t.id = s.id \ + WHEN NOT MATCHED THEN INSERT (id, n, at) \ + VALUES (s.id, '1', 1583402400000)", + ) + .await + .expect("MERGE INSERT of a coercible literal must succeed"); + + let rows = server + .query_rows("SELECT n, at FROM merge_typed_target") + .await + .expect("select inserted row"); + assert_eq!(rows.len(), 1, "one row inserted: {rows:?}"); + assert_eq!( + rows[0], + vec!["1".to_string(), "2020-03-05T10:00:00.000000Z".to_string()], + "n and at must take the declared column types: {rows:?}" + ); + + let doubled = server + .query_rows("SELECT n * 2 FROM merge_typed_target") + .await + .expect("select n * 2"); + assert_eq!( + doubled, + vec![vec!["2".to_string()]], + "n must be numeric, not text, for arithmetic to succeed: {doubled:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn merge_insert_literals_take_the_declared_column_types_document_strict() { + merge_insert_literals_take_the_declared_column_types_on("document_strict", "INT").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn merge_insert_literals_take_the_declared_column_types_document_schemaless() { + merge_insert_literals_take_the_declared_column_types_on("document_schemaless", "INT").await; +} + +/// `MERGE ... WHEN MATCHED THEN UPDATE SET` runs its literals through the +/// same declared-type coercion, on the row it rewrites rather than the row it +/// inserts, on every engine whose rule plans MERGE. +async fn merge_update_literals_take_the_declared_column_types_on(engine: &str) { + let server = TestServer::start().await; + create_typed_target(&server, "merge_upd_target", engine, "INT").await; + create_typed_source(&server, "merge_upd_source", engine, "INT").await; + + server + .exec("INSERT INTO merge_upd_target (id, n, at) VALUES (1, 0, 1583402400000)") + .await + .expect("seed target"); + server + .exec("INSERT INTO merge_upd_source (id) VALUES (1)") + .await + .expect("seed source"); + + server + .exec( + "MERGE INTO merge_upd_target t \ + USING merge_upd_source s ON t.id = s.id \ + WHEN MATCHED THEN UPDATE SET n = '2', at = 1583402400001", + ) + .await + .expect("MERGE UPDATE of coercible literals must succeed"); + + let rows = server + .query_rows("SELECT at FROM merge_upd_target") + .await + .expect("select updated row"); + assert_eq!( + rows, + vec![vec!["2020-03-05T10:00:00.001000Z".to_string()]], + "at must take the declared TIMESTAMP type: {rows:?}" + ); + + let doubled = server + .query_rows("SELECT n * 2 FROM merge_upd_target") + .await + .expect("select n * 2"); + assert_eq!( + doubled, + vec![vec!["4".to_string()]], + "n must be numeric, not text, for arithmetic to succeed: {doubled:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn merge_update_literals_take_the_declared_column_types_document_strict() { + merge_update_literals_take_the_declared_column_types_on("document_strict").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn merge_update_literals_take_the_declared_column_types_document_schemaless() { + merge_update_literals_take_the_declared_column_types_on("document_schemaless").await; +} + +/// A literal the declared column type cannot hold is refused, naming the +/// column, on both a MERGE INSERT arm and a MERGE UPDATE arm — exactly as an +/// ordinary `INSERT VALUES` / `UPDATE SET` refuses it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn merge_refuses_a_literal_the_column_cannot_hold() { + let server = TestServer::start().await; + create_typed_target(&server, "merge_bad_target", "document_strict", "INT").await; + create_typed_source(&server, "merge_bad_source", "document_strict", "INT").await; + + server + .exec("INSERT INTO merge_bad_source (id) VALUES (1)") + .await + .expect("seed source"); + + let insert_err = server + .exec( + "MERGE INTO merge_bad_target t \ + USING merge_bad_source s ON t.id = s.id \ + WHEN NOT MATCHED THEN INSERT (id, n, at) \ + VALUES (s.id, 'abc', 1583402400000)", + ) + .await + .expect_err("MERGE INSERT of 'abc' into an INT column must be refused"); + assert!( + insert_err.contains("'n'"), + "error must name column 'n': {insert_err}" + ); + + server + .exec("INSERT INTO merge_bad_target (id, n, at) VALUES (1, 0, 1583402400000)") + .await + .expect("seed target for the UPDATE arm"); + + let update_err = server + .exec( + "MERGE INTO merge_bad_target t \ + USING merge_bad_source s ON t.id = s.id \ + WHEN MATCHED THEN UPDATE SET at = true", + ) + .await + .expect_err("MERGE UPDATE of a boolean into a TIMESTAMP column must be refused"); + assert!( + update_err.contains("'at'"), + "error must name column 'at': {update_err}" + ); +} diff --git a/nodedb/tests/wire/cases/sql_where_instant_literals.rs b/nodedb/tests/wire/cases/sql_where_instant_literals.rs new file mode 100644 index 000000000..45a283a34 --- /dev/null +++ b/nodedb/tests/wire/cases/sql_where_instant_literals.rs @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A literal compared against a declared `TIMESTAMP`/`TIMESTAMPTZ` column +//! follows one rule on every engine and every path: a string literal parses +//! as a datetime, a numeric literal is epoch milliseconds, and any other +//! literal kind is a typed error. The comparison then compares instants, and +//! `SELECT` renders the column as `2020-03-05T10:00:00.000000Z` style +//! ISO-8601. + +use crate::harness::TestServer; + +/// `2020-03-05 10:00:00` as a bare datetime literal. Its epoch-millisecond +/// form is `1583402400000`, which the predicate tables below spell out +/// directly in each SQL clause. +const EARLY: &str = "2020-03-05 10:00:00"; +/// `EARLY` rendered by a `SELECT` of a `TIMESTAMP` column. +const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; + +/// A predicate against `at`, and the row ids it must select, in `ORDER BY +/// id` order. +struct Predicate { + label: &'static str, + clause: &'static str, + expected: &'static [&'static str], +} + +/// One family per operator, expressed with a numeric-ms literal. Callers +/// substitute the literal for a string form to cover the other literal kind. +const NUMERIC_PREDICATES: &[Predicate] = &[ + Predicate { + label: "= ms", + clause: "at = 1583402400000", + expected: &["r2_at"], + }, + Predicate { + label: "> ms", + clause: "at > 1583402400000", + expected: &["r3_after"], + }, + Predicate { + label: ">= ms", + clause: "at >= 1583402400000", + expected: &["r2_at", "r3_after"], + }, + Predicate { + label: "< ms", + clause: "at < 1583402400000", + expected: &["r1_before"], + }, + Predicate { + label: "<= ms", + clause: "at <= 1583402400000", + expected: &["r1_before", "r2_at"], + }, + Predicate { + label: "BETWEEN ms", + clause: "at BETWEEN 1583402400000 AND 1583406000000", + expected: &["r2_at", "r3_after"], + }, + Predicate { + label: "IN ms", + clause: "at IN (1583402400000, 1583406000000)", + expected: &["r2_at", "r3_after"], + }, +]; + +/// The same seven predicates with a space-separated datetime string literal. +const STRING_SPACE_PREDICATES: &[Predicate] = &[ + Predicate { + label: "= string", + clause: "at = '2020-03-05 10:00:00'", + expected: &["r2_at"], + }, + Predicate { + label: "> string", + clause: "at > '2020-03-05 10:00:00'", + expected: &["r3_after"], + }, + Predicate { + label: ">= string", + clause: "at >= '2020-03-05 10:00:00'", + expected: &["r2_at", "r3_after"], + }, + Predicate { + label: "< string", + clause: "at < '2020-03-05 10:00:00'", + expected: &["r1_before"], + }, + Predicate { + label: "<= string", + clause: "at <= '2020-03-05 10:00:00'", + expected: &["r1_before", "r2_at"], + }, + Predicate { + label: "BETWEEN string", + clause: "at BETWEEN '2020-03-05 10:00:00' AND '2020-03-05 11:00:00'", + expected: &["r2_at", "r3_after"], + }, + Predicate { + label: "IN string", + clause: "at IN ('2020-03-05 10:00:00', '2020-03-05 11:00:00')", + expected: &["r2_at", "r3_after"], + }, +]; + +/// The same seven predicates with an ISO-8601 `T`/`Z` string literal. +const STRING_ISO_PREDICATES: &[Predicate] = &[ + Predicate { + label: "= iso", + clause: "at = '2020-03-05T10:00:00Z'", + expected: &["r2_at"], + }, + Predicate { + label: "> iso", + clause: "at > '2020-03-05T10:00:00Z'", + expected: &["r3_after"], + }, + Predicate { + label: ">= iso", + clause: "at >= '2020-03-05T10:00:00Z'", + expected: &["r2_at", "r3_after"], + }, + Predicate { + label: "< iso", + clause: "at < '2020-03-05T10:00:00Z'", + expected: &["r1_before"], + }, + Predicate { + label: "<= iso", + clause: "at <= '2020-03-05T10:00:00Z'", + expected: &["r1_before", "r2_at"], + }, + Predicate { + label: "BETWEEN iso", + clause: "at BETWEEN '2020-03-05T10:00:00Z' AND '2020-03-05T11:00:00Z'", + expected: &["r2_at", "r3_after"], + }, + Predicate { + label: "IN iso", + clause: "at IN ('2020-03-05T10:00:00Z', '2020-03-05T11:00:00Z')", + expected: &["r2_at", "r3_after"], + }, +]; + +/// Create `name` on `engine` with `(id TEXT PRIMARY KEY, at TIMESTAMP, v +/// FLOAT)` and seed `r1_before`/`r2_at`/`r3_after` an hour apart around `EARLY`. +/// `engine` is `None` for a schemaless document collection (no `WITH` +/// clause). +async fn seed(server: &TestServer, name: &str, engine: Option<&str>) { + let with_clause = match engine { + Some(e) => format!(" WITH (engine='{e}')"), + None => String::new(), + }; + server + .exec(&format!( + "CREATE COLLECTION {name} (id TEXT PRIMARY KEY, at TIMESTAMP, v FLOAT){with_clause}" + )) + .await + .unwrap_or_else(|e| panic!("create {name}: {e}")); + server + .exec(&format!( + "INSERT INTO {name} (id, at, v) VALUES \ + ('r1_before', '2020-03-05 09:00:00', 1.0), \ + ('r2_at', '{EARLY}', 2.0), \ + ('r3_after', '2020-03-05 11:00:00', 3.0)" + )) + .await + .unwrap_or_else(|e| panic!("insert into {name}: {e}")); +} + +/// Create a timeseries collection `(at TIMESTAMP TIME_KEY, id TEXT, v FLOAT) +/// WITH (engine='timeseries')` and seed the same three rows. +async fn seed_timeseries(server: &TestServer, name: &str) { + server + .exec(&format!( + "CREATE COLLECTION {name} (at TIMESTAMP TIME_KEY, id TEXT, v FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .unwrap_or_else(|e| panic!("create {name}: {e}")); + server + .exec(&format!( + "INSERT INTO {name} (at, id, v) VALUES \ + ('2020-03-05 09:00:00', 'r1_before', 1.0), \ + ('{EARLY}', 'r2_at', 2.0), \ + ('2020-03-05 11:00:00', 'r3_after', 3.0)" + )) + .await + .unwrap_or_else(|e| panic!("insert into {name}: {e}")); +} + +/// Run every predicate in `table` against `collection` and assert the row +/// ids each one selects, each assertion message naming the predicate. +async fn assert_predicates(server: &TestServer, collection: &str, table: &[Predicate]) { + for p in table { + let rows = server + .query_text(&format!( + "SELECT id FROM {collection} WHERE {} ORDER BY id", + p.clause + )) + .await + .unwrap_or_else(|e| panic!("{collection} [{}]: {e}", p.label)); + assert_eq!( + rows, p.expected, + "{collection} [{}]: expected {:?}, got {rows:?}", + p.label, p.expected + ); + } +} + +/// Run the full literal-kind suite (numeric ms, space-separated string, +/// ISO-8601 string) against `collection`, plus the typed-error case for a +/// non-instant literal kind. +async fn assert_full_suite(server: &TestServer, collection: &str) { + assert_predicates(server, collection, NUMERIC_PREDICATES).await; + assert_predicates(server, collection, STRING_SPACE_PREDICATES).await; + assert_predicates(server, collection, STRING_ISO_PREDICATES).await; + server + .expect_error( + &format!("SELECT id FROM {collection} WHERE at = true"), + "at", + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn document_strict_instant_literal_predicates() { + let server = TestServer::start().await; + let name = "swi_strict"; + seed(&server, name, Some("document_strict")).await; + assert_full_suite(&server, name).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn schemaless_document_instant_literal_predicates() { + let server = TestServer::start().await; + let name = "swi_schemaless"; + seed(&server, name, None).await; + assert_full_suite(&server, name).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn kv_instant_literal_predicates() { + let server = TestServer::start().await; + let name = "swi_kv"; + seed(&server, name, Some("kv")).await; + assert_full_suite(&server, name).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn columnar_instant_literal_predicates() { + let server = TestServer::start().await; + let name = "swi_columnar"; + seed(&server, name, Some("columnar")).await; + assert_full_suite(&server, name).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn timeseries_instant_literal_predicates() { + let server = TestServer::start().await; + let name = "swi_timeseries"; + seed_timeseries(&server, name).await; + assert_full_suite(&server, name).await; +} + +/// Rows flushed to a timeseries partition (a low memtable budget forces the +/// flush) follow the same instant-literal rule as an unflushed memtable read. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn timeseries_instant_literal_predicates_after_flush() { + let server = TestServer::start_with_timeseries_memtable_budget(1).await; + let name = "swi_ts_flushed"; + seed_timeseries(&server, name).await; + assert_full_suite(&server, name).await; +} + +/// Rows on a flushed columnar segment follow the same instant-literal rule +/// as rows still in the write buffer. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn columnar_instant_literal_predicates_after_flush() { + let server = TestServer::start_with_columnar_flush_threshold(2).await; + let name = "swi_columnar_flushed"; + seed(&server, name, Some("columnar")).await; + assert_full_suite(&server, name).await; +} + +/// `UPDATE ... WHERE at > ` and `DELETE ... WHERE at < ` scope by the +/// same instant the read path selects by, for every DML-capable engine. +async fn assert_update_delete_scoped_by_instant(server: &TestServer, collection: &str) { + server + .exec(&format!( + "UPDATE {collection} SET v = 9 WHERE at > 1583402400000" + )) + .await + .unwrap_or_else(|e| panic!("update {collection}: {e}")); + let updated = server + .query_text(&format!("SELECT id FROM {collection} WHERE v = 9")) + .await + .unwrap_or_else(|e| panic!("select updated {collection}: {e}")); + assert_eq!( + updated, + vec!["r3_after".to_string()], + "UPDATE ... WHERE at > must touch only r3_after, got {updated:?}" + ); + + server + .exec(&format!( + "DELETE FROM {collection} WHERE at < 1583402400000" + )) + .await + .unwrap_or_else(|e| panic!("delete {collection}: {e}")); + let remaining = server + .query_text(&format!("SELECT id FROM {collection} ORDER BY id")) + .await + .unwrap_or_else(|e| panic!("select remaining {collection}: {e}")); + assert_eq!( + remaining, + vec!["r2_at".to_string(), "r3_after".to_string()], + "DELETE ... WHERE at < must remove only r1_before, got {remaining:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn document_strict_update_delete_scoped_by_instant_literal() { + let server = TestServer::start().await; + let name = "swi_strict_dml"; + seed(&server, name, Some("document_strict")).await; + assert_update_delete_scoped_by_instant(&server, name).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn columnar_update_delete_scoped_by_instant_literal() { + let server = TestServer::start().await; + let name = "swi_columnar_dml"; + seed(&server, name, Some("columnar")).await; + assert_update_delete_scoped_by_instant(&server, name).await; +} + +/// A JOIN `ON` clause comparing a timeseries time key against a numeric-ms +/// literal follows the same rule as a top-level `WHERE`: the literal is an +/// instant, so the predicate scopes the join to rows after `EARLY`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn join_on_clause_with_instant_ms_literal_scopes_the_match() { + let server = TestServer::start().await; + let ts_name = "swi_join_ts"; + let strict_name = "swi_join_strict"; + seed_timeseries(&server, ts_name).await; + seed(&server, strict_name, Some("document_strict")).await; + + let joined = server + .query_text(&format!( + "SELECT a.id FROM {ts_name} a JOIN {strict_name} b \ + ON a.id = b.id AND a.at > 1583402400000" + )) + .await + .unwrap_or_else(|e| panic!("join {ts_name}/{strict_name}: {e}")); + assert_eq!( + joined, + vec!["r3_after".to_string()], + "the ON clause's ms literal must scope the join to rows after EARLY: {joined:?}" + ); +} + +/// A `SELECT` of the declared `at` column renders `EARLY` as ISO-8601 UTC, +/// independent of which literal kind selected the row. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn selecting_the_matched_row_renders_iso8601() { + let server = TestServer::start().await; + let name = "swi_render"; + seed(&server, name, Some("document_strict")).await; + + let rows = server + .query_text(&format!("SELECT at FROM {name} WHERE at = 1583402400000")) + .await + .unwrap_or_else(|e| panic!("select at from {name}: {e}")); + assert_eq!( + rows, + vec![EARLY_ISO.to_string()], + "SELECT of a TIMESTAMP column must render ISO-8601 UTC: {rows:?}" + ); +} diff --git a/nodedb/tests/wire/cases/strict_typed_column_rendering.rs b/nodedb/tests/wire/cases/strict_typed_column_rendering.rs new file mode 100644 index 000000000..3dfa4c5b8 --- /dev/null +++ b/nodedb/tests/wire/cases/strict_typed_column_rendering.rs @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A `document_strict` `TIMESTAMP` or `TIMESTAMPTZ` column renders the +//! stored instant as ISO-8601 whichever route reads it, and the same way a +//! timeseries time key does. A `columnar` `TIMESTAMP` column renders the +//! same instant from the live memtable and from a flushed segment. An +//! integer literal written into a `TIMESTAMP` column is epoch milliseconds +//! on every engine, and a literal that carries no instant is refused. + +use crate::harness::TestServer; + +/// The instant every test in this file stores. +const EARLY: &str = "2020-03-05 10:00:00"; +/// `EARLY` as a declared timestamp column renders it: the engine stores the +/// instant as epoch microseconds and hands the encoder a typed instant, +/// which renders as ISO-8601 UTC. +const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; +/// `EARLY` as seconds since the Unix epoch. +const EARLY_UNIX_SECS: u64 = 1_583_402_400; +/// `EARLY` as milliseconds since the Unix epoch, the unit an integer literal +/// written into a timestamp column denotes. +const EARLY_UNIX_MILLIS: i64 = 1_583_402_400_000; + +/// A strict `document_strict` collection carrying a `TIMESTAMP` column, read +/// back with a direct `SELECT`, renders the stored instant as ISO-8601 — +/// never as an epoch integer. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_strict_timestamp_column_renders_the_stored_instant() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION strict_ts_direct \ + (id TEXT PRIMARY KEY, created_at TIMESTAMP) \ + WITH (engine='document_strict')", + ) + .await + .expect("create strict_ts_direct"); + server + .exec(&format!( + "INSERT INTO strict_ts_direct (id, created_at) VALUES ('r1', '{EARLY}')" + )) + .await + .expect("insert into strict_ts_direct"); + + let rows = server + .query_text("SELECT created_at FROM strict_ts_direct WHERE id = 'r1'") + .await + .expect("SELECT of a strict TIMESTAMP column must succeed"); + assert_eq!( + rows, + vec![EARLY_ISO.to_string()], + "a strict TIMESTAMP column must render {EARLY} as {EARLY_ISO}" + ); +} + +/// The `TIMESTAMPTZ` sibling renders the same instant the same way. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_strict_timestamptz_column_renders_the_stored_instant() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION strict_tstz_direct \ + (id TEXT PRIMARY KEY, created_at TIMESTAMPTZ) \ + WITH (engine='document_strict')", + ) + .await + .expect("create strict_tstz_direct"); + server + .exec(&format!( + "INSERT INTO strict_tstz_direct (id, created_at) VALUES ('r1', '{EARLY}')" + )) + .await + .expect("insert into strict_tstz_direct"); + + let rows = server + .query_text("SELECT created_at FROM strict_tstz_direct WHERE id = 'r1'") + .await + .expect("SELECT of a strict TIMESTAMPTZ column must succeed"); + assert_eq!( + rows, + vec![EARLY_ISO.to_string()], + "a strict TIMESTAMPTZ column must render {EARLY} as {EARLY_ISO}" + ); +} + +/// Over the extended protocol the driver requests binary results, and a +/// timestamp column honours that: both `TIMESTAMP` and `TIMESTAMPTZ` decode +/// through the driver's `SystemTime` reader to the stored instant. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_strict_timestamp_column_decodes_from_binary() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION strict_ts_binary \ + (id TEXT PRIMARY KEY, at TIMESTAMP, at_tz TIMESTAMPTZ) \ + WITH (engine='document_strict')", + ) + .await + .expect("create strict_ts_binary"); + server + .exec(&format!( + "INSERT INTO strict_ts_binary (id, at, at_tz) VALUES ('r1', '{EARLY}', '{EARLY}')" + )) + .await + .expect("insert into strict_ts_binary"); + + let rows = server + .client + .query( + "SELECT at, at_tz FROM strict_ts_binary WHERE id = $1", + &[&"r1"], + ) + .await + .expect("extended-protocol SELECT of timestamp columns must succeed"); + assert_eq!(rows.len(), 1, "one stored row"); + let expected = std::time::UNIX_EPOCH + std::time::Duration::from_secs(EARLY_UNIX_SECS); + assert_eq!( + rows[0].get::<_, std::time::SystemTime>("at"), + expected, + "binary TIMESTAMP must decode to {EARLY}" + ); + assert_eq!( + rows[0].get::<_, std::time::SystemTime>("at_tz"), + expected, + "binary TIMESTAMPTZ must decode to {EARLY}" + ); +} + +/// The same column, read back through `INSERT ... RETURNING` rather than a +/// follow-up `SELECT`, denotes the same stored instant. `RETURNING` runs its +/// own encoder over the just-written row, so it is a second route to the +/// same cell. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_strict_timestamp_column_returned_by_insert_renders_the_stored_instant() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION strict_ts_returning \ + (id TEXT PRIMARY KEY, created_at TIMESTAMP) \ + WITH (engine='document_strict')", + ) + .await + .expect("create strict_ts_returning"); + + let rows = server + .query_text(&format!( + "INSERT INTO strict_ts_returning (id, created_at) VALUES ('r2', '{EARLY}') \ + RETURNING created_at" + )) + .await + .expect("INSERT ... RETURNING of a strict TIMESTAMP column must succeed"); + assert_eq!( + rows, + vec![EARLY_ISO.to_string()], + "a strict TIMESTAMP column returned by INSERT must render {EARLY} as {EARLY_ISO}" + ); +} + +/// A `document_strict` `TIMESTAMP` column and a timeseries `TIMESTAMP` time +/// key holding the same instant render identically. One instant renders one +/// way whichever engine stores it, so a divergence between the two engines' +/// encoders is a rendering defect, not an engine-specific choice. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_strict_timestamp_column_renders_the_same_as_a_timeseries_time_key() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION strict_ts_parity_doc \ + (id TEXT PRIMARY KEY, created_at TIMESTAMP) \ + WITH (engine='document_strict')", + ) + .await + .expect("create strict_ts_parity_doc"); + server + .exec(&format!( + "INSERT INTO strict_ts_parity_doc (id, created_at) VALUES ('r3', '{EARLY}')" + )) + .await + .expect("insert into strict_ts_parity_doc"); + + server + .exec( + "CREATE COLLECTION strict_ts_parity_ts \ + (captured_at TIMESTAMP TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')", + ) + .await + .expect("create strict_ts_parity_ts"); + server + .exec(&format!( + "INSERT INTO strict_ts_parity_ts (captured_at, host, v) VALUES ('{EARLY}', 'h1', 1.5)" + )) + .await + .expect("insert into strict_ts_parity_ts"); + + let strict_reading = server + .query_text("SELECT created_at FROM strict_ts_parity_doc WHERE id = 'r3'") + .await + .expect("SELECT of the strict TIMESTAMP column must succeed"); + assert_eq!( + strict_reading.len(), + 1, + "one stored row: {strict_reading:?}" + ); + + let timeseries_reading = server + .query_text("SELECT captured_at FROM strict_ts_parity_ts") + .await + .expect("SELECT of the timeseries time key must succeed"); + assert_eq!( + timeseries_reading.len(), + 1, + "one stored point: {timeseries_reading:?}" + ); + + assert_eq!( + strict_reading[0], timeseries_reading[0], + "a strict TIMESTAMP column must render the same instant as a timeseries time \ + key: strict={strict_reading:?} timeseries={timeseries_reading:?}" + ); +} + +/// A `columnar` collection carrying a `TIMESTAMP` column, read back with a +/// direct `SELECT` while the row is still in the live memtable, renders the +/// stored instant as ISO-8601. The columnar engine stores the cell as epoch +/// microseconds and reads it back as a typed instant, so the millisecond and +/// microsecond integer forms are both rendering defects here. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_columnar_timestamp_column_renders_the_stored_instant() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION columnar_ts_direct \ + (id TEXT PRIMARY KEY, created_at TIMESTAMP) \ + WITH (engine='columnar')", + ) + .await + .expect("create columnar_ts_direct"); + server + .exec(&format!( + "INSERT INTO columnar_ts_direct (id, created_at) VALUES ('r1', '{EARLY}')" + )) + .await + .expect("insert into columnar_ts_direct"); + + let rows = server + .query_text("SELECT created_at FROM columnar_ts_direct WHERE id = 'r1'") + .await + .expect("SELECT of a columnar TIMESTAMP column must succeed"); + assert_eq!(rows.len(), 1, "one stored row: {rows:?}"); + assert_eq!( + rows[0], EARLY_ISO, + "a columnar TIMESTAMP column must render {EARLY} as {EARLY_ISO}, got {rows:?}" + ); +} + +/// The columnar engine has two read paths for one column: the live memtable +/// and the flushed segments a full memtable drains into. With a flush +/// threshold of two, the first insert is read from the memtable, and after +/// two more inserts the first rows live only in a flushed segment while the +/// last stays in the memtable. Every row renders the one instant identically +/// on both paths. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_columnar_timestamp_column_renders_the_same_before_and_after_flush() { + let server = TestServer::start_with_columnar_flush_threshold(2).await; + server + .exec( + "CREATE COLLECTION columnar_ts_flush \ + (id TEXT PRIMARY KEY, created_at TIMESTAMP) \ + WITH (engine='columnar')", + ) + .await + .expect("create columnar_ts_flush"); + server + .exec(&format!( + "INSERT INTO columnar_ts_flush (id, created_at) VALUES ('r1', '{EARLY}')" + )) + .await + .expect("insert r1 into columnar_ts_flush"); + + let before_flush = server + .query_text("SELECT created_at FROM columnar_ts_flush WHERE id = 'r1'") + .await + .expect("SELECT from the live memtable must succeed"); + assert_eq!( + before_flush, + vec![EARLY_ISO.to_string()], + "the live-memtable read must render {EARLY} as {EARLY_ISO}" + ); + + for id in ["r2", "r3"] { + server + .exec(&format!( + "INSERT INTO columnar_ts_flush (id, created_at) VALUES ('{id}', '{EARLY}')" + )) + .await + .unwrap_or_else(|e| panic!("insert {id} into columnar_ts_flush: {e}")); + } + + let after_flush = server + .query_text("SELECT created_at FROM columnar_ts_flush ORDER BY id") + .await + .expect("SELECT across a flushed segment and the memtable must succeed"); + assert_eq!( + after_flush, + vec![EARLY_ISO.to_string(); 3], + "a flushed-segment cell and a live-memtable cell must render the one \ + instant identically: before={before_flush:?} after={after_flush:?}" + ); +} + +/// An integer literal written into a `TIMESTAMP` column is epoch +/// milliseconds, resolved once in the planner, so the four engines that +/// store a declared column — strict, key-value, columnar, and schemaless +/// document — all render the one instant it denotes. No engine stores the +/// bare integer, and no read path picks a unit for it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_integer_literal_into_a_timestamp_column_is_epoch_milliseconds() { + let server = TestServer::start().await; + let collections = [ + ("int_ts_strict", "id", "document_strict"), + ("int_ts_kv", "key", "kv"), + ("int_ts_columnar", "id", "columnar"), + ("int_ts_document", "id", "document_schemaless"), + ]; + for (name, key, engine) in collections { + server + .exec(&format!( + "CREATE COLLECTION {name} \ + ({key} TEXT PRIMARY KEY, created_at TIMESTAMP) \ + WITH (engine='{engine}')" + )) + .await + .unwrap_or_else(|e| panic!("create {name} on {engine}: {e}")); + server + .exec(&format!( + "INSERT INTO {name} ({key}, created_at) VALUES ('r1', {EARLY_UNIX_MILLIS})" + )) + .await + .unwrap_or_else(|e| panic!("insert an integer literal into {name}: {e}")); + + let rows = server + .query_text(&format!("SELECT created_at FROM {name} WHERE {key} = 'r1'")) + .await + .unwrap_or_else(|e| panic!("SELECT of {name}.created_at: {e}")); + assert_eq!( + rows, + vec![EARLY_ISO.to_string()], + "{engine}: an integer literal into a TIMESTAMP column must render \ + {EARLY_UNIX_MILLIS} epoch milliseconds as {EARLY_ISO}" + ); + } +} + +/// A literal that carries no instant is refused at the statement, naming +/// the column, rather than stored under the `TIMESTAMP` column for a read +/// to fail on later. Text that spells no date and a boolean are both +/// refused, on an engine that persists the planner's value verbatim. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_non_datetime_literal_into_a_timestamp_column_is_refused() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION refused_ts \ + (id TEXT PRIMARY KEY, created_at TIMESTAMP) \ + WITH (engine='document_schemaless')", + ) + .await + .expect("create refused_ts"); + + let text_error = server + .exec("INSERT INTO refused_ts (id, created_at) VALUES ('r2', 'not a date')") + .await + .expect_err("text that spells no date must be refused"); + assert!( + text_error.contains("created_at"), + "the refusal must name the column: {text_error}" + ); + + let bool_error = server + .exec("INSERT INTO refused_ts (id, created_at) VALUES ('r3', true)") + .await + .expect_err("a boolean carries no instant and must be refused"); + assert!( + bool_error.contains("created_at"), + "the refusal must name the column: {bool_error}" + ); + + let rows = server + .query_text("SELECT id FROM refused_ts") + .await + .expect("SELECT from refused_ts must succeed"); + assert!( + rows.is_empty(), + "a refused INSERT must store nothing: {rows:?}" + ); +} diff --git a/nodedb/tests/wire/cases/timeseries_declared_time_key.rs b/nodedb/tests/wire/cases/timeseries_declared_time_key.rs index 464ecd0f6..9950f45f1 100644 --- a/nodedb/tests/wire/cases/timeseries_declared_time_key.rs +++ b/nodedb/tests/wire/cases/timeseries_declared_time_key.rs @@ -18,10 +18,10 @@ //! (`ts`, `timestamp`, `time`) is not special: it is the user's column. //! //! A declared `TIMESTAMP` time key reads back as the instant that was -//! inserted: the engine stores epoch milliseconds and a `TIMESTAMP` cell is -//! read as epoch microseconds, so the two units meet as the row leaves the -//! scan. A `BIGINT` time key shares that storage column and is not a -//! timestamp, so it reads back as the number that was inserted. +//! inserted: the engine stores epoch milliseconds and emits the cell as a +//! typed instant from the column's own kind. A `BIGINT` time key shares +//! that storage column and is not a timestamp, so it reads back as the +//! number that was inserted. use crate::harness::TestServer; @@ -34,13 +34,10 @@ const LATE: &str = "2020-03-05 13:00:00"; /// wall-clock time ever will. const AFTER_BOTH: &str = "2021-01-01 00:00:00"; /// `EARLY` as a declared `TIMESTAMP` column renders it. The engine stores -/// 1583402400000 epoch milliseconds; a `TIMESTAMP` cell carries epoch -/// microseconds, which the pgwire encoder writes as ISO-8601 UTC. +/// 1583402400000 epoch milliseconds and emits the cell as a typed instant, +/// which the pgwire encoder writes as ISO-8601 UTC. A star projection reads +/// the same typed cell, so it renders the same. const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; -/// `EARLY` as `SELECT *` renders it: a star projection announces no catalog -/// type, so its cells stay the raw stored number — here the same instant in -/// epoch microseconds. -const EARLY_MICROS: &str = "1583402400000000"; #[tokio::test] async fn time_key_named_ts_round_trips() { @@ -126,7 +123,7 @@ async fn select_star_projects_declared_columns_only() { ); assert_eq!( rows[0].get("ts").map(String::as_str), - Some(EARLY_MICROS), + Some(EARLY_ISO), "`SELECT *` must carry the inserted event time under `ts`: {rows:?}" ); } diff --git a/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs b/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs index ecc4d4e71..ee390111a 100644 --- a/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs +++ b/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs @@ -20,12 +20,9 @@ use crate::harness::TestServer; /// ingest (wall-clock "now") is separable from the value the INSERT supplied. const EARLY: &str = "2020-03-05 10:00:00"; /// `EARLY` as a declared `TIMESTAMP` column renders it. The engine stores -/// 1583402400000 epoch milliseconds; a `TIMESTAMP` cell carries epoch -/// microseconds, which the pgwire encoder writes as ISO-8601 UTC. +/// 1583402400000 epoch milliseconds and emits the cell as a typed instant, +/// which the pgwire encoder writes as ISO-8601 UTC. const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; -/// `EARLY` as epoch microseconds — 1583402400000 milliseconds times 1000. -/// A projection that announces no catalog type leaves its cells this number. -const EARLY_MICROS: &str = "1583402400000000"; /// Create a timeseries collection and a document collection that join on the /// event's `host`, then insert exactly one row into each so the join yields @@ -142,17 +139,12 @@ async fn a_joined_time_key_denotes_the_stored_instant() { .expect("the time key projected through a JOIN must succeed"); assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); - // Two renderings denote 2020-03-05T10:00:00Z, and the expected value is - // whichever one the join announces a type for. EARLY_ISO is that instant - // written as ISO-8601 UTC, which a cell typed TIMESTAMP produces. - // EARLY_MICROS is the same instant in epoch microseconds — the unit a - // TIMESTAMP cell carries — which an untyped cell leaves as a number. - // Epoch MILLISECONDS denote 1970-01-19 read either way, so a millisecond - // value fails both arms. - assert!( - joined[0] == EARLY_ISO || joined[0] == EARLY_MICROS, - "a joined time key must denote {EARLY}: expected {EARLY_ISO} \ - or {EARLY_MICROS}, got {joined:?}" + // The cell is typed at emission, so the join carries the instant itself + // and renders it as ISO-8601 UTC. The stored 1583402400000 milliseconds + // read as a number would denote 1970-01-19. + assert_eq!( + joined[0], EARLY_ISO, + "a joined time key must denote {EARLY}: got {joined:?}" ); } @@ -202,15 +194,298 @@ async fn an_aliased_joined_time_key_denotes_the_stored_instant() { .expect("an aliased time key projected through a JOIN must succeed"); assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); - // The expected value is EARLY, the inserted instant, in whichever unit the - // alias announces a type for. EARLY_ISO is 2020-03-05T10:00:00Z as a cell - // typed TIMESTAMP renders it; EARLY_MICROS is the same instant in the epoch - // microseconds a TIMESTAMP cell carries, which an untyped cell leaves as a - // number. The stored 1583402400000 milliseconds denote 1970-01-19 read - // either way, so a millisecond value fails both arms. - assert!( - joined[0] == EARLY_ISO || joined[0] == EARLY_MICROS, - "an aliased joined time key must denote {EARLY}: expected {EARLY_ISO} \ - or {EARLY_MICROS}, got {joined:?}" + // The alias renames a cell that is already a typed instant, so it renders + // as ISO-8601 UTC under the new name. + assert_eq!( + joined[0], EARLY_ISO, + "an aliased joined time key must denote {EARLY}: got {joined:?}" + ); +} + +/// `EARLY` truncated to its hour, as `time_bucket('1 hour', ...)` denotes it. +/// `EARLY` sits on the hour, so the bucket is the same instant. +const EARLY_BUCKET_ISO: &str = EARLY_ISO; + +/// A transforming computed projection of a time key renders the same through +/// a JOIN as through a direct read. `time_bucket` arithmetic depends on the +/// unit the expression sees, so the two routes agree only when the cell +/// reaches the expression in one unit. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_time_bucket_of_a_time_key_matches_between_a_join_and_a_direct_read() { + let server = TestServer::start().await; + setup(&server, "tsj_bucket_events", "tsj_bucket_hosts").await; + + let direct = server + .query_text("SELECT time_bucket('1 hour', captured_at) AS bucket FROM tsj_bucket_events") + .await + .expect("time_bucket over the time key must succeed"); + assert_eq!(direct.len(), 1, "one stored point: {direct:?}"); + + let joined = server + .query_text( + "SELECT time_bucket('1 hour', tsj_bucket_events.captured_at) AS bucket \ + FROM tsj_bucket_events \ + INNER JOIN tsj_bucket_hosts ON tsj_bucket_events.host = tsj_bucket_hosts.id", + ) + .await + .expect("time_bucket over the time key through a JOIN must succeed"); + assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); + + assert_eq!( + joined[0], direct[0], + "time_bucket of the time key must render the same through a JOIN as through \ + a SELECT: joined={joined:?} direct={direct:?}" + ); +} + +/// `time_bucket` of a time key read directly denotes the bucket of the stored +/// instant. A bucket computed in one unit and rendered in another lands +/// decades away from `EARLY`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_time_bucket_of_a_time_key_denotes_the_stored_instant() { + let server = TestServer::start().await; + setup(&server, "tsj_bucket_abs_events", "tsj_bucket_abs_hosts").await; + + let direct = server + .query_text( + "SELECT time_bucket('1 hour', captured_at) AS bucket FROM tsj_bucket_abs_events", + ) + .await + .expect("time_bucket over the time key must succeed"); + assert_eq!(direct.len(), 1, "one stored point: {direct:?}"); + + assert_eq!( + direct[0], EARLY_BUCKET_ISO, + "time_bucket of the time key must denote {EARLY}: got {direct:?}" + ); +} + +/// `time_bucket` of a time key read through a JOIN denotes the bucket of the +/// stored instant. This anchors the comparison above. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_joined_time_bucket_of_a_time_key_denotes_the_stored_instant() { + let server = TestServer::start().await; + setup(&server, "tsj_bucket_jabs_events", "tsj_bucket_jabs_hosts").await; + + let joined = server + .query_text( + "SELECT time_bucket('1 hour', tsj_bucket_jabs_events.captured_at) AS bucket \ + FROM tsj_bucket_jabs_events \ + INNER JOIN tsj_bucket_jabs_hosts \ + ON tsj_bucket_jabs_events.host = tsj_bucket_jabs_hosts.id", + ) + .await + .expect("time_bucket over the time key through a JOIN must succeed"); + assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); + + assert_eq!( + joined[0], EARLY_BUCKET_ISO, + "a joined time_bucket of the time key must denote {EARLY}: got {joined:?}" + ); +} + +/// `date_part` reads the calendar year of a time key directly. The docs list +/// `EXTRACT(field FROM ts)` (`date_part` is its function form) as a scalar over any timestamp, so a declared +/// timeseries instant must be readable by it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn date_part_year_of_a_time_key_reads_the_stored_year() { + let server = TestServer::start().await; + setup(&server, "tsj_extract_events", "tsj_extract_hosts").await; + + let direct = server + .query_text("SELECT date_part('year', captured_at) AS y FROM tsj_extract_events") + .await + .expect("date_part over the time key must succeed"); + assert_eq!(direct.len(), 1, "one stored point: {direct:?}"); + assert_eq!( + direct[0], "2020", + "the stored instant is in 2020: {direct:?}" + ); +} + +/// `date_part` reads the calendar year of a time key through a JOIN, the same +/// as directly. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn date_part_year_of_a_joined_time_key_reads_the_stored_year() { + let server = TestServer::start().await; + setup(&server, "tsj_extract_j_events", "tsj_extract_j_hosts").await; + + let joined = server + .query_text( + "SELECT date_part('year', tsj_extract_j_events.captured_at) AS y \ + FROM tsj_extract_j_events \ + INNER JOIN tsj_extract_j_hosts \ + ON tsj_extract_j_events.host = tsj_extract_j_hosts.id", + ) + .await + .expect("date_part over the time key through a JOIN must succeed"); + assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); + assert_eq!( + joined[0], "2020", + "the stored instant is in 2020: {joined:?}" + ); +} + +/// A JOIN whose `ON` compares a timeseries time key against a `TIMESTAMP` +/// column of a document collection matches the row that holds the same +/// instant. Both cells name `EARLY`, so the predicate holds only when the two +/// sides reach the comparison in one unit. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_join_on_a_time_key_against_a_document_timestamp_matches_the_same_instant() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION tsj_xe_events \ + (captured_at TIMESTAMP TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')", + ) + .await + .expect("create events"); + server + .exec( + "CREATE COLLECTION tsj_xe_marks (id TEXT PRIMARY KEY, seen_at TIMESTAMP) \ + WITH (engine='document_strict')", + ) + .await + .expect("create marks"); + server + .exec(&format!( + "INSERT INTO tsj_xe_events (captured_at, host, v) VALUES ('{EARLY}', 'h1', 1.5)" + )) + .await + .expect("insert event"); + server + .exec(&format!( + "INSERT INTO tsj_xe_marks (id, seen_at) VALUES ('m1', '{EARLY}')" + )) + .await + .expect("insert mark"); + + let joined = server + .query_text( + "SELECT tsj_xe_marks.id FROM tsj_xe_events \ + INNER JOIN tsj_xe_marks ON tsj_xe_events.captured_at = tsj_xe_marks.seen_at", + ) + .await + .expect("a JOIN comparing two instants must succeed"); + assert_eq!( + joined, + vec!["m1".to_string()], + "the event and the mark hold the same instant, so the join matches: {joined:?}" + ); +} + +/// The feature-store JOIN the docs advertise: a feature row joins an event +/// row when the feature's time key is at or before the event's. Two +/// timeseries collections compare time keys in the `ON` clause, so the +/// predicate holds only when both sides reach it in one unit. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_join_comparing_two_time_keys_matches_when_the_feature_precedes_the_event() { + let server = TestServer::start().await; + for name in ["tsj_tt_events", "tsj_tt_features"] { + server + .exec(&format!( + "CREATE COLLECTION {name} \ + (captured_at TIMESTAMP TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .unwrap_or_else(|e| panic!("create {name}: {e}")); + } + server + .exec(&format!( + "INSERT INTO tsj_tt_events (captured_at, host, v) VALUES ('{EARLY}', 'h1', 1.5)" + )) + .await + .expect("insert event"); + server + .exec( + "INSERT INTO tsj_tt_features (captured_at, host, v) \ + VALUES ('2020-03-05 09:00:00', 'h1', 2.5)", + ) + .await + .expect("insert feature"); + + let joined = server + .query_text( + "SELECT tsj_tt_features.v FROM tsj_tt_events \ + INNER JOIN tsj_tt_features \ + ON tsj_tt_events.host = tsj_tt_features.host \ + AND tsj_tt_features.captured_at <= tsj_tt_events.captured_at", + ) + .await + .expect("a JOIN comparing two time keys must succeed"); + assert_eq!( + joined, + vec!["2.5".to_string()], + "the feature precedes the event, so the join matches it: {joined:?}" + ); +} + +/// `time_bucket` of a time key used as a `GROUP BY` key denotes the bucket of +/// the stored instant. `EARLY` sits on the hour, so its bucket is `EARLY` +/// itself; the aggregate encoder writes the group key, which is a fourth +/// route to the same stored instant alongside a direct read, a join, and a +/// plain `GROUP BY` on the column. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_grouped_time_bucket_denotes_the_stored_instant() { + let server = TestServer::start().await; + setup(&server, "tsj_bucket_grp_events", "tsj_bucket_grp_hosts").await; + + let rows = server + .query_rows( + "SELECT time_bucket('1 hour', captured_at) AS bucket, host, COUNT(*) \ + FROM tsj_bucket_grp_events GROUP BY bucket, host", + ) + .await + .expect("GROUP BY a time_bucket key must succeed"); + assert_eq!( + rows.len(), + 1, + "one stored point falls in one group: {rows:?}" + ); + + assert_eq!( + rows[0][0], EARLY_ISO, + "a grouped time_bucket key must denote {EARLY}: got {rows:?}" + ); +} + +/// A time key grouped through `time_bucket` renders the same as the same +/// column grouped directly. `EARLY` sits on the hour, so the bucket of that +/// instant is that instant, and a divergence between the two group-key +/// encoders is a rendering defect. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_grouped_time_bucket_renders_as_the_time_key_grouped_directly_does() { + let server = TestServer::start().await; + setup(&server, "tsj_bucket_cmp_events", "tsj_bucket_cmp_hosts").await; + + let bucketed = server + .query_text( + "SELECT time_bucket('1 hour', captured_at) AS bucket, host, COUNT(*) \ + FROM tsj_bucket_cmp_events GROUP BY bucket, host", + ) + .await + .expect("GROUP BY a time_bucket key must succeed"); + assert_eq!( + bucketed.len(), + 1, + "one stored point falls in one group: {bucketed:?}" + ); + + let direct = server + .query_text("SELECT captured_at, COUNT(*) FROM tsj_bucket_cmp_events GROUP BY captured_at") + .await + .expect("GROUP BY the time key directly must succeed"); + assert_eq!( + direct.len(), + 1, + "one stored point falls in one group: {direct:?}" + ); + + assert_eq!( + bucketed[0], direct[0], + "a time_bucket GROUP BY key must render the same as the time key grouped \ + directly: bucketed={bucketed:?} direct={direct:?}" ); } diff --git a/nodedb/tests/wire/cases/timeseries_read_row_level_security.rs b/nodedb/tests/wire/cases/timeseries_read_row_level_security.rs new file mode 100644 index 000000000..af9512198 --- /dev/null +++ b/nodedb/tests/wire/cases/timeseries_read_row_level_security.rs @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Row-level security on timeseries reads. +//! +//! A `FOR READ` policy on a timeseries collection governs every row a +//! `SELECT` returns, from the live memtable and from flushed partitions +//! alike. The raw scan applies the policy after time-range pruning and the +//! query's own WHERE, before computed columns, sort, and limit, so a row the +//! policy excludes never reaches the client, never consumes a `LIMIT` slot, +//! and never joins a partner. The aggregate path pushes the policy into the +//! grouped scan, so an excluded row never reaches an accumulator. + +use crate::harness::TestServer; + +const PASSWORD: &str = "ts-read-rls-secret-7"; + +/// Epoch-millisecond time keys, one per seeded row. +const TS_MINE: i64 = 1_700_000_000_000; +const TS_A: i64 = 1_700_000_001_000; +const TS_B: i64 = 1_700_000_002_000; + +/// Create a timeseries `collection` holding one row owned by `user` and two +/// owned by someone else, plus `user` with the readwrite role and a read +/// policy admitting only the caller's own rows. +async fn seed(server: &TestServer, collection: &str, user: &str) { + server + .exec(&format!( + "CREATE COLLECTION {collection} \ + (ts BIGINT TIME_KEY, owner TEXT, value FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .unwrap_or_else(|e| panic!("create {collection}: {e}")); + for (ts, owner, value) in [ + (TS_MINE, user, 1.0), + (TS_A, "someone_else", 2.0), + (TS_B, "someone_else", 3.0), + ] { + server + .exec(&format!( + "INSERT INTO {collection} (ts, owner, value) \ + VALUES ({ts}, '{owner}', {value})" + )) + .await + .unwrap_or_else(|e| panic!("seed {collection} row {ts}: {e}")); + } + server + .exec(&format!("CREATE USER {user} PASSWORD '{PASSWORD}'")) + .await + .unwrap_or_else(|e| panic!("create user {user}: {e}")); + server + .exec(&format!("GRANT ROLE readwrite TO {user}")) + .await + .unwrap_or_else(|e| panic!("grant readwrite to {user}: {e}")); + server + .exec(&format!( + "CREATE RLS POLICY {collection}_owner ON {collection} FOR READ \ + USING (owner = $auth.username)" + )) + .await + .unwrap_or_else(|e| panic!("create read policy on {collection}: {e}")); +} + +/// Run `sql` as `user` and return each row's cells joined by `|`. +async fn rows_as(server: &TestServer, user: &str, sql: &str) -> Vec { + let (client, handle) = server + .connect_as(user, PASSWORD) + .await + .unwrap_or_else(|e| panic!("connect as {user}: {e}")); + let messages = client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("{user} runs {sql}: {e}")); + let mut out = Vec::new(); + for message in messages { + if let tokio_postgres::SimpleQueryMessage::Row(row) = message { + let mut cells = Vec::new(); + for i in 0..row.len() { + cells.push(row.get(i).unwrap_or("").to_string()); + } + out.push(cells.join("|")); + } + } + drop(client); + handle.abort(); + out +} + +/// A `SELECT` over the live memtable returns only the rows the read policy +/// admits for the caller. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_timeseries_select_returns_only_policy_admitted_rows() { + let server = TestServer::start().await; + let user = "ts_rls_reader"; + seed(&server, "ts_rls_read", user).await; + + let rows = rows_as( + &server, + user, + "SELECT ts, owner FROM ts_rls_read ORDER BY ts", + ) + .await; + assert_eq!( + rows, + vec![format!("{TS_MINE}|{user}")], + "the read policy admits one row for this caller: {rows:?}" + ); +} + +/// A `SELECT` over flushed partitions returns only the rows the read policy +/// admits for the caller. The policy applies inside the partition reader, +/// so a flushed row is governed the same as a live one. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_timeseries_select_over_flushed_partitions_returns_only_policy_admitted_rows() { + let server = TestServer::start_with_timeseries_memtable_budget(1).await; + let user = "ts_rls_flushed_reader"; + seed(&server, "ts_rls_read_flushed", user).await; + + let rows = rows_as( + &server, + user, + "SELECT ts, owner FROM ts_rls_read_flushed ORDER BY ts", + ) + .await; + assert_eq!( + rows, + vec![format!("{TS_MINE}|{user}")], + "the read policy admits one row for this caller: {rows:?}" + ); +} + +/// A `LIMIT` counts admitted rows only: with one admitted row and two +/// excluded rows seeded, `LIMIT 1` returns the admitted row, never an +/// excluded row that happened to be scanned first. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_timeseries_select_with_a_limit_counts_only_admitted_rows() { + let server = TestServer::start().await; + let user = "ts_rls_limiter"; + seed(&server, "ts_rls_limit", user).await; + + let rows = rows_as( + &server, + user, + "SELECT ts, owner FROM ts_rls_limit ORDER BY ts LIMIT 1", + ) + .await; + assert_eq!( + rows, + vec![format!("{TS_MINE}|{user}")], + "the limit applies to admitted rows only: {rows:?}" + ); +} + +/// A `COUNT(*)` over a governed timeseries collection leaves the metadata +/// fast path and counts only the rows the read policy admits. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_timeseries_count_counts_only_policy_admitted_rows() { + let server = TestServer::start().await; + let user = "ts_rls_counter"; + seed(&server, "ts_rls_count", user).await; + + let rows = rows_as(&server, user, "SELECT COUNT(*) FROM ts_rls_count").await; + assert_eq!( + rows, + vec!["1".to_string()], + "the read policy admits one row for this caller: {rows:?}" + ); +} + +/// A `time_bucket` aggregate counts only admitted rows: the three seeded +/// rows share one hourly bucket, and the bucket counts one. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_timeseries_time_bucket_aggregate_counts_only_policy_admitted_rows() { + let server = TestServer::start().await; + let user = "ts_rls_bucketer"; + seed(&server, "ts_rls_bucket", user).await; + + let rows = rows_as( + &server, + user, + "SELECT time_bucket('1 hour', ts) AS b, COUNT(*) FROM ts_rls_bucket GROUP BY b", + ) + .await; + assert_eq!(rows.len(), 1, "one bucket holds every seeded row: {rows:?}"); + assert!( + rows[0].ends_with("|1"), + "the bucket counts the one admitted row: {rows:?}" + ); +} + +/// A `GROUP BY` aggregate groups only admitted rows: the excluded owner's +/// group never appears. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_timeseries_group_by_aggregate_groups_only_policy_admitted_rows() { + let server = TestServer::start().await; + let user = "ts_rls_grouper"; + seed(&server, "ts_rls_group", user).await; + + let rows = rows_as( + &server, + user, + "SELECT owner, AVG(value) FROM ts_rls_group GROUP BY owner", + ) + .await; + assert_eq!(rows.len(), 1, "one group is admitted: {rows:?}"); + assert!( + rows[0].starts_with(&format!("{user}|")), + "the admitted group is the caller's own: {rows:?}" + ); +} + +/// A join reads the governed timeseries side on the caller's behalf, so its +/// policy applies to that side before the join: excluded rows neither match +/// a partner nor reach the client. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_join_over_a_governed_timeseries_collection_excludes_policy_filtered_rows() { + let server = TestServer::start().await; + let user = "ts_rls_joiner"; + seed(&server, "ts_rls_join_t", user).await; + server + .exec( + "CREATE COLLECTION ts_rls_join_d (id TEXT PRIMARY KEY, tag TEXT) \ + WITH (engine='document_strict')", + ) + .await + .expect("create document side"); + server + .exec(&format!( + "INSERT INTO ts_rls_join_d (id, tag) VALUES \ + ('{user}', 't_mine'), ('someone_else', 't_theirs')" + )) + .await + .expect("seed document side"); + + let rows = rows_as( + &server, + user, + "SELECT t.ts, d.tag FROM ts_rls_join_t t \ + JOIN ts_rls_join_d d ON t.owner = d.id ORDER BY t.ts", + ) + .await; + assert_eq!( + rows, + vec![format!("{TS_MINE}|t_mine")], + "the join surfaced timeseries rows the read policy excludes: {rows:?}" + ); +} + +/// `2020-03-05T10:00:00Z` as epoch milliseconds: the instant the policies +/// below compare against. +const CUTOFF_MS: i64 = 1_583_402_400_000; +const CUTOFF_TEXT: &str = "2020-03-05 10:00:00"; + +/// Create a timeseries `collection` with a declared `TIMESTAMP` time key +/// holding one row before, one at, and one after the cutoff, plus `user` +/// with the readwrite role. No policy: each test creates its own. +async fn seed_instant(server: &TestServer, collection: &str, user: &str) { + server + .exec(&format!( + "CREATE COLLECTION {collection} \ + (captured_at TIMESTAMP TIME_KEY, label TEXT, value FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .unwrap_or_else(|e| panic!("create {collection}: {e}")); + for (at, label) in [ + ("2020-03-05 09:00:00", "before"), + (CUTOFF_TEXT, "at"), + ("2020-03-05 11:00:00", "after"), + ] { + server + .exec(&format!( + "INSERT INTO {collection} (captured_at, label, value) \ + VALUES ('{at}', '{label}', 1.0)" + )) + .await + .unwrap_or_else(|e| panic!("seed {collection} row {label}: {e}")); + } + server + .exec(&format!("CREATE USER {user} PASSWORD '{PASSWORD}'")) + .await + .unwrap_or_else(|e| panic!("create user {user}: {e}")); + server + .exec(&format!("GRANT ROLE readwrite TO {user}")) + .await + .unwrap_or_else(|e| panic!("grant readwrite to {user}: {e}")); +} + +/// A numeric policy literal compared against a declared `TIMESTAMP` time +/// key is epoch milliseconds, typed at `CREATE RLS POLICY` by the rule a +/// query predicate follows: the governed user sees the rows at and after the +/// cutoff, and the row before it is excluded. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_numeric_policy_literal_on_a_timestamp_time_key_is_an_instant() { + let server = TestServer::start().await; + let user = "ts_rls_ms_reader"; + seed_instant(&server, "ts_rls_ms", user).await; + server + .exec(&format!( + "CREATE RLS POLICY ts_rls_ms_recent ON ts_rls_ms FOR READ \ + USING (captured_at >= {CUTOFF_MS})" + )) + .await + .expect("a numeric literal on a TIMESTAMP column is epoch milliseconds"); + + let rows = rows_as( + &server, + user, + "SELECT label FROM ts_rls_ms ORDER BY captured_at", + ) + .await; + assert_eq!( + rows, + vec!["at".to_string(), "after".to_string()], + "the policy admits the rows at and after the cutoff: {rows:?}" + ); +} + +/// A text policy literal compared against a declared `TIMESTAMP` time key +/// is parsed as ISO-8601 and enforced as the same instant. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_text_policy_literal_on_a_timestamp_time_key_is_an_instant() { + let server = TestServer::start().await; + let user = "ts_rls_text_reader"; + seed_instant(&server, "ts_rls_text", user).await; + server + .exec(&format!( + "CREATE RLS POLICY ts_rls_text_recent ON ts_rls_text FOR READ \ + USING (captured_at >= '{CUTOFF_TEXT}')" + )) + .await + .expect("a text literal on a TIMESTAMP column is parsed as an instant"); + + let rows = rows_as( + &server, + user, + "SELECT label FROM ts_rls_text ORDER BY captured_at", + ) + .await; + assert_eq!( + rows, + vec!["at".to_string(), "after".to_string()], + "the policy admits the rows at and after the cutoff: {rows:?}" + ); +} + +/// A policy literal no instant can be read from is refused at +/// `CREATE RLS POLICY`, naming the column, so an unenforceable policy is +/// never stored. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_non_instant_policy_literal_on_a_timestamp_time_key_is_refused() { + let server = TestServer::start().await; + seed_instant(&server, "ts_rls_bool", "ts_rls_bool_reader").await; + let error = server + .exec( + "CREATE RLS POLICY ts_rls_bool_bad ON ts_rls_bool FOR READ \ + USING (captured_at >= true)", + ) + .await + .expect_err("a boolean is not an instant"); + assert!( + error.contains("captured_at"), + "the refusal must name the column: {error}" + ); +} diff --git a/nodedb/tests/wire/harness/config_toml.rs b/nodedb/tests/wire/harness/config_toml.rs index 20582a762..9a14446c4 100644 --- a/nodedb/tests/wire/harness/config_toml.rs +++ b/nodedb/tests/wire/harness/config_toml.rs @@ -43,6 +43,9 @@ pub(super) struct TuningOverrides { /// Overrides `[tuning.query] stream_chunk_size` so a test can drive the /// chunked-streaming scan path without seeding 1000 rows. pub(super) stream_chunk_size: Option, + /// Overrides `[tuning.timeseries] memtable_budget_bytes` so a test can + /// observe timeseries partition flushes on a handful of rows. + pub(super) timeseries_memtable_budget_bytes: Option, } impl TuningOverrides { @@ -74,6 +77,14 @@ impl TuningOverrides { ..Self::default() } } + + /// Boot with a lowered timeseries memtable budget. + pub(super) fn timeseries_memtable_budget(bytes: usize) -> Self { + Self { + timeseries_memtable_budget_bytes: Some(bytes), + ..Self::default() + } + } } /// Write `nodedb.toml` into `dir` and return its path. @@ -111,6 +122,11 @@ pub(super) fn write_config(dir: &Path, auth_mode: AuthMode, tuning: TuningOverri if let Some(rows) = tuning.stream_chunk_size { toml.push_str(&format!("\n[tuning.query]\nstream_chunk_size = {rows}\n")); } + if let Some(bytes) = tuning.timeseries_memtable_budget_bytes { + toml.push_str(&format!( + "\n[tuning.timeseries]\nmemtable_budget_bytes = {bytes}\n" + )); + } toml.push_str(&format!( "\n[backup_encryption]\nkey_path = {}\n", toml_quote(&write_backup_kek(dir)) diff --git a/nodedb/tests/wire/harness/lifecycle.rs b/nodedb/tests/wire/harness/lifecycle.rs index 9c13179a7..683f1c54f 100644 --- a/nodedb/tests/wire/harness/lifecycle.rs +++ b/nodedb/tests/wire/harness/lifecycle.rs @@ -101,6 +101,20 @@ impl TestServer { Self::connect_and_build(spawned, dir, AuthMode::Trust).await } + /// Spawn a single-core NodeDB server with a lowered timeseries memtable + /// budget so every ingest flushes its rows to a partition, and a read + /// exercises the partition path on a handful of rows. + pub async fn start_with_timeseries_memtable_budget(bytes: usize) -> Self { + let dir = tempfile::tempdir().expect("tempdir"); + let spawned = process::spawn( + dir.path(), + AuthMode::Trust, + TuningOverrides::timeseries_memtable_budget(bytes), + 1, + ); + Self::connect_and_build(spawned, dir, AuthMode::Trust).await + } + /// Open a server backed by an existing data directory, reopened in place /// so a previous server's data is visible after boot. `dir` is not /// consumed — ownership stays with the caller.