From b187251a359a515857b5381b6cac44036b5578d8 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 16 Sep 2026 11:48:17 +0800 Subject: [PATCH 01/21] feat(msgpack): add a typed instant ext encoding for DateTime/NaiveDateTime Value::DateTime and Value::NaiveDateTime now serialize to a dedicated fixext8 ext type (UTC vs naive) carrying epoch microseconds, instead of a formatted string. This gives byte-decodable comparison and lets the JSON transcoder render the value as ISO 8601 without a full msgpack-to-Value decode. - nodedb-types::json_msgpack::instant_ext defines the encode/decode and InstantKind, re-exported as write_instant/read_instant/InstantKind from nodedb-types; Value gains as_instant() to extract either variant. - The transcoder recognizes the instant ext and writes an ISO 8601 string; any other ext type still renders as null. - nodedb-query's msgpack_scan writer/compare gain matching write_instant/write_kv_instant helpers and an instant type rank so raw-byte field comparison orders instants correctly (by kind, then signed micros) instead of falling into the generic ext bucket. - Split the single-file json_msgpack/reader.rs and msgpack_scan/reader.rs into per-concern submodules (cursor/json/native and tags/scalar/skip/value respectively); no behavior change beyond the module boundary. --- nodedb-query/src/msgpack_scan/compare.rs | 153 ++- nodedb-query/src/msgpack_scan/mod.rs | 5 +- nodedb-query/src/msgpack_scan/reader.rs | 1145 ----------------- nodedb-query/src/msgpack_scan/reader/mod.rs | 19 + .../src/msgpack_scan/reader/scalar.rs | 470 +++++++ nodedb-query/src/msgpack_scan/reader/skip.rs | 376 ++++++ nodedb-query/src/msgpack_scan/reader/tags.rs | 70 + nodedb-query/src/msgpack_scan/reader/value.rs | 299 +++++ nodedb-query/src/msgpack_scan/writer.rs | 31 + nodedb-types/src/json_msgpack/instant_ext.rs | 181 +++ nodedb-types/src/json_msgpack/mod.rs | 5 + nodedb-types/src/json_msgpack/reader.rs | 682 ---------- .../src/json_msgpack/reader/cursor.rs | 132 ++ nodedb-types/src/json_msgpack/reader/json.rs | 355 +++++ nodedb-types/src/json_msgpack/reader/mod.rs | 15 + .../src/json_msgpack/reader/native.rs | 295 +++++ nodedb-types/src/json_msgpack/transcoder.rs | 36 +- nodedb-types/src/json_msgpack/writer.rs | 11 +- nodedb-types/src/lib.rs | 5 +- nodedb-types/src/value/core.rs | 10 + 20 files changed, 2433 insertions(+), 1862 deletions(-) delete mode 100644 nodedb-query/src/msgpack_scan/reader.rs create mode 100644 nodedb-query/src/msgpack_scan/reader/mod.rs create mode 100644 nodedb-query/src/msgpack_scan/reader/scalar.rs create mode 100644 nodedb-query/src/msgpack_scan/reader/skip.rs create mode 100644 nodedb-query/src/msgpack_scan/reader/tags.rs create mode 100644 nodedb-query/src/msgpack_scan/reader/value.rs create mode 100644 nodedb-types/src/json_msgpack/instant_ext.rs delete mode 100644 nodedb-types/src/json_msgpack/reader.rs create mode 100644 nodedb-types/src/json_msgpack/reader/cursor.rs create mode 100644 nodedb-types/src/json_msgpack/reader/json.rs create mode 100644 nodedb-types/src/json_msgpack/reader/mod.rs create mode 100644 nodedb-types/src/json_msgpack/reader/native.rs diff --git a/nodedb-query/src/msgpack_scan/compare.rs b/nodedb-query/src/msgpack_scan/compare.rs index dc0769592..b48122876 100644 --- a/nodedb-query/src/msgpack_scan/compare.rs +++ b/nodedb-query/src/msgpack_scan/compare.rs @@ -8,6 +8,8 @@ use std::cmp::Ordering; use std::hash::{BuildHasher, Hasher}; +use nodedb_types::read_instant; + use crate::msgpack_scan::reader::{read_f64, read_i64, read_null, str_bounds}; /// Hash the raw bytes of a MessagePack value at `range` within `buf`. @@ -46,11 +48,12 @@ pub fn hash_field_bytes_with( /// Compare two MessagePack values by their decoded content. /// /// Comparison order: -/// 1. Null < Bool < Number < String < Binary < Array < Map +/// 1. Null < Bool < Number < Instant < String < Binary < Array < Map < Ext /// 2. Within numbers: compare as f64 -/// 3. Within strings: lexicographic on raw bytes (valid UTF-8 guarantees +/// 3. Within instants: by kind (UTC before naive), then signed epoch micros +/// 4. Within strings: lexicographic on raw bytes (valid UTF-8 guarantees /// byte order = Unicode code-point order for ASCII/Latin-1) -/// 4. Fallback: raw byte comparison +/// 5. Fallback: raw byte comparison pub fn compare_field_bytes( a_buf: &[u8], a_range: (usize, usize), @@ -69,23 +72,22 @@ pub fn compare_field_bytes( None => return Ordering::Greater, }; - let a_type = type_rank(a_tag); - let b_type = type_rank(b_tag); + let a_type = type_rank(a_buf, a_off, a_tag); + let b_type = type_rank(b_buf, b_off, b_tag); if a_type != b_type { return a_type.cmp(&b_type); } match a_type { - 0 => Ordering::Equal, // both null - 1 => { - // bool + RANK_NULL => Ordering::Equal, + RANK_BOOL => { let a_val = a_tag == 0xc3; // true let b_val = b_tag == 0xc3; a_val.cmp(&b_val) } - 2 => { - // number — compare as f64 + RANK_NUMBER => { + // compare as f64 match (read_f64(a_buf, a_off), read_f64(b_buf, b_off)) { (Some(a), Some(b)) => a.partial_cmp(&b).unwrap_or(Ordering::Equal), (Some(_), None) => Ordering::Greater, @@ -93,7 +95,20 @@ pub fn compare_field_bytes( (None, None) => Ordering::Equal, } } - 3 => { + RANK_INSTANT => { + // Both sides decoded as instants by `type_rank`. Kind first, then + // signed micros: the raw payload is not byte-comparable below 0. + match (read_instant(a_buf, a_off), read_instant(b_buf, b_off)) { + (Some((a_kind, a_us)), Some((b_kind, b_us))) => a_kind + .ext_type() + .cmp(&b_kind.ext_type()) + .then(a_us.cmp(&b_us)), + (Some(_), None) => Ordering::Greater, + (None, Some(_)) => Ordering::Less, + (None, None) => Ordering::Equal, + } + } + RANK_STRING => { // string — compare raw bytes match (str_bounds(a_buf, a_off), str_bounds(b_buf, b_off)) { (Some((a_s, a_l)), Some((b_s, b_l))) => { @@ -148,20 +163,41 @@ pub fn is_field_null(buf: &[u8], range: (usize, usize)) -> bool { read_null(buf, range.0) } +const RANK_NULL: u8 = 0; +const RANK_BOOL: u8 = 1; +const RANK_NUMBER: u8 = 2; +const RANK_INSTANT: u8 = 3; +const RANK_STRING: u8 = 4; +const RANK_BINARY: u8 = 5; +const RANK_ARRAY: u8 = 6; +const RANK_MAP: u8 = 7; +const RANK_EXT: u8 = 8; +const RANK_UNKNOWN: u8 = 9; + /// Type rank for cross-type ordering. Lower rank = sorts first. -/// Null(0) < Bool(1) < Number(2) < String(3) < Binary(4) < Array(5) < Map(6) < Ext(7) -fn type_rank(tag: u8) -> u8 { +/// Null < Bool < Number < Instant < String < Binary < Array < Map < Ext +/// +/// An instant is a complete `fixext8` of type 1 / 2 at `off`. A `fixext8` +/// of any other type, or a truncated one, ranks as ext. +fn type_rank(buf: &[u8], off: usize, tag: u8) -> u8 { match tag { - 0xc0 => 0, // nil - 0xc2 | 0xc3 => 1, // bool - 0x00..=0x7f | 0xe0..=0xff => 2, // fixint - 0xca..=0xd3 => 2, // float/uint/int - 0xa0..=0xbf | 0xd9..=0xdb => 3, // string - 0xc4..=0xc6 => 4, // binary - 0x90..=0x9f | 0xdc | 0xdd => 5, // array - 0x80..=0x8f | 0xde | 0xdf => 6, // map - 0xc7..=0xc9 | 0xd4..=0xd8 => 7, // ext - _ => 8, // unknown + 0xc0 => RANK_NULL, + 0xc2 | 0xc3 => RANK_BOOL, + 0x00..=0x7f | 0xe0..=0xff => RANK_NUMBER, // fixint + 0xca..=0xd3 => RANK_NUMBER, // float/uint/int + 0xa0..=0xbf | 0xd9..=0xdb => RANK_STRING, + 0xc4..=0xc6 => RANK_BINARY, + 0x90..=0x9f | 0xdc | 0xdd => RANK_ARRAY, + 0x80..=0x8f | 0xde | 0xdf => RANK_MAP, + 0xd7 => { + if read_instant(buf, off).is_some() { + RANK_INSTANT + } else { + RANK_EXT + } + } + 0xc7..=0xc9 | 0xd4..=0xd6 | 0xd8 => RANK_EXT, + _ => RANK_UNKNOWN, } } @@ -292,6 +328,77 @@ mod tests { ); } + #[test] + fn compare_instants_negative_micros() { + use nodedb_types::{InstantKind, write_instant}; + let mut a = Vec::new(); + write_instant(&mut a, InstantKind::Utc, -10); + let mut b = Vec::new(); + write_instant(&mut b, InstantKind::Utc, -5); + let mut c = Vec::new(); + write_instant(&mut c, InstantKind::Utc, 1); + assert_eq!( + compare_field_bytes(&a, val_range(&a), &b, val_range(&b)), + Ordering::Less + ); + assert_eq!( + compare_field_bytes(&a, val_range(&a), &c, val_range(&c)), + Ordering::Less + ); + assert_eq!( + compare_field_bytes(&c, val_range(&c), &b, val_range(&b)), + Ordering::Greater + ); + assert_eq!( + compare_field_bytes(&a, val_range(&a), &a, val_range(&a)), + Ordering::Equal + ); + } + + #[test] + fn compare_instant_kinds_order_utc_first() { + use nodedb_types::{InstantKind, write_instant}; + let mut utc = Vec::new(); + write_instant(&mut utc, InstantKind::Utc, 100); + let mut naive = Vec::new(); + write_instant(&mut naive, InstantKind::Naive, 1); + assert_eq!( + compare_field_bytes(&utc, val_range(&utc), &naive, val_range(&naive)), + Ordering::Less + ); + } + + #[test] + fn compare_instant_vs_integer_uses_rank() { + use nodedb_types::{InstantKind, write_instant}; + // A negative instant has a payload starting 0xff, above any fixint + // byte. Rank places it after every number and before every string. + let mut inst = Vec::new(); + write_instant(&mut inst, InstantKind::Naive, -1); + let int = encode(&json!(i64::MAX)); + let s = encode(&json!("")); + assert_eq!( + compare_field_bytes(&int, val_range(&int), &inst, val_range(&inst)), + Ordering::Less + ); + assert_eq!( + compare_field_bytes(&inst, val_range(&inst), &s, val_range(&s)), + Ordering::Less + ); + } + + #[test] + fn unknown_fixext8_ranks_as_ext() { + use nodedb_types::{InstantKind, write_instant}; + let mut inst = Vec::new(); + write_instant(&mut inst, InstantKind::Utc, 0); + let other = [0xd7, 0x09, 0, 0, 0, 0, 0, 0, 0, 0]; + assert_eq!( + compare_field_bytes(&inst, val_range(&inst), &other, val_range(&other)), + Ordering::Less + ); + } + #[test] fn hash_from_extracted_field() { let buf = encode(&json!({"id": 42})); diff --git a/nodedb-query/src/msgpack_scan/mod.rs b/nodedb-query/src/msgpack_scan/mod.rs index e306b6b44..62c227ee0 100644 --- a/nodedb-query/src/msgpack_scan/mod.rs +++ b/nodedb-query/src/msgpack_scan/mod.rs @@ -34,6 +34,7 @@ pub use sidecar::{ }; pub use writer::{ build_str_map, inject_str_field, merge_fields, write_array_header, write_bin, write_bool, - write_f64, write_i64, write_kv_bool, write_kv_f64, write_kv_i64, write_kv_null, write_kv_raw, - write_kv_str, write_map_header, write_null, write_str, + write_f64, write_i64, write_instant, write_kv_bool, write_kv_f64, write_kv_i64, + write_kv_instant, write_kv_null, write_kv_raw, write_kv_str, write_map_header, write_null, + write_str, }; diff --git a/nodedb-query/src/msgpack_scan/reader.rs b/nodedb-query/src/msgpack_scan/reader.rs deleted file mode 100644 index afaeaca5f..000000000 --- a/nodedb-query/src/msgpack_scan/reader.rs +++ /dev/null @@ -1,1145 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Low-level MessagePack binary reader: tag parsing, value skipping, and typed reads. -//! -//! All functions operate on `&[u8]` with explicit offsets. Zero allocation, -//! zero copy. Returns `None` on truncated/invalid data — never panics. - -use std::str; - -// ── Tag constants ────────────────────────────────────────────────────── - -const NIL: u8 = 0xc0; -const FALSE: u8 = 0xc2; -const TRUE: u8 = 0xc3; -const BIN8: u8 = 0xc4; -const BIN16: u8 = 0xc5; -const BIN32: u8 = 0xc6; -const EXT8: u8 = 0xc7; -const EXT16: u8 = 0xc8; -const EXT32: u8 = 0xc9; -const FLOAT32: u8 = 0xca; -const FLOAT64: u8 = 0xcb; -const UINT8: u8 = 0xcc; -const UINT16: u8 = 0xcd; -const UINT32: u8 = 0xce; -const UINT64: u8 = 0xcf; -const INT8: u8 = 0xd0; -const INT16: u8 = 0xd1; -const INT32: u8 = 0xd2; -const INT64: u8 = 0xd3; -const FIXEXT1: u8 = 0xd4; -const FIXEXT2: u8 = 0xd5; -const FIXEXT4: u8 = 0xd6; -const FIXEXT8: u8 = 0xd7; -const FIXEXT16: u8 = 0xd8; -const STR8: u8 = 0xd9; -const STR16: u8 = 0xda; -const STR32: u8 = 0xdb; -const ARRAY16: u8 = 0xdc; -const ARRAY32: u8 = 0xdd; -const MAP16: u8 = 0xde; -const MAP32: u8 = 0xdf; - -/// Maximum nesting depth to prevent stack overflow on malicious payloads. -const MAX_DEPTH: u16 = 128; - -// ── Inline helpers ───────────────────────────────────────────────────── - -#[inline(always)] -fn get(buf: &[u8], pos: usize) -> Option { - buf.get(pos).copied() -} - -#[inline(always)] -fn read_u16_be(buf: &[u8], pos: usize) -> Option { - let bytes = buf.get(pos..pos + 2)?; - Some(u16::from_be_bytes([bytes[0], bytes[1]])) -} - -#[inline(always)] -fn read_u32_be(buf: &[u8], pos: usize) -> Option { - let bytes = buf.get(pos..pos + 4)?; - Some(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) -} - -#[inline(always)] -fn read_u64_be(buf: &[u8], pos: usize) -> Option { - let bytes = buf.get(pos..pos + 8)?; - Some(u64::from_be_bytes([ - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], - ])) -} - -/// Return `Some(offset + size)` only if the buffer has enough bytes. -#[inline(always)] -fn checked_advance(buf: &[u8], offset: usize, size: usize) -> Option { - let end = offset + size; - if end <= buf.len() { Some(end) } else { None } -} - -// ── skip_value ───────────────────────────────────────────────────────── - -/// Advance past the MessagePack value starting at `offset`, returning the -/// offset of the next value. Returns `None` if the buffer is truncated or -/// nesting exceeds `MAX_DEPTH`. -/// -/// This is the performance-critical primitive. It never allocates. -pub fn skip_value(buf: &[u8], offset: usize) -> Option { - skip_value_depth(buf, offset, 0) -} - -fn skip_value_depth(buf: &[u8], offset: usize, depth: u16) -> Option { - if depth > MAX_DEPTH { - return None; - } - let tag = get(buf, offset)?; - match tag { - // positive fixint (0x00..=0x7f) - 0x00..=0x7f => Some(offset + 1), - // negative fixint (0xe0..=0xff) - 0xe0..=0xff => Some(offset + 1), - // nil, false, true - NIL | FALSE | TRUE => Some(offset + 1), - - // fixmap (0x80..=0x8f) - 0x80..=0x8f => { - let count = (tag & 0x0f) as usize; - skip_n_pairs(buf, offset + 1, count, depth) - } - MAP16 => { - let count = read_u16_be(buf, offset + 1)? as usize; - skip_n_pairs(buf, offset + 3, count, depth) - } - MAP32 => { - let count = read_u32_be(buf, offset + 1)? as usize; - skip_n_pairs(buf, offset + 5, count, depth) - } - - // fixarray (0x90..=0x9f) - 0x90..=0x9f => { - let count = (tag & 0x0f) as usize; - skip_n_values(buf, offset + 1, count, depth) - } - ARRAY16 => { - let count = read_u16_be(buf, offset + 1)? as usize; - skip_n_values(buf, offset + 3, count, depth) - } - ARRAY32 => { - let count = read_u32_be(buf, offset + 1)? as usize; - skip_n_values(buf, offset + 5, count, depth) - } - - // fixstr (0xa0..=0xbf) - 0xa0..=0xbf => { - let len = (tag & 0x1f) as usize; - checked_advance(buf, offset, 1 + len) - } - STR8 => { - let len = get(buf, offset + 1)? as usize; - checked_advance(buf, offset, 2 + len) - } - STR16 => { - let len = read_u16_be(buf, offset + 1)? as usize; - checked_advance(buf, offset, 3 + len) - } - STR32 => { - let len = read_u32_be(buf, offset + 1)? as usize; - checked_advance(buf, offset, 5 + len) - } - - // bin - BIN8 => { - let len = get(buf, offset + 1)? as usize; - checked_advance(buf, offset, 2 + len) - } - BIN16 => { - let len = read_u16_be(buf, offset + 1)? as usize; - checked_advance(buf, offset, 3 + len) - } - BIN32 => { - let len = read_u32_be(buf, offset + 1)? as usize; - checked_advance(buf, offset, 5 + len) - } - - // fixed-width numerics (bounds-check against buffer length) - FLOAT32 => checked_advance(buf, offset, 5), - FLOAT64 => checked_advance(buf, offset, 9), - UINT8 | INT8 => checked_advance(buf, offset, 2), - UINT16 | INT16 => checked_advance(buf, offset, 3), - UINT32 | INT32 => checked_advance(buf, offset, 5), - UINT64 | INT64 => checked_advance(buf, offset, 9), - - // ext - FIXEXT1 => checked_advance(buf, offset, 3), - FIXEXT2 => checked_advance(buf, offset, 4), - FIXEXT4 => checked_advance(buf, offset, 6), - FIXEXT8 => checked_advance(buf, offset, 10), - FIXEXT16 => checked_advance(buf, offset, 18), - EXT8 => { - let len = get(buf, offset + 1)? as usize; - checked_advance(buf, offset, 3 + len) - } - EXT16 => { - let len = read_u16_be(buf, offset + 1)? as usize; - checked_advance(buf, offset, 4 + len) - } - EXT32 => { - let len = read_u32_be(buf, offset + 1)? as usize; - checked_advance(buf, offset, 6 + len) - } - - // 0xc1 is never used in the spec - _ => None, - } -} - -fn skip_n_values(buf: &[u8], mut pos: usize, count: usize, depth: u16) -> Option { - for _ in 0..count { - pos = skip_value_depth(buf, pos, depth + 1)?; - } - Some(pos) -} - -fn skip_n_pairs(buf: &[u8], mut pos: usize, count: usize, depth: u16) -> Option { - for _ in 0..count { - pos = skip_value_depth(buf, pos, depth + 1)?; // key - pos = skip_value_depth(buf, pos, depth + 1)?; // value - } - Some(pos) -} - -// ── Typed reads ──────────────────────────────────────────────────────── - -/// Read an f64 from the value at `offset`. Handles float32, float64, -/// and all integer types (coerced to f64). -pub fn read_f64(buf: &[u8], offset: usize) -> Option { - let tag = get(buf, offset)?; - match tag { - // positive fixint - 0x00..=0x7f => Some(tag as f64), - // negative fixint - 0xe0..=0xff => Some((tag as i8) as f64), - FLOAT64 => { - let bits = read_u64_be(buf, offset + 1)?; - Some(f64::from_bits(bits)) - } - FLOAT32 => { - let bits = read_u32_be(buf, offset + 1)?; - Some(f32::from_bits(bits) as f64) - } - UINT8 => Some(get(buf, offset + 1)? as f64), - UINT16 => Some(read_u16_be(buf, offset + 1)? as f64), - UINT32 => Some(read_u32_be(buf, offset + 1)? as f64), - UINT64 => Some(read_u64_be(buf, offset + 1)? as f64), - INT8 => Some(get(buf, offset + 1)? as i8 as f64), - INT16 => Some(read_u16_be(buf, offset + 1)? as i16 as f64), - INT32 => Some(read_u32_be(buf, offset + 1)? as i32 as f64), - INT64 => Some(read_u64_be(buf, offset + 1)? as i64 as f64), - _ => None, - } -} - -/// Read an i64 from the value at `offset`. Handles all integer types. -/// Floats return `None` — use `read_f64` for those. -pub fn read_i64(buf: &[u8], offset: usize) -> Option { - let tag = get(buf, offset)?; - match tag { - 0x00..=0x7f => Some(tag as i64), - 0xe0..=0xff => Some((tag as i8) as i64), - UINT8 => Some(get(buf, offset + 1)? as i64), - UINT16 => Some(read_u16_be(buf, offset + 1)? as i64), - UINT32 => Some(read_u32_be(buf, offset + 1)? as i64), - UINT64 => { - let v = read_u64_be(buf, offset + 1)?; - Some(v as i64) - } - INT8 => Some(get(buf, offset + 1)? as i8 as i64), - INT16 => Some(read_u16_be(buf, offset + 1)? as i16 as i64), - INT32 => Some(read_u32_be(buf, offset + 1)? as i32 as i64), - INT64 => { - let v = read_u64_be(buf, offset + 1)?; - Some(v as i64) - } - _ => None, - } -} - -/// Read a string slice from the value at `offset`. Zero-copy — borrows -/// directly from the input buffer. Returns `None` for non-string types -/// or invalid UTF-8. -pub fn read_str(buf: &[u8], offset: usize) -> Option<&str> { - let (start, len) = str_bounds(buf, offset)?; - let bytes = buf.get(start..start + len)?; - str::from_utf8(bytes).ok() -} - -/// Read a string slice at `*off`, advancing `*off` past it. Zero-copy. -/// Returns `None` for non-string types, invalid UTF-8, or truncated input. -pub fn read_str_advance<'a>(buf: &'a [u8], off: &mut usize) -> Option<&'a str> { - let (start, len) = str_bounds(buf, *off)?; - let bytes = buf.get(start..start + len)?; - let s = str::from_utf8(bytes).ok()?; - *off = start + len; - Some(s) -} - -/// Read a `bin` value at `*off`, advancing `*off` past it. Zero-copy — -/// the returned slice borrows from `buf`. Returns `None` for non-bin tags -/// or truncated input. -pub fn read_bin_advance<'a>(buf: &'a [u8], off: &mut usize) -> Option<&'a [u8]> { - let tag = get(buf, *off)?; - let (len, header) = match tag { - BIN8 => (get(buf, *off + 1)? as usize, 2), - BIN16 => (read_u16_be(buf, *off + 1)? as usize, 3), - BIN32 => (read_u32_be(buf, *off + 1)? as usize, 5), - _ => return None, - }; - let start = *off + header; - let end = start + len; - let data = buf.get(start..end)?; - *off = end; - Some(data) -} - -/// Read an unsigned integer that fits in a `u32` at `*off`, advancing `*off` -/// past it. Accepts positive fixint, uint8, uint16, uint32. Returns `None` -/// for negative, signed-typed, oversized (uint64), or non-integer values. -pub fn read_u32_advance(buf: &[u8], off: &mut usize) -> Option { - let tag = get(buf, *off)?; - match tag { - 0x00..=0x7f => { - *off += 1; - Some(tag as u32) - } - UINT8 => { - let v = get(buf, *off + 1)? as u32; - *off += 2; - Some(v) - } - UINT16 => { - let v = read_u16_be(buf, *off + 1)? as u32; - *off += 3; - Some(v) - } - UINT32 => { - let v = read_u32_be(buf, *off + 1)?; - *off += 5; - Some(v) - } - _ => None, - } -} - -/// Return `(data_start, byte_len)` for the string at `offset` without -/// validating UTF-8. Used internally for key comparison. -pub(crate) fn str_bounds(buf: &[u8], offset: usize) -> Option<(usize, usize)> { - let tag = get(buf, offset)?; - match tag { - 0xa0..=0xbf => { - let len = (tag & 0x1f) as usize; - Some((offset + 1, len)) - } - STR8 => { - let len = get(buf, offset + 1)? as usize; - Some((offset + 2, len)) - } - STR16 => { - let len = read_u16_be(buf, offset + 1)? as usize; - Some((offset + 3, len)) - } - STR32 => { - let len = read_u32_be(buf, offset + 1)? as usize; - Some((offset + 5, len)) - } - _ => None, - } -} - -/// Read a boolean from the value at `offset`. -pub fn read_bool(buf: &[u8], offset: usize) -> Option { - match get(buf, offset)? { - TRUE => Some(true), - FALSE => Some(false), - _ => None, - } -} - -/// Check if the value at `offset` is nil. -pub fn read_null(buf: &[u8], offset: usize) -> bool { - get(buf, offset) == Some(NIL) -} - -/// Read a scalar msgpack value at `offset` into `nodedb_types::Value`. -/// -/// Handles null, bool, integers, floats, and strings. For complex types -/// (array, map, bin, ext), returns `None` — caller should use -/// `json_from_msgpack` for those. -pub fn read_value(buf: &[u8], offset: usize) -> Option { - let tag = get(buf, offset)?; - match tag { - NIL => Some(nodedb_types::Value::Null), - TRUE => Some(nodedb_types::Value::Bool(true)), - FALSE => Some(nodedb_types::Value::Bool(false)), - // Integers - 0x00..=0x7f => Some(nodedb_types::Value::Integer(tag as i64)), - 0xe0..=0xff => Some(nodedb_types::Value::Integer((tag as i8) as i64)), - UINT8 => Some(nodedb_types::Value::Integer(get(buf, offset + 1)? as i64)), - UINT16 => Some(nodedb_types::Value::Integer( - read_u16_be(buf, offset + 1)? as i64 - )), - UINT32 => Some(nodedb_types::Value::Integer( - read_u32_be(buf, offset + 1)? as i64 - )), - UINT64 => Some(nodedb_types::Value::Integer( - read_u64_be(buf, offset + 1)? as i64 - )), - INT8 => Some(nodedb_types::Value::Integer( - get(buf, offset + 1)? as i8 as i64 - )), - INT16 => Some(nodedb_types::Value::Integer( - read_u16_be(buf, offset + 1)? as i16 as i64, - )), - INT32 => Some(nodedb_types::Value::Integer( - read_u32_be(buf, offset + 1)? as i32 as i64, - )), - INT64 => Some(nodedb_types::Value::Integer( - read_u64_be(buf, offset + 1)? as i64 - )), - // Floats - FLOAT32 => { - let bits = read_u32_be(buf, offset + 1)?; - Some(nodedb_types::Value::Float(f32::from_bits(bits) as f64)) - } - FLOAT64 => { - let bits = read_u64_be(buf, offset + 1)?; - Some(nodedb_types::Value::Float(f64::from_bits(bits))) - } - // Strings - 0xa0..=0xbf | STR8 | STR16 | STR32 => { - read_str(buf, offset).map(|s| nodedb_types::Value::String(s.to_string())) - } - _ => None, - } -} - -/// Return the number of key-value pairs and the offset of the first pair, -/// for the map starting at `offset`. Returns `None` if not a map. -pub fn map_header(buf: &[u8], offset: usize) -> Option<(usize, usize)> { - let tag = get(buf, offset)?; - match tag { - 0x80..=0x8f => Some(((tag & 0x0f) as usize, offset + 1)), - MAP16 => Some((read_u16_be(buf, offset + 1)? as usize, offset + 3)), - MAP32 => Some((read_u32_be(buf, offset + 1)? as usize, offset + 5)), - _ => None, - } -} - -/// Return the number of elements and the offset of the first element, -/// for the array starting at `offset`. Returns `None` if not an array. -pub fn array_header(buf: &[u8], offset: usize) -> Option<(usize, usize)> { - let tag = get(buf, offset)?; - match tag { - 0x90..=0x9f => Some(((tag & 0x0f) as usize, offset + 1)), - ARRAY16 => Some((read_u16_be(buf, offset + 1)? as usize, offset + 3)), - ARRAY32 => Some((read_u32_be(buf, offset + 1)? as usize, offset + 5)), - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - use serde_json::json; - - /// Helper: encode a serde_json::Value to MessagePack bytes. - fn encode(v: &serde_json::Value) -> Vec { - nodedb_types::json_msgpack::json_to_msgpack(v).expect("encode") - } - - #[test] - fn skip_positive_fixint() { - let buf = [0x05, 0xff]; - assert_eq!(skip_value(&buf, 0), Some(1)); - } - - #[test] - fn skip_negative_fixint() { - let buf = [0xe0, 0x00]; - assert_eq!(skip_value(&buf, 0), Some(1)); - } - - #[test] - fn skip_nil_bool() { - assert_eq!(skip_value(&[NIL], 0), Some(1)); - assert_eq!(skip_value(&[TRUE], 0), Some(1)); - assert_eq!(skip_value(&[FALSE], 0), Some(1)); - } - - #[test] - fn skip_float64() { - let buf = encode(&json!(9.81)); - assert_eq!(skip_value(&buf, 0), Some(buf.len())); - } - - #[test] - fn skip_string() { - let buf = encode(&json!("hello")); - assert_eq!(skip_value(&buf, 0), Some(buf.len())); - } - - #[test] - fn skip_map() { - let buf = encode(&json!({"a": 1, "b": 2})); - assert_eq!(skip_value(&buf, 0), Some(buf.len())); - } - - #[test] - fn skip_nested_array() { - let buf = encode(&json!([[1, 2], [3, 4, 5]])); - assert_eq!(skip_value(&buf, 0), Some(buf.len())); - } - - #[test] - fn skip_truncated_returns_none() { - let buf = [FLOAT64, 0x40]; // truncated float64 - assert_eq!(skip_value(&buf, 0), None); - } - - #[test] - fn read_f64_fixint() { - assert_eq!(read_f64(&[42u8], 0), Some(42.0)); - } - - #[test] - fn read_f64_negative_fixint() { - assert_eq!(read_f64(&[0xffu8], 0), Some(-1.0)); - } - - #[test] - fn read_f64_float64() { - let buf = encode(&json!(std::f64::consts::PI)); - assert_eq!(read_f64(&buf, 0), Some(std::f64::consts::PI)); - } - - #[test] - fn read_f64_uint16() { - let buf = encode(&json!(1000)); - assert_eq!(read_f64(&buf, 0), Some(1000.0)); - } - - #[test] - fn read_i64_values() { - assert_eq!(read_i64(&[42u8], 0), Some(42)); - assert_eq!(read_i64(&[0xffu8], 0), Some(-1)); - - let buf = encode(&json!(300)); - assert_eq!(read_i64(&buf, 0), Some(300)); - - let buf = encode(&json!(-500)); - assert_eq!(read_i64(&buf, 0), Some(-500)); - } - - #[test] - fn read_str_fixstr() { - let buf = encode(&json!("hi")); - assert_eq!(read_str(&buf, 0), Some("hi")); - } - - #[test] - fn read_str_str8() { - let long = "a".repeat(40); - let buf = encode(&json!(long)); - assert_eq!(read_str(&buf, 0), Some(long.as_str())); - } - - #[test] - fn read_bool_values() { - assert_eq!(read_bool(&[TRUE], 0), Some(true)); - assert_eq!(read_bool(&[FALSE], 0), Some(false)); - assert_eq!(read_bool(&[NIL], 0), None); - } - - #[test] - fn read_null_check() { - assert!(read_null(&[NIL], 0)); - assert!(!read_null(&[TRUE], 0)); - } - - #[test] - fn map_header_fixmap() { - let buf = encode(&json!({"x": 1})); - let (count, _data_offset) = map_header(&buf, 0).unwrap(); - assert_eq!(count, 1); - } - - #[test] - fn skip_bin() { - // bin8: 0xc4, len=3, 3 bytes of data - let buf = [BIN8, 3, 0xde, 0xad, 0xbe, 0xff]; - assert_eq!(skip_value(&buf, 0), Some(5)); - } - - #[test] - fn skip_ext() { - // fixext1: 0xd4, type byte, 1 data byte - let buf = [FIXEXT1, 0x01, 0xab, 0xff]; - assert_eq!(skip_value(&buf, 0), Some(3)); - } - - #[test] - fn read_f64_float32() { - // json! always produces f64, so test float32 with raw bytes - // float32 tag (0xca) + 1.5 in IEEE 754 big-endian - let buf = [0xca, 0x3f, 0xc0, 0x00, 0x00]; - let val = read_f64(&buf, 0).unwrap(); - assert!((val - 1.5).abs() < 1e-6); - } - - #[test] - fn skip_empty_containers() { - // empty fixmap - assert_eq!(skip_value(&[0x80], 0), Some(1)); - // empty fixarray - assert_eq!(skip_value(&[0x90], 0), Some(1)); - } - - #[test] - fn array_header_fixarray() { - let buf = encode(&json!([10, 20, 30])); - let (count, data_offset) = array_header(&buf, 0).unwrap(); - assert_eq!(count, 3); - assert_eq!(read_i64(&buf, data_offset), Some(10)); - } - - // ── Canonical encoding guarantee tests ───────────────────────────── - - #[test] - fn canonical_integer_smallest_representation() { - // fixint (0-127): single byte - let buf = encode(&json!(42)); - assert_eq!(buf.len(), 1); - assert_eq!(buf[0], 42); - - // 0 as fixint - let buf = encode(&json!(0)); - assert_eq!(buf.len(), 1); - assert_eq!(buf[0], 0); - - // 127 as fixint - let buf = encode(&json!(127)); - assert_eq!(buf.len(), 1); - assert_eq!(buf[0], 127); - - // 128 should NOT be fixint. JSON parses as i64, so zerompk uses - // int16 (0xd1) since 128 > i8::MAX. This is canonical for signed path. - let buf = encode(&json!(128)); - assert_eq!(buf[0], 0xd1); // int16 tag - assert_eq!(buf.len(), 3); // tag + 2 bytes - - // negative fixint (-32 to -1) - let buf = encode(&json!(-1)); - assert_eq!(buf.len(), 1); - assert_eq!(buf[0], 0xff); // -1 as negative fixint - - let buf = encode(&json!(-32)); - assert_eq!(buf.len(), 1); - assert_eq!(buf[0], 0xe0); // -32 as negative fixint - } - - #[test] - fn canonical_map_keys_sorted() { - // Keys should be lexicographically sorted in msgpack output. - // Encode with keys in non-sorted order in JSON source. - let buf = encode(&json!({"z": 1, "a": 2, "m": 3})); - - // Parse map and verify keys come out sorted - let (count, mut pos) = map_header(&buf, 0).unwrap(); - assert_eq!(count, 3); - - let mut keys = Vec::new(); - for _ in 0..count { - let key = read_str(&buf, pos).unwrap(); - keys.push(key.to_string()); - pos = skip_value(&buf, pos).unwrap(); // skip key - pos = skip_value(&buf, pos).unwrap(); // skip value - } - assert_eq!(keys, vec!["a", "m", "z"]); - } - - #[test] - fn canonical_deterministic_bytes() { - // Same logical document encoded twice must produce identical bytes. - let doc1 = encode(&json!({"name": "alice", "age": 30, "active": true})); - let doc2 = encode(&json!({"age": 30, "active": true, "name": "alice"})); - assert_eq!( - doc1, doc2, - "same logical doc must produce identical msgpack bytes" - ); - } - - #[test] - fn canonical_nested_map_keys_sorted() { - let buf = encode(&json!({"outer": {"z": 1, "a": 2}})); - // Extract the inner map - let (start, _end) = crate::msgpack_scan::field::extract_field(&buf, 0, "outer").unwrap(); - - let (count, mut pos) = map_header(&buf, start).unwrap(); - assert_eq!(count, 2); - - let key1 = read_str(&buf, pos).unwrap(); - pos = skip_value(&buf, pos).unwrap(); - pos = skip_value(&buf, pos).unwrap(); - let key2 = read_str(&buf, pos).unwrap(); - - assert_eq!(key1, "a"); - assert_eq!(key2, "z"); - } - - // ── Fuzz-style tests ─────────────────────────────────────────────────── - - /// Feed every single-byte sequence through all reader functions. None may - /// panic — they must return `None` or a valid result. - #[test] - fn fuzz_all_single_byte_sequences() { - for byte in 0u8..=255 { - let buf = [byte]; - // None of these must panic - let _ = skip_value(&buf, 0); - let _ = read_f64(&buf, 0); - let _ = read_i64(&buf, 0); - let _ = read_str(&buf, 0); - let _ = read_bool(&buf, 0); - let _ = read_null(&buf, 0); - let _ = map_header(&buf, 0); - let _ = array_header(&buf, 0); - let _ = read_value(&buf, 0); - } - } - - /// Feed two-byte patterns to cover tag + partial payload (truncated). - #[test] - fn fuzz_two_byte_patterns() { - // Tags that expect more bytes than we provide - let tags_need_extra: &[u8] = &[ - 0xca, // FLOAT32 needs 4 more - 0xcb, // FLOAT64 needs 8 more - 0xcc, // UINT8 needs 1 more - 0xcd, // UINT16 needs 2 more - 0xce, // UINT32 needs 4 more - 0xcf, // UINT64 needs 8 more - 0xd0, // INT8 needs 1 more - 0xd1, // INT16 needs 2 more - 0xd2, // INT32 needs 4 more - 0xd3, // INT64 needs 8 more - 0xd9, // STR8 length byte then data - 0xda, // STR16 2-byte length then data - 0xdb, // STR32 4-byte length then data - 0xdc, // ARRAY16 2-byte count then elements - 0xdd, // ARRAY32 4-byte count then elements - 0xde, // MAP16 2-byte count then pairs - 0xdf, // MAP32 4-byte count then pairs - 0xc4, // BIN8 - 0xc5, // BIN16 - 0xc6, // BIN32 - 0xd4, // FIXEXT1 - 0xd5, // FIXEXT2 - 0xd6, // FIXEXT4 - 0xd7, // FIXEXT8 - 0xd8, // FIXEXT16 - ]; - for &tag in tags_need_extra { - // Single byte (completely truncated payload) - let buf = [tag]; - let _ = skip_value(&buf, 0); - let _ = read_f64(&buf, 0); - let _ = read_i64(&buf, 0); - let _ = read_value(&buf, 0); - - // Tag + one garbage byte - for second in [0x00u8, 0x01, 0x7f, 0x80, 0xff] { - let buf = [tag, second]; - let _ = skip_value(&buf, 0); - let _ = read_f64(&buf, 0); - let _ = read_i64(&buf, 0); - let _ = read_value(&buf, 0); - } - } - } - - /// Deterministic pseudo-random byte sequences must not cause panics. - #[test] - fn fuzz_deterministic_random_payloads() { - // Generate deterministic sequences without external crates using a - // simple LCG (Knuth multiplicative hash). - let mut state: u64 = 0xdeadbeef_cafebabe; - let next = |s: &mut u64| -> u8 { - *s = s - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); - (*s >> 33) as u8 - }; - - let mut buf = vec![0u8; 256]; - for _ in 0..2000 { - // Randomize buffer length (1..=256) and contents - let len = (next(&mut state) as usize % 256) + 1; - for b in buf[..len].iter_mut() { - *b = next(&mut state); - } - let slice = &buf[..len]; - - // Try reading from multiple offsets - for offset in [0, 1, len / 2, len.saturating_sub(1)] { - let _ = skip_value(slice, offset); - let _ = read_f64(slice, offset); - let _ = read_i64(slice, offset); - let _ = read_str(slice, offset); - let _ = read_bool(slice, offset); - let _ = read_null(slice, offset); - let _ = map_header(slice, offset); - let _ = array_header(slice, offset); - let _ = read_value(slice, offset); - } - } - } - - /// Truncate a valid msgpack buffer at every byte position. - /// All reader functions must return `None` — never panic. - #[test] - fn fuzz_truncated_valid_payloads() { - let docs = [ - json!({"key": "value", "num": 42, "flag": true}), - json!({"nested": {"a": 1, "b": [1, 2, 3]}}), - json!([1, "two", 3.0, null, false]), - json!({"large": 9999999999_i64}), - json!({"float": 1.23456789}), - ]; - - for doc in &docs { - let full = encode(doc); - // Truncate at every position from 0 to full.len()-1 - for truncate_at in 0..full.len() { - let slice = &full[..truncate_at]; - // None of these may panic; result doesn't matter - let _ = skip_value(slice, 0); - let _ = read_f64(slice, 0); - let _ = read_i64(slice, 0); - let _ = read_str(slice, 0); - let _ = read_bool(slice, 0); - let _ = map_header(slice, 0); - let _ = array_header(slice, 0); - let _ = read_value(slice, 0); - } - } - } - - /// The never-used 0xc1 tag must return `None` for all functions. - #[test] - fn fuzz_never_used_tag_c1() { - // 0xc1 is explicitly "never used" in the msgpack spec - let buf = [0xc1u8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; - assert_eq!( - skip_value(&buf, 0), - None, - "0xc1 must return None from skip_value" - ); - assert_eq!(read_f64(&buf, 0), None); - assert_eq!(read_i64(&buf, 0), None); - assert_eq!(read_str(&buf, 0), None); - assert_eq!(read_bool(&buf, 0), None); - assert_eq!(map_header(&buf, 0), None); - assert_eq!(array_header(&buf, 0), None); - assert_eq!(read_value(&buf, 0), None); - } - - /// All tag boundary bytes — test transitions at fixint/fixmap/fixarray/fixstr edges. - #[test] - fn fuzz_tag_boundaries() { - // Each entry: (tag, expected_skip_result) - // For tags that are self-contained single bytes, skip returns Some(1). - // For tags requiring more data we just verify no panic with empty tail. - let boundary_tags: &[(u8, bool)] = &[ - (0x00, true), // positive fixint 0 - (0x7f, true), // positive fixint 127 - (0x80, true), // fixmap length 0 (empty map) - (0x8f, false), // fixmap length 15 — needs 15 pairs - (0x90, true), // fixarray length 0 (empty array) - (0x9f, false), // fixarray length 15 — needs 15 elements - (0xa0, true), // fixstr length 0 (empty string) - (0xbf, false), // fixstr length 31 — needs 31 bytes after - (0xc0, true), // nil - (0xc1, false), // never used — must return None - (0xc2, true), // false - (0xc3, true), // true - (0xe0, true), // negative fixint -32 - (0xff, true), // negative fixint -1 - ]; - for &(tag, self_contained) in boundary_tags { - let buf = [tag; 64]; // fill with the same tag as padding - let result = skip_value(&buf, 0); - if self_contained { - assert!(result.is_some(), "tag 0x{tag:02x} should skip OK"); - } else if tag == 0xc1 { - assert_eq!(result, None, "0xc1 must always return None"); - } - // For non-self-contained tags with valid padding we just verify no panic. - } - } - - /// Buffers where length fields claim enormous sizes but the buffer is tiny. - #[test] - fn fuzz_adversarial_length_fields() { - // STR32: tag 0xdb + 4-byte big-endian length claiming 0xffffffff bytes - let buf = [0xdbu8, 0xff, 0xff, 0xff, 0xff, b'x', b'y']; - assert_eq!(skip_value(&buf, 0), None); - assert_eq!(read_str(&buf, 0), None); - - // STR16: tag 0xda + 2-byte length claiming 0xffff bytes - let buf = [0xdau8, 0xff, 0xff, b'x']; - assert_eq!(skip_value(&buf, 0), None); - - // ARRAY32: claims 0xffffffff elements but buffer is empty after header - let buf = [0xddu8, 0xff, 0xff, 0xff, 0xff]; - assert_eq!(skip_value(&buf, 0), None); - - // MAP32: claims 0xffffffff pairs but buffer is empty after header - let buf = [0xdfu8, 0xff, 0xff, 0xff, 0xff]; - assert_eq!(skip_value(&buf, 0), None); - - // ARRAY16: claims 0xffff elements - let buf = [0xdcu8, 0xff, 0xff]; - assert_eq!(skip_value(&buf, 0), None); - - // MAP16: claims 0xffff pairs - let buf = [0xdeu8, 0xff, 0xff]; - assert_eq!(skip_value(&buf, 0), None); - - // BIN32: claims max length - let buf = [0xc6u8, 0xff, 0xff, 0xff, 0xff, 0x00]; - assert_eq!(skip_value(&buf, 0), None); - - // EXT32: claims max length - let buf = [0xc9u8, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00]; - assert_eq!(skip_value(&buf, 0), None); - } - - /// Deeply nested maps/arrays must cause `skip_value` to return `None` - /// once nesting exceeds MAX_DEPTH (128). - #[test] - fn fuzz_malicious_nesting_depth() { - // Build a buffer with 200 levels of fixarray (each containing 1 element) - // fixarray tag for 1 element = 0x91 - let depth = 200usize; - let mut buf = vec![0x91u8; depth]; // fixarray(1) — opens 1-element array - buf.push(0xc0u8); // nil at the innermost leaf - - // skip_value must return None because nesting > MAX_DEPTH - assert_eq!( - skip_value(&buf, 0), - None, - "deeply nested arrays must return None to guard against stack overflow" - ); - - // Same with maps: fixmap(1) = 0x81, then a fixstr(1) key + value - // Build 200 levels of fixmap(1) — each pair is (fixstr key, next map) - let mut map_buf: Vec = Vec::new(); - for i in 0..(depth as u8) { - map_buf.push(0x81); // fixmap(1) - map_buf.push(0xa1); // fixstr(1) key - map_buf.push(b'a'.wrapping_add(i % 26)); - // value = next map (already pushed in next iteration), or nil at end - } - map_buf.push(0xc0); // nil leaf - - assert_eq!( - skip_value(&map_buf, 0), - None, - "deeply nested maps must return None" - ); - } - - /// Verify skip_value correctly consumes exactly the right number of bytes - /// for all fixed-width numeric types and returns the correct next offset. - #[test] - fn fuzz_fixed_width_numeric_skip_offsets() { - // (tag, expected_total_bytes_consumed) - let cases: &[(u8, usize)] = &[ - (0xca, 5), // FLOAT32: 1 tag + 4 data - (0xcb, 9), // FLOAT64: 1 tag + 8 data - (0xcc, 2), // UINT8 - (0xcd, 3), // UINT16 - (0xce, 5), // UINT32 - (0xcf, 9), // UINT64 - (0xd0, 2), // INT8 - (0xd1, 3), // INT16 - (0xd2, 5), // INT32 - (0xd3, 9), // INT64 - ]; - for &(tag, size) in cases { - let mut buf = vec![0u8; size + 4]; // extra padding - buf[0] = tag; - let result = skip_value(&buf, 0); - assert_eq!( - result, - Some(size), - "tag 0x{tag:02x} should advance by {size} bytes" - ); - } - } - - /// Verify all fixext types consume the correct byte count. - #[test] - fn fuzz_fixext_skip_offsets() { - // (tag, expected_bytes_consumed) - let cases: &[(u8, usize)] = &[ - (0xd4, 3), // FIXEXT1: 1+1+1 - (0xd5, 4), // FIXEXT2: 1+1+2 - (0xd6, 6), // FIXEXT4: 1+1+4 - (0xd7, 10), // FIXEXT8: 1+1+8 - (0xd8, 18), // FIXEXT16: 1+1+16 - ]; - for &(tag, size) in cases { - let mut buf = vec![0u8; size + 4]; - buf[0] = tag; - let result = skip_value(&buf, 0); - assert_eq!( - result, - Some(size), - "fixext tag 0x{tag:02x} should advance by {size} bytes" - ); - } - } - - /// Out-of-bounds offset must return `None` — not panic. - #[test] - fn fuzz_out_of_bounds_offset() { - let buf = encode(&json!({"x": 1})); - let way_out = buf.len() + 1000; - assert_eq!(skip_value(&buf, way_out), None); - assert_eq!(read_f64(&buf, way_out), None); - assert_eq!(read_i64(&buf, way_out), None); - assert_eq!(read_str(&buf, way_out), None); - assert_eq!(read_bool(&buf, way_out), None); - assert_eq!(map_header(&buf, way_out), None); - assert_eq!(array_header(&buf, way_out), None); - assert_eq!(read_value(&buf, way_out), None); - } - - #[test] - fn read_bin_advance_all_widths() { - // bin8: 0xc4, len=3 - let mut off = 0; - let buf = [BIN8, 3, 0xde, 0xad, 0xbe, 0xff]; - assert_eq!( - read_bin_advance(&buf, &mut off), - Some(&[0xde, 0xad, 0xbe][..]) - ); - assert_eq!(off, 5); - - // bin16: 0xc5, big-endian len=4 - let mut off = 0; - let buf = [BIN16, 0x00, 0x04, 0x01, 0x02, 0x03, 0x04]; - assert_eq!( - read_bin_advance(&buf, &mut off), - Some(&[0x01, 0x02, 0x03, 0x04][..]) - ); - assert_eq!(off, 7); - - // bin32: 0xc6, big-endian len=2 - let mut off = 0; - let buf = [BIN32, 0x00, 0x00, 0x00, 0x02, 0xaa, 0xbb]; - assert_eq!(read_bin_advance(&buf, &mut off), Some(&[0xaa, 0xbb][..])); - assert_eq!(off, 7); - - // Non-bin tag returns None and does not advance. - let mut off = 0; - let buf = [0xc0u8]; // nil - assert_eq!(read_bin_advance(&buf, &mut off), None); - assert_eq!(off, 0); - - // Truncated returns None. - let mut off = 0; - let buf = [BIN8, 5, 0x01]; // claims 5 bytes, only 1 present - assert_eq!(read_bin_advance(&buf, &mut off), None); - } - - #[test] - fn read_u32_advance_all_widths() { - // positive fixint - let mut off = 0; - assert_eq!(read_u32_advance(&[42u8], &mut off), Some(42)); - assert_eq!(off, 1); - - // uint8 - let mut off = 0; - assert_eq!(read_u32_advance(&[UINT8, 200], &mut off), Some(200)); - assert_eq!(off, 2); - - // uint16 - let mut off = 0; - let buf = [UINT16, 0x12, 0x34]; - assert_eq!(read_u32_advance(&buf, &mut off), Some(0x1234)); - assert_eq!(off, 3); - - // uint32 - let mut off = 0; - let buf = [UINT32, 0xde, 0xad, 0xbe, 0xef]; - assert_eq!(read_u32_advance(&buf, &mut off), Some(0xdeadbeef)); - assert_eq!(off, 5); - - // negative fixint, int*, uint64, float, etc. all rejected - let mut off = 0; - assert_eq!(read_u32_advance(&[0xffu8], &mut off), None); // negative fixint - assert_eq!(off, 0); - let mut off = 0; - assert_eq!(read_u32_advance(&[INT8, 5], &mut off), None); - let mut off = 0; - assert_eq!( - read_u32_advance(&[UINT64, 0, 0, 0, 0, 0, 0, 0, 1], &mut off), - None - ); - - // Truncated returns None. - let mut off = 0; - assert_eq!(read_u32_advance(&[UINT16, 0x12], &mut off), None); - } - - #[test] - fn read_str_advance_basic() { - // fixstr "hi" - let mut off = 0; - let buf = encode(&json!("hi")); - assert_eq!(read_str_advance(&buf, &mut off), Some("hi")); - assert_eq!(off, buf.len()); - - // Sequential reads - let buf = encode(&json!(["one", "two"])); - let (count, mut off) = array_header(&buf, 0).unwrap(); - assert_eq!(count, 2); - assert_eq!(read_str_advance(&buf, &mut off), Some("one")); - assert_eq!(read_str_advance(&buf, &mut off), Some("two")); - assert_eq!(off, buf.len()); - - // Non-string returns None. - let mut off = 0; - assert_eq!(read_str_advance(&[NIL], &mut off), None); - assert_eq!(off, 0); - } - - /// Empty buffer must return `None` for all functions that can. - #[test] - fn fuzz_empty_buffer() { - let buf: &[u8] = &[]; - assert_eq!(skip_value(buf, 0), None); - assert_eq!(read_f64(buf, 0), None); - assert_eq!(read_i64(buf, 0), None); - assert_eq!(read_str(buf, 0), None); - assert_eq!(read_bool(buf, 0), None); - assert!(!read_null(buf, 0)); // returns bool, not Option - assert_eq!(map_header(buf, 0), None); - assert_eq!(array_header(buf, 0), None); - assert_eq!(read_value(buf, 0), None); - } -} diff --git a/nodedb-query/src/msgpack_scan/reader/mod.rs b/nodedb-query/src/msgpack_scan/reader/mod.rs new file mode 100644 index 000000000..1a19f8c94 --- /dev/null +++ b/nodedb-query/src/msgpack_scan/reader/mod.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Low-level MessagePack binary reader: tag parsing, value skipping, and typed reads. +//! +//! All functions operate on `&[u8]` with explicit offsets. Zero allocation, +//! zero copy. Returns `None` on truncated/invalid data — never panics. + +pub mod scalar; +pub mod skip; +pub mod tags; +pub mod value; + +pub(crate) use scalar::str_bounds; +pub use scalar::{ + array_header, map_header, read_bin_advance, read_bool, read_f64, read_i64, read_null, read_str, + read_str_advance, read_u32_advance, +}; +pub use skip::skip_value; +pub use value::read_value; diff --git a/nodedb-query/src/msgpack_scan/reader/scalar.rs b/nodedb-query/src/msgpack_scan/reader/scalar.rs new file mode 100644 index 000000000..1067b7e04 --- /dev/null +++ b/nodedb-query/src/msgpack_scan/reader/scalar.rs @@ -0,0 +1,470 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Typed scalar reads and container headers. + +use std::str; + +use super::tags::*; + +/// Read an f64 from the value at `offset`. Handles float32, float64, +/// and all integer types (coerced to f64). +pub fn read_f64(buf: &[u8], offset: usize) -> Option { + let tag = get(buf, offset)?; + match tag { + // positive fixint + 0x00..=0x7f => Some(tag as f64), + // negative fixint + 0xe0..=0xff => Some((tag as i8) as f64), + FLOAT64 => { + let bits = read_u64_be(buf, offset + 1)?; + Some(f64::from_bits(bits)) + } + FLOAT32 => { + let bits = read_u32_be(buf, offset + 1)?; + Some(f32::from_bits(bits) as f64) + } + UINT8 => Some(get(buf, offset + 1)? as f64), + UINT16 => Some(read_u16_be(buf, offset + 1)? as f64), + UINT32 => Some(read_u32_be(buf, offset + 1)? as f64), + UINT64 => Some(read_u64_be(buf, offset + 1)? as f64), + INT8 => Some(get(buf, offset + 1)? as i8 as f64), + INT16 => Some(read_u16_be(buf, offset + 1)? as i16 as f64), + INT32 => Some(read_u32_be(buf, offset + 1)? as i32 as f64), + INT64 => Some(read_u64_be(buf, offset + 1)? as i64 as f64), + _ => None, + } +} + +/// Read an i64 from the value at `offset`. Handles all integer types. +/// Floats return `None` — use `read_f64` for those. +pub fn read_i64(buf: &[u8], offset: usize) -> Option { + let tag = get(buf, offset)?; + match tag { + 0x00..=0x7f => Some(tag as i64), + 0xe0..=0xff => Some((tag as i8) as i64), + UINT8 => Some(get(buf, offset + 1)? as i64), + UINT16 => Some(read_u16_be(buf, offset + 1)? as i64), + UINT32 => Some(read_u32_be(buf, offset + 1)? as i64), + UINT64 => { + let v = read_u64_be(buf, offset + 1)?; + Some(v as i64) + } + INT8 => Some(get(buf, offset + 1)? as i8 as i64), + INT16 => Some(read_u16_be(buf, offset + 1)? as i16 as i64), + INT32 => Some(read_u32_be(buf, offset + 1)? as i32 as i64), + INT64 => { + let v = read_u64_be(buf, offset + 1)?; + Some(v as i64) + } + _ => None, + } +} + +/// Read a string slice from the value at `offset`. Zero-copy — borrows +/// directly from the input buffer. Returns `None` for non-string types +/// or invalid UTF-8. +pub fn read_str(buf: &[u8], offset: usize) -> Option<&str> { + let (start, len) = str_bounds(buf, offset)?; + let bytes = buf.get(start..start + len)?; + str::from_utf8(bytes).ok() +} + +/// Read a string slice at `*off`, advancing `*off` past it. Zero-copy. +/// Returns `None` for non-string types, invalid UTF-8, or truncated input. +pub fn read_str_advance<'a>(buf: &'a [u8], off: &mut usize) -> Option<&'a str> { + let (start, len) = str_bounds(buf, *off)?; + let bytes = buf.get(start..start + len)?; + let s = str::from_utf8(bytes).ok()?; + *off = start + len; + Some(s) +} + +/// Read a `bin` value at `*off`, advancing `*off` past it. Zero-copy — +/// the returned slice borrows from `buf`. Returns `None` for non-bin tags +/// or truncated input. +pub fn read_bin_advance<'a>(buf: &'a [u8], off: &mut usize) -> Option<&'a [u8]> { + let tag = get(buf, *off)?; + let (len, header) = match tag { + BIN8 => (get(buf, *off + 1)? as usize, 2), + BIN16 => (read_u16_be(buf, *off + 1)? as usize, 3), + BIN32 => (read_u32_be(buf, *off + 1)? as usize, 5), + _ => return None, + }; + let start = *off + header; + let end = start + len; + let data = buf.get(start..end)?; + *off = end; + Some(data) +} + +/// Read an unsigned integer that fits in a `u32` at `*off`, advancing `*off` +/// past it. Accepts positive fixint, uint8, uint16, uint32. Returns `None` +/// for negative, signed-typed, oversized (uint64), or non-integer values. +pub fn read_u32_advance(buf: &[u8], off: &mut usize) -> Option { + let tag = get(buf, *off)?; + match tag { + 0x00..=0x7f => { + *off += 1; + Some(tag as u32) + } + UINT8 => { + let v = get(buf, *off + 1)? as u32; + *off += 2; + Some(v) + } + UINT16 => { + let v = read_u16_be(buf, *off + 1)? as u32; + *off += 3; + Some(v) + } + UINT32 => { + let v = read_u32_be(buf, *off + 1)?; + *off += 5; + Some(v) + } + _ => None, + } +} + +/// Return `(data_start, byte_len)` for the string at `offset` without +/// validating UTF-8. Used internally for key comparison. +pub(crate) fn str_bounds(buf: &[u8], offset: usize) -> Option<(usize, usize)> { + let tag = get(buf, offset)?; + match tag { + 0xa0..=0xbf => { + let len = (tag & 0x1f) as usize; + Some((offset + 1, len)) + } + STR8 => { + let len = get(buf, offset + 1)? as usize; + Some((offset + 2, len)) + } + STR16 => { + let len = read_u16_be(buf, offset + 1)? as usize; + Some((offset + 3, len)) + } + STR32 => { + let len = read_u32_be(buf, offset + 1)? as usize; + Some((offset + 5, len)) + } + _ => None, + } +} + +/// Read a boolean from the value at `offset`. +pub fn read_bool(buf: &[u8], offset: usize) -> Option { + match get(buf, offset)? { + TRUE => Some(true), + FALSE => Some(false), + _ => None, + } +} + +/// Check if the value at `offset` is nil. +pub fn read_null(buf: &[u8], offset: usize) -> bool { + get(buf, offset) == Some(NIL) +} + +/// Return the number of key-value pairs and the offset of the first pair, +/// for the map starting at `offset`. Returns `None` if not a map. +pub fn map_header(buf: &[u8], offset: usize) -> Option<(usize, usize)> { + let tag = get(buf, offset)?; + match tag { + 0x80..=0x8f => Some(((tag & 0x0f) as usize, offset + 1)), + MAP16 => Some((read_u16_be(buf, offset + 1)? as usize, offset + 3)), + MAP32 => Some((read_u32_be(buf, offset + 1)? as usize, offset + 5)), + _ => None, + } +} + +/// Return the number of elements and the offset of the first element, +/// for the array starting at `offset`. Returns `None` if not an array. +pub fn array_header(buf: &[u8], offset: usize) -> Option<(usize, usize)> { + let tag = get(buf, offset)?; + match tag { + 0x90..=0x9f => Some(((tag & 0x0f) as usize, offset + 1)), + ARRAY16 => Some((read_u16_be(buf, offset + 1)? as usize, offset + 3)), + ARRAY32 => Some((read_u32_be(buf, offset + 1)? as usize, offset + 5)), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::msgpack_scan::reader::skip_value; + + use serde_json::json; + + /// Helper: encode a serde_json::Value to MessagePack bytes. + fn encode(v: &serde_json::Value) -> Vec { + nodedb_types::json_msgpack::json_to_msgpack(v).expect("encode") + } + + #[test] + fn read_f64_fixint() { + assert_eq!(read_f64(&[42u8], 0), Some(42.0)); + } + + #[test] + fn read_f64_negative_fixint() { + assert_eq!(read_f64(&[0xffu8], 0), Some(-1.0)); + } + + #[test] + fn read_f64_float64() { + let buf = encode(&json!(std::f64::consts::PI)); + assert_eq!(read_f64(&buf, 0), Some(std::f64::consts::PI)); + } + + #[test] + fn read_f64_uint16() { + let buf = encode(&json!(1000)); + assert_eq!(read_f64(&buf, 0), Some(1000.0)); + } + + #[test] + fn read_f64_float32() { + // json! always produces f64, so test float32 with raw bytes + // float32 tag (0xca) + 1.5 in IEEE 754 big-endian + let buf = [0xca, 0x3f, 0xc0, 0x00, 0x00]; + let val = read_f64(&buf, 0).unwrap(); + assert!((val - 1.5).abs() < 1e-6); + } + + #[test] + fn read_i64_values() { + assert_eq!(read_i64(&[42u8], 0), Some(42)); + assert_eq!(read_i64(&[0xffu8], 0), Some(-1)); + + let buf = encode(&json!(300)); + assert_eq!(read_i64(&buf, 0), Some(300)); + + let buf = encode(&json!(-500)); + assert_eq!(read_i64(&buf, 0), Some(-500)); + } + + #[test] + fn read_str_fixstr() { + let buf = encode(&json!("hi")); + assert_eq!(read_str(&buf, 0), Some("hi")); + } + + #[test] + fn read_str_str8() { + let long = "a".repeat(40); + let buf = encode(&json!(long)); + assert_eq!(read_str(&buf, 0), Some(long.as_str())); + } + + #[test] + fn read_bool_values() { + assert_eq!(read_bool(&[TRUE], 0), Some(true)); + assert_eq!(read_bool(&[FALSE], 0), Some(false)); + assert_eq!(read_bool(&[NIL], 0), None); + } + + #[test] + fn read_null_check() { + assert!(read_null(&[NIL], 0)); + assert!(!read_null(&[TRUE], 0)); + } + + #[test] + fn map_header_fixmap() { + let buf = encode(&json!({"x": 1})); + let (count, _data_offset) = map_header(&buf, 0).unwrap(); + assert_eq!(count, 1); + } + + #[test] + fn array_header_fixarray() { + let buf = encode(&json!([10, 20, 30])); + let (count, data_offset) = array_header(&buf, 0).unwrap(); + assert_eq!(count, 3); + assert_eq!(read_i64(&buf, data_offset), Some(10)); + } + + #[test] + fn canonical_integer_smallest_representation() { + // fixint (0-127): single byte + let buf = encode(&json!(42)); + assert_eq!(buf.len(), 1); + assert_eq!(buf[0], 42); + + // 0 as fixint + let buf = encode(&json!(0)); + assert_eq!(buf.len(), 1); + assert_eq!(buf[0], 0); + + // 127 as fixint + let buf = encode(&json!(127)); + assert_eq!(buf.len(), 1); + assert_eq!(buf[0], 127); + + // 128 should NOT be fixint. JSON parses as i64, so zerompk uses + // int16 (0xd1) since 128 > i8::MAX. This is canonical for signed path. + let buf = encode(&json!(128)); + assert_eq!(buf[0], 0xd1); // int16 tag + assert_eq!(buf.len(), 3); // tag + 2 bytes + + // negative fixint (-32 to -1) + let buf = encode(&json!(-1)); + assert_eq!(buf.len(), 1); + assert_eq!(buf[0], 0xff); // -1 as negative fixint + + let buf = encode(&json!(-32)); + assert_eq!(buf.len(), 1); + assert_eq!(buf[0], 0xe0); // -32 as negative fixint + } + + #[test] + fn canonical_map_keys_sorted() { + // Keys should be lexicographically sorted in msgpack output. + // Encode with keys in non-sorted order in JSON source. + let buf = encode(&json!({"z": 1, "a": 2, "m": 3})); + + // Parse map and verify keys come out sorted + let (count, mut pos) = map_header(&buf, 0).unwrap(); + assert_eq!(count, 3); + + let mut keys = Vec::new(); + for _ in 0..count { + let key = read_str(&buf, pos).unwrap(); + keys.push(key.to_string()); + pos = skip_value(&buf, pos).unwrap(); // skip key + pos = skip_value(&buf, pos).unwrap(); // skip value + } + assert_eq!(keys, vec!["a", "m", "z"]); + } + + #[test] + fn canonical_deterministic_bytes() { + // Same logical document encoded twice must produce identical bytes. + let doc1 = encode(&json!({"name": "alice", "age": 30, "active": true})); + let doc2 = encode(&json!({"age": 30, "active": true, "name": "alice"})); + assert_eq!( + doc1, doc2, + "same logical doc must produce identical msgpack bytes" + ); + } + + #[test] + fn canonical_nested_map_keys_sorted() { + let buf = encode(&json!({"outer": {"z": 1, "a": 2}})); + // Extract the inner map + let (start, _end) = crate::msgpack_scan::field::extract_field(&buf, 0, "outer").unwrap(); + + let (count, mut pos) = map_header(&buf, start).unwrap(); + assert_eq!(count, 2); + + let key1 = read_str(&buf, pos).unwrap(); + pos = skip_value(&buf, pos).unwrap(); + pos = skip_value(&buf, pos).unwrap(); + let key2 = read_str(&buf, pos).unwrap(); + + assert_eq!(key1, "a"); + assert_eq!(key2, "z"); + } + + #[test] + fn read_bin_advance_all_widths() { + // bin8: 0xc4, len=3 + let mut off = 0; + let buf = [BIN8, 3, 0xde, 0xad, 0xbe, 0xff]; + assert_eq!( + read_bin_advance(&buf, &mut off), + Some(&[0xde, 0xad, 0xbe][..]) + ); + assert_eq!(off, 5); + + // bin16: 0xc5, big-endian len=4 + let mut off = 0; + let buf = [BIN16, 0x00, 0x04, 0x01, 0x02, 0x03, 0x04]; + assert_eq!( + read_bin_advance(&buf, &mut off), + Some(&[0x01, 0x02, 0x03, 0x04][..]) + ); + assert_eq!(off, 7); + + // bin32: 0xc6, big-endian len=2 + let mut off = 0; + let buf = [BIN32, 0x00, 0x00, 0x00, 0x02, 0xaa, 0xbb]; + assert_eq!(read_bin_advance(&buf, &mut off), Some(&[0xaa, 0xbb][..])); + assert_eq!(off, 7); + + // Non-bin tag returns None and does not advance. + let mut off = 0; + let buf = [0xc0u8]; // nil + assert_eq!(read_bin_advance(&buf, &mut off), None); + assert_eq!(off, 0); + + // Truncated returns None. + let mut off = 0; + let buf = [BIN8, 5, 0x01]; // claims 5 bytes, only 1 present + assert_eq!(read_bin_advance(&buf, &mut off), None); + } + + #[test] + fn read_u32_advance_all_widths() { + // positive fixint + let mut off = 0; + assert_eq!(read_u32_advance(&[42u8], &mut off), Some(42)); + assert_eq!(off, 1); + + // uint8 + let mut off = 0; + assert_eq!(read_u32_advance(&[UINT8, 200], &mut off), Some(200)); + assert_eq!(off, 2); + + // uint16 + let mut off = 0; + let buf = [UINT16, 0x12, 0x34]; + assert_eq!(read_u32_advance(&buf, &mut off), Some(0x1234)); + assert_eq!(off, 3); + + // uint32 + let mut off = 0; + let buf = [UINT32, 0xde, 0xad, 0xbe, 0xef]; + assert_eq!(read_u32_advance(&buf, &mut off), Some(0xdeadbeef)); + assert_eq!(off, 5); + + // negative fixint, int*, uint64, float, etc. all rejected + let mut off = 0; + assert_eq!(read_u32_advance(&[0xffu8], &mut off), None); // negative fixint + assert_eq!(off, 0); + let mut off = 0; + assert_eq!(read_u32_advance(&[INT8, 5], &mut off), None); + let mut off = 0; + assert_eq!( + read_u32_advance(&[UINT64, 0, 0, 0, 0, 0, 0, 0, 1], &mut off), + None + ); + + // Truncated returns None. + let mut off = 0; + assert_eq!(read_u32_advance(&[UINT16, 0x12], &mut off), None); + } + + #[test] + fn read_str_advance_basic() { + // fixstr "hi" + let mut off = 0; + let buf = encode(&json!("hi")); + assert_eq!(read_str_advance(&buf, &mut off), Some("hi")); + assert_eq!(off, buf.len()); + + // Sequential reads + let buf = encode(&json!(["one", "two"])); + let (count, mut off) = array_header(&buf, 0).unwrap(); + assert_eq!(count, 2); + assert_eq!(read_str_advance(&buf, &mut off), Some("one")); + assert_eq!(read_str_advance(&buf, &mut off), Some("two")); + assert_eq!(off, buf.len()); + + // Non-string returns None. + let mut off = 0; + assert_eq!(read_str_advance(&[NIL], &mut off), None); + assert_eq!(off, 0); + } +} diff --git a/nodedb-query/src/msgpack_scan/reader/skip.rs b/nodedb-query/src/msgpack_scan/reader/skip.rs new file mode 100644 index 000000000..9f63ee35b --- /dev/null +++ b/nodedb-query/src/msgpack_scan/reader/skip.rs @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! `skip_value`: advance past one MessagePack value without decoding it. + +use super::tags::*; + +/// Advance past the MessagePack value starting at `offset`, returning the +/// offset of the next value. Returns `None` if the buffer is truncated or +/// nesting exceeds `MAX_DEPTH`. +/// +/// This is the performance-critical primitive. It never allocates. +pub fn skip_value(buf: &[u8], offset: usize) -> Option { + skip_value_depth(buf, offset, 0) +} + +fn skip_value_depth(buf: &[u8], offset: usize, depth: u16) -> Option { + if depth > MAX_DEPTH { + return None; + } + let tag = get(buf, offset)?; + match tag { + // positive fixint (0x00..=0x7f) + 0x00..=0x7f => Some(offset + 1), + // negative fixint (0xe0..=0xff) + 0xe0..=0xff => Some(offset + 1), + // nil, false, true + NIL | FALSE | TRUE => Some(offset + 1), + + // fixmap (0x80..=0x8f) + 0x80..=0x8f => { + let count = (tag & 0x0f) as usize; + skip_n_pairs(buf, offset + 1, count, depth) + } + MAP16 => { + let count = read_u16_be(buf, offset + 1)? as usize; + skip_n_pairs(buf, offset + 3, count, depth) + } + MAP32 => { + let count = read_u32_be(buf, offset + 1)? as usize; + skip_n_pairs(buf, offset + 5, count, depth) + } + + // fixarray (0x90..=0x9f) + 0x90..=0x9f => { + let count = (tag & 0x0f) as usize; + skip_n_values(buf, offset + 1, count, depth) + } + ARRAY16 => { + let count = read_u16_be(buf, offset + 1)? as usize; + skip_n_values(buf, offset + 3, count, depth) + } + ARRAY32 => { + let count = read_u32_be(buf, offset + 1)? as usize; + skip_n_values(buf, offset + 5, count, depth) + } + + // fixstr (0xa0..=0xbf) + 0xa0..=0xbf => { + let len = (tag & 0x1f) as usize; + checked_advance(buf, offset, 1 + len) + } + STR8 => { + let len = get(buf, offset + 1)? as usize; + checked_advance(buf, offset, 2 + len) + } + STR16 => { + let len = read_u16_be(buf, offset + 1)? as usize; + checked_advance(buf, offset, 3 + len) + } + STR32 => { + let len = read_u32_be(buf, offset + 1)? as usize; + checked_advance(buf, offset, 5 + len) + } + + // bin + BIN8 => { + let len = get(buf, offset + 1)? as usize; + checked_advance(buf, offset, 2 + len) + } + BIN16 => { + let len = read_u16_be(buf, offset + 1)? as usize; + checked_advance(buf, offset, 3 + len) + } + BIN32 => { + let len = read_u32_be(buf, offset + 1)? as usize; + checked_advance(buf, offset, 5 + len) + } + + // fixed-width numerics (bounds-check against buffer length) + FLOAT32 => checked_advance(buf, offset, 5), + FLOAT64 => checked_advance(buf, offset, 9), + UINT8 | INT8 => checked_advance(buf, offset, 2), + UINT16 | INT16 => checked_advance(buf, offset, 3), + UINT32 | INT32 => checked_advance(buf, offset, 5), + UINT64 | INT64 => checked_advance(buf, offset, 9), + + // ext + FIXEXT1 => checked_advance(buf, offset, 3), + FIXEXT2 => checked_advance(buf, offset, 4), + FIXEXT4 => checked_advance(buf, offset, 6), + FIXEXT8 => checked_advance(buf, offset, 10), + FIXEXT16 => checked_advance(buf, offset, 18), + EXT8 => { + let len = get(buf, offset + 1)? as usize; + checked_advance(buf, offset, 3 + len) + } + EXT16 => { + let len = read_u16_be(buf, offset + 1)? as usize; + checked_advance(buf, offset, 4 + len) + } + EXT32 => { + let len = read_u32_be(buf, offset + 1)? as usize; + checked_advance(buf, offset, 6 + len) + } + + // 0xc1 is never used in the spec + _ => None, + } +} + +fn skip_n_values(buf: &[u8], mut pos: usize, count: usize, depth: u16) -> Option { + for _ in 0..count { + pos = skip_value_depth(buf, pos, depth + 1)?; + } + Some(pos) +} + +fn skip_n_pairs(buf: &[u8], mut pos: usize, count: usize, depth: u16) -> Option { + for _ in 0..count { + pos = skip_value_depth(buf, pos, depth + 1)?; // key + pos = skip_value_depth(buf, pos, depth + 1)?; // value + } + Some(pos) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::msgpack_scan::reader::read_str; + + use serde_json::json; + + /// Helper: encode a serde_json::Value to MessagePack bytes. + fn encode(v: &serde_json::Value) -> Vec { + nodedb_types::json_msgpack::json_to_msgpack(v).expect("encode") + } + + #[test] + fn skip_positive_fixint() { + let buf = [0x05, 0xff]; + assert_eq!(skip_value(&buf, 0), Some(1)); + } + + #[test] + fn skip_negative_fixint() { + let buf = [0xe0, 0x00]; + assert_eq!(skip_value(&buf, 0), Some(1)); + } + + #[test] + fn skip_nil_bool() { + assert_eq!(skip_value(&[NIL], 0), Some(1)); + assert_eq!(skip_value(&[TRUE], 0), Some(1)); + assert_eq!(skip_value(&[FALSE], 0), Some(1)); + } + + #[test] + fn skip_float64() { + let buf = encode(&json!(9.81)); + assert_eq!(skip_value(&buf, 0), Some(buf.len())); + } + + #[test] + fn skip_string() { + let buf = encode(&json!("hello")); + assert_eq!(skip_value(&buf, 0), Some(buf.len())); + } + + #[test] + fn skip_map() { + let buf = encode(&json!({"a": 1, "b": 2})); + assert_eq!(skip_value(&buf, 0), Some(buf.len())); + } + + #[test] + fn skip_nested_array() { + let buf = encode(&json!([[1, 2], [3, 4, 5]])); + assert_eq!(skip_value(&buf, 0), Some(buf.len())); + } + + #[test] + fn skip_truncated_returns_none() { + let buf = [FLOAT64, 0x40]; // truncated float64 + assert_eq!(skip_value(&buf, 0), None); + } + + #[test] + fn skip_bin() { + // bin8: 0xc4, len=3, 3 bytes of data + let buf = [BIN8, 3, 0xde, 0xad, 0xbe, 0xff]; + assert_eq!(skip_value(&buf, 0), Some(5)); + } + + #[test] + fn skip_ext() { + // fixext1: 0xd4, type byte, 1 data byte + let buf = [FIXEXT1, 0x01, 0xab, 0xff]; + assert_eq!(skip_value(&buf, 0), Some(3)); + } + + #[test] + fn skip_empty_containers() { + // empty fixmap + assert_eq!(skip_value(&[0x80], 0), Some(1)); + // empty fixarray + assert_eq!(skip_value(&[0x90], 0), Some(1)); + } + + /// All tag boundary bytes — test transitions at fixint/fixmap/fixarray/fixstr edges. + #[test] + fn fuzz_tag_boundaries() { + // Each entry: (tag, expected_skip_result) + // For tags that are self-contained single bytes, skip returns Some(1). + // For tags requiring more data we just verify no panic with empty tail. + let boundary_tags: &[(u8, bool)] = &[ + (0x00, true), // positive fixint 0 + (0x7f, true), // positive fixint 127 + (0x80, true), // fixmap length 0 (empty map) + (0x8f, false), // fixmap length 15 — needs 15 pairs + (0x90, true), // fixarray length 0 (empty array) + (0x9f, false), // fixarray length 15 — needs 15 elements + (0xa0, true), // fixstr length 0 (empty string) + (0xbf, false), // fixstr length 31 — needs 31 bytes after + (0xc0, true), // nil + (0xc1, false), // never used — must return None + (0xc2, true), // false + (0xc3, true), // true + (0xe0, true), // negative fixint -32 + (0xff, true), // negative fixint -1 + ]; + for &(tag, self_contained) in boundary_tags { + let buf = [tag; 64]; // fill with the same tag as padding + let result = skip_value(&buf, 0); + if self_contained { + assert!(result.is_some(), "tag 0x{tag:02x} should skip OK"); + } else if tag == 0xc1 { + assert_eq!(result, None, "0xc1 must always return None"); + } + // For non-self-contained tags with valid padding we just verify no panic. + } + } + + /// Buffers where length fields claim enormous sizes but the buffer is tiny. + #[test] + fn fuzz_adversarial_length_fields() { + // STR32: tag 0xdb + 4-byte big-endian length claiming 0xffffffff bytes + let buf = [0xdbu8, 0xff, 0xff, 0xff, 0xff, b'x', b'y']; + assert_eq!(skip_value(&buf, 0), None); + assert_eq!(read_str(&buf, 0), None); + + // STR16: tag 0xda + 2-byte length claiming 0xffff bytes + let buf = [0xdau8, 0xff, 0xff, b'x']; + assert_eq!(skip_value(&buf, 0), None); + + // ARRAY32: claims 0xffffffff elements but buffer is empty after header + let buf = [0xddu8, 0xff, 0xff, 0xff, 0xff]; + assert_eq!(skip_value(&buf, 0), None); + + // MAP32: claims 0xffffffff pairs but buffer is empty after header + let buf = [0xdfu8, 0xff, 0xff, 0xff, 0xff]; + assert_eq!(skip_value(&buf, 0), None); + + // ARRAY16: claims 0xffff elements + let buf = [0xdcu8, 0xff, 0xff]; + assert_eq!(skip_value(&buf, 0), None); + + // MAP16: claims 0xffff pairs + let buf = [0xdeu8, 0xff, 0xff]; + assert_eq!(skip_value(&buf, 0), None); + + // BIN32: claims max length + let buf = [0xc6u8, 0xff, 0xff, 0xff, 0xff, 0x00]; + assert_eq!(skip_value(&buf, 0), None); + + // EXT32: claims max length + let buf = [0xc9u8, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00]; + assert_eq!(skip_value(&buf, 0), None); + } + + /// Deeply nested maps/arrays must cause `skip_value` to return `None` + /// once nesting exceeds MAX_DEPTH (128). + #[test] + fn fuzz_malicious_nesting_depth() { + // Build a buffer with 200 levels of fixarray (each containing 1 element) + // fixarray tag for 1 element = 0x91 + let depth = 200usize; + let mut buf = vec![0x91u8; depth]; // fixarray(1) — opens 1-element array + buf.push(0xc0u8); // nil at the innermost leaf + + // skip_value must return None because nesting > MAX_DEPTH + assert_eq!( + skip_value(&buf, 0), + None, + "deeply nested arrays must return None to guard against stack overflow" + ); + + // Same with maps: fixmap(1) = 0x81, then a fixstr(1) key + value + // Build 200 levels of fixmap(1) — each pair is (fixstr key, next map) + let mut map_buf: Vec = Vec::new(); + for i in 0..(depth as u8) { + map_buf.push(0x81); // fixmap(1) + map_buf.push(0xa1); // fixstr(1) key + map_buf.push(b'a'.wrapping_add(i % 26)); + // value = next map (already pushed in next iteration), or nil at end + } + map_buf.push(0xc0); // nil leaf + + assert_eq!( + skip_value(&map_buf, 0), + None, + "deeply nested maps must return None" + ); + } + + /// Verify skip_value correctly consumes exactly the right number of bytes + /// for all fixed-width numeric types and returns the correct next offset. + #[test] + fn fuzz_fixed_width_numeric_skip_offsets() { + // (tag, expected_total_bytes_consumed) + let cases: &[(u8, usize)] = &[ + (0xca, 5), // FLOAT32: 1 tag + 4 data + (0xcb, 9), // FLOAT64: 1 tag + 8 data + (0xcc, 2), // UINT8 + (0xcd, 3), // UINT16 + (0xce, 5), // UINT32 + (0xcf, 9), // UINT64 + (0xd0, 2), // INT8 + (0xd1, 3), // INT16 + (0xd2, 5), // INT32 + (0xd3, 9), // INT64 + ]; + for &(tag, size) in cases { + let mut buf = vec![0u8; size + 4]; // extra padding + buf[0] = tag; + let result = skip_value(&buf, 0); + assert_eq!( + result, + Some(size), + "tag 0x{tag:02x} should advance by {size} bytes" + ); + } + } + + /// Verify all fixext types consume the correct byte count. + #[test] + fn fuzz_fixext_skip_offsets() { + // (tag, expected_bytes_consumed) + let cases: &[(u8, usize)] = &[ + (0xd4, 3), // FIXEXT1: 1+1+1 + (0xd5, 4), // FIXEXT2: 1+1+2 + (0xd6, 6), // FIXEXT4: 1+1+4 + (0xd7, 10), // FIXEXT8: 1+1+8 + (0xd8, 18), // FIXEXT16: 1+1+16 + ]; + for &(tag, size) in cases { + let mut buf = vec![0u8; size + 4]; + buf[0] = tag; + let result = skip_value(&buf, 0); + assert_eq!( + result, + Some(size), + "fixext tag 0x{tag:02x} should advance by {size} bytes" + ); + } + } +} diff --git a/nodedb-query/src/msgpack_scan/reader/tags.rs b/nodedb-query/src/msgpack_scan/reader/tags.rs new file mode 100644 index 000000000..4b1daf45c --- /dev/null +++ b/nodedb-query/src/msgpack_scan/reader/tags.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! MessagePack tag constants and bounds-checked byte helpers. + +pub(super) const NIL: u8 = 0xc0; +pub(super) const FALSE: u8 = 0xc2; +pub(super) const TRUE: u8 = 0xc3; +pub(super) const BIN8: u8 = 0xc4; +pub(super) const BIN16: u8 = 0xc5; +pub(super) const BIN32: u8 = 0xc6; +pub(super) const EXT8: u8 = 0xc7; +pub(super) const EXT16: u8 = 0xc8; +pub(super) const EXT32: u8 = 0xc9; +pub(super) const FLOAT32: u8 = 0xca; +pub(super) const FLOAT64: u8 = 0xcb; +pub(super) const UINT8: u8 = 0xcc; +pub(super) const UINT16: u8 = 0xcd; +pub(super) const UINT32: u8 = 0xce; +pub(super) const UINT64: u8 = 0xcf; +pub(super) const INT8: u8 = 0xd0; +pub(super) const INT16: u8 = 0xd1; +pub(super) const INT32: u8 = 0xd2; +pub(super) const INT64: u8 = 0xd3; +pub(super) const FIXEXT1: u8 = 0xd4; +pub(super) const FIXEXT2: u8 = 0xd5; +pub(super) const FIXEXT4: u8 = 0xd6; +pub(super) const FIXEXT8: u8 = 0xd7; +pub(super) const FIXEXT16: u8 = 0xd8; +pub(super) const STR8: u8 = 0xd9; +pub(super) const STR16: u8 = 0xda; +pub(super) const STR32: u8 = 0xdb; +pub(super) const ARRAY16: u8 = 0xdc; +pub(super) const ARRAY32: u8 = 0xdd; +pub(super) const MAP16: u8 = 0xde; +pub(super) const MAP32: u8 = 0xdf; + +/// Maximum nesting depth to prevent stack overflow on malicious payloads. +pub(super) const MAX_DEPTH: u16 = 128; + +#[inline(always)] +pub(super) fn get(buf: &[u8], pos: usize) -> Option { + buf.get(pos).copied() +} + +#[inline(always)] +pub(super) fn read_u16_be(buf: &[u8], pos: usize) -> Option { + let bytes = buf.get(pos..pos + 2)?; + Some(u16::from_be_bytes([bytes[0], bytes[1]])) +} + +#[inline(always)] +pub(super) fn read_u32_be(buf: &[u8], pos: usize) -> Option { + let bytes = buf.get(pos..pos + 4)?; + Some(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) +} + +#[inline(always)] +pub(super) fn read_u64_be(buf: &[u8], pos: usize) -> Option { + let bytes = buf.get(pos..pos + 8)?; + Some(u64::from_be_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ])) +} + +/// Return `Some(offset + size)` only if the buffer has enough bytes. +#[inline(always)] +pub(super) fn checked_advance(buf: &[u8], offset: usize, size: usize) -> Option { + let end = offset + size; + if end <= buf.len() { Some(end) } else { None } +} diff --git a/nodedb-query/src/msgpack_scan/reader/value.rs b/nodedb-query/src/msgpack_scan/reader/value.rs new file mode 100644 index 000000000..74b830d59 --- /dev/null +++ b/nodedb-query/src/msgpack_scan/reader/value.rs @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! `read_value`: decode one scalar into `nodedb_types::Value`. + +use nodedb_types::read_instant; + +use super::scalar::read_str; +use super::tags::*; + +/// Read a scalar msgpack value at `offset` into `nodedb_types::Value`. +/// +/// Handles null, bool, integers, floats, strings, and instant ext +/// (`fixext8` type 1 / 2 → `Value::DateTime` / `Value::NaiveDateTime`). +/// For complex types (array, map, bin, other ext), returns `None` — caller +/// should use `json_from_msgpack` for those. +pub fn read_value(buf: &[u8], offset: usize) -> Option { + let tag = get(buf, offset)?; + match tag { + NIL => Some(nodedb_types::Value::Null), + TRUE => Some(nodedb_types::Value::Bool(true)), + FALSE => Some(nodedb_types::Value::Bool(false)), + // Integers + 0x00..=0x7f => Some(nodedb_types::Value::Integer(tag as i64)), + 0xe0..=0xff => Some(nodedb_types::Value::Integer((tag as i8) as i64)), + UINT8 => Some(nodedb_types::Value::Integer(get(buf, offset + 1)? as i64)), + UINT16 => Some(nodedb_types::Value::Integer( + read_u16_be(buf, offset + 1)? as i64 + )), + UINT32 => Some(nodedb_types::Value::Integer( + read_u32_be(buf, offset + 1)? as i64 + )), + UINT64 => Some(nodedb_types::Value::Integer( + read_u64_be(buf, offset + 1)? as i64 + )), + INT8 => Some(nodedb_types::Value::Integer( + get(buf, offset + 1)? as i8 as i64 + )), + INT16 => Some(nodedb_types::Value::Integer( + read_u16_be(buf, offset + 1)? as i16 as i64, + )), + INT32 => Some(nodedb_types::Value::Integer( + read_u32_be(buf, offset + 1)? as i32 as i64, + )), + INT64 => Some(nodedb_types::Value::Integer( + read_u64_be(buf, offset + 1)? as i64 + )), + // Floats + FLOAT32 => { + let bits = read_u32_be(buf, offset + 1)?; + Some(nodedb_types::Value::Float(f32::from_bits(bits) as f64)) + } + FLOAT64 => { + let bits = read_u64_be(buf, offset + 1)?; + Some(nodedb_types::Value::Float(f64::from_bits(bits))) + } + // Strings + 0xa0..=0xbf | STR8 | STR16 | STR32 => { + read_str(buf, offset).map(|s| nodedb_types::Value::String(s.to_string())) + } + // Instants + FIXEXT8 => read_instant(buf, offset).map(|(kind, micros)| kind.from_micros(micros)), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::msgpack_scan::reader::{ + array_header, map_header, read_bool, read_f64, read_i64, read_null, skip_value, + }; + use nodedb_types::{InstantKind, NdbDateTime, Value, write_instant}; + + use serde_json::json; + + /// Helper: encode a serde_json::Value to MessagePack bytes. + fn encode(v: &serde_json::Value) -> Vec { + nodedb_types::json_msgpack::json_to_msgpack(v).expect("encode") + } + + #[test] + fn read_value_instants() { + let mut buf = vec![0xc0]; + write_instant(&mut buf, InstantKind::Utc, 1_710_498_600_000_000); + write_instant(&mut buf, InstantKind::Naive, -5); + assert_eq!( + read_value(&buf, 1), + Some(Value::DateTime(NdbDateTime::from_micros( + 1_710_498_600_000_000 + ))) + ); + assert_eq!( + read_value(&buf, 11), + Some(Value::NaiveDateTime(NdbDateTime::from_micros(-5))) + ); + assert_eq!(skip_value(&buf, 1), Some(11)); + } + + #[test] + fn read_value_unknown_ext_is_none() { + let buf = [FIXEXT8, 0x09, 0, 0, 0, 0, 0, 0, 0, 1]; + assert_eq!(read_value(&buf, 0), None); + let buf = [FIXEXT1, 0x01, 0xab]; + assert_eq!(read_value(&buf, 0), None); + } + + #[test] + fn read_value_truncated_instant_is_none() { + let mut buf = Vec::new(); + write_instant(&mut buf, InstantKind::Utc, 42); + buf.truncate(9); + assert_eq!(read_value(&buf, 0), None); + } + + /// Feed every single-byte sequence through all reader functions. None may + /// panic — they must return `None` or a valid result. + #[test] + fn fuzz_all_single_byte_sequences() { + for byte in 0u8..=255 { + let buf = [byte]; + // None of these must panic + let _ = skip_value(&buf, 0); + let _ = read_f64(&buf, 0); + let _ = read_i64(&buf, 0); + let _ = read_str(&buf, 0); + let _ = read_bool(&buf, 0); + let _ = read_null(&buf, 0); + let _ = map_header(&buf, 0); + let _ = array_header(&buf, 0); + let _ = read_value(&buf, 0); + } + } + + /// Feed two-byte patterns to cover tag + partial payload (truncated). + #[test] + fn fuzz_two_byte_patterns() { + // Tags that expect more bytes than we provide + let tags_need_extra: &[u8] = &[ + 0xca, // FLOAT32 needs 4 more + 0xcb, // FLOAT64 needs 8 more + 0xcc, // UINT8 needs 1 more + 0xcd, // UINT16 needs 2 more + 0xce, // UINT32 needs 4 more + 0xcf, // UINT64 needs 8 more + 0xd0, // INT8 needs 1 more + 0xd1, // INT16 needs 2 more + 0xd2, // INT32 needs 4 more + 0xd3, // INT64 needs 8 more + 0xd9, // STR8 length byte then data + 0xda, // STR16 2-byte length then data + 0xdb, // STR32 4-byte length then data + 0xdc, // ARRAY16 2-byte count then elements + 0xdd, // ARRAY32 4-byte count then elements + 0xde, // MAP16 2-byte count then pairs + 0xdf, // MAP32 4-byte count then pairs + 0xc4, // BIN8 + 0xc5, // BIN16 + 0xc6, // BIN32 + 0xd4, // FIXEXT1 + 0xd5, // FIXEXT2 + 0xd6, // FIXEXT4 + 0xd7, // FIXEXT8 + 0xd8, // FIXEXT16 + ]; + for &tag in tags_need_extra { + // Single byte (completely truncated payload) + let buf = [tag]; + let _ = skip_value(&buf, 0); + let _ = read_f64(&buf, 0); + let _ = read_i64(&buf, 0); + let _ = read_value(&buf, 0); + + // Tag + one garbage byte + for second in [0x00u8, 0x01, 0x7f, 0x80, 0xff] { + let buf = [tag, second]; + let _ = skip_value(&buf, 0); + let _ = read_f64(&buf, 0); + let _ = read_i64(&buf, 0); + let _ = read_value(&buf, 0); + } + } + } + + /// Deterministic pseudo-random byte sequences must not cause panics. + #[test] + fn fuzz_deterministic_random_payloads() { + // Generate deterministic sequences without external crates using a + // simple LCG (Knuth multiplicative hash). + let mut state: u64 = 0xdeadbeef_cafebabe; + let next = |s: &mut u64| -> u8 { + *s = s + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + (*s >> 33) as u8 + }; + + let mut buf = vec![0u8; 256]; + for _ in 0..2000 { + // Randomize buffer length (1..=256) and contents + let len = (next(&mut state) as usize % 256) + 1; + for b in buf[..len].iter_mut() { + *b = next(&mut state); + } + let slice = &buf[..len]; + + // Try reading from multiple offsets + for offset in [0, 1, len / 2, len.saturating_sub(1)] { + let _ = skip_value(slice, offset); + let _ = read_f64(slice, offset); + let _ = read_i64(slice, offset); + let _ = read_str(slice, offset); + let _ = read_bool(slice, offset); + let _ = read_null(slice, offset); + let _ = map_header(slice, offset); + let _ = array_header(slice, offset); + let _ = read_value(slice, offset); + } + } + } + + /// Truncate a valid msgpack buffer at every byte position. + /// All reader functions must return `None` — never panic. + #[test] + fn fuzz_truncated_valid_payloads() { + let docs = [ + json!({"key": "value", "num": 42, "flag": true}), + json!({"nested": {"a": 1, "b": [1, 2, 3]}}), + json!([1, "two", 3.0, null, false]), + json!({"large": 9999999999_i64}), + json!({"float": 1.23456789}), + ]; + + for doc in &docs { + let full = encode(doc); + // Truncate at every position from 0 to full.len()-1 + for truncate_at in 0..full.len() { + let slice = &full[..truncate_at]; + // None of these may panic; result doesn't matter + let _ = skip_value(slice, 0); + let _ = read_f64(slice, 0); + let _ = read_i64(slice, 0); + let _ = read_str(slice, 0); + let _ = read_bool(slice, 0); + let _ = map_header(slice, 0); + let _ = array_header(slice, 0); + let _ = read_value(slice, 0); + } + } + } + + /// The never-used 0xc1 tag must return `None` for all functions. + #[test] + fn fuzz_never_used_tag_c1() { + // 0xc1 is explicitly "never used" in the msgpack spec + let buf = [0xc1u8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; + assert_eq!( + skip_value(&buf, 0), + None, + "0xc1 must return None from skip_value" + ); + assert_eq!(read_f64(&buf, 0), None); + assert_eq!(read_i64(&buf, 0), None); + assert_eq!(read_str(&buf, 0), None); + assert_eq!(read_bool(&buf, 0), None); + assert_eq!(map_header(&buf, 0), None); + assert_eq!(array_header(&buf, 0), None); + assert_eq!(read_value(&buf, 0), None); + } + + /// Out-of-bounds offset must return `None` — not panic. + #[test] + fn fuzz_out_of_bounds_offset() { + let buf = encode(&json!({"x": 1})); + let way_out = buf.len() + 1000; + assert_eq!(skip_value(&buf, way_out), None); + assert_eq!(read_f64(&buf, way_out), None); + assert_eq!(read_i64(&buf, way_out), None); + assert_eq!(read_str(&buf, way_out), None); + assert_eq!(read_bool(&buf, way_out), None); + assert_eq!(map_header(&buf, way_out), None); + assert_eq!(array_header(&buf, way_out), None); + assert_eq!(read_value(&buf, way_out), None); + } + + /// Empty buffer must return `None` for all functions that can. + #[test] + fn fuzz_empty_buffer() { + let buf: &[u8] = &[]; + assert_eq!(skip_value(buf, 0), None); + assert_eq!(read_f64(buf, 0), None); + assert_eq!(read_i64(buf, 0), None); + assert_eq!(read_str(buf, 0), None); + assert_eq!(read_bool(buf, 0), None); + assert!(!read_null(buf, 0)); // returns bool, not Option + assert_eq!(map_header(buf, 0), None); + assert_eq!(array_header(buf, 0), None); + assert_eq!(read_value(buf, 0), None); + } +} diff --git a/nodedb-query/src/msgpack_scan/writer.rs b/nodedb-query/src/msgpack_scan/writer.rs index 876eee582..622f88c12 100644 --- a/nodedb-query/src/msgpack_scan/writer.rs +++ b/nodedb-query/src/msgpack_scan/writer.rs @@ -7,6 +7,8 @@ //! Follow the timeseries ingest pattern: `SqlValue → row_to_msgpack()` at ingress, //! raw msgpack throughout, `msgpack_to_json_string()` at outermost pgwire/HTTP layer. +use nodedb_types::InstantKind; + /// Write a msgpack map header. #[inline] pub fn write_map_header(buf: &mut Vec, len: usize) { @@ -96,6 +98,12 @@ pub fn write_null(buf: &mut Vec) { buf.push(0xC0); } +/// Write a typed instant as the ten-byte instant ext (`fixext8` type 1 / 2). +#[inline] +pub fn write_instant(buf: &mut Vec, kind: InstantKind, micros: i64) { + nodedb_types::write_instant(buf, kind, micros); +} + /// Write a msgpack binary blob (bin 8/16/32). #[inline] pub fn write_bin(buf: &mut Vec, data: &[u8]) { @@ -161,6 +169,13 @@ pub fn write_kv_null(buf: &mut Vec, key: &str) { write_null(buf); } +/// Write a key-value pair (string key, instant value). +#[inline] +pub fn write_kv_instant(buf: &mut Vec, key: &str, kind: InstantKind, micros: i64) { + write_str(buf, key); + write_instant(buf, kind, micros); +} + /// Inject a string field into a msgpack map without full decode. /// /// If `value` is a valid msgpack map that already contains `field_name`, @@ -352,4 +367,20 @@ mod tests { write_i64(&mut buf, -100); assert_eq!(buf, vec![0xD0, (-100i8) as u8]); } + + #[test] + fn instant_cell_reads_back() { + let mut buf = Vec::new(); + write_map_header(&mut buf, 1); + write_kv_instant(&mut buf, "ts", InstantKind::Naive, -42); + + let field = crate::msgpack_scan::field::extract_field(&buf, 0, "ts").unwrap(); + assert_eq!(field.1 - field.0, 10); + assert_eq!( + crate::msgpack_scan::reader::read_value(&buf, field.0), + Some(nodedb_types::Value::NaiveDateTime( + nodedb_types::NdbDateTime::from_micros(-42) + )) + ); + } } diff --git a/nodedb-types/src/json_msgpack/instant_ext.rs b/nodedb-types/src/json_msgpack/instant_ext.rs new file mode 100644 index 000000000..3b5cf99bf --- /dev/null +++ b/nodedb-types/src/json_msgpack/instant_ext.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Msgpack ext encoding for typed instants. +//! +//! A typed instant is written to plain msgpack as `fixext8`: marker `0xD7`, +//! one ext type byte, then epoch microseconds as big-endian `i64`. Ten bytes +//! in total. Ext type `1` is a UTC instant (`Value::DateTime`), ext type `2` +//! is a naive instant (`Value::NaiveDateTime`). +//! +//! The payload is byte-comparable within one kind for non-negative micros +//! only. Comparators decode and compare `(kind, i64)` signed. + +use serde::{Deserialize, Serialize}; + +use crate::datetime::{NdbDateTime, NdbDateTimeError}; +use crate::value::Value; + +/// Ext type byte for a UTC instant. +pub const EXT_INSTANT_UTC: i8 = 1; +/// Ext type byte for a naive (timezone-less) instant. +pub const EXT_INSTANT_NAIVE: i8 = 2; +/// Encoded length of an instant: marker + ext type + `i64` payload. +pub const INSTANT_EXT_LEN: usize = 10; + +/// Msgpack `fixext8` marker. +const FIXEXT8: u8 = 0xD7; + +/// Which instant variant an ext payload carries. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum InstantKind { + /// Timezone-aware instant, `Value::DateTime`. + Utc, + /// Timezone-less instant, `Value::NaiveDateTime`. + Naive, +} + +impl InstantKind { + /// Ext type byte for this kind. + pub fn ext_type(self) -> i8 { + match self { + Self::Utc => EXT_INSTANT_UTC, + Self::Naive => EXT_INSTANT_NAIVE, + } + } + + /// Kind for an ext type byte. `None` when the byte is not an instant type. + pub fn from_ext_type(t: i8) -> Option { + match t { + EXT_INSTANT_UTC => Some(Self::Utc), + EXT_INSTANT_NAIVE => Some(Self::Naive), + _ => None, + } + } + + /// Wrap a timestamp in the `Value` variant for this kind. + pub fn value(self, dt: NdbDateTime) -> Value { + match self { + Self::Utc => Value::DateTime(dt), + Self::Naive => Value::NaiveDateTime(dt), + } + } + + /// Build the `Value` for this kind from epoch milliseconds. + pub fn from_millis(self, ms: i64) -> Result { + Ok(self.value(NdbDateTime::from_millis(ms)?)) + } + + /// Build the `Value` for this kind from epoch microseconds. + pub fn from_micros(self, micros: i64) -> Value { + self.value(NdbDateTime::from_micros(micros)) + } +} + +/// Append the ten-byte `fixext8` encoding of an instant to `buf`. +pub fn write_instant(buf: &mut Vec, kind: InstantKind, micros: i64) { + buf.push(FIXEXT8); + buf.push(kind.ext_type() as u8); + buf.extend_from_slice(µs.to_be_bytes()); +} + +/// Decode an instant from its ext type byte and eight-byte payload. +/// +/// `None` when the type byte is not an instant type or the payload is not +/// exactly eight bytes. +pub fn instant_from_ext(ext_type: i8, payload: &[u8]) -> Option<(InstantKind, i64)> { + let kind = InstantKind::from_ext_type(ext_type)?; + let bytes: [u8; 8] = payload.try_into().ok()?; + Some((kind, i64::from_be_bytes(bytes))) +} + +/// Decode the instant at `offset` in `bytes`. +/// +/// `None` when the bytes at `offset` are not a `fixext8` of an instant type +/// or the buffer is truncated. +pub fn read_instant(bytes: &[u8], offset: usize) -> Option<(InstantKind, i64)> { + let ext = bytes.get(offset..offset + INSTANT_EXT_LEN)?; + if ext[0] != FIXEXT8 { + return None; + } + instant_from_ext(ext[1] as i8, &ext[2..]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip_both_kinds() { + for kind in [InstantKind::Utc, InstantKind::Naive] { + let mut buf = Vec::new(); + write_instant(&mut buf, kind, 1_700_000_000_123_456); + assert_eq!(buf.len(), INSTANT_EXT_LEN); + assert_eq!(buf[0], 0xD7); + assert_eq!(buf[1] as i8, kind.ext_type()); + assert_eq!(read_instant(&buf, 0), Some((kind, 1_700_000_000_123_456))); + } + } + + #[test] + fn negative_micros_round_trip() { + let mut buf = vec![0xC0]; + write_instant(&mut buf, InstantKind::Naive, -86_400_000_000); + assert_eq!( + read_instant(&buf, 1), + Some((InstantKind::Naive, -86_400_000_000)) + ); + } + + #[test] + fn non_ext_byte_is_none() { + let buf = [0x2A, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + assert_eq!(read_instant(&buf, 0), None); + } + + #[test] + fn unknown_ext_type_is_none() { + let buf = [0xD7, 0x07, 0, 0, 0, 0, 0, 0, 0, 0]; + assert_eq!(read_instant(&buf, 0), None); + } + + #[test] + fn truncated_is_none() { + let mut buf = Vec::new(); + write_instant(&mut buf, InstantKind::Utc, 42); + buf.truncate(9); + assert_eq!(read_instant(&buf, 0), None); + assert_eq!(instant_from_ext(EXT_INSTANT_UTC, &buf[2..]), None); + } + + #[test] + fn kind_value_mapping() { + let dt = NdbDateTime::from_micros(5); + assert_eq!(InstantKind::Utc.value(dt), Value::DateTime(dt)); + assert_eq!(InstantKind::Naive.value(dt), Value::NaiveDateTime(dt)); + assert_eq!( + Value::DateTime(dt).as_instant(), + Some((InstantKind::Utc, dt)) + ); + assert_eq!( + Value::NaiveDateTime(dt).as_instant(), + Some((InstantKind::Naive, dt)) + ); + assert_eq!(Value::Integer(5).as_instant(), None); + assert_eq!( + InstantKind::Utc.from_millis(3).unwrap(), + Value::DateTime(NdbDateTime::from_micros(3_000)) + ); + assert!(InstantKind::Utc.from_millis(i64::MAX).is_err()); + } +} diff --git a/nodedb-types/src/json_msgpack/mod.rs b/nodedb-types/src/json_msgpack/mod.rs index 5f67b9789..4a0122149 100644 --- a/nodedb-types/src/json_msgpack/mod.rs +++ b/nodedb-types/src/json_msgpack/mod.rs @@ -1,12 +1,17 @@ // SPDX-License-Identifier: Apache-2.0 pub mod error; +pub mod instant_ext; pub mod json_value; pub mod reader; pub mod transcoder; pub mod writer; pub use error::{MsgpackError, MsgpackResult}; +pub use instant_ext::{ + EXT_INSTANT_NAIVE, EXT_INSTANT_UTC, INSTANT_EXT_LEN, InstantKind, instant_from_ext, + read_instant, write_instant, +}; pub use json_value::JsonValue; pub use reader::{json_from_msgpack, value_from_msgpack}; pub use transcoder::msgpack_to_json_string; diff --git a/nodedb-types/src/json_msgpack/reader.rs b/nodedb-types/src/json_msgpack/reader.rs deleted file mode 100644 index cdec98d6d..000000000 --- a/nodedb-types/src/json_msgpack/reader.rs +++ /dev/null @@ -1,682 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Cursor-based msgpack → `serde_json::Value` and `nodedb_types::Value` readers. -//! -//! Deterministic raw byte parser — the first byte of each msgpack value -//! unambiguously identifies its type per the msgpack specification. - -use super::error::{MsgpackError, MsgpackResult}; - -pub(crate) struct Cursor<'a> { - pub(crate) data: &'a [u8], - pub(crate) pos: usize, - pub(crate) depth: usize, -} - -impl<'a> Cursor<'a> { - pub(crate) fn new(data: &'a [u8]) -> Self { - Self { - data, - pos: 0, - depth: 0, - } - } - - #[inline] - pub(crate) fn peek(&self) -> zerompk::Result { - self.data - .get(self.pos) - .copied() - .ok_or(zerompk::Error::BufferTooSmall) - } - - #[inline] - pub(crate) fn take(&mut self) -> zerompk::Result { - let b = self.peek()?; - self.pos += 1; - Ok(b) - } - - #[inline] - pub(crate) fn take_n(&mut self, n: usize) -> zerompk::Result<&'a [u8]> { - if self.pos + n > self.data.len() { - return Err(zerompk::Error::BufferTooSmall); - } - let slice = &self.data[self.pos..self.pos + n]; - self.pos += n; - Ok(slice) - } - - pub(crate) fn read_u16_be(&mut self) -> zerompk::Result { - let b = self.take_n(2)?; - Ok(u16::from_be_bytes([b[0], b[1]])) - } - - pub(crate) fn read_u32_be(&mut self) -> zerompk::Result { - let b = self.take_n(4)?; - Ok(u32::from_be_bytes([b[0], b[1], b[2], b[3]])) - } - - /// Assert the whole input belonged to the value just read. - /// - /// A body carries exactly one top-level value, so anything left over means - /// the bytes are not the value they claim to be — a stray suffix, a - /// truncated body with another appended, or two values in one slot. - /// Returning the leading value and dropping the rest would report success - /// on corrupt bytes, so the remainder is an error. - pub(crate) fn finish(&self) -> MsgpackResult<()> { - if self.pos == self.data.len() { - Ok(()) - } else { - Err(MsgpackError::TrailingBytes { - consumed: self.pos, - total: self.data.len(), - }) - } - } -} - -/// Deserialize a `serde_json::Value` from MessagePack bytes. -/// -/// The input must contain exactly one top-level value and nothing else; -/// trailing bytes are rejected. -pub fn json_from_msgpack(bytes: &[u8]) -> MsgpackResult { - let mut cursor = Cursor::new(bytes); - let value = read_json_value(&mut cursor)?; - cursor.finish()?; - Ok(value) -} - -/// Deserialize a `nodedb_types::Value` from standard MessagePack bytes. -/// -/// The input must contain exactly one top-level value and nothing else; -/// trailing bytes are rejected. -pub fn value_from_msgpack(bytes: &[u8]) -> MsgpackResult { - let mut cursor = Cursor::new(bytes); - let value = read_native_value(&mut cursor)?; - cursor.finish()?; - Ok(value) -} - -// ── JSON value reader ── - -fn read_json_value(c: &mut Cursor<'_>) -> zerompk::Result { - if c.depth > 500 { - return Err(zerompk::Error::DepthLimitExceeded { max: 500 }); - } - - let marker = c.take()?; - match marker { - 0xC0 => Ok(serde_json::Value::Null), - 0xC2 => Ok(serde_json::Value::Bool(false)), - 0xC3 => Ok(serde_json::Value::Bool(true)), - - 0x00..=0x7F => Ok(serde_json::Value::Number((marker as i64).into())), - 0xE0..=0xFF => Ok(serde_json::Value::Number((marker as i8 as i64).into())), - - 0xCC => Ok(serde_json::Value::Number(c.take()?.into())), - 0xCD => Ok(serde_json::Value::Number(c.read_u16_be()?.into())), - 0xCE => Ok(serde_json::Value::Number(c.read_u32_be()?.into())), - 0xCF => { - let b = c.take_n(8)?; - let v = u64::from_be_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]); - Ok(serde_json::Value::Number(v.into())) - } - - 0xD0 => Ok(serde_json::Value::Number((c.take()? as i8 as i64).into())), - 0xD1 => { - let b = c.take_n(2)?; - Ok(serde_json::Value::Number( - (i16::from_be_bytes([b[0], b[1]]) as i64).into(), - )) - } - 0xD2 => { - let b = c.take_n(4)?; - Ok(serde_json::Value::Number( - (i32::from_be_bytes([b[0], b[1], b[2], b[3]]) as i64).into(), - )) - } - 0xD3 => { - let b = c.take_n(8)?; - Ok(serde_json::Value::Number( - i64::from_be_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]).into(), - )) - } - - 0xCA => { - let b = c.take_n(4)?; - Ok(serde_json::json!( - f32::from_be_bytes([b[0], b[1], b[2], b[3]]) as f64 - )) - } - 0xCB => { - let b = c.take_n(8)?; - Ok(serde_json::json!(f64::from_be_bytes([ - b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7] - ]))) - } - - m @ 0xA0..=0xBF => read_json_str(c, (m & 0x1F) as usize), - 0xD9 => { - let l = c.take()? as usize; - read_json_str(c, l) - } - 0xDA => { - let l = c.read_u16_be()? as usize; - read_json_str(c, l) - } - 0xDB => { - let l = c.read_u32_be()? as usize; - read_json_str(c, l) - } - - 0xC4 => { - let l = c.take()? as usize; - Ok(serde_json::Value::String(base64_encode(c.take_n(l)?))) - } - 0xC5 => { - let l = c.read_u16_be()? as usize; - Ok(serde_json::Value::String(base64_encode(c.take_n(l)?))) - } - 0xC6 => { - let l = c.read_u32_be()? as usize; - Ok(serde_json::Value::String(base64_encode(c.take_n(l)?))) - } - - m @ 0x90..=0x9F => read_json_array(c, (m & 0x0F) as usize), - 0xDC => { - let l = c.read_u16_be()? as usize; - read_json_array(c, l) - } - 0xDD => { - let l = c.read_u32_be()? as usize; - read_json_array(c, l) - } - - m @ 0x80..=0x8F => read_json_map(c, (m & 0x0F) as usize), - 0xDE => { - let l = c.read_u16_be()? as usize; - read_json_map(c, l) - } - 0xDF => { - let l = c.read_u32_be()? as usize; - read_json_map(c, l) - } - - // ext types — skip - 0xD4 => { - c.take_n(2)?; - Ok(serde_json::Value::Null) - } - 0xD5 => { - c.take_n(3)?; - Ok(serde_json::Value::Null) - } - 0xD6 => { - c.take_n(5)?; - Ok(serde_json::Value::Null) - } - 0xD7 => { - c.take_n(9)?; - Ok(serde_json::Value::Null) - } - 0xD8 => { - c.take_n(17)?; - Ok(serde_json::Value::Null) - } - 0xC7 => { - let l = c.take()? as usize; - c.take_n(1 + l)?; - Ok(serde_json::Value::Null) - } - 0xC8 => { - let l = c.read_u16_be()? as usize; - c.take_n(1 + l)?; - Ok(serde_json::Value::Null) - } - 0xC9 => { - let l = c.read_u32_be()? as usize; - c.take_n(1 + l)?; - Ok(serde_json::Value::Null) - } - - _ => Err(zerompk::Error::InvalidMarker(marker)), - } -} - -fn read_json_str(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - let bytes = c.take_n(len)?; - let s = String::from_utf8(bytes.to_vec()).map_err(|_| zerompk::Error::InvalidMarker(0))?; - Ok(serde_json::Value::String(s)) -} - -fn read_json_array(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - c.depth += 1; - let mut arr = Vec::with_capacity(len.min(4096)); - for _ in 0..len { - arr.push(read_json_value(c)?); - } - c.depth -= 1; - Ok(serde_json::Value::Array(arr)) -} - -fn read_json_map(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - c.depth += 1; - let mut map = serde_json::Map::with_capacity(len.min(4096)); - for _ in 0..len { - let key_marker = c.peek()?; - let key = if (0xA0..=0xBF).contains(&key_marker) - || key_marker == 0xD9 - || key_marker == 0xDA - || key_marker == 0xDB - { - match read_json_value(c)? { - serde_json::Value::String(s) => s, - other => other.to_string(), - } - } else { - read_json_value(c)?.to_string() - }; - let val = read_json_value(c)?; - map.insert(key, val); - } - c.depth -= 1; - Ok(serde_json::Value::Object(map)) -} - -// ── Native value reader ── - -fn read_native_value(c: &mut Cursor<'_>) -> zerompk::Result { - if c.depth > 500 { - return Err(zerompk::Error::DepthLimitExceeded { max: 500 }); - } - - let marker = c.take()?; - match marker { - 0xC0 => Ok(crate::Value::Null), - 0xC2 => Ok(crate::Value::Bool(false)), - 0xC3 => Ok(crate::Value::Bool(true)), - - 0x00..=0x7F => Ok(crate::Value::Integer(marker as i64)), - 0xE0..=0xFF => Ok(crate::Value::Integer(marker as i8 as i64)), - - 0xCC => Ok(crate::Value::Integer(c.take()? as i64)), - 0xCD => Ok(crate::Value::Integer(c.read_u16_be()? as i64)), - 0xCE => Ok(crate::Value::Integer(c.read_u32_be()? as i64)), - 0xCF => { - let b = c.take_n(8)?; - Ok(crate::Value::Integer(u64::from_be_bytes([ - b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], - ]) as i64)) - } - - 0xD0 => Ok(crate::Value::Integer(c.take()? as i8 as i64)), - 0xD1 => { - let b = c.take_n(2)?; - Ok(crate::Value::Integer( - i16::from_be_bytes([b[0], b[1]]) as i64 - )) - } - 0xD2 => { - let b = c.take_n(4)?; - Ok(crate::Value::Integer( - i32::from_be_bytes([b[0], b[1], b[2], b[3]]) as i64, - )) - } - 0xD3 => { - let b = c.take_n(8)?; - Ok(crate::Value::Integer(i64::from_be_bytes([ - b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], - ]))) - } - - 0xCA => { - let b = c.take_n(4)?; - Ok(crate::Value::Float( - f32::from_be_bytes([b[0], b[1], b[2], b[3]]) as f64, - )) - } - 0xCB => { - let b = c.take_n(8)?; - Ok(crate::Value::Float(f64::from_be_bytes([ - b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], - ]))) - } - - m @ 0xA0..=0xBF => read_native_str(c, (m & 0x1F) as usize), - 0xD9 => { - let l = c.take()? as usize; - read_native_str(c, l) - } - 0xDA => { - let l = c.read_u16_be()? as usize; - read_native_str(c, l) - } - 0xDB => { - let l = c.read_u32_be()? as usize; - read_native_str(c, l) - } - - 0xC4 => { - let l = c.take()? as usize; - Ok(crate::Value::Bytes(c.take_n(l)?.to_vec())) - } - 0xC5 => { - let l = c.read_u16_be()? as usize; - Ok(crate::Value::Bytes(c.take_n(l)?.to_vec())) - } - 0xC6 => { - let l = c.read_u32_be()? as usize; - Ok(crate::Value::Bytes(c.take_n(l)?.to_vec())) - } - - m @ 0x90..=0x9F => read_native_array(c, (m & 0x0F) as usize), - 0xDC => { - let l = c.read_u16_be()? as usize; - read_native_array(c, l) - } - 0xDD => { - let l = c.read_u32_be()? as usize; - read_native_array(c, l) - } - - m @ 0x80..=0x8F => read_native_map(c, (m & 0x0F) as usize), - 0xDE => { - let l = c.read_u16_be()? as usize; - read_native_map(c, l) - } - 0xDF => { - let l = c.read_u32_be()? as usize; - read_native_map(c, l) - } - - // ext types — skip - 0xD4 => { - c.take_n(2)?; - Ok(crate::Value::Null) - } - 0xD5 => { - c.take_n(3)?; - Ok(crate::Value::Null) - } - 0xD6 => { - c.take_n(5)?; - Ok(crate::Value::Null) - } - 0xD7 => { - c.take_n(9)?; - Ok(crate::Value::Null) - } - 0xD8 => { - c.take_n(17)?; - Ok(crate::Value::Null) - } - 0xC7 => { - let l = c.take()? as usize; - c.take_n(1 + l)?; - Ok(crate::Value::Null) - } - 0xC8 => { - let l = c.read_u16_be()? as usize; - c.take_n(1 + l)?; - Ok(crate::Value::Null) - } - 0xC9 => { - let l = c.read_u32_be()? as usize; - c.take_n(1 + l)?; - Ok(crate::Value::Null) - } - - _ => Err(zerompk::Error::InvalidMarker(marker)), - } -} - -fn read_native_str(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - let bytes = c.take_n(len)?; - let s = String::from_utf8(bytes.to_vec()).map_err(|_| zerompk::Error::InvalidMarker(0))?; - Ok(crate::Value::String(s)) -} - -fn read_native_array(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - c.depth += 1; - let mut arr = Vec::with_capacity(len.min(4096)); - for _ in 0..len { - arr.push(read_native_value(c)?); - } - c.depth -= 1; - Ok(crate::Value::Array(arr)) -} - -fn read_native_map(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - c.depth += 1; - let mut map = std::collections::HashMap::with_capacity(len.min(4096)); - for _ in 0..len { - let key_marker = c.peek()?; - let key = if (0xA0..=0xBF).contains(&key_marker) - || key_marker == 0xD9 - || key_marker == 0xDA - || key_marker == 0xDB - { - match read_native_value(c)? { - crate::Value::String(s) => s, - other => format!("{other:?}"), - } - } else { - let v = read_native_value(c)?; - format!("{v:?}") - }; - let val = read_native_value(c)?; - map.insert(key, val); - } - c.depth -= 1; - Ok(crate::Value::Object(map)) -} - -// ── Shared helpers ── - -pub(crate) fn base64_encode(data: &[u8]) -> String { - use std::fmt::Write; - const CHARS: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - let mut out = String::with_capacity(data.len().div_ceil(3) * 4); - for chunk in data.chunks(3) { - let b0 = chunk[0] as u32; - let b1 = chunk.get(1).copied().unwrap_or(0) as u32; - let b2 = chunk.get(2).copied().unwrap_or(0) as u32; - let triple = (b0 << 16) | (b1 << 8) | b2; - let _ = write!(out, "{}", CHARS[((triple >> 18) & 0x3F) as usize] as char); - let _ = write!(out, "{}", CHARS[((triple >> 12) & 0x3F) as usize] as char); - if chunk.len() > 1 { - let _ = write!(out, "{}", CHARS[((triple >> 6) & 0x3F) as usize] as char); - } - if chunk.len() > 2 { - let _ = write!(out, "{}", CHARS[(triple & 0x3F) as usize] as char); - } - } - out -} - -#[cfg(test)] -mod tests { - //! Roundtrip tests for json_msgpack reader/writer. - - use super::*; - use crate::json_msgpack::msgpack_to_json_string; - use crate::json_msgpack::writer::{json_to_msgpack, value_to_msgpack}; - use serde_json::json; - - #[test] - fn roundtrip_null() { - let val = json!(null); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn roundtrip_bool() { - for val in [json!(true), json!(false)] { - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - } - - #[test] - fn roundtrip_integers() { - for val in [ - json!(0), - json!(42), - json!(-1), - json!(i64::MAX), - json!(i64::MIN), - ] { - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - } - - #[test] - fn roundtrip_float() { - let val = json!(9.81); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn roundtrip_string() { - let val = json!("hello world"); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn roundtrip_array() { - let val = json!([1, "two", true, null, 2.72]); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn roundtrip_nested_object() { - let val = json!({"a": 1, "b": {"c": [2, 3]}, "d": null}); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn roundtrip_empty_map() { - let val = json!({}); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn roundtrip_empty_array() { - let val = json!([]); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn roundtrip_large_string() { - let s = "x".repeat(300); - let val = json!(s); - let bytes = json_to_msgpack(&val).unwrap(); - let restored = json_from_msgpack(&bytes).unwrap(); - assert_eq!(val, restored); - } - - #[test] - fn native_value_roundtrip() { - let mut map = std::collections::HashMap::new(); - map.insert("id".to_string(), crate::Value::String("host1".into())); - map.insert("cpu".to_string(), crate::Value::Float(0.75)); - map.insert("mem".to_string(), crate::Value::Float(0.5)); - - let row = crate::Value::Object(map); - let arr = crate::Value::Array(vec![row]); - - let bytes = value_to_msgpack(&arr).unwrap(); - let decoded = value_from_msgpack(&bytes).unwrap(); - - match &decoded { - crate::Value::Array(items) => { - assert_eq!(items.len(), 1); - match &items[0] { - crate::Value::Object(m) => { - assert_eq!(m.len(), 3); - assert_eq!(m.get("id"), Some(&crate::Value::String("host1".into()))); - assert_eq!(m.get("cpu"), Some(&crate::Value::Float(0.75))); - assert_eq!(m.get("mem"), Some(&crate::Value::Float(0.5))); - } - other => panic!("expected Object, got {other:?}"), - } - } - other => panic!("expected Array, got {other:?}"), - } - } - - /// The readers decode exactly one top-level value. Bytes left over mean the - /// input is not the value it claims to be, so returning the leading value would - /// report success on corrupt input. - #[test] - fn trailing_byte_is_rejected() { - let val = json!({"a": 1, "b": "two"}); - let mut bytes = json_to_msgpack(&val).unwrap(); - assert_eq!(json_from_msgpack(&bytes).unwrap(), val); - - bytes.push(0xC0); - match json_from_msgpack(&bytes) { - Err(MsgpackError::TrailingBytes { consumed, total }) => { - assert_eq!(total, consumed + 1); - } - other => panic!("expected TrailingBytes, got {other:?}"), - } - assert!(value_from_msgpack(&bytes).is_err()); - assert!(msgpack_to_json_string(&bytes).is_err()); - } - - #[test] - fn two_concatenated_values_are_rejected() { - let first = json_to_msgpack(&json!({"a": 1})).unwrap(); - let second = json_to_msgpack(&json!({"b": 2})).unwrap(); - let mut joined = first.clone(); - joined.extend_from_slice(&second); - - assert!(json_from_msgpack(&first).is_ok()); - assert!(json_from_msgpack(&joined).is_err()); - assert!(value_from_msgpack(&joined).is_err()); - assert!(msgpack_to_json_string(&joined).is_err()); - } - - #[test] - fn empty_input_is_unchanged() { - // Readers still fail on empty input; the transcoder still yields "". - assert!(json_from_msgpack(&[]).is_err()); - assert!(value_from_msgpack(&[]).is_err()); - assert_eq!(msgpack_to_json_string(&[]).unwrap(), ""); - } - - #[test] - fn native_value_scalars() { - let cases: Vec = vec![ - crate::Value::Null, - crate::Value::Bool(true), - crate::Value::Integer(42), - crate::Value::Float(2.72), - crate::Value::String("hello".into()), - ]; - for val in cases { - let bytes = value_to_msgpack(&val).unwrap(); - let decoded = value_from_msgpack(&bytes).unwrap(); - assert_eq!(val, decoded); - } - } -} diff --git a/nodedb-types/src/json_msgpack/reader/cursor.rs b/nodedb-types/src/json_msgpack/reader/cursor.rs new file mode 100644 index 000000000..4ee11edb1 --- /dev/null +++ b/nodedb-types/src/json_msgpack/reader/cursor.rs @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Byte cursor shared by the msgpack readers and the JSON transcoder. + +use super::super::error::{MsgpackError, MsgpackResult}; + +pub(crate) struct Cursor<'a> { + pub(crate) data: &'a [u8], + pub(crate) pos: usize, + pub(crate) depth: usize, +} + +impl<'a> Cursor<'a> { + pub(crate) fn new(data: &'a [u8]) -> Self { + Self { + data, + pos: 0, + depth: 0, + } + } + + #[inline] + pub(crate) fn peek(&self) -> zerompk::Result { + self.data + .get(self.pos) + .copied() + .ok_or(zerompk::Error::BufferTooSmall) + } + + #[inline] + pub(crate) fn take(&mut self) -> zerompk::Result { + let b = self.peek()?; + self.pos += 1; + Ok(b) + } + + #[inline] + pub(crate) fn take_n(&mut self, n: usize) -> zerompk::Result<&'a [u8]> { + if self.pos + n > self.data.len() { + return Err(zerompk::Error::BufferTooSmall); + } + let slice = &self.data[self.pos..self.pos + n]; + self.pos += n; + Ok(slice) + } + + pub(crate) fn read_u16_be(&mut self) -> zerompk::Result { + let b = self.take_n(2)?; + Ok(u16::from_be_bytes([b[0], b[1]])) + } + + pub(crate) fn read_u32_be(&mut self) -> zerompk::Result { + let b = self.take_n(4)?; + Ok(u32::from_be_bytes([b[0], b[1], b[2], b[3]])) + } + + /// Read the ext type byte and eight-byte payload of a `fixext8` whose + /// marker is already consumed. + pub(crate) fn take_fixext8(&mut self) -> zerompk::Result<(i8, &'a [u8])> { + let ext_type = self.take()? as i8; + let payload = self.take_n(8)?; + Ok((ext_type, payload)) + } + + /// Assert the whole input belonged to the value just read. + /// + /// A body carries exactly one top-level value, so anything left over means + /// the bytes are not the value they claim to be — a stray suffix, a + /// truncated body with another appended, or two values in one slot. + /// Returning the leading value and dropping the rest would report success + /// on corrupt bytes, so the remainder is an error. + pub(crate) fn finish(&self) -> MsgpackResult<()> { + if self.pos == self.data.len() { + Ok(()) + } else { + Err(MsgpackError::TrailingBytes { + consumed: self.pos, + total: self.data.len(), + }) + } + } +} + +#[cfg(test)] +mod tests { + use crate::json_msgpack::error::MsgpackError; + use crate::json_msgpack::reader::{json_from_msgpack, value_from_msgpack}; + use crate::json_msgpack::transcoder::msgpack_to_json_string; + use crate::json_msgpack::writer::json_to_msgpack; + use serde_json::json; + + /// The readers decode exactly one top-level value. Bytes left over mean the + /// input is not the value it claims to be, so returning the leading value would + /// report success on corrupt input. + #[test] + fn trailing_byte_is_rejected() { + let val = json!({"a": 1, "b": "two"}); + let mut bytes = json_to_msgpack(&val).unwrap(); + assert_eq!(json_from_msgpack(&bytes).unwrap(), val); + + bytes.push(0xC0); + match json_from_msgpack(&bytes) { + Err(MsgpackError::TrailingBytes { consumed, total }) => { + assert_eq!(total, consumed + 1); + } + other => panic!("expected TrailingBytes, got {other:?}"), + } + assert!(value_from_msgpack(&bytes).is_err()); + assert!(msgpack_to_json_string(&bytes).is_err()); + } + + #[test] + fn two_concatenated_values_are_rejected() { + let first = json_to_msgpack(&json!({"a": 1})).unwrap(); + let second = json_to_msgpack(&json!({"b": 2})).unwrap(); + let mut joined = first.clone(); + joined.extend_from_slice(&second); + + assert!(json_from_msgpack(&first).is_ok()); + assert!(json_from_msgpack(&joined).is_err()); + assert!(value_from_msgpack(&joined).is_err()); + assert!(msgpack_to_json_string(&joined).is_err()); + } + + #[test] + fn empty_input_is_unchanged() { + // Readers fail on empty input; the transcoder yields "". + assert!(json_from_msgpack(&[]).is_err()); + assert!(value_from_msgpack(&[]).is_err()); + assert_eq!(msgpack_to_json_string(&[]).unwrap(), ""); + } +} diff --git a/nodedb-types/src/json_msgpack/reader/json.rs b/nodedb-types/src/json_msgpack/reader/json.rs new file mode 100644 index 000000000..61f449e60 --- /dev/null +++ b/nodedb-types/src/json_msgpack/reader/json.rs @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Msgpack → `serde_json::Value` reader. +//! +//! Instant ext values (fixext8 type 1 / 2) become ISO 8601 strings. Every +//! other ext type becomes `null`. + +use super::super::error::MsgpackResult; +use super::super::instant_ext::instant_from_ext; +use super::cursor::Cursor; +use crate::datetime::NdbDateTime; + +/// Deserialize a `serde_json::Value` from MessagePack bytes. +/// +/// The input must contain exactly one top-level value and nothing else; +/// trailing bytes are rejected. +pub fn json_from_msgpack(bytes: &[u8]) -> MsgpackResult { + let mut cursor = Cursor::new(bytes); + let value = read_json_value(&mut cursor)?; + cursor.finish()?; + Ok(value) +} + +fn read_json_value(c: &mut Cursor<'_>) -> zerompk::Result { + if c.depth > 500 { + return Err(zerompk::Error::DepthLimitExceeded { max: 500 }); + } + + let marker = c.take()?; + match marker { + 0xC0 => Ok(serde_json::Value::Null), + 0xC2 => Ok(serde_json::Value::Bool(false)), + 0xC3 => Ok(serde_json::Value::Bool(true)), + + 0x00..=0x7F => Ok(serde_json::Value::Number((marker as i64).into())), + 0xE0..=0xFF => Ok(serde_json::Value::Number((marker as i8 as i64).into())), + + 0xCC => Ok(serde_json::Value::Number(c.take()?.into())), + 0xCD => Ok(serde_json::Value::Number(c.read_u16_be()?.into())), + 0xCE => Ok(serde_json::Value::Number(c.read_u32_be()?.into())), + 0xCF => { + let b = c.take_n(8)?; + let v = u64::from_be_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]); + Ok(serde_json::Value::Number(v.into())) + } + + 0xD0 => Ok(serde_json::Value::Number((c.take()? as i8 as i64).into())), + 0xD1 => { + let b = c.take_n(2)?; + Ok(serde_json::Value::Number( + (i16::from_be_bytes([b[0], b[1]]) as i64).into(), + )) + } + 0xD2 => { + let b = c.take_n(4)?; + Ok(serde_json::Value::Number( + (i32::from_be_bytes([b[0], b[1], b[2], b[3]]) as i64).into(), + )) + } + 0xD3 => { + let b = c.take_n(8)?; + Ok(serde_json::Value::Number( + i64::from_be_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]).into(), + )) + } + + 0xCA => { + let b = c.take_n(4)?; + Ok(serde_json::json!( + f32::from_be_bytes([b[0], b[1], b[2], b[3]]) as f64 + )) + } + 0xCB => { + let b = c.take_n(8)?; + Ok(serde_json::json!(f64::from_be_bytes([ + b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7] + ]))) + } + + m @ 0xA0..=0xBF => read_json_str(c, (m & 0x1F) as usize), + 0xD9 => { + let l = c.take()? as usize; + read_json_str(c, l) + } + 0xDA => { + let l = c.read_u16_be()? as usize; + read_json_str(c, l) + } + 0xDB => { + let l = c.read_u32_be()? as usize; + read_json_str(c, l) + } + + 0xC4 => { + let l = c.take()? as usize; + Ok(serde_json::Value::String(base64_encode(c.take_n(l)?))) + } + 0xC5 => { + let l = c.read_u16_be()? as usize; + Ok(serde_json::Value::String(base64_encode(c.take_n(l)?))) + } + 0xC6 => { + let l = c.read_u32_be()? as usize; + Ok(serde_json::Value::String(base64_encode(c.take_n(l)?))) + } + + m @ 0x90..=0x9F => read_json_array(c, (m & 0x0F) as usize), + 0xDC => { + let l = c.read_u16_be()? as usize; + read_json_array(c, l) + } + 0xDD => { + let l = c.read_u32_be()? as usize; + read_json_array(c, l) + } + + m @ 0x80..=0x8F => read_json_map(c, (m & 0x0F) as usize), + 0xDE => { + let l = c.read_u16_be()? as usize; + read_json_map(c, l) + } + 0xDF => { + let l = c.read_u32_be()? as usize; + read_json_map(c, l) + } + + // fixext8: instants render as ISO 8601, other types as null + 0xD7 => { + let (ext_type, payload) = c.take_fixext8()?; + Ok(match instant_from_ext(ext_type, payload) { + Some((_, micros)) => { + serde_json::Value::String(NdbDateTime::from_micros(micros).to_iso8601()) + } + None => serde_json::Value::Null, + }) + } + + // other ext types — skip + 0xD4 => { + c.take_n(2)?; + Ok(serde_json::Value::Null) + } + 0xD5 => { + c.take_n(3)?; + Ok(serde_json::Value::Null) + } + 0xD6 => { + c.take_n(5)?; + Ok(serde_json::Value::Null) + } + 0xD8 => { + c.take_n(17)?; + Ok(serde_json::Value::Null) + } + 0xC7 => { + let l = c.take()? as usize; + c.take_n(1 + l)?; + Ok(serde_json::Value::Null) + } + 0xC8 => { + let l = c.read_u16_be()? as usize; + c.take_n(1 + l)?; + Ok(serde_json::Value::Null) + } + 0xC9 => { + let l = c.read_u32_be()? as usize; + c.take_n(1 + l)?; + Ok(serde_json::Value::Null) + } + + _ => Err(zerompk::Error::InvalidMarker(marker)), + } +} + +fn read_json_str(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { + let bytes = c.take_n(len)?; + let s = String::from_utf8(bytes.to_vec()).map_err(|_| zerompk::Error::InvalidMarker(0))?; + Ok(serde_json::Value::String(s)) +} + +fn read_json_array(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { + c.depth += 1; + let mut arr = Vec::with_capacity(len.min(4096)); + for _ in 0..len { + arr.push(read_json_value(c)?); + } + c.depth -= 1; + Ok(serde_json::Value::Array(arr)) +} + +fn read_json_map(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { + c.depth += 1; + let mut map = serde_json::Map::with_capacity(len.min(4096)); + for _ in 0..len { + let key_marker = c.peek()?; + let key = if (0xA0..=0xBF).contains(&key_marker) + || key_marker == 0xD9 + || key_marker == 0xDA + || key_marker == 0xDB + { + match read_json_value(c)? { + serde_json::Value::String(s) => s, + other => other.to_string(), + } + } else { + read_json_value(c)?.to_string() + }; + let val = read_json_value(c)?; + map.insert(key, val); + } + c.depth -= 1; + Ok(serde_json::Value::Object(map)) +} + +pub(crate) fn base64_encode(data: &[u8]) -> String { + use std::fmt::Write; + const CHARS: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::with_capacity(data.len().div_ceil(3) * 4); + for chunk in data.chunks(3) { + let b0 = chunk[0] as u32; + let b1 = chunk.get(1).copied().unwrap_or(0) as u32; + let b2 = chunk.get(2).copied().unwrap_or(0) as u32; + let triple = (b0 << 16) | (b1 << 8) | b2; + let _ = write!(out, "{}", CHARS[((triple >> 18) & 0x3F) as usize] as char); + let _ = write!(out, "{}", CHARS[((triple >> 12) & 0x3F) as usize] as char); + if chunk.len() > 1 { + let _ = write!(out, "{}", CHARS[((triple >> 6) & 0x3F) as usize] as char); + } + if chunk.len() > 2 { + let _ = write!(out, "{}", CHARS[(triple & 0x3F) as usize] as char); + } + } + out +} + +#[cfg(test)] +mod tests { + //! Roundtrip tests for the JSON reader against the JSON writer. + + use super::*; + use crate::json_msgpack::instant_ext::{InstantKind, write_instant}; + use crate::json_msgpack::writer::json_to_msgpack; + use serde_json::json; + + #[test] + fn roundtrip_null() { + let val = json!(null); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn roundtrip_bool() { + for val in [json!(true), json!(false)] { + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + } + + #[test] + fn roundtrip_integers() { + for val in [ + json!(0), + json!(42), + json!(-1), + json!(i64::MAX), + json!(i64::MIN), + ] { + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + } + + #[test] + fn roundtrip_float() { + let val = json!(9.81); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn roundtrip_string() { + let val = json!("hello world"); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn roundtrip_array() { + let val = json!([1, "two", true, null, 2.72]); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn roundtrip_nested_object() { + let val = json!({"a": 1, "b": {"c": [2, 3]}, "d": null}); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn roundtrip_empty_map() { + let val = json!({}); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn roundtrip_empty_array() { + let val = json!([]); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn roundtrip_large_string() { + let s = "x".repeat(300); + let val = json!(s); + let bytes = json_to_msgpack(&val).unwrap(); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(val, restored); + } + + #[test] + fn instant_ext_renders_iso8601() { + let mut bytes = vec![0x91]; + write_instant(&mut bytes, InstantKind::Utc, 1_710_498_600_000_000); + let restored = json_from_msgpack(&bytes).unwrap(); + assert_eq!(restored, json!(["2024-03-15T10:30:00.000000Z"])); + } + + #[test] + fn unknown_ext_is_null() { + let bytes = [0xD7, 0x09, 0, 0, 0, 0, 0, 0, 0, 1]; + assert_eq!(json_from_msgpack(&bytes).unwrap(), json!(null)); + } + + #[test] + fn truncated_instant_ext_is_error() { + let mut bytes = Vec::new(); + write_instant(&mut bytes, InstantKind::Naive, 7); + bytes.truncate(6); + assert!(json_from_msgpack(&bytes).is_err()); + } +} diff --git a/nodedb-types/src/json_msgpack/reader/mod.rs b/nodedb-types/src/json_msgpack/reader/mod.rs new file mode 100644 index 000000000..a08621de8 --- /dev/null +++ b/nodedb-types/src/json_msgpack/reader/mod.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Cursor-based msgpack → `serde_json::Value` and `nodedb_types::Value` readers. +//! +//! Deterministic raw byte parser — the first byte of each msgpack value +//! unambiguously identifies its type per the msgpack specification. + +pub mod cursor; +pub mod json; +pub mod native; + +pub(crate) use cursor::Cursor; +pub(crate) use json::base64_encode; +pub use json::json_from_msgpack; +pub use native::value_from_msgpack; diff --git a/nodedb-types/src/json_msgpack/reader/native.rs b/nodedb-types/src/json_msgpack/reader/native.rs new file mode 100644 index 000000000..424ebbcf5 --- /dev/null +++ b/nodedb-types/src/json_msgpack/reader/native.rs @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Msgpack → `nodedb_types::Value` reader. +//! +//! Instant ext values (fixext8 type 1 / 2) become `Value::DateTime` / +//! `Value::NaiveDateTime`. Every other ext type becomes `Value::Null`. + +use super::super::error::MsgpackResult; +use super::super::instant_ext::instant_from_ext; +use super::cursor::Cursor; + +/// Deserialize a `nodedb_types::Value` from standard MessagePack bytes. +/// +/// The input must contain exactly one top-level value and nothing else; +/// trailing bytes are rejected. +pub fn value_from_msgpack(bytes: &[u8]) -> MsgpackResult { + let mut cursor = Cursor::new(bytes); + let value = read_native_value(&mut cursor)?; + cursor.finish()?; + Ok(value) +} + +fn read_native_value(c: &mut Cursor<'_>) -> zerompk::Result { + if c.depth > 500 { + return Err(zerompk::Error::DepthLimitExceeded { max: 500 }); + } + + let marker = c.take()?; + match marker { + 0xC0 => Ok(crate::Value::Null), + 0xC2 => Ok(crate::Value::Bool(false)), + 0xC3 => Ok(crate::Value::Bool(true)), + + 0x00..=0x7F => Ok(crate::Value::Integer(marker as i64)), + 0xE0..=0xFF => Ok(crate::Value::Integer(marker as i8 as i64)), + + 0xCC => Ok(crate::Value::Integer(c.take()? as i64)), + 0xCD => Ok(crate::Value::Integer(c.read_u16_be()? as i64)), + 0xCE => Ok(crate::Value::Integer(c.read_u32_be()? as i64)), + 0xCF => { + let b = c.take_n(8)?; + Ok(crate::Value::Integer(u64::from_be_bytes([ + b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], + ]) as i64)) + } + + 0xD0 => Ok(crate::Value::Integer(c.take()? as i8 as i64)), + 0xD1 => { + let b = c.take_n(2)?; + Ok(crate::Value::Integer( + i16::from_be_bytes([b[0], b[1]]) as i64 + )) + } + 0xD2 => { + let b = c.take_n(4)?; + Ok(crate::Value::Integer( + i32::from_be_bytes([b[0], b[1], b[2], b[3]]) as i64, + )) + } + 0xD3 => { + let b = c.take_n(8)?; + Ok(crate::Value::Integer(i64::from_be_bytes([ + b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], + ]))) + } + + 0xCA => { + let b = c.take_n(4)?; + Ok(crate::Value::Float( + f32::from_be_bytes([b[0], b[1], b[2], b[3]]) as f64, + )) + } + 0xCB => { + let b = c.take_n(8)?; + Ok(crate::Value::Float(f64::from_be_bytes([ + b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], + ]))) + } + + m @ 0xA0..=0xBF => read_native_str(c, (m & 0x1F) as usize), + 0xD9 => { + let l = c.take()? as usize; + read_native_str(c, l) + } + 0xDA => { + let l = c.read_u16_be()? as usize; + read_native_str(c, l) + } + 0xDB => { + let l = c.read_u32_be()? as usize; + read_native_str(c, l) + } + + 0xC4 => { + let l = c.take()? as usize; + Ok(crate::Value::Bytes(c.take_n(l)?.to_vec())) + } + 0xC5 => { + let l = c.read_u16_be()? as usize; + Ok(crate::Value::Bytes(c.take_n(l)?.to_vec())) + } + 0xC6 => { + let l = c.read_u32_be()? as usize; + Ok(crate::Value::Bytes(c.take_n(l)?.to_vec())) + } + + m @ 0x90..=0x9F => read_native_array(c, (m & 0x0F) as usize), + 0xDC => { + let l = c.read_u16_be()? as usize; + read_native_array(c, l) + } + 0xDD => { + let l = c.read_u32_be()? as usize; + read_native_array(c, l) + } + + m @ 0x80..=0x8F => read_native_map(c, (m & 0x0F) as usize), + 0xDE => { + let l = c.read_u16_be()? as usize; + read_native_map(c, l) + } + 0xDF => { + let l = c.read_u32_be()? as usize; + read_native_map(c, l) + } + + // fixext8: instants decode to their Value variant, other types to Null + 0xD7 => { + let (ext_type, payload) = c.take_fixext8()?; + Ok(match instant_from_ext(ext_type, payload) { + Some((kind, micros)) => kind.from_micros(micros), + None => crate::Value::Null, + }) + } + + // other ext types — skip + 0xD4 => { + c.take_n(2)?; + Ok(crate::Value::Null) + } + 0xD5 => { + c.take_n(3)?; + Ok(crate::Value::Null) + } + 0xD6 => { + c.take_n(5)?; + Ok(crate::Value::Null) + } + 0xD8 => { + c.take_n(17)?; + Ok(crate::Value::Null) + } + 0xC7 => { + let l = c.take()? as usize; + c.take_n(1 + l)?; + Ok(crate::Value::Null) + } + 0xC8 => { + let l = c.read_u16_be()? as usize; + c.take_n(1 + l)?; + Ok(crate::Value::Null) + } + 0xC9 => { + let l = c.read_u32_be()? as usize; + c.take_n(1 + l)?; + Ok(crate::Value::Null) + } + + _ => Err(zerompk::Error::InvalidMarker(marker)), + } +} + +fn read_native_str(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { + let bytes = c.take_n(len)?; + let s = String::from_utf8(bytes.to_vec()).map_err(|_| zerompk::Error::InvalidMarker(0))?; + Ok(crate::Value::String(s)) +} + +fn read_native_array(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { + c.depth += 1; + let mut arr = Vec::with_capacity(len.min(4096)); + for _ in 0..len { + arr.push(read_native_value(c)?); + } + c.depth -= 1; + Ok(crate::Value::Array(arr)) +} + +fn read_native_map(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { + c.depth += 1; + let mut map = std::collections::HashMap::with_capacity(len.min(4096)); + for _ in 0..len { + let key_marker = c.peek()?; + let key = if (0xA0..=0xBF).contains(&key_marker) + || key_marker == 0xD9 + || key_marker == 0xDA + || key_marker == 0xDB + { + match read_native_value(c)? { + crate::Value::String(s) => s, + other => format!("{other:?}"), + } + } else { + let v = read_native_value(c)?; + format!("{v:?}") + }; + let val = read_native_value(c)?; + map.insert(key, val); + } + c.depth -= 1; + Ok(crate::Value::Object(map)) +} + +#[cfg(test)] +mod tests { + //! Roundtrip tests for the native reader against the native writer. + + use super::*; + use crate::NdbDateTime; + use crate::json_msgpack::transcoder::msgpack_to_json_string; + use crate::json_msgpack::writer::value_to_msgpack; + + #[test] + fn native_value_roundtrip() { + let mut map = std::collections::HashMap::new(); + map.insert("id".to_string(), crate::Value::String("host1".into())); + map.insert("cpu".to_string(), crate::Value::Float(0.75)); + map.insert("mem".to_string(), crate::Value::Float(0.5)); + + let row = crate::Value::Object(map); + let arr = crate::Value::Array(vec![row]); + + let bytes = value_to_msgpack(&arr).unwrap(); + let decoded = value_from_msgpack(&bytes).unwrap(); + + match &decoded { + crate::Value::Array(items) => { + assert_eq!(items.len(), 1); + match &items[0] { + crate::Value::Object(m) => { + assert_eq!(m.len(), 3); + assert_eq!(m.get("id"), Some(&crate::Value::String("host1".into()))); + assert_eq!(m.get("cpu"), Some(&crate::Value::Float(0.75))); + assert_eq!(m.get("mem"), Some(&crate::Value::Float(0.5))); + } + other => panic!("expected Object, got {other:?}"), + } + } + other => panic!("expected Array, got {other:?}"), + } + } + + #[test] + fn native_value_scalars() { + let cases: Vec = vec![ + crate::Value::Null, + crate::Value::Bool(true), + crate::Value::Integer(42), + crate::Value::Float(2.72), + crate::Value::String("hello".into()), + ]; + for val in cases { + let bytes = value_to_msgpack(&val).unwrap(); + let decoded = value_from_msgpack(&bytes).unwrap(); + assert_eq!(val, decoded); + } + } + + #[test] + fn instant_roundtrip_both_kinds() { + let dt = NdbDateTime::from_micros(1_710_498_600_000_000); + for val in [ + crate::Value::DateTime(dt), + crate::Value::NaiveDateTime(dt), + crate::Value::DateTime(NdbDateTime::from_micros(-1)), + ] { + let bytes = value_to_msgpack(&val).unwrap(); + assert_eq!(bytes.len(), 10); + assert_eq!(bytes[0], 0xD7); + assert_eq!(value_from_msgpack(&bytes).unwrap(), val); + } + + let bytes = value_to_msgpack(&crate::Value::NaiveDateTime(dt)).unwrap(); + assert_eq!( + msgpack_to_json_string(&bytes).unwrap(), + "\"2024-03-15T10:30:00.000000Z\"" + ); + } + + #[test] + fn unknown_ext_is_null() { + let bytes = [0xD7, 0x09, 0, 0, 0, 0, 0, 0, 0, 1]; + assert_eq!(value_from_msgpack(&bytes).unwrap(), crate::Value::Null); + } +} diff --git a/nodedb-types/src/json_msgpack/transcoder.rs b/nodedb-types/src/json_msgpack/transcoder.rs index ddb4f099d..228669b59 100644 --- a/nodedb-types/src/json_msgpack/transcoder.rs +++ b/nodedb-types/src/json_msgpack/transcoder.rs @@ -5,11 +5,16 @@ //! Walks msgpack bytes and writes JSON text directly into a String. //! No intermediate `serde_json::Value` or `nodedb_types::Value`. //! Used ONLY at the outermost pgwire/HTTP layer for client compatibility. +//! +//! Instant ext values (fixext8 type 1 / 2) render as ISO 8601 strings. +//! Every other ext type renders as `null`. use std::fmt::Write as _; use super::error::MsgpackResult; +use super::instant_ext::instant_from_ext; use super::reader::{Cursor, base64_encode}; +use crate::datetime::NdbDateTime; /// Transcode raw msgpack bytes to a JSON string without intermediate types. /// @@ -139,7 +144,20 @@ fn transcode_value(c: &mut Cursor<'_>, out: &mut String) -> zerompk::Result<()> transcode_map(c, out, l)?; } - // ext types — render as null + // fixext8: instants render as ISO 8601, other types as null + 0xD7 => { + let (ext_type, payload) = c.take_fixext8()?; + match instant_from_ext(ext_type, payload) { + Some((_, micros)) => { + out.push('"'); + out.push_str(&NdbDateTime::from_micros(micros).to_iso8601()); + out.push('"'); + } + None => out.push_str("null"), + } + } + + // other ext types — render as null 0xD4 => { c.take_n(2)?; out.push_str("null"); @@ -152,10 +170,6 @@ fn transcode_value(c: &mut Cursor<'_>, out: &mut String) -> zerompk::Result<()> c.take_n(5)?; out.push_str("null"); } - 0xD7 => { - c.take_n(9)?; - out.push_str("null"); - } 0xD8 => { c.take_n(17)?; out.push_str("null"); @@ -310,6 +324,18 @@ mod tests { assert_eq!(msgpack_to_json_string(&[]).unwrap(), ""); } + #[test] + fn instant_ext_renders_iso8601() { + use crate::json_msgpack::instant_ext::{InstantKind, write_instant}; + let mut mp = vec![0x81, 0xA2, b't', b's']; + write_instant(&mut mp, InstantKind::Naive, 1_710_498_600_000_000); + let json_str = msgpack_to_json_string(&mp).unwrap(); + assert_eq!(json_str, "{\"ts\":\"2024-03-15T10:30:00.000000Z\"}"); + + let unknown = [0xD7, 0x09, 0, 0, 0, 0, 0, 0, 0, 1]; + assert_eq!(msgpack_to_json_string(&unknown).unwrap(), "null"); + } + #[test] fn nested() { let val = serde_json::json!({"a": {"b": [1, 2, {"c": 3}]}}); diff --git a/nodedb-types/src/json_msgpack/writer.rs b/nodedb-types/src/json_msgpack/writer.rs index e557b580d..daaa680cb 100644 --- a/nodedb-types/src/json_msgpack/writer.rs +++ b/nodedb-types/src/json_msgpack/writer.rs @@ -1,7 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 //! Msgpack serialization for `serde_json::Value` and `nodedb_types::Value`. +//! +//! `Value::DateTime` and `Value::NaiveDateTime` are written as the instant +//! ext (`fixext8`, see `instant_ext`). `Duration`, `Decimal`, and `Geometry` +//! are written as strings. `Vector` is a float64 array, `ArrayCell` a map, +//! and `Range` / `Record` are `nil`. +use super::instant_ext::{InstantKind, write_instant}; use super::json_value::JsonValue; /// Serialize a `serde_json::Value` to MessagePack bytes. @@ -60,9 +66,8 @@ fn write_native_value(buf: &mut Vec, value: &crate::Value) { write_native_value(buf, v); } } - crate::Value::DateTime(dt) | crate::Value::NaiveDateTime(dt) => { - write_native_str(buf, &dt.to_string()) - } + crate::Value::DateTime(dt) => write_instant(buf, InstantKind::Utc, dt.micros), + crate::Value::NaiveDateTime(dt) => write_instant(buf, InstantKind::Naive, dt.micros), crate::Value::Duration(d) => write_native_str(buf, &d.to_string()), crate::Value::Decimal(d) => write_native_str(buf, &d.to_string()), crate::Value::Geometry(g) => { diff --git a/nodedb-types/src/lib.rs b/nodedb-types/src/lib.rs index c7f9ec841..4640dce48 100644 --- a/nodedb-types/src/lib.rs +++ b/nodedb-types/src/lib.rs @@ -105,8 +105,9 @@ pub use id::{ }; pub use identity::KeyRepr; pub use json_msgpack::{ - JsonValue, MsgpackError, MsgpackResult, json_from_msgpack, json_to_msgpack, - json_to_msgpack_or_empty, msgpack_to_json_string, value_from_msgpack, value_to_msgpack, + InstantKind, JsonValue, MsgpackError, MsgpackResult, json_from_msgpack, json_to_msgpack, + json_to_msgpack_or_empty, msgpack_to_json_string, read_instant, value_from_msgpack, + value_to_msgpack, write_instant, }; pub use kv::{KV_DEFAULT_INLINE_THRESHOLD, KvConfig, KvTtlPolicy, is_valid_kv_key_type}; pub use lsn::Lsn; diff --git a/nodedb-types/src/value/core.rs b/nodedb-types/src/value/core.rs index 32fd5528f..81ecab911 100644 --- a/nodedb-types/src/value/core.rs +++ b/nodedb-types/src/value/core.rs @@ -13,6 +13,7 @@ use serde::{Deserialize, Serialize}; use crate::array_cell::ArrayCell; use crate::datetime::{NdbDateTime, NdbDuration}; use crate::geometry::Geometry; +use crate::json_msgpack::InstantKind; /// A dynamic value that can represent any field type in a document /// or any parameter in a SQL query. @@ -195,6 +196,15 @@ impl Value { } } + /// Try to extract as an instant of either kind. + pub fn as_instant(&self) -> Option<(InstantKind, NdbDateTime)> { + match self { + Value::DateTime(dt) => Some((InstantKind::Utc, *dt)), + Value::NaiveDateTime(dt) => Some((InstantKind::Naive, *dt)), + _ => None, + } + } + /// Try to extract as Duration. pub fn as_duration(&self) -> Option<&NdbDuration> { match self { From 3f89535d385ad2dacfe1a040bc6ab9c11d8f72de Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 16 Sep 2026 11:48:29 +0800 Subject: [PATCH 02/21] test(wire): cover time-key rendering across joins, buckets, and units Add coverage that a TIMESTAMP time key or column denotes the same stored instant regardless of the route that reads it: direct SELECT, INSERT ... RETURNING, an INNER JOIN (local, shuffle, and cross-node), time_bucket, date_part, and GROUP BY. Cases span a document_strict TIMESTAMP column (strict_typed_column_rendering.rs, new), a timeseries TIME_KEY joined against another timeseries collection or a document_strict TIMESTAMP column, and single-node vs 3-node cluster cross-node joins comparing TIME_KEY columns in their ON predicate. --- .../cases/shuffle_join_end_to_end.rs | 141 +++++++++ .../join_cross_node.rs | 158 ++++++++++ nodedb/tests/wire/cases/mod.rs | 1 + .../cases/strict_typed_column_rendering.rs | 147 +++++++++ .../cases/timeseries_join_time_rendering.rs | 292 ++++++++++++++++++ 5 files changed, 739 insertions(+) create mode 100644 nodedb/tests/wire/cases/strict_typed_column_rendering.rs diff --git a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_end_to_end.rs b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_end_to_end.rs index 1f6881a87..2ce02e070 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_end_to_end.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_end_to_end.rs @@ -24,6 +24,17 @@ use nodedb_types::DatabaseId; use crate::common::cluster_harness::{TestCluster, wait_for}; +/// The instant [`a_shuffle_join_renders_a_time_key_as_the_stored_instant`] +/// stores in its timeseries collection. +const EARLY: &str = "2020-03-05 10:00:00"; +/// `EARLY` as a declared `TIMESTAMP` time key renders it. The engine stores +/// 1583402400000 epoch milliseconds; a `TIMESTAMP` cell carries epoch +/// microseconds, which the pgwire encoder writes as ISO-8601 UTC. +const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; +/// `EARLY` as epoch microseconds — 1583402400000 milliseconds times 1000. +/// A projection that announces no catalog type leaves its cells this number. +const EARLY_MICROS: &str = "1583402400000000"; + /// Run `sql` and collect the `id` column of every returned data row, sorted, so /// the result is order-independent for equality assertions. async fn collect_ids(client: &tokio_postgres::Client, sql: &str, col: &str) -> Vec { @@ -211,3 +222,133 @@ async fn distributed_shuffle_join_matches_inner_join() { cluster.shutdown().await; } + +/// A timeseries `TIMESTAMP` time key projected through a shuffle join denotes +/// the stored instant, the same as a direct read does. The shuffle path +/// repartitions rows through its own coordinator-side encode/decode hop, so +/// it is a route to the cell distinct from the local join and the direct +/// `SELECT` both. +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_shuffle_join_renders_a_time_key_as_the_stored_instant() { + const LEFT: &str = "ts_shuffle_events"; + const RIGHT: &str = "ts_shuffle_hosts"; + assert_ne!( + vshard_for_collection(DatabaseId::DEFAULT, LEFT), + vshard_for_collection(DatabaseId::DEFAULT, RIGHT), + "test collections must hash to different vShards to exercise cross-node shuffle" + ); + + let cluster = TestCluster::spawn_three().await.expect("3-node cluster"); + + cluster + .exec_ddl_on_any_leader(&format!( + "CREATE COLLECTION {LEFT} \ + (captured_at TIMESTAMP TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .expect("CREATE COLLECTION for the timeseries side"); + cluster + .exec_ddl_on_any_leader(&format!( + "CREATE COLLECTION {RIGHT} (id TEXT PRIMARY KEY, region TEXT) \ + WITH (engine='document_strict')" + )) + .await + .expect("CREATE COLLECTION for the document side"); + + wait_for( + "all 3 nodes see both collections", + Duration::from_secs(10), + Duration::from_millis(50), + || { + cluster + .nodes + .iter() + .all(|n| n.cached_collection_count() >= 2) + }, + ) + .await; + + cluster.nodes[0] + .client + .simple_query(&format!( + "INSERT INTO {LEFT} (captured_at, host, v) VALUES ('{EARLY}', 'h1', 1.5)" + )) + .await + .expect("insert timeseries row"); + cluster.nodes[0] + .client + .simple_query(&format!( + "INSERT INTO {RIGHT} (id, region) VALUES ('h1', 'eu')" + )) + .await + .expect("insert document row"); + + for (idx, node) in cluster.nodes.iter().enumerate() { + wait_for( + &format!("node {idx} sees the timeseries row"), + Duration::from_secs(15), + Duration::from_millis(50), + || { + let n = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(count_rows( + &node.client, + &format!("SELECT host FROM {LEFT}"), + )) + }); + n >= 1 + }, + ) + .await; + wait_for( + &format!("node {idx} sees the document row"), + Duration::from_secs(15), + Duration::from_millis(50), + || { + let n = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current() + .block_on(count_rows(&node.client, &format!("SELECT id FROM {RIGHT}"))) + }); + n >= 1 + }, + ) + .await; + } + + cluster.nodes[1] + .client + .simple_query("SET nodedb.force_shuffle_join = on") + .await + .expect("SET nodedb.force_shuffle_join"); + cluster.nodes[1] + .client + .simple_query("SET nodedb.shuffle_num_parts = 4") + .await + .expect("SET nodedb.shuffle_num_parts"); + + let join_sql = format!( + "SELECT {LEFT}.captured_at FROM {LEFT} \ + INNER JOIN {RIGHT} ON {LEFT}.host = {RIGHT}.id" + ); + let msgs = cluster.nodes[1] + .client + .simple_query(&join_sql) + .await + .expect("a shuffle join over a time key must succeed"); + let values: Vec = msgs + .into_iter() + .filter_map(|m| match m { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_string), + _ => None, + }) + .collect(); + + assert_eq!(values.len(), 1, "one event matches one host: {values:?}"); + assert!( + values[0] == EARLY_ISO || values[0] == EARLY_MICROS, + "a shuffle-joined time key must denote {EARLY}: expected {EARLY_ISO} \ + or {EARLY_MICROS}, got {values:?}" + ); + + cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/join_cross_node.rs b/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/join_cross_node.rs index be92431a2..59beca7e7 100644 --- a/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/join_cross_node.rs +++ b/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/join_cross_node.rs @@ -26,6 +26,18 @@ async fn count_rows(client: &tokio_postgres::Client, sql: &str) -> usize { .count() } +/// Helper: run a simple query and return the first column of every data row, +/// as the pgwire text encoder rendered it. +async fn first_column_text(client: &tokio_postgres::Client, sql: &str) -> Vec { + let msgs = client.simple_query(sql).await.expect("simple_query"); + msgs.into_iter() + .filter_map(|m| match m { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_string), + _ => None, + }) + .collect() +} + /// A cross-node inner join between two single-vShard-homed collections must /// return every matching row, regardless of which node the SELECT is issued /// from. Before the build-side gather fix this returned 0 rows when the build @@ -151,3 +163,149 @@ async fn cross_node_join_returns_all_matches() { cluster.shutdown().await; } + +/// A distributed join gathers the remote (build) side through the coordinator +/// while scanning the local (probe) side directly. When the `ON` predicate +/// compares two TIME_KEY columns, both sides must reach that comparison +/// expressed in the same unit — the gathered side must not arrive pre-decoded +/// into a different representation than the locally scanned side. This holds +/// for every node the query runs from, whichever side that node hosts. +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn cross_node_join_compares_time_keys_in_one_unit() { + use nodedb_cluster::routing::vshard_for_collection; + use nodedb_types::DatabaseId; + const EVENTS: &str = "ts_events"; + const FEATURES: &str = "ts_features"; + assert_ne!( + vshard_for_collection(DatabaseId::DEFAULT, EVENTS), + vshard_for_collection(DatabaseId::DEFAULT, FEATURES), + "test collections must hash to different vShards to exercise cross-node join" + ); + + let cluster = TestCluster::spawn_three().await.expect("3-node cluster"); + + cluster + .exec_ddl_on_any_leader(&format!( + "CREATE COLLECTION {EVENTS} \ + (captured_at TIMESTAMP TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .expect("CREATE COLLECTION ts_events"); + cluster + .exec_ddl_on_any_leader(&format!( + "CREATE COLLECTION {FEATURES} \ + (captured_at TIMESTAMP TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .expect("CREATE COLLECTION ts_features"); + + wait_for( + "all 3 nodes see both collections", + Duration::from_secs(10), + Duration::from_millis(50), + || { + cluster + .nodes + .iter() + .all(|n| n.cached_collection_count() >= 2) + }, + ) + .await; + + cluster.nodes[0] + .client + .simple_query(&format!( + "INSERT INTO {EVENTS} (captured_at, host, v) VALUES ('2020-03-05 10:00:00', 'h1', 1.5)" + )) + .await + .expect("insert event row"); + cluster.nodes[0] + .client + .simple_query(&format!( + "INSERT INTO {FEATURES} (captured_at, host, v) VALUES ('2020-03-05 09:00:00', 'h1', 2.5)" + )) + .await + .expect("insert feature row"); + + for (idx, node) in cluster.nodes.iter().enumerate() { + wait_for( + &format!("node {idx} sees the event row"), + Duration::from_secs(15), + Duration::from_millis(50), + || { + let n = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(count_rows( + &node.client, + &format!("SELECT host FROM {EVENTS}"), + )) + }); + n >= 1 + }, + ) + .await; + wait_for( + &format!("node {idx} sees the feature row"), + Duration::from_secs(15), + Duration::from_millis(50), + || { + let n = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(count_rows( + &node.client, + &format!("SELECT host FROM {FEATURES}"), + )) + }); + n >= 1 + }, + ) + .await; + } + + // The join's ON predicate compares the two TIME_KEY columns directly + // (`FEATURES.captured_at <= EVENTS.captured_at`). This only matches the + // single feature row when both sides land in the comparison expressed + // in the same unit, regardless of which node the coordinator gathers the + // build side from. + let value_sql = format!( + "SELECT {FEATURES}.v FROM {EVENTS} INNER JOIN {FEATURES} \ + ON {EVENTS}.host = {FEATURES}.host \ + AND {FEATURES}.captured_at <= {EVENTS}.captured_at" + ); + for (idx, node) in cluster.nodes.iter().enumerate() { + let rows = first_column_text(&node.client, &value_sql).await; + assert_eq!( + rows, + vec!["2.5".to_string()], + "node {idx}: join comparing time keys must return the one matching feature row" + ); + } + + // The joined TIME_KEY itself must denote the stored instant, whichever + // unit the coordinator's gather step renders it in: ISO-8601 UTC (a + // typed TIMESTAMP cell) or epoch microseconds (an untyped cell). Epoch + // MILLISECONDS denote 1970-01-19 under either reading, so a millisecond + // value fails both arms and reveals the gather step decoded the wrong + // unit. + const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; + const EARLY_MICROS: &str = "1583402400000000"; + let time_key_sql = format!( + "SELECT {EVENTS}.captured_at FROM {EVENTS} INNER JOIN {FEATURES} \ + ON {EVENTS}.host = {FEATURES}.host" + ); + for (idx, node) in cluster.nodes.iter().enumerate() { + let rows = first_column_text(&node.client, &time_key_sql).await; + assert_eq!( + rows.len(), + 1, + "node {idx}: the join must yield one row: {rows:?}" + ); + assert!( + rows[0] == EARLY_ISO || rows[0] == EARLY_MICROS, + "node {idx}: joined time key must denote 2020-03-05T10:00:00Z: \ + expected {EARLY_ISO} or {EARLY_MICROS}, got {rows:?}" + ); + } + + cluster.shutdown().await; +} diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index e47f21052..958c12265 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -267,6 +267,7 @@ mod streaming_select; mod strict_bitemporal_audit_query; mod strict_bitemporal_select_star; mod strict_schema_restart; +mod strict_typed_column_rendering; mod timeseries_declared_time_key; mod timeseries_join_time_rendering; mod timeseries_write_row_level_security; diff --git a/nodedb/tests/wire/cases/strict_typed_column_rendering.rs b/nodedb/tests/wire/cases/strict_typed_column_rendering.rs new file mode 100644 index 000000000..9886c65c0 --- /dev/null +++ b/nodedb/tests/wire/cases/strict_typed_column_rendering.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A `document_strict` `TIMESTAMP` column renders the stored instant the same +//! way whichever route reads it, and the same way a timeseries time key does. + +use crate::harness::TestServer; + +/// The instant every test in this file stores. +const EARLY: &str = "2020-03-05 10:00:00"; +/// `EARLY` as a declared `TIMESTAMP` column renders it. The engine stores +/// 1583402400000 epoch milliseconds; a `TIMESTAMP` cell carries epoch +/// microseconds, which the pgwire encoder writes as ISO-8601 UTC. +const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; +/// `EARLY` as epoch microseconds — 1583402400000 milliseconds times 1000. +/// A projection that announces no catalog type leaves its cells this number. +const EARLY_MICROS: &str = "1583402400000000"; + +/// A strict `document_strict` collection carrying a `TIMESTAMP` column, read +/// back with a direct `SELECT`, denotes the stored instant. Epoch +/// milliseconds — 1583402400000 — read as microseconds denote 1970-01-19, so +/// a millisecond value fails both arms. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_strict_timestamp_column_renders_the_stored_instant() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION strict_ts_direct \ + (id TEXT PRIMARY KEY, created_at TIMESTAMP) \ + WITH (engine='document_strict')", + ) + .await + .expect("create strict_ts_direct"); + server + .exec(&format!( + "INSERT INTO strict_ts_direct (id, created_at) VALUES ('r1', '{EARLY}')" + )) + .await + .expect("insert into strict_ts_direct"); + + let rows = server + .query_text("SELECT created_at FROM strict_ts_direct WHERE id = 'r1'") + .await + .expect("SELECT of a strict TIMESTAMP column must succeed"); + assert_eq!(rows.len(), 1, "one stored row: {rows:?}"); + + assert!( + rows[0] == EARLY_ISO || rows[0] == EARLY_MICROS, + "a strict TIMESTAMP column must denote {EARLY}: expected {EARLY_ISO} \ + or {EARLY_MICROS}, got {rows:?}" + ); +} + +/// The same column, read back through `INSERT ... RETURNING` rather than a +/// follow-up `SELECT`, denotes the same stored instant. `RETURNING` runs its +/// own encoder over the just-written row, so it is a second route to the +/// same cell. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_strict_timestamp_column_returned_by_insert_renders_the_stored_instant() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION strict_ts_returning \ + (id TEXT PRIMARY KEY, created_at TIMESTAMP) \ + WITH (engine='document_strict')", + ) + .await + .expect("create strict_ts_returning"); + + let rows = server + .query_text(&format!( + "INSERT INTO strict_ts_returning (id, created_at) VALUES ('r2', '{EARLY}') \ + RETURNING created_at" + )) + .await + .expect("INSERT ... RETURNING of a strict TIMESTAMP column must succeed"); + assert_eq!(rows.len(), 1, "one inserted row: {rows:?}"); + + assert!( + rows[0] == EARLY_ISO || rows[0] == EARLY_MICROS, + "a strict TIMESTAMP column returned by INSERT must denote {EARLY}: expected \ + {EARLY_ISO} or {EARLY_MICROS}, got {rows:?}" + ); +} + +/// A `document_strict` `TIMESTAMP` column and a timeseries `TIMESTAMP` time +/// key holding the same instant render identically. One instant renders one +/// way whichever engine stores it, so a divergence between the two engines' +/// encoders is a rendering defect, not an engine-specific choice. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_strict_timestamp_column_renders_the_same_as_a_timeseries_time_key() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION strict_ts_parity_doc \ + (id TEXT PRIMARY KEY, created_at TIMESTAMP) \ + WITH (engine='document_strict')", + ) + .await + .expect("create strict_ts_parity_doc"); + server + .exec(&format!( + "INSERT INTO strict_ts_parity_doc (id, created_at) VALUES ('r3', '{EARLY}')" + )) + .await + .expect("insert into strict_ts_parity_doc"); + + server + .exec( + "CREATE COLLECTION strict_ts_parity_ts \ + (captured_at TIMESTAMP TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')", + ) + .await + .expect("create strict_ts_parity_ts"); + server + .exec(&format!( + "INSERT INTO strict_ts_parity_ts (captured_at, host, v) VALUES ('{EARLY}', 'h1', 1.5)" + )) + .await + .expect("insert into strict_ts_parity_ts"); + + let strict_reading = server + .query_text("SELECT created_at FROM strict_ts_parity_doc WHERE id = 'r3'") + .await + .expect("SELECT of the strict TIMESTAMP column must succeed"); + assert_eq!( + strict_reading.len(), + 1, + "one stored row: {strict_reading:?}" + ); + + let timeseries_reading = server + .query_text("SELECT captured_at FROM strict_ts_parity_ts") + .await + .expect("SELECT of the timeseries time key must succeed"); + assert_eq!( + timeseries_reading.len(), + 1, + "one stored point: {timeseries_reading:?}" + ); + + assert_eq!( + strict_reading[0], timeseries_reading[0], + "a strict TIMESTAMP column must render the same instant as a timeseries time \ + key: strict={strict_reading:?} timeseries={timeseries_reading:?}" + ); +} diff --git a/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs b/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs index ecc4d4e71..52a686157 100644 --- a/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs +++ b/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs @@ -214,3 +214,295 @@ async fn an_aliased_joined_time_key_denotes_the_stored_instant() { or {EARLY_MICROS}, got {joined:?}" ); } + +/// `EARLY` truncated to its hour, as `time_bucket('1 hour', ...)` denotes it. +/// `EARLY` sits on the hour, so the bucket is the same instant. +const EARLY_BUCKET_ISO: &str = EARLY_ISO; +const EARLY_BUCKET_MICROS: &str = EARLY_MICROS; + +/// A transforming computed projection of a time key renders the same through +/// a JOIN as through a direct read. `time_bucket` arithmetic depends on the +/// unit the expression sees, so the two routes agree only when the cell +/// reaches the expression in one unit. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_time_bucket_of_a_time_key_matches_between_a_join_and_a_direct_read() { + let server = TestServer::start().await; + setup(&server, "tsj_bucket_events", "tsj_bucket_hosts").await; + + let direct = server + .query_text("SELECT time_bucket('1 hour', captured_at) AS bucket FROM tsj_bucket_events") + .await + .expect("time_bucket over the time key must succeed"); + assert_eq!(direct.len(), 1, "one stored point: {direct:?}"); + + let joined = server + .query_text( + "SELECT time_bucket('1 hour', tsj_bucket_events.captured_at) AS bucket \ + FROM tsj_bucket_events \ + INNER JOIN tsj_bucket_hosts ON tsj_bucket_events.host = tsj_bucket_hosts.id", + ) + .await + .expect("time_bucket over the time key through a JOIN must succeed"); + assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); + + assert_eq!( + joined[0], direct[0], + "time_bucket of the time key must render the same through a JOIN as through \ + a SELECT: joined={joined:?} direct={direct:?}" + ); +} + +/// `time_bucket` of a time key read directly denotes the bucket of the stored +/// instant. A bucket computed in one unit and rendered in another lands +/// decades away from `EARLY`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_time_bucket_of_a_time_key_denotes_the_stored_instant() { + let server = TestServer::start().await; + setup(&server, "tsj_bucket_abs_events", "tsj_bucket_abs_hosts").await; + + let direct = server + .query_text( + "SELECT time_bucket('1 hour', captured_at) AS bucket FROM tsj_bucket_abs_events", + ) + .await + .expect("time_bucket over the time key must succeed"); + assert_eq!(direct.len(), 1, "one stored point: {direct:?}"); + + assert!( + direct[0] == EARLY_BUCKET_ISO || direct[0] == EARLY_BUCKET_MICROS, + "time_bucket of the time key must denote {EARLY}: expected {EARLY_BUCKET_ISO} \ + or {EARLY_BUCKET_MICROS}, got {direct:?}" + ); +} + +/// `time_bucket` of a time key read through a JOIN denotes the bucket of the +/// stored instant. This anchors the comparison above. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_joined_time_bucket_of_a_time_key_denotes_the_stored_instant() { + let server = TestServer::start().await; + setup(&server, "tsj_bucket_jabs_events", "tsj_bucket_jabs_hosts").await; + + let joined = server + .query_text( + "SELECT time_bucket('1 hour', tsj_bucket_jabs_events.captured_at) AS bucket \ + FROM tsj_bucket_jabs_events \ + INNER JOIN tsj_bucket_jabs_hosts \ + ON tsj_bucket_jabs_events.host = tsj_bucket_jabs_hosts.id", + ) + .await + .expect("time_bucket over the time key through a JOIN must succeed"); + assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); + + assert!( + joined[0] == EARLY_BUCKET_ISO || joined[0] == EARLY_BUCKET_MICROS, + "a joined time_bucket of the time key must denote {EARLY}: expected \ + {EARLY_BUCKET_ISO} or {EARLY_BUCKET_MICROS}, got {joined:?}" + ); +} + +/// `date_part` reads the calendar year of a time key directly. The docs list +/// `EXTRACT(field FROM ts)` (`date_part` is its function form) as a scalar over any timestamp, so a declared +/// timeseries instant must be readable by it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn date_part_year_of_a_time_key_reads_the_stored_year() { + let server = TestServer::start().await; + setup(&server, "tsj_extract_events", "tsj_extract_hosts").await; + + let direct = server + .query_text("SELECT date_part('year', captured_at) AS y FROM tsj_extract_events") + .await + .expect("date_part over the time key must succeed"); + assert_eq!(direct.len(), 1, "one stored point: {direct:?}"); + assert_eq!( + direct[0], "2020", + "the stored instant is in 2020: {direct:?}" + ); +} + +/// `date_part` reads the calendar year of a time key through a JOIN, the same +/// as directly. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn date_part_year_of_a_joined_time_key_reads_the_stored_year() { + let server = TestServer::start().await; + setup(&server, "tsj_extract_j_events", "tsj_extract_j_hosts").await; + + let joined = server + .query_text( + "SELECT date_part('year', tsj_extract_j_events.captured_at) AS y \ + FROM tsj_extract_j_events \ + INNER JOIN tsj_extract_j_hosts \ + ON tsj_extract_j_events.host = tsj_extract_j_hosts.id", + ) + .await + .expect("date_part over the time key through a JOIN must succeed"); + assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); + assert_eq!( + joined[0], "2020", + "the stored instant is in 2020: {joined:?}" + ); +} + +/// A JOIN whose `ON` compares a timeseries time key against a `TIMESTAMP` +/// column of a document collection matches the row that holds the same +/// instant. Both cells name `EARLY`, so the predicate holds only when the two +/// sides reach the comparison in one unit. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_join_on_a_time_key_against_a_document_timestamp_matches_the_same_instant() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION tsj_xe_events \ + (captured_at TIMESTAMP TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')", + ) + .await + .expect("create events"); + server + .exec( + "CREATE COLLECTION tsj_xe_marks (id TEXT PRIMARY KEY, seen_at TIMESTAMP) \ + WITH (engine='document_strict')", + ) + .await + .expect("create marks"); + server + .exec(&format!( + "INSERT INTO tsj_xe_events (captured_at, host, v) VALUES ('{EARLY}', 'h1', 1.5)" + )) + .await + .expect("insert event"); + server + .exec(&format!( + "INSERT INTO tsj_xe_marks (id, seen_at) VALUES ('m1', '{EARLY}')" + )) + .await + .expect("insert mark"); + + let joined = server + .query_text( + "SELECT tsj_xe_marks.id FROM tsj_xe_events \ + INNER JOIN tsj_xe_marks ON tsj_xe_events.captured_at = tsj_xe_marks.seen_at", + ) + .await + .expect("a JOIN comparing two instants must succeed"); + assert_eq!( + joined, + vec!["m1".to_string()], + "the event and the mark hold the same instant, so the join matches: {joined:?}" + ); +} + +/// The feature-store JOIN the docs advertise: a feature row joins an event +/// row when the feature's time key is at or before the event's. Two +/// timeseries collections compare time keys in the `ON` clause, so the +/// predicate holds only when both sides reach it in one unit. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_join_comparing_two_time_keys_matches_when_the_feature_precedes_the_event() { + let server = TestServer::start().await; + for name in ["tsj_tt_events", "tsj_tt_features"] { + server + .exec(&format!( + "CREATE COLLECTION {name} \ + (captured_at TIMESTAMP TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .unwrap_or_else(|e| panic!("create {name}: {e}")); + } + server + .exec(&format!( + "INSERT INTO tsj_tt_events (captured_at, host, v) VALUES ('{EARLY}', 'h1', 1.5)" + )) + .await + .expect("insert event"); + server + .exec( + "INSERT INTO tsj_tt_features (captured_at, host, v) \ + VALUES ('2020-03-05 09:00:00', 'h1', 2.5)", + ) + .await + .expect("insert feature"); + + let joined = server + .query_text( + "SELECT tsj_tt_features.v FROM tsj_tt_events \ + INNER JOIN tsj_tt_features \ + ON tsj_tt_events.host = tsj_tt_features.host \ + AND tsj_tt_features.captured_at <= tsj_tt_events.captured_at", + ) + .await + .expect("a JOIN comparing two time keys must succeed"); + assert_eq!( + joined, + vec!["2.5".to_string()], + "the feature precedes the event, so the join matches it: {joined:?}" + ); +} + +/// `time_bucket` of a time key used as a `GROUP BY` key denotes the bucket of +/// the stored instant. `EARLY` sits on the hour, so its bucket is `EARLY` +/// itself; the aggregate encoder writes the group key, which is a fourth +/// route to the same stored instant alongside a direct read, a join, and a +/// plain `GROUP BY` on the column. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_grouped_time_bucket_denotes_the_stored_instant() { + let server = TestServer::start().await; + setup(&server, "tsj_bucket_grp_events", "tsj_bucket_grp_hosts").await; + + let rows = server + .query_rows( + "SELECT time_bucket('1 hour', captured_at) AS bucket, host, COUNT(*) \ + FROM tsj_bucket_grp_events GROUP BY bucket, host", + ) + .await + .expect("GROUP BY a time_bucket key must succeed"); + assert_eq!( + rows.len(), + 1, + "one stored point falls in one group: {rows:?}" + ); + + assert!( + rows[0][0] == EARLY_ISO || rows[0][0] == EARLY_MICROS, + "a grouped time_bucket key must denote {EARLY}: expected {EARLY_ISO} \ + or {EARLY_MICROS}, got {rows:?}" + ); +} + +/// A time key grouped through `time_bucket` renders the same as the same +/// column grouped directly. `EARLY` sits on the hour, so the bucket of that +/// instant is that instant, and a divergence between the two group-key +/// encoders is a rendering defect. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_grouped_time_bucket_renders_as_the_time_key_grouped_directly_does() { + let server = TestServer::start().await; + setup(&server, "tsj_bucket_cmp_events", "tsj_bucket_cmp_hosts").await; + + let bucketed = server + .query_text( + "SELECT time_bucket('1 hour', captured_at) AS bucket, host, COUNT(*) \ + FROM tsj_bucket_cmp_events GROUP BY bucket, host", + ) + .await + .expect("GROUP BY a time_bucket key must succeed"); + assert_eq!( + bucketed.len(), + 1, + "one stored point falls in one group: {bucketed:?}" + ); + + let direct = server + .query_text("SELECT captured_at, COUNT(*) FROM tsj_bucket_cmp_events GROUP BY captured_at") + .await + .expect("GROUP BY the time key directly must succeed"); + assert_eq!( + direct.len(), + 1, + "one stored point falls in one group: {direct:?}" + ); + + assert_eq!( + bucketed[0], direct[0], + "a time_bucket GROUP BY key must render the same as the time key grouped \ + directly: bucketed={bucketed:?} direct={direct:?}" + ); +} From 3af7e9eca8a1316b2b99f178e2643cf7dd83b44d Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 16 Sep 2026 12:11:37 +0800 Subject: [PATCH 03/21] fix(query): preserve instant kind across datetime functions Datetime scalar functions (datetime, extract, date_trunc, date_add, date_sub, date_diff, time_bucket) previously required string arguments and always parsed/returned via string ISO 8601, discarding whether the value was a UTC or naive instant. Add instant_arg to accept Value::DateTime, Value::NaiveDateTime, or a parseable string, and have each function return the same instant kind it received. time_bucket now buckets typed instants on epoch microseconds (floored toward negative infinity) while still supporting integer millisecond timestamps bucketed on epoch milliseconds. --- nodedb-query/src/functions/datetime.rs | 297 ++++++++++++++++++------- 1 file changed, 212 insertions(+), 85 deletions(-) diff --git a/nodedb-query/src/functions/datetime.rs b/nodedb-query/src/functions/datetime.rs index ef245a4b1..6837eba0a 100644 --- a/nodedb-query/src/functions/datetime.rs +++ b/nodedb-query/src/functions/datetime.rs @@ -1,52 +1,61 @@ // SPDX-License-Identifier: Apache-2.0 //! DateTime and duration scalar functions. +//! +//! Instant-typed arguments (`Value::DateTime`, `Value::NaiveDateTime`, or a +//! parseable ISO 8601 string) are accepted through [`instant_arg`]. A +//! function that returns an instant preserves the kind of its input: +//! `Utc` in, `DateTime` out; `Naive` in, `NaiveDateTime` out. use crate::value_ops::{to_value_number, value_to_display_string}; -use nodedb_types::Value; +use nodedb_types::{InstantKind, NdbDateTime, Value}; + +/// Resolve an argument to a typed instant. +/// +/// `Value::DateTime`/`Value::NaiveDateTime` pass through their own kind. A +/// `Value::String` that parses via [`NdbDateTime::parse`] resolves to +/// `InstantKind::Utc` when it ends in `Z`/`z` (the only zone marker +/// `NdbDateTime::parse` recognizes) and `InstantKind::Naive` otherwise. +/// Anything else, or a string that fails to parse, is `None`. +fn instant_arg(v: &Value) -> Option<(InstantKind, NdbDateTime)> { + if let Some(pair) = v.as_instant() { + return Some(pair); + } + let s = v.as_str()?; + let dt = NdbDateTime::parse(s)?; + let trimmed = s.trim(); + let kind = if trimmed.ends_with('Z') || trimmed.ends_with('z') { + InstantKind::Utc + } else { + InstantKind::Naive + }; + Some((kind, dt)) +} pub(super) fn try_eval(name: &str, args: &[Value]) -> Option { let v = match name { "now" | "current_timestamp" => { - let dt = nodedb_types::NdbDateTime::now(); + let dt = NdbDateTime::now(); Value::DateTime(dt) } - "datetime" | "to_datetime" => args - .first() - .and_then(|v| match v { - Value::String(s) => { - nodedb_types::NdbDateTime::parse(s).map(|dt| Value::String(dt.to_iso8601())) - } - Value::Integer(micros) => Some(Value::String( - nodedb_types::NdbDateTime::from_micros(*micros).to_iso8601(), - )), - Value::Float(f) => Some(Value::String( - nodedb_types::NdbDateTime::from_micros(*f as i64).to_iso8601(), - )), - Value::DateTime(dt) | Value::NaiveDateTime(dt) => { - Some(Value::String(dt.to_iso8601())) - } - _ => None, - }) - .unwrap_or(Value::Null), + "datetime" | "to_datetime" => args.first().map_or(Value::Null, |v| match v { + Value::Integer(micros) => InstantKind::Utc.from_micros(*micros), + Value::Float(f) => InstantKind::Utc.from_micros(*f as i64), + _ => instant_arg(v).map_or(Value::Null, |(kind, dt)| kind.value(dt)), + }), "unix_secs" | "epoch_secs" => args .first() - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse) - .map_or(Value::Null, |dt| Value::Integer(dt.unix_secs())), + .and_then(instant_arg) + .map_or(Value::Null, |(_, dt)| Value::Integer(dt.unix_secs())), "unix_millis" | "epoch_millis" => args .first() - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse) - .map_or(Value::Null, |dt| Value::Integer(dt.unix_millis())), + .and_then(instant_arg) + .map_or(Value::Null, |(_, dt)| Value::Integer(dt.unix_millis())), "extract" | "date_part" => { let part = args.first().and_then(|v| v.as_str()).unwrap_or(""); - let dt = args - .get(1) - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse); + let dt = args.get(1).and_then(instant_arg); match dt { - Some(dt) => { + Some((_, dt)) => { let c = dt.components(); let val: i64 = match part.to_lowercase().as_str() { "year" | "y" => c.year as i64, @@ -70,88 +79,70 @@ pub(super) fn try_eval(name: &str, args: &[Value]) -> Option { } "date_trunc" | "datetrunc" => { let part = args.first().and_then(|v| v.as_str()).unwrap_or(""); - let dt = args - .get(1) - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse); + let dt = args.get(1).and_then(instant_arg); match dt { - Some(dt) => { + Some((kind, dt)) => { let c = dt.components(); let truncated = match part.to_lowercase().as_str() { - "year" => nodedb_types::NdbDateTime::parse(&format!( - "{:04}-01-01T00:00:00Z", - c.year - )), - "month" => nodedb_types::NdbDateTime::parse(&format!( + "year" => NdbDateTime::parse(&format!("{:04}-01-01T00:00:00Z", c.year)), + "month" => NdbDateTime::parse(&format!( "{:04}-{:02}-01T00:00:00Z", c.year, c.month )), - "day" => nodedb_types::NdbDateTime::parse(&format!( + "day" => NdbDateTime::parse(&format!( "{:04}-{:02}-{:02}T00:00:00Z", c.year, c.month, c.day )), - "hour" => nodedb_types::NdbDateTime::parse(&format!( + "hour" => NdbDateTime::parse(&format!( "{:04}-{:02}-{:02}T{:02}:00:00Z", c.year, c.month, c.day, c.hour )), - "minute" => nodedb_types::NdbDateTime::parse(&format!( + "minute" => NdbDateTime::parse(&format!( "{:04}-{:02}-{:02}T{:02}:{:02}:00Z", c.year, c.month, c.day, c.hour, c.minute )), - "second" => nodedb_types::NdbDateTime::parse(&format!( + "second" => NdbDateTime::parse(&format!( "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", c.year, c.month, c.day, c.hour, c.minute, c.second )), _ => None, }; - truncated.map_or(Value::Null, |t| Value::String(t.to_iso8601())) + truncated.map_or(Value::Null, |t| kind.value(t)) } None => Value::Null, } } "date_add" | "datetime_add" => { - let dt = args - .first() - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse); + let dt = args.first().and_then(instant_arg); let dur = args .get(1) .and_then(|v| v.as_str()) .and_then(nodedb_types::NdbDuration::parse); match (dt, dur) { - (Some(dt), Some(dur)) => dt + (Some((kind, dt)), Some(dur)) => dt .add_duration(dur) - .map(|r| Value::String(r.to_iso8601())) + .map(|r| kind.value(r)) .unwrap_or(Value::Null), _ => Value::Null, } } "date_sub" | "datetime_sub" => { - let dt = args - .first() - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse); + let dt = args.first().and_then(instant_arg); let dur = args .get(1) .and_then(|v| v.as_str()) .and_then(nodedb_types::NdbDuration::parse); match (dt, dur) { - (Some(dt), Some(dur)) => dt + (Some((kind, dt)), Some(dur)) => dt .sub_duration(dur) - .map(|r| Value::String(r.to_iso8601())) + .map(|r| kind.value(r)) .unwrap_or(Value::Null), _ => Value::Null, } } "date_diff" | "datediff" => { - let dt1 = args - .first() - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse); - let dt2 = args - .get(1) - .and_then(|v| v.as_str()) - .and_then(nodedb_types::NdbDateTime::parse); + let dt1 = args.first().and_then(instant_arg).map(|(_, dt)| dt); + let dt2 = args.get(1).and_then(instant_arg).map(|(_, dt)| dt); match (dt1, dt2) { (Some(a), Some(b)) => a .duration_since(&b) @@ -178,39 +169,51 @@ pub(super) fn try_eval(name: &str, args: &[Value]) -> Option { Some(v) } -/// `time_bucket(interval, timestamp)` — truncate a millisecond timestamp -/// to the start of the given interval bucket. +/// `time_bucket(interval, timestamp)` — truncate a timestamp to the start +/// of the given interval bucket. /// /// Accepts two argument orders (both common in SQL): /// - `time_bucket('1 hour', timestamp_col)` — interval first /// - `time_bucket(timestamp_col, '1 hour')` — timestamp first /// -/// The interval is a string like `'1h'`, `'5m'`, `'1 hour'`, `'30 seconds'`. -/// The timestamp is an integer (epoch milliseconds). +/// The interval is a string like `'1h'`, `'5m'`, `'1 hour'`, `'30 seconds'`, +/// or an integer number of seconds. +/// +/// The timestamp is either: +/// - a typed instant (`Value::DateTime`, `Value::NaiveDateTime`, or a +/// parseable instant string) — bucketed on epoch microseconds, floored +/// toward negative infinity, and returned as the same instant kind +/// - an `Value::Integer`/`Value::Float` epoch-millisecond value — bucketed +/// on epoch milliseconds and returned as `Value::Integer` milliseconds, +/// truncated toward zero fn eval_time_bucket(args: &[Value]) -> Value { if args.len() < 2 { return Value::Null; } - // Detect which arg is the interval string and which is the timestamp. + let instant_and_interval = instant_arg(&args[0]) + .map(|pair| (pair, &args[1])) + .or_else(|| instant_arg(&args[1]).map(|pair| (pair, &args[0]))); + + if let Some(((kind, dt), interval)) = instant_and_interval { + let interval_us = interval_arg_ms(interval).and_then(|ms| ms.checked_mul(1000)); + return match interval_us { + Some(i) if i > 0 => { + let bucket = dt.micros.div_euclid(i) * i; + kind.value(NdbDateTime::from_micros(bucket)) + } + _ => Value::Null, + }; + } + + // Detect which arg is the interval and which is the timestamp. let (interval_ms, timestamp_ms) = match (&args[0], &args[1]) { // time_bucket('1 hour', timestamp) - (Value::String(s), ts_val) => { - let interval = parse_interval_to_ms(s); - let ts = value_to_timestamp_ms(ts_val); - (interval, ts) - } + (Value::String(_), ts_val) => (interval_arg_ms(&args[0]), value_to_timestamp_ms(ts_val)), // time_bucket(timestamp, '1 hour') - (ts_val, Value::String(s)) => { - let interval = parse_interval_to_ms(s); - let ts = value_to_timestamp_ms(ts_val); - (interval, ts) - } + (ts_val, Value::String(_)) => (interval_arg_ms(&args[1]), value_to_timestamp_ms(ts_val)), // time_bucket(3600, timestamp) — interval as integer seconds - (Value::Integer(interval_secs), ts_val) => { - let ts = value_to_timestamp_ms(ts_val); - (Some((*interval_secs) * 1000), ts) - } + (Value::Integer(_), ts_val) => (interval_arg_ms(&args[0]), value_to_timestamp_ms(ts_val)), _ => return Value::Null, }; @@ -220,6 +223,16 @@ fn eval_time_bucket(args: &[Value]) -> Value { } } +/// The bucket interval in milliseconds: an interval string such as +/// `'1 hour'`, or an integer count of seconds. +fn interval_arg_ms(v: &Value) -> Option { + match v { + Value::String(s) => parse_interval_to_ms(s), + Value::Integer(secs) => secs.checked_mul(1000), + _ => None, + } +} + fn value_to_timestamp_ms(v: &Value) -> Option { match v { Value::Integer(n) => Some(*n), @@ -237,3 +250,117 @@ fn parse_interval_to_ms(s: &str) -> Option { .map(|ms| ms as i64) .filter(|&ms| ms > 0) } + +#[cfg(test)] +mod tests { + use super::*; + + fn naive(s: &str) -> Value { + Value::NaiveDateTime(NdbDateTime::parse(s).expect("valid test timestamp")) + } + + fn utc(s: &str) -> Value { + Value::DateTime(NdbDateTime::parse(s).expect("valid test timestamp")) + } + + #[test] + fn date_part_year_accepts_naive_and_utc() { + let naive_year = try_eval( + "date_part", + &[Value::String("year".into()), naive("2020-03-05T10:00:00")], + ); + assert_eq!(naive_year, Some(Value::Integer(2020))); + + let utc_year = try_eval( + "date_part", + &[Value::String("year".into()), utc("2020-03-05T10:00:00Z")], + ); + assert_eq!(utc_year, Some(Value::Integer(2020))); + } + + #[test] + fn date_trunc_preserves_utc_kind() { + let truncated = try_eval( + "date_trunc", + &[Value::String("hour".into()), utc("2020-03-05T10:42:17Z")], + ); + assert_eq!( + truncated, + Some(Value::DateTime( + NdbDateTime::parse("2020-03-05T10:00:00Z").expect("valid test timestamp") + )) + ); + } + + #[test] + fn time_bucket_naive_instant_floors_to_hour() { + let bucketed = try_eval( + "time_bucket", + &[Value::String("1 hour".into()), naive("2020-03-05T10:00:00")], + ); + assert_eq!(bucketed, Some(naive("2020-03-05T10:00:00"))); + + let bucketed = try_eval( + "time_bucket", + &[Value::String("1 hour".into()), naive("2020-03-05T10:59:59")], + ); + assert_eq!(bucketed, Some(naive("2020-03-05T10:00:00"))); + } + + #[test] + fn time_bucket_negative_micros_floors_toward_negative_infinity() { + let before_epoch = Value::NaiveDateTime(NdbDateTime::from_micros(-1_800_000_000)); + let bucketed = try_eval( + "time_bucket", + &[Value::String("1 hour".into()), before_epoch], + ); + assert_eq!( + bucketed, + Some(Value::NaiveDateTime(NdbDateTime::from_micros( + -3_600_000_000 + ))) + ); + } + + #[test] + fn time_bucket_integer_ms_unchanged() { + let bucketed = try_eval( + "time_bucket", + &[Value::String("1 hour".into()), Value::Integer(3_661_000)], + ); + assert_eq!(bucketed, Some(Value::Integer(3_600_000))); + } + + #[test] + fn datetime_of_naive_instant_keeps_kind() { + let result = try_eval("datetime", &[naive("2020-03-05T10:00:00")]); + assert_eq!(result, Some(naive("2020-03-05T10:00:00"))); + } + + #[test] + fn extract_on_integer_returns_null() { + let result = try_eval( + "extract", + &[Value::String("year".into()), Value::Integer(1_583_400_000)], + ); + assert_eq!(result, Some(Value::Null)); + } + + #[test] + fn time_bucket_integer_seconds_interval_accepts_an_instant() { + let bucketed = try_eval( + "time_bucket", + &[Value::Integer(3600), naive("2020-03-05T10:59:59")], + ); + assert_eq!(bucketed, Some(naive("2020-03-05T10:00:00"))); + } + + #[test] + fn date_diff_accepts_typed_instants() { + let diff = try_eval( + "date_diff", + &[naive("2020-03-05T10:00:00"), naive("2020-03-05T09:00:00")], + ); + assert_eq!(diff, Some(Value::Integer(3600))); + } +} From c22a0b58627a339d4319c90054550fe4a091e9d8 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 16 Sep 2026 13:41:16 +0800 Subject: [PATCH 04/21] refactor(msgpack): carry RowsPayload cells as typed Value, not TEXT strings RowsPayload cells (RETURNING, RLS row values, trigger batches, bulk DML, merge, timeseries scan/sort/merge) move from pre-rendered Option TEXT cells to NativeCell, a Value written in plain (untagged) msgpack. NativeCell shares its plain-msgpack reader/writer with value_to_msgpack/value_from_msgpack via a native decoder now generic over zerompk::Read, so both entry points produce identical bytes and agree on decode. RETURNING now retypes only text cells found under a numeric, bool or timestamp column instead of every cell, since typed cells already carry their number/bool/instant form. nodedb/src/util/rmpv_value.rs centralizes the rmpv::Value <-> nodedb_types::Value conversion (including instant ext round-tripping) used by timeseries rows, trigger batches and RETURNING projections, replacing the ad-hoc converter that lived in the trigger batch collector. --- nodedb-types/src/json_msgpack/mod.rs | 2 + nodedb-types/src/json_msgpack/native_cell.rs | 100 +++++ nodedb-types/src/json_msgpack/reader/mod.rs | 6 +- .../src/json_msgpack/reader/native.rs | 405 +++++++++--------- nodedb-types/src/json_msgpack/writer.rs | 186 +++----- nodedb-types/src/lib.rs | 6 +- .../server/response_shape/returning.rs | 118 +++-- nodedb/src/control/trigger/batch/collector.rs | 39 +- .../data/executor/handlers/bulk_dml/delete.rs | 54 +-- .../handlers/bulk_dml/delete_cascade.rs | 4 +- .../data/executor/handlers/bulk_dml/update.rs | 9 +- .../handlers/bulk_dml/update_project.rs | 2 +- .../handlers/document/resolve/bulk.rs | 4 +- .../merge_orchestrated/apply/insert_rows.rs | 2 +- .../merge_orchestrated/apply/orchestrate.rs | 2 +- .../merge_orchestrated/apply/update_rows.rs | 2 +- .../merge_orchestrated/apply_support.rs | 8 +- .../merge_orchestrated/delete_arms.rs | 2 +- .../data/executor/handlers/returning_doc.rs | 83 ++-- .../data/executor/handlers/returning_rows.rs | 184 ++++---- nodedb/src/data/executor/handlers/rls_eval.rs | 64 +-- .../data/executor/handlers/rls_write_gate.rs | 71 +-- .../handlers/timeseries/raw_scan/row_emit.rs | 50 +-- .../data/executor/handlers/timeseries/sort.rs | 59 +-- .../transaction/overlay/timeseries_merge.rs | 21 +- .../handlers/update_from_join_write.rs | 11 +- .../data/executor/response_codec/encode.rs | 25 +- .../src/data/executor/response_codec/hits.rs | 42 +- .../src/data/executor/strict_format/coerce.rs | 6 +- .../src/data/executor/strict_format/decode.rs | 52 ++- nodedb/src/data/executor/strict_format/mod.rs | 3 +- nodedb/src/util.rs | 1 + nodedb/src/util/rmpv_value.rs | 195 +++++++++ 33 files changed, 1070 insertions(+), 748 deletions(-) create mode 100644 nodedb-types/src/json_msgpack/native_cell.rs create mode 100644 nodedb/src/util/rmpv_value.rs diff --git a/nodedb-types/src/json_msgpack/mod.rs b/nodedb-types/src/json_msgpack/mod.rs index 4a0122149..8e76ce082 100644 --- a/nodedb-types/src/json_msgpack/mod.rs +++ b/nodedb-types/src/json_msgpack/mod.rs @@ -3,6 +3,7 @@ pub mod error; pub mod instant_ext; pub mod json_value; +pub mod native_cell; pub mod reader; pub mod transcoder; pub mod writer; @@ -13,6 +14,7 @@ pub use instant_ext::{ read_instant, write_instant, }; pub use json_value::JsonValue; +pub use native_cell::NativeCell; pub use reader::{json_from_msgpack, value_from_msgpack}; pub use transcoder::msgpack_to_json_string; pub use writer::{json_to_msgpack, json_to_msgpack_or_empty, value_to_msgpack}; diff --git a/nodedb-types/src/json_msgpack/native_cell.rs b/nodedb-types/src/json_msgpack/native_cell.rs new file mode 100644 index 000000000..2a79279a0 --- /dev/null +++ b/nodedb-types/src/json_msgpack/native_cell.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! `NativeCell`: a `Value` carried in plain msgpack inside a zerompk struct. +//! +//! `Value`'s own zerompk impl writes the tagged `[tag, payload]` form. A +//! struct that derives `ToMessagePack` with a `Value` field therefore ships +//! `"alice"` as `[4, "alice"]`, which the JSON transcoder renders verbatim. +//! Wrapping the field in `NativeCell` writes the same bytes +//! `value_to_msgpack` produces, so the transcoder renders a plain cell and +//! `value_from_msgpack` reads it back typed. + +use zerompk::{FromMessagePack, Read, ToMessagePack, Write}; + +use super::reader::native::read_native_value; +use super::writer::write_native_value; +use crate::Value; + +/// A `Value` written and read as plain msgpack. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct NativeCell(pub Value); + +impl From for NativeCell { + #[inline] + fn from(v: Value) -> Self { + Self(v) + } +} + +impl From for Value { + #[inline] + fn from(c: NativeCell) -> Self { + c.0 + } +} + +impl ToMessagePack for NativeCell { + fn write(&self, writer: &mut W) -> zerompk::Result<()> { + write_native_value(writer, &self.0) + } +} + +impl<'a> FromMessagePack<'a> for NativeCell { + fn read>(reader: &mut R) -> zerompk::Result { + read_native_value(reader).map(NativeCell) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::NdbDateTime; + use crate::json_msgpack::{value_from_msgpack, value_to_msgpack}; + + #[derive(ToMessagePack, FromMessagePack, PartialEq, Debug)] + #[msgpack(map)] + struct Row { + cells: Vec, + } + + fn sample() -> Vec { + let mut obj = std::collections::HashMap::new(); + obj.insert("n".to_string(), Value::Integer(-200)); + obj.insert( + "arr".to_string(), + Value::Array(vec![Value::Float(1.5), Value::Null]), + ); + vec![ + Value::Null, + Value::Bool(true), + Value::Integer(300), + Value::Integer(-5), + Value::Float(2.25), + Value::String("alice".into()), + Value::Bytes(vec![1, 2, 3]), + Value::DateTime(NdbDateTime::from_micros(1_583_402_400_000_000)), + Value::NaiveDateTime(NdbDateTime::from_micros(-86_400_000_000)), + Value::Object(obj), + ] + } + + #[test] + fn cells_round_trip_through_a_derived_struct() { + let row = Row { + cells: sample().into_iter().map(NativeCell).collect(), + }; + let bytes = zerompk::to_msgpack_vec(&row).expect("encode"); + let back: Row = zerompk::from_msgpack(&bytes).expect("decode"); + assert_eq!(back, row); + } + + #[test] + fn cell_bytes_match_value_to_msgpack() { + for v in sample() { + let plain = value_to_msgpack(&v).expect("plain"); + let cell = zerompk::to_msgpack_vec(&NativeCell(v.clone())).expect("cell"); + assert_eq!(cell, plain, "{v:?}"); + assert_eq!(value_from_msgpack(&cell).expect("read"), v); + } + } +} diff --git a/nodedb-types/src/json_msgpack/reader/mod.rs b/nodedb-types/src/json_msgpack/reader/mod.rs index a08621de8..afd926e62 100644 --- a/nodedb-types/src/json_msgpack/reader/mod.rs +++ b/nodedb-types/src/json_msgpack/reader/mod.rs @@ -1,8 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 -//! Cursor-based msgpack → `serde_json::Value` and `nodedb_types::Value` readers. +//! Msgpack → `serde_json::Value` and `nodedb_types::Value` readers. //! -//! Deterministic raw byte parser — the first byte of each msgpack value +//! The JSON reader walks the crate's own `Cursor`. The native reader is +//! generic over `zerompk::Read` and is shared with `NativeCell`. Both are +//! deterministic raw byte parsers — the first byte of each msgpack value //! unambiguously identifies its type per the msgpack specification. pub mod cursor; diff --git a/nodedb-types/src/json_msgpack/reader/native.rs b/nodedb-types/src/json_msgpack/reader/native.rs index 424ebbcf5..ba0cc4386 100644 --- a/nodedb-types/src/json_msgpack/reader/native.rs +++ b/nodedb-types/src/json_msgpack/reader/native.rs @@ -2,246 +2,159 @@ //! Msgpack → `nodedb_types::Value` reader. //! -//! Instant ext values (fixext8 type 1 / 2) become `Value::DateTime` / -//! `Value::NaiveDateTime`. Every other ext type becomes `Value::Null`. +//! [`read_native_value`] is the one decoder of a plain (untagged) msgpack +//! value. It is generic over `zerompk::Read`, so `value_from_msgpack` (slice +//! entry point) and `NativeCell::read` (inside a zerompk derive) share it. +//! +//! Instant ext values (`fixext8` type 1 / 2) become `Value::DateTime` / +//! `Value::NaiveDateTime`. Every other ext marker, and a `fixext8` of any +//! other type, becomes `Value::Null`. Map keys that are not strings are +//! rendered with `Debug`. + +use zerompk::{Read, SliceReader}; -use super::super::error::MsgpackResult; +use super::super::error::{MsgpackError, MsgpackResult}; use super::super::instant_ext::instant_from_ext; -use super::cursor::Cursor; +use crate::Value; + +const FIXEXT8: u8 = 0xD7; /// Deserialize a `nodedb_types::Value` from standard MessagePack bytes. /// /// The input must contain exactly one top-level value and nothing else; /// trailing bytes are rejected. -pub fn value_from_msgpack(bytes: &[u8]) -> MsgpackResult { - let mut cursor = Cursor::new(bytes); - let value = read_native_value(&mut cursor)?; - cursor.finish()?; +pub fn value_from_msgpack(bytes: &[u8]) -> MsgpackResult { + let mut reader = SliceReader::new(bytes); + let value = read_native_value(&mut reader)?; + if reader.peek_marker().is_ok() { + return Err(MsgpackError::TrailingBytes { + consumed: consumed_len(bytes), + total: bytes.len(), + }); + } Ok(value) } -fn read_native_value(c: &mut Cursor<'_>) -> zerompk::Result { - if c.depth > 500 { - return Err(zerompk::Error::DepthLimitExceeded { max: 500 }); +/// Length of the top-level value at the start of `bytes`. +/// +/// `SliceReader` exposes no position, so the length is found by binary +/// search over prefix lengths: decoding a prefix shorter than the value fails +/// on the missing bytes, and decoding any prefix at least as long as the value +/// succeeds. Called only on the trailing-bytes error path, after a full decode +/// of `bytes` succeeded. +fn consumed_len(bytes: &[u8]) -> usize { + let (mut lo, mut hi) = (0usize, bytes.len()); + while lo < hi { + let mid = lo + (hi - lo) / 2; + match read_native_value(&mut SliceReader::new(&bytes[..mid])) { + Ok(_) => hi = mid, + Err(_) => lo = mid + 1, + } } + lo +} - let marker = c.take()?; +/// Read one plain msgpack value, dispatching on the peeked marker. +pub(crate) fn read_native_value<'de, R: Read<'de>>(reader: &mut R) -> zerompk::Result { + let marker = reader.peek_marker()?; match marker { - 0xC0 => Ok(crate::Value::Null), - 0xC2 => Ok(crate::Value::Bool(false)), - 0xC3 => Ok(crate::Value::Bool(true)), - - 0x00..=0x7F => Ok(crate::Value::Integer(marker as i64)), - 0xE0..=0xFF => Ok(crate::Value::Integer(marker as i8 as i64)), - - 0xCC => Ok(crate::Value::Integer(c.take()? as i64)), - 0xCD => Ok(crate::Value::Integer(c.read_u16_be()? as i64)), - 0xCE => Ok(crate::Value::Integer(c.read_u32_be()? as i64)), - 0xCF => { - let b = c.take_n(8)?; - Ok(crate::Value::Integer(u64::from_be_bytes([ - b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], - ]) as i64)) - } - - 0xD0 => Ok(crate::Value::Integer(c.take()? as i8 as i64)), - 0xD1 => { - let b = c.take_n(2)?; - Ok(crate::Value::Integer( - i16::from_be_bytes([b[0], b[1]]) as i64 - )) - } - 0xD2 => { - let b = c.take_n(4)?; - Ok(crate::Value::Integer( - i32::from_be_bytes([b[0], b[1], b[2], b[3]]) as i64, - )) - } - 0xD3 => { - let b = c.take_n(8)?; - Ok(crate::Value::Integer(i64::from_be_bytes([ - b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], - ]))) - } - - 0xCA => { - let b = c.take_n(4)?; - Ok(crate::Value::Float( - f32::from_be_bytes([b[0], b[1], b[2], b[3]]) as f64, - )) - } - 0xCB => { - let b = c.take_n(8)?; - Ok(crate::Value::Float(f64::from_be_bytes([ - b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], - ]))) - } - - m @ 0xA0..=0xBF => read_native_str(c, (m & 0x1F) as usize), - 0xD9 => { - let l = c.take()? as usize; - read_native_str(c, l) + 0xC0 => { + reader.read_nil()?; + Ok(Value::Null) } - 0xDA => { - let l = c.read_u16_be()? as usize; - read_native_str(c, l) - } - 0xDB => { - let l = c.read_u32_be()? as usize; - read_native_str(c, l) - } - - 0xC4 => { - let l = c.take()? as usize; - Ok(crate::Value::Bytes(c.take_n(l)?.to_vec())) - } - 0xC5 => { - let l = c.read_u16_be()? as usize; - Ok(crate::Value::Bytes(c.take_n(l)?.to_vec())) - } - 0xC6 => { - let l = c.read_u32_be()? as usize; - Ok(crate::Value::Bytes(c.take_n(l)?.to_vec())) - } - - m @ 0x90..=0x9F => read_native_array(c, (m & 0x0F) as usize), - 0xDC => { - let l = c.read_u16_be()? as usize; - read_native_array(c, l) - } - 0xDD => { - let l = c.read_u32_be()? as usize; - read_native_array(c, l) - } - - m @ 0x80..=0x8F => read_native_map(c, (m & 0x0F) as usize), - 0xDE => { - let l = c.read_u16_be()? as usize; - read_native_map(c, l) + 0xC2 | 0xC3 => Ok(Value::Bool(reader.read_boolean()?)), + 0x00..=0x7F | 0xE0..=0xFF | 0xD0..=0xD3 => Ok(Value::Integer(reader.read_i64()?)), + 0xCC..=0xCF => Ok(Value::Integer(reader.read_u64()? as i64)), + 0xCA => Ok(Value::Float(f64::from(reader.read_f32()?))), + 0xCB => Ok(Value::Float(reader.read_f64()?)), + 0xA0..=0xBF | 0xD9..=0xDB => Ok(Value::String(reader.read_string()?.into_owned())), + 0xC4..=0xC6 => Ok(Value::Bytes(reader.read_binary()?.into_owned())), + 0x90..=0x9F | 0xDC | 0xDD => { + let len = reader.read_array_len()?; + reader.increment_depth()?; + let mut arr = Vec::with_capacity(len.min(4096)); + for _ in 0..len { + arr.push(read_native_value(reader)?); + } + reader.decrement_depth(); + Ok(Value::Array(arr)) } - 0xDF => { - let l = c.read_u32_be()? as usize; - read_native_map(c, l) + 0x80..=0x8F | 0xDE | 0xDF => { + let len = reader.read_map_len()?; + reader.increment_depth()?; + let mut map = std::collections::HashMap::with_capacity(len.min(4096)); + for _ in 0..len { + let key = read_native_key(reader)?; + map.insert(key, read_native_value(reader)?); + } + reader.decrement_depth(); + Ok(Value::Object(map)) } - - // fixext8: instants decode to their Value variant, other types to Null - 0xD7 => { - let (ext_type, payload) = c.take_fixext8()?; - Ok(match instant_from_ext(ext_type, payload) { + 0xC7..=0xC9 | 0xD4..=0xD8 => { + let (ext_type, payload) = reader.read_ext()?; + let instant = if marker == FIXEXT8 { + instant_from_ext(ext_type, &payload) + } else { + None + }; + Ok(match instant { Some((kind, micros)) => kind.from_micros(micros), - None => crate::Value::Null, + None => Value::Null, }) } - - // other ext types — skip - 0xD4 => { - c.take_n(2)?; - Ok(crate::Value::Null) - } - 0xD5 => { - c.take_n(3)?; - Ok(crate::Value::Null) - } - 0xD6 => { - c.take_n(5)?; - Ok(crate::Value::Null) - } - 0xD8 => { - c.take_n(17)?; - Ok(crate::Value::Null) - } - 0xC7 => { - let l = c.take()? as usize; - c.take_n(1 + l)?; - Ok(crate::Value::Null) - } - 0xC8 => { - let l = c.read_u16_be()? as usize; - c.take_n(1 + l)?; - Ok(crate::Value::Null) - } - 0xC9 => { - let l = c.read_u32_be()? as usize; - c.take_n(1 + l)?; - Ok(crate::Value::Null) - } - - _ => Err(zerompk::Error::InvalidMarker(marker)), - } -} - -fn read_native_str(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - let bytes = c.take_n(len)?; - let s = String::from_utf8(bytes.to_vec()).map_err(|_| zerompk::Error::InvalidMarker(0))?; - Ok(crate::Value::String(s)) -} - -fn read_native_array(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - c.depth += 1; - let mut arr = Vec::with_capacity(len.min(4096)); - for _ in 0..len { - arr.push(read_native_value(c)?); + other => Err(zerompk::Error::InvalidMarker(other)), } - c.depth -= 1; - Ok(crate::Value::Array(arr)) } -fn read_native_map(c: &mut Cursor<'_>, len: usize) -> zerompk::Result { - c.depth += 1; - let mut map = std::collections::HashMap::with_capacity(len.min(4096)); - for _ in 0..len { - let key_marker = c.peek()?; - let key = if (0xA0..=0xBF).contains(&key_marker) - || key_marker == 0xD9 - || key_marker == 0xDA - || key_marker == 0xDB - { - match read_native_value(c)? { - crate::Value::String(s) => s, - other => format!("{other:?}"), - } - } else { - let v = read_native_value(c)?; - format!("{v:?}") - }; - let val = read_native_value(c)?; - map.insert(key, val); +/// Read a map key. A string key is taken as-is; any other value is rendered +/// with `Debug`. +fn read_native_key<'de, R: Read<'de>>(reader: &mut R) -> zerompk::Result { + match reader.peek_marker()? { + 0xA0..=0xBF | 0xD9..=0xDB => Ok(reader.read_string()?.into_owned()), + _ => { + let v = read_native_value(reader)?; + Ok(format!("{v:?}")) + } } - c.depth -= 1; - Ok(crate::Value::Object(map)) } #[cfg(test)] mod tests { - //! Roundtrip tests for the native reader against the native writer. + //! Roundtrip tests for the native reader against the native writer, and + //! agreement between the `value_from_msgpack` and `NativeCell` entry points. use super::*; use crate::NdbDateTime; + use crate::json_msgpack::NativeCell; use crate::json_msgpack::transcoder::msgpack_to_json_string; use crate::json_msgpack::writer::value_to_msgpack; + fn through_cell(bytes: &[u8]) -> zerompk::Result { + zerompk::from_msgpack::(bytes).map(Value::from) + } + #[test] fn native_value_roundtrip() { let mut map = std::collections::HashMap::new(); - map.insert("id".to_string(), crate::Value::String("host1".into())); - map.insert("cpu".to_string(), crate::Value::Float(0.75)); - map.insert("mem".to_string(), crate::Value::Float(0.5)); + map.insert("id".to_string(), Value::String("host1".into())); + map.insert("cpu".to_string(), Value::Float(0.75)); + map.insert("mem".to_string(), Value::Float(0.5)); - let row = crate::Value::Object(map); - let arr = crate::Value::Array(vec![row]); + let row = Value::Object(map); + let arr = Value::Array(vec![row]); let bytes = value_to_msgpack(&arr).unwrap(); let decoded = value_from_msgpack(&bytes).unwrap(); match &decoded { - crate::Value::Array(items) => { + Value::Array(items) => { assert_eq!(items.len(), 1); match &items[0] { - crate::Value::Object(m) => { + Value::Object(m) => { assert_eq!(m.len(), 3); - assert_eq!(m.get("id"), Some(&crate::Value::String("host1".into()))); - assert_eq!(m.get("cpu"), Some(&crate::Value::Float(0.75))); - assert_eq!(m.get("mem"), Some(&crate::Value::Float(0.5))); + assert_eq!(m.get("id"), Some(&Value::String("host1".into()))); + assert_eq!(m.get("cpu"), Some(&Value::Float(0.75))); + assert_eq!(m.get("mem"), Some(&Value::Float(0.5))); } other => panic!("expected Object, got {other:?}"), } @@ -252,12 +165,12 @@ mod tests { #[test] fn native_value_scalars() { - let cases: Vec = vec![ - crate::Value::Null, - crate::Value::Bool(true), - crate::Value::Integer(42), - crate::Value::Float(2.72), - crate::Value::String("hello".into()), + let cases: Vec = vec![ + Value::Null, + Value::Bool(true), + Value::Integer(42), + Value::Float(2.72), + Value::String("hello".into()), ]; for val in cases { let bytes = value_to_msgpack(&val).unwrap(); @@ -270,9 +183,9 @@ mod tests { fn instant_roundtrip_both_kinds() { let dt = NdbDateTime::from_micros(1_710_498_600_000_000); for val in [ - crate::Value::DateTime(dt), - crate::Value::NaiveDateTime(dt), - crate::Value::DateTime(NdbDateTime::from_micros(-1)), + Value::DateTime(dt), + Value::NaiveDateTime(dt), + Value::DateTime(NdbDateTime::from_micros(-1)), ] { let bytes = value_to_msgpack(&val).unwrap(); assert_eq!(bytes.len(), 10); @@ -280,7 +193,7 @@ mod tests { assert_eq!(value_from_msgpack(&bytes).unwrap(), val); } - let bytes = value_to_msgpack(&crate::Value::NaiveDateTime(dt)).unwrap(); + let bytes = value_to_msgpack(&Value::NaiveDateTime(dt)).unwrap(); assert_eq!( msgpack_to_json_string(&bytes).unwrap(), "\"2024-03-15T10:30:00.000000Z\"" @@ -290,6 +203,86 @@ mod tests { #[test] fn unknown_ext_is_null() { let bytes = [0xD7, 0x09, 0, 0, 0, 0, 0, 0, 0, 1]; - assert_eq!(value_from_msgpack(&bytes).unwrap(), crate::Value::Null); + assert_eq!(value_from_msgpack(&bytes).unwrap(), Value::Null); + } + + #[test] + fn unsigned_64_reads_as_integer() { + let bytes = [0xCF, 0, 0, 0, 0, 0, 0, 0x01, 0x00]; + assert_eq!(through_cell(&bytes).unwrap(), Value::Integer(256)); + assert_eq!(value_from_msgpack(&bytes).unwrap(), Value::Integer(256)); + } + + #[test] + fn unknown_ext_reads_as_null() { + let bytes = [0xD4, 0x07, 0x00]; + assert_eq!(through_cell(&bytes).unwrap(), Value::Null); + assert_eq!(value_from_msgpack(&bytes).unwrap(), Value::Null); + } + + #[test] + fn unused_marker_is_an_error() { + assert!(through_cell(&[0xC1]).is_err()); + assert!(value_from_msgpack(&[0xC1]).is_err()); + } + + #[test] + fn non_string_map_key_is_debug_rendered() { + // {1: "a"} + let bytes = [0x81, 0x01, 0xA1, b'a']; + let expected = Value::Object(std::collections::HashMap::from([( + "Integer(1)".to_string(), + Value::String("a".into()), + )])); + assert_eq!(value_from_msgpack(&bytes).unwrap(), expected); + assert_eq!(through_cell(&bytes).unwrap(), expected); + } + + #[test] + fn trailing_bytes_report_consumed_length() { + let mut bytes = value_to_msgpack(&Value::Array(vec![ + Value::Integer(1), + Value::String("ab".into()), + ])) + .unwrap(); + let len = bytes.len(); + bytes.extend_from_slice(&[0xC0, 0xC0]); + match value_from_msgpack(&bytes) { + Err(MsgpackError::TrailingBytes { consumed, total }) => { + assert_eq!(consumed, len); + assert_eq!(total, len + 2); + } + other => panic!("expected TrailingBytes, got {other:?}"), + } + } + + #[test] + fn both_entry_points_agree() { + let mut inner = std::collections::HashMap::new(); + inner.insert("k".to_string(), Value::Array(vec![Value::Integer(1)])); + let mut outer = std::collections::HashMap::new(); + outer.insert("nested".to_string(), Value::Object(inner)); + outer.insert("list".to_string(), Value::Array(vec![Value::Null])); + + let mut cases: Vec> = [ + Value::Object(outer), + Value::DateTime(NdbDateTime::from_micros(1_583_402_400_000_000)), + Value::NaiveDateTime(NdbDateTime::from_micros(-86_400_000_000)), + Value::Integer(-70_000), + Value::Bytes(vec![0, 255, 7]), + ] + .iter() + .map(|v| value_to_msgpack(v).unwrap()) + .collect(); + // u64 above i64::MAX + cases.push(vec![0xCF, 0x80, 0, 0, 0, 0, 0, 0, 0x01]); + // unknown ext (ext8, type 9, one payload byte) + cases.push(vec![0xC7, 0x01, 0x09, 0x2A]); + + for bytes in cases { + let a = value_from_msgpack(&bytes).unwrap(); + let b = through_cell(&bytes).unwrap(); + assert_eq!(a, b, "{bytes:02X?}"); + } } } diff --git a/nodedb-types/src/json_msgpack/writer.rs b/nodedb-types/src/json_msgpack/writer.rs index daaa680cb..2cd85ae50 100644 --- a/nodedb-types/src/json_msgpack/writer.rs +++ b/nodedb-types/src/json_msgpack/writer.rs @@ -7,7 +7,9 @@ //! are written as strings. `Vector` is a float64 array, `ArrayCell` a map, //! and `Range` / `Record` are `nil`. -use super::instant_ext::{InstantKind, write_instant}; +use zerompk::Write; + +use super::instant_ext::InstantKind; use super::json_value::JsonValue; /// Serialize a `serde_json::Value` to MessagePack bytes. @@ -32,60 +34,68 @@ pub fn json_to_msgpack_or_empty(value: &serde_json::Value) -> Vec { /// Writes standard msgpack format (fixmap 0x80-0x8F, fixstr 0xA0-0xBF, etc.) /// directly from `Value` — no zerompk tagged encoding. pub fn value_to_msgpack(value: &crate::Value) -> zerompk::Result> { - let mut buf = Vec::with_capacity(128); - write_native_value(&mut buf, value); - Ok(buf) + zerompk::to_msgpack_vec(&NativeRef(value)) +} + +/// A borrowed `Value` written in its plain msgpack form. +struct NativeRef<'a>(&'a crate::Value); + +impl zerompk::ToMessagePack for NativeRef<'_> { + fn write(&self, writer: &mut W) -> zerompk::Result<()> { + write_native_value(writer, self.0) + } } -/// Write a `nodedb_types::Value` as standard msgpack bytes. -fn write_native_value(buf: &mut Vec, value: &crate::Value) { +/// Write a `nodedb_types::Value` as standard msgpack. +/// +/// `Duration`, `Decimal`, and `Geometry` are strings. `Vector` is a float64 +/// array, `ArrayCell` a map, and `Range` / `Record` are `nil`. +pub(crate) fn write_native_value( + writer: &mut W, + value: &crate::Value, +) -> zerompk::Result<()> { match value { - crate::Value::Null => buf.push(0xC0), - crate::Value::Bool(false) => buf.push(0xC2), - crate::Value::Bool(true) => buf.push(0xC3), - crate::Value::Integer(i) => write_native_int(buf, *i), - crate::Value::Float(f) => { - buf.push(0xCB); - buf.extend_from_slice(&f.to_be_bytes()); - } + crate::Value::Null => writer.write_nil(), + crate::Value::Bool(b) => writer.write_boolean(*b), + crate::Value::Integer(i) => writer.write_i64(*i), + crate::Value::Float(f) => writer.write_f64(*f), crate::Value::String(s) | crate::Value::Uuid(s) | crate::Value::Ulid(s) - | crate::Value::Regex(s) => write_native_str(buf, s), - crate::Value::Bytes(b) => write_native_bin(buf, b), + | crate::Value::Regex(s) => writer.write_string(s), + crate::Value::Bytes(b) => writer.write_binary(b), crate::Value::Array(arr) | crate::Value::Set(arr) => { - write_native_array_header(buf, arr.len()); + writer.write_array_len(arr.len())?; for v in arr { - write_native_value(buf, v); + write_native_value(writer, v)?; } + Ok(()) } crate::Value::Object(map) => { - write_native_map_header(buf, map.len()); + writer.write_map_len(map.len())?; for (k, v) in map { - write_native_str(buf, k); - write_native_value(buf, v); - } - } - crate::Value::DateTime(dt) => write_instant(buf, InstantKind::Utc, dt.micros), - crate::Value::NaiveDateTime(dt) => write_instant(buf, InstantKind::Naive, dt.micros), - crate::Value::Duration(d) => write_native_str(buf, &d.to_string()), - crate::Value::Decimal(d) => write_native_str(buf, &d.to_string()), - crate::Value::Geometry(g) => { - if let Ok(s) = sonic_rs::to_string(g) { - write_native_str(buf, &s); - } else { - buf.push(0xC0); + writer.write_string(k)?; + write_native_value(writer, v)?; } + Ok(()) } - crate::Value::Range { .. } | crate::Value::Record { .. } => buf.push(0xC0), + crate::Value::DateTime(dt) => write_instant_ext(writer, InstantKind::Utc, dt.micros), + crate::Value::NaiveDateTime(dt) => write_instant_ext(writer, InstantKind::Naive, dt.micros), + crate::Value::Duration(d) => writer.write_string(&d.to_string()), + crate::Value::Decimal(d) => writer.write_string(&d.to_string()), + crate::Value::Geometry(g) => match sonic_rs::to_string(g) { + Ok(s) => writer.write_string(&s), + Err(_) => writer.write_nil(), + }, + crate::Value::Range { .. } | crate::Value::Record { .. } => writer.write_nil(), crate::Value::Vector(v) => { - // Encode as a standard msgpack array of float64 values so that - // pgwire clients receive a plain JSON number array. - write_native_array_header(buf, v.len()); + // A standard msgpack array of float64 values, so pgwire clients + // receive a plain JSON number array. + writer.write_array_len(v.len())?; for f in v.iter() { - buf.push(0xCB); - buf.extend_from_slice(&(*f as f64).to_be_bytes()); + writer.write_f64(f64::from(*f))?; } + Ok(()) } // ArrayCell is encoded as a `{coords:[...], attrs:[...]}` map so the // pgwire `msgpack_to_json_string` transcoder produces clean JSON for @@ -94,97 +104,31 @@ fn write_native_value(buf: &mut Vec, value: &crate::Value) { // `_ts_system` column (mirrors the document-engine audit-log shape). crate::Value::ArrayCell(cell) => { let map_len = if cell.system_time.is_some() { 3 } else { 2 }; - write_native_map_header(buf, map_len); - write_native_str(buf, "coords"); - write_native_array_header(buf, cell.coords.len()); + writer.write_map_len(map_len)?; + writer.write_string("coords")?; + writer.write_array_len(cell.coords.len())?; for v in &cell.coords { - write_native_value(buf, v); + write_native_value(writer, v)?; } - write_native_str(buf, "attrs"); - write_native_array_header(buf, cell.attrs.len()); + writer.write_string("attrs")?; + writer.write_array_len(cell.attrs.len())?; for v in &cell.attrs { - write_native_value(buf, v); + write_native_value(writer, v)?; } if let Some(ts) = cell.system_time { - write_native_str(buf, "_ts_system"); - write_native_int(buf, ts); + writer.write_string("_ts_system")?; + writer.write_i64(ts)?; } + Ok(()) } } } -fn write_native_int(buf: &mut Vec, i: i64) { - if (0..=0x7F).contains(&i) { - buf.push(i as u8); - } else if (-32..0).contains(&i) { - buf.push(i as u8); // negative fixint - } else if i >= i8::MIN as i64 && i <= i8::MAX as i64 { - buf.push(0xD0); - buf.push(i as i8 as u8); - } else if i >= i16::MIN as i64 && i <= i16::MAX as i64 { - buf.push(0xD1); - buf.extend_from_slice(&(i as i16).to_be_bytes()); - } else if i >= i32::MIN as i64 && i <= i32::MAX as i64 { - buf.push(0xD2); - buf.extend_from_slice(&(i as i32).to_be_bytes()); - } else { - buf.push(0xD3); - buf.extend_from_slice(&i.to_be_bytes()); - } -} - -fn write_native_str(buf: &mut Vec, s: &str) { - let len = s.len(); - if len < 32 { - buf.push(0xA0 | len as u8); - } else if len <= u8::MAX as usize { - buf.push(0xD9); - buf.push(len as u8); - } else if len <= u16::MAX as usize { - buf.push(0xDA); - buf.extend_from_slice(&(len as u16).to_be_bytes()); - } else { - buf.push(0xDB); - buf.extend_from_slice(&(len as u32).to_be_bytes()); - } - buf.extend_from_slice(s.as_bytes()); -} - -fn write_native_bin(buf: &mut Vec, b: &[u8]) { - let len = b.len(); - if len <= u8::MAX as usize { - buf.push(0xC4); - buf.push(len as u8); - } else if len <= u16::MAX as usize { - buf.push(0xC5); - buf.extend_from_slice(&(len as u16).to_be_bytes()); - } else { - buf.push(0xC6); - buf.extend_from_slice(&(len as u32).to_be_bytes()); - } - buf.extend_from_slice(b); -} - -fn write_native_array_header(buf: &mut Vec, len: usize) { - if len < 16 { - buf.push(0x90 | len as u8); - } else if len <= u16::MAX as usize { - buf.push(0xDC); - buf.extend_from_slice(&(len as u16).to_be_bytes()); - } else { - buf.push(0xDD); - buf.extend_from_slice(&(len as u32).to_be_bytes()); - } -} - -fn write_native_map_header(buf: &mut Vec, len: usize) { - if len < 16 { - buf.push(0x80 | len as u8); - } else if len <= u16::MAX as usize { - buf.push(0xDE); - buf.extend_from_slice(&(len as u16).to_be_bytes()); - } else { - buf.push(0xDF); - buf.extend_from_slice(&(len as u32).to_be_bytes()); - } +/// Write an instant as the `fixext8` ext `write_instant` produces. +fn write_instant_ext( + writer: &mut W, + kind: InstantKind, + micros: i64, +) -> zerompk::Result<()> { + writer.write_ext(kind.ext_type(), µs.to_be_bytes()) } diff --git a/nodedb-types/src/lib.rs b/nodedb-types/src/lib.rs index 4640dce48..1342db40e 100644 --- a/nodedb-types/src/lib.rs +++ b/nodedb-types/src/lib.rs @@ -105,9 +105,9 @@ pub use id::{ }; pub use identity::KeyRepr; pub use json_msgpack::{ - InstantKind, JsonValue, MsgpackError, MsgpackResult, json_from_msgpack, json_to_msgpack, - json_to_msgpack_or_empty, msgpack_to_json_string, read_instant, value_from_msgpack, - value_to_msgpack, write_instant, + InstantKind, JsonValue, MsgpackError, MsgpackResult, NativeCell, json_from_msgpack, + json_to_msgpack, json_to_msgpack_or_empty, msgpack_to_json_string, read_instant, + value_from_msgpack, value_to_msgpack, write_instant, }; pub use kv::{KV_DEFAULT_INLINE_THRESHOLD, KvConfig, KvTtlPolicy, is_valid_kv_key_type}; pub use lsn::Lsn; diff --git a/nodedb/src/control/server/response_shape/returning.rs b/nodedb/src/control/server/response_shape/returning.rs index 82af689ec..eb3395391 100644 --- a/nodedb/src/control/server/response_shape/returning.rs +++ b/nodedb/src/control/server/response_shape/returning.rs @@ -3,7 +3,7 @@ //! Shaping for DML `RETURNING` responses. //! //! A `RETURNING` payload is a [`RowsPayload`]: the Data Plane's own column list -//! plus already-TEXT-formatted cells. For `RETURNING *` that column list is +//! plus typed `Value` cells. For `RETURNING *` that column list is //! derived from the STORED row, so a schemaless collection can carry fields no //! catalog column declares — the list is only knowable once the rows exist. //! @@ -36,7 +36,7 @@ use serde_json::{Map, Value as JsonValue}; -use nodedb_types::NodeDbError; +use nodedb_types::{NativeCell, NodeDbError}; use crate::data::executor::response_codec::{RowsPayload, decode_payload_to_json}; @@ -91,16 +91,17 @@ pub fn shape_returning_rows( } }; - let mut rows = rows_keyed_by_column(&rp); + let RowsPayload { columns, rows } = rp; + let mut rows = rows_keyed_by_column(&columns, rows); // `RETURNING` delivers stored column values to the client just as a SELECT // does, so the same redaction applies — and it runs on the payload's own // names, before any projection renames or drops them. redact_rows(redaction.as_ref(), &mut rows); let Some(schema) = announced else { - let column_types = ShapedRows::text_types(rp.columns.len()); + let column_types = ShapedRows::text_types(columns.len()); return Ok(ShapedRows { - columns: rp.columns, + columns, column_types, rows, notice: None, @@ -116,19 +117,19 @@ fn announced_columns(projection: Option<&OutputSchema>) -> Option<&OutputSchema> projection.filter(|schema| !schema.is_star && !schema.columns.is_empty()) } -/// Re-key each payload row from positional cells to a name-keyed map, with -/// JSON `null` for the cells the Data Plane marked SQL NULL. -fn rows_keyed_by_column(rp: &RowsPayload) -> Vec> { - rp.rows - .iter() +/// Re-key each payload row from positional typed cells to a name-keyed JSON +/// map. This is the one step where a cell leaves its `Value` form: an +/// instant renders as ISO-8601 text, `Value::Null` (SQL NULL) as JSON `null`, +/// numbers and booleans as themselves. +fn rows_keyed_by_column( + columns: &[String], + rows: Vec>, +) -> Vec> { + rows.into_iter() .map(|row_vals| { let mut map = Map::new(); - for (col, cell) in rp.columns.iter().zip(row_vals.iter()) { - let v = match cell { - Some(s) => JsonValue::String(s.clone()), - None => JsonValue::Null, - }; - map.insert(col.clone(), v); + for (col, cell) in columns.iter().zip(row_vals) { + map.insert(col.clone(), JsonValue::from(cell.0)); } map }) @@ -175,15 +176,15 @@ fn project_onto_announced(schema: &OutputSchema, rows: &[Map] } } -/// Re-read a `RETURNING` cell's TEXT form as the column's announced type. +/// Re-read a text `RETURNING` cell as the column's announced type. /// -/// `RowsPayload` cells arrive already rendered as text, but the RowDescription -/// this response is held to announces each column's real catalog type — and a -/// client that asked for a column in BINARY result format is handed the -/// scalar's wire bytes, which have to come from a number or a bool, not from -/// the digits of its text form. Retyping here also makes a `RETURNING` -/// timestamp render in the same ISO-8601 text a `SELECT` of that column -/// renders, instead of raw epoch microseconds. +/// A typed cell is already a number, bool or instant and passes through. A +/// text cell under a numeric, bool or timestamp column is a stored string +/// (a schemaless row can hold `"42"` under a declared `INT`), and the +/// RowDescription this response is held to announces the catalog type — a +/// client that asked for BINARY result format is handed the scalar's wire +/// bytes, which have to come from a number or a bool, not from the digits +/// of its text form. /// /// A cell that does not parse as its announced type is left as text, which /// both encoders render verbatim. @@ -251,18 +252,37 @@ fn single_result_column_empty() -> ShapedRows { mod tests { use super::*; use crate::control::server::response_shape::schema::OutputColumn; + use nodedb_types::{NdbDateTime, Value}; - fn payload(columns: &[&str], rows: &[&[Option<&str>]]) -> Vec { + fn typed_payload(columns: &[&str], rows: Vec>) -> Vec { let rp = RowsPayload { columns: columns.iter().map(|c| (*c).to_string()).collect(), rows: rows - .iter() - .map(|row| row.iter().map(|cell| cell.map(|c| c.to_string())).collect()) + .into_iter() + .map(|row| row.into_iter().map(NativeCell).collect()) .collect(), }; zerompk::to_msgpack_vec(&rp).expect("encode RowsPayload") } + /// Text cells, the form a schemaless row's string fields arrive in; + /// `None` is SQL NULL. + fn payload(columns: &[&str], rows: &[&[Option<&str>]]) -> Vec { + typed_payload( + columns, + rows.iter() + .map(|row| { + row.iter() + .map(|cell| match cell { + Some(text) => Value::String((*text).to_string()), + None => Value::Null, + }) + .collect() + }) + .collect(), + ) + } + fn announced(columns: &[(&str, DdlColType)]) -> OutputSchema { OutputSchema { columns: columns @@ -348,6 +368,50 @@ mod tests { assert_eq!(shaped.rows[0]["s"], JsonValue::String("42".into())); } + /// A typed cell passes through as itself: an integer stays a number under + /// an INT column, an instant renders as the ISO-8601 text a `SELECT` of + /// the same column renders, and SQL NULL is JSON `null`. + #[test] + fn typed_cells_reach_the_row_as_themselves() { + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + let bytes = typed_payload( + &["n", "at", "f", "gone"], + vec![vec![ + Value::Integer(7), + Value::NaiveDateTime(at), + Value::Float(1.5), + Value::Null, + ]], + ); + let schema = announced(&[ + ("n", DdlColType::Int8), + ("at", DdlColType::Timestamp), + ("f", DdlColType::Float8), + ("gone", DdlColType::Text), + ]); + let shaped = shape_returning_rows(&bytes, Some(&schema), None).expect("shape"); + assert_eq!(shaped.rows[0]["n"], JsonValue::from(7i64)); + assert_eq!( + shaped.rows[0]["at"], + JsonValue::String("2020-03-05T10:00:00.000000Z".into()) + ); + assert_eq!(shaped.rows[0]["f"], JsonValue::from(1.5f64)); + assert_eq!(shaped.rows[0]["gone"], JsonValue::Null); + } + + /// The same instant renders as ISO-8601 when nothing was announced, so the + /// simple-query protocol shows what the extended one does. + #[test] + fn an_instant_cell_renders_iso8601_without_a_projection() { + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + let bytes = typed_payload(&["at"], vec![vec![Value::DateTime(at)]]); + let shaped = shape_returning_rows(&bytes, None, None).expect("shape"); + assert_eq!( + shaped.rows[0]["at"], + JsonValue::String("2020-03-05T10:00:00.000000Z".into()) + ); + } + /// A value that does not parse as its announced type stays text rather /// than becoming NULL — the encoder renders it verbatim. #[test] diff --git a/nodedb/src/control/trigger/batch/collector.rs b/nodedb/src/control/trigger/batch/collector.rs index 7eda62a17..ae87e3d4e 100644 --- a/nodedb/src/control/trigger/batch/collector.rs +++ b/nodedb/src/control/trigger/batch/collector.rs @@ -20,6 +20,9 @@ use std::sync::OnceLock; use nodedb_types::Value; +use super::super::row_identity::inject_row_identity; +use crate::util::rmpv_value::rmpv_to_value; + /// A single row in a trigger batch. /// /// Stores raw MessagePack bytes from the WriteEvent and decodes directly @@ -176,42 +179,6 @@ fn decode_msgpack_to_value_map(bytes: &[u8], row_id: &str) -> Option Value { - match val { - rmpv::Value::Nil => Value::Null, - rmpv::Value::Boolean(b) => Value::Bool(*b), - rmpv::Value::Integer(i) => { - if let Some(n) = i.as_i64() { - Value::Integer(n) - } else if let Some(n) = i.as_u64() { - Value::Integer(n as i64) - } else { - Value::Null - } - } - rmpv::Value::F32(f) => Value::Float(*f as f64), - rmpv::Value::F64(f) => Value::Float(*f), - rmpv::Value::String(s) => Value::String(s.as_str().unwrap_or("").to_string()), - rmpv::Value::Binary(b) => Value::Bytes(b.clone()), - rmpv::Value::Array(arr) => Value::Array(arr.iter().map(rmpv_to_value).collect()), - rmpv::Value::Map(pairs) => { - let mut map = HashMap::new(); - for (k, v) in pairs { - let key = match k { - rmpv::Value::String(s) => s.as_str().unwrap_or("").to_string(), - other => format!("{other}"), - }; - map.insert(key, rmpv_to_value(v)); - } - Value::Object(map) - } - rmpv::Value::Ext(_, _) => Value::Null, - } -} - /// A complete batch of rows for trigger dispatch. #[derive(Debug)] pub struct TriggerBatch { diff --git a/nodedb/src/data/executor/handlers/bulk_dml/delete.rs b/nodedb/src/data/executor/handlers/bulk_dml/delete.rs index 1c5d0e7b3..d4810a54e 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/delete.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/delete.rs @@ -225,7 +225,7 @@ impl CoreLoop { // replay soft-deletes the HNSW node through `apply_point_delete`. Only // populated when the collection has a vector index. let mut write_set: Vec = Vec::new(); - let mut returned_docs: Vec = if returning.is_some() { + let mut returned_docs: Vec = if returning.is_some() { Vec::with_capacity(apply_ids.len()) } else { Vec::new() @@ -241,33 +241,35 @@ impl CoreLoop { // will not decode is a different answer: it would silently drop out // of RETURNING and, worse, contribute no removed index tuples, so // its old secondary-index entries would survive the delete. - let pre_delete_doc: Option = if returning.is_some() - || !index_paths.is_empty() - { - match self - .sparse - .get( - task.request.database_id.as_u64(), - tid, - collection, - storage_key, - ) - .ok() - .flatten() - { - Some(bytes) => { - let identity = storage_key.to_identity(); - match returning_doc::from_stored(&bytes, &identity, strict_schema.as_ref()) - { - Ok(doc) => Some(doc), - Err(e) => return self.response_error(task, e), + let pre_delete_doc: Option = + if returning.is_some() || !index_paths.is_empty() { + match self + .sparse + .get( + task.request.database_id.as_u64(), + tid, + collection, + storage_key, + ) + .ok() + .flatten() + { + Some(bytes) => { + let identity = storage_key.to_identity(); + match returning_doc::from_stored_json( + &bytes, + &identity, + strict_schema.as_ref(), + ) { + Ok(doc) => Some(doc), + Err(e) => return self.response_error(task, e), + } } + None => None, } - None => None, - } - } else { - None - }; + } else { + None + }; // The removal and the materialized-sum deltas it owes share ONE // transaction, so a debited target row can never outlive a removal diff --git a/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs b/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs index 7e5c3949a..19f851396 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs @@ -51,7 +51,7 @@ impl CoreLoop { &mut self, cascade: BulkDeleteRowCascade<'_>, write_set: &mut Vec, - returned_docs: &mut Vec, + returned_docs: &mut Vec, ) { let BulkDeleteRowCascade { task, @@ -185,7 +185,7 @@ impl CoreLoop { Some(old_converted.as_deref().unwrap_or(deleted_bytes)), ); if returning && let Some(doc) = pre_delete_doc { - returned_docs.push(doc); + returned_docs.push(nodedb_types::Value::from(doc)); } } } diff --git a/nodedb/src/data/executor/handlers/bulk_dml/update.rs b/nodedb/src/data/executor/handlers/bulk_dml/update.rs index 12a0b029d..cbe7a232f 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/update.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/update.rs @@ -186,7 +186,7 @@ impl CoreLoop { // lets the Control Plane mint a durable `Put` redo per row. Only populated // when the collection has a vector index. let mut write_set: Vec = Vec::new(); - let mut returned_docs: Vec = if returning.is_some() { + let mut returned_docs: Vec = if returning.is_some() { Vec::with_capacity(apply_ids.len()) } else { Vec::new() @@ -230,7 +230,7 @@ impl CoreLoop { key: storage_key, current_bytes, old_doc: old_doc_json, - mut doc, + doc, updated_bytes, } = row; // Period lock, both images — matching `execute_point_update`: a @@ -391,8 +391,9 @@ impl CoreLoop { // `row_identity` only stands in as `id` for a row that // declares no primary key of its own — overwriting a // declared key would return a value the client never wrote. - returning_doc::attach_row_id(&mut doc, &row_identity); - returned_docs.push(doc); + let mut row = nodedb_types::Value::from(doc); + returning_doc::attach_row_id(&mut row, &row_identity); + returned_docs.push(row); } // Carry the surrogate + post-image back for a post-apply // `Put` redo. `updated_bytes` is moved as its last use; diff --git a/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs b/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs index 833476975..e2aed6754 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs @@ -100,7 +100,7 @@ impl CoreLoop { // decoded document's `id` is the row's client-visible // identity, not the storage key. let identity = key.to_identity(); - crate::data::executor::handlers::returning_doc::from_stored( + crate::data::executor::handlers::returning_doc::from_stored_json( ¤t_bytes, &identity, None, diff --git a/nodedb/src/data/executor/handlers/document/resolve/bulk.rs b/nodedb/src/data/executor/handlers/document/resolve/bulk.rs index e513ed705..cc4d2e7fa 100644 --- a/nodedb/src/data/executor/handlers/document/resolve/bulk.rs +++ b/nodedb/src/data/executor/handlers/document/resolve/bulk.rs @@ -103,7 +103,7 @@ impl CoreLoop { .map_err(ErrorCode::from)?; let mut mutations = Vec::with_capacity(projected.len()); - let mut returned_docs: Vec = Vec::new(); + let mut returned_docs: Vec = Vec::new(); for row in projected { let ProjectedUpdateRow { key: storage_key, @@ -129,7 +129,7 @@ impl CoreLoop { resolved_sum_targets, })); if returning.is_some() { - returned_docs.push(doc); + returned_docs.push(nodedb_types::Value::from(doc)); } } diff --git a/nodedb/src/data/executor/handlers/merge_orchestrated/apply/insert_rows.rs b/nodedb/src/data/executor/handlers/merge_orchestrated/apply/insert_rows.rs index e79998529..0946ea9bd 100644 --- a/nodedb/src/data/executor/handlers/merge_orchestrated/apply/insert_rows.rs +++ b/nodedb/src/data/executor/handlers/merge_orchestrated/apply/insert_rows.rs @@ -49,7 +49,7 @@ pub(super) struct InsertRowsTally<'a, 'p> { pub(super) put_events: &'a mut Vec>, pub(super) write_set: &'a mut Vec, pub(super) balanced_entries: &'a mut Vec, - pub(super) returned_docs: &'a mut Vec, + pub(super) returned_docs: &'a mut Vec, } impl CoreLoop { diff --git a/nodedb/src/data/executor/handlers/merge_orchestrated/apply/orchestrate.rs b/nodedb/src/data/executor/handlers/merge_orchestrated/apply/orchestrate.rs index 9129c39b8..b09af3f77 100644 --- a/nodedb/src/data/executor/handlers/merge_orchestrated/apply/orchestrate.rs +++ b/nodedb/src/data/executor/handlers/merge_orchestrated/apply/orchestrate.rs @@ -120,7 +120,7 @@ impl CoreLoop { // than carried in, because an attempt that ends in `OllpRetryRequired` // is fully re-resolved and re-applied by the orchestrator — rows from a // failed attempt describe a snapshot that never committed. - let mut returned_docs: Vec = Vec::new(); + let mut returned_docs: Vec = Vec::new(); if let Err(response) = self.apply_merge_update_arm( &txn, diff --git a/nodedb/src/data/executor/handlers/merge_orchestrated/apply/update_rows.rs b/nodedb/src/data/executor/handlers/merge_orchestrated/apply/update_rows.rs index 54024e2fe..54e897884 100644 --- a/nodedb/src/data/executor/handlers/merge_orchestrated/apply/update_rows.rs +++ b/nodedb/src/data/executor/handlers/merge_orchestrated/apply/update_rows.rs @@ -44,7 +44,7 @@ pub(super) struct UpdateRowsTally<'a, 'p> { pub(super) put_events: &'a mut Vec>, pub(super) write_set: &'a mut Vec, pub(super) balanced_entries: &'a mut Vec, - pub(super) returned_docs: &'a mut Vec, + pub(super) returned_docs: &'a mut Vec, } impl CoreLoop { diff --git a/nodedb/src/data/executor/handlers/merge_orchestrated/apply_support.rs b/nodedb/src/data/executor/handlers/merge_orchestrated/apply_support.rs index 1eb0bb74a..883d6c037 100644 --- a/nodedb/src/data/executor/handlers/merge_orchestrated/apply_support.rs +++ b/nodedb/src/data/executor/handlers/merge_orchestrated/apply_support.rs @@ -90,9 +90,9 @@ pub(super) fn gate_merge_arms( Ok(()) } -/// Decode one merge row body into the JSON document a RETURNING projection -/// reads. Same shape the point and bulk DML RETURNING paths emit, so a MERGE -/// row projects identically. +/// Decode one merge row body into the `Value` document a RETURNING +/// projection reads. Same shape the point and bulk DML RETURNING paths +/// emit, so a MERGE row projects identically. /// /// `key` is the row's storage key: every caller's `MergeUpdate::key`, /// `MergeDelete::key`, or a freshly minted insert key. This function converts @@ -102,7 +102,7 @@ pub(super) fn gate_merge_arms( /// bodies are MessagePack for BOTH storage modes (`collect_merge_plan` decodes /// a strict target's Binary Tuple and re-encodes the resolved row before the /// apply pass ever sees it), so the strict decoder would have nothing to read. -pub(super) fn returning_doc(body: &[u8], key: &StorageKey) -> crate::Result { +pub(super) fn returning_doc(body: &[u8], key: &StorageKey) -> crate::Result { let identity = key.to_identity(); super::super::returning_doc::from_stored(body, &identity, None) } diff --git a/nodedb/src/data/executor/handlers/merge_orchestrated/delete_arms.rs b/nodedb/src/data/executor/handlers/merge_orchestrated/delete_arms.rs index 81ce590c8..d8b1f6327 100644 --- a/nodedb/src/data/executor/handlers/merge_orchestrated/delete_arms.rs +++ b/nodedb/src/data/executor/handlers/merge_orchestrated/delete_arms.rs @@ -46,7 +46,7 @@ pub(super) struct MergeDeleteArms<'a> { pub(super) struct MergeDeleteTally<'a> { pub(super) affected: &'a mut u64, pub(super) write_set: &'a mut Vec, - pub(super) returned_docs: &'a mut Vec, + pub(super) returned_docs: &'a mut Vec, } impl CoreLoop { diff --git a/nodedb/src/data/executor/handlers/returning_doc.rs b/nodedb/src/data/executor/handlers/returning_doc.rs index 0a01f12c3..09586282b 100644 --- a/nodedb/src/data/executor/handlers/returning_doc.rs +++ b/nodedb/src/data/executor/handlers/returning_doc.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Construction of the JSON document a DML `RETURNING` projection reads. +//! Construction of the `Value` document a DML `RETURNING` projection reads. //! //! Every `RETURNING` path — point and bulk UPDATE/DELETE, `UPDATE ... FROM`, //! MERGE, CRDT document DML — builds its rows from a stored row body, and two @@ -16,6 +16,7 @@ //! key; overwriting it with the surrogate hex storage key returns a value the //! client never wrote and cannot use to address the row. +use nodedb_types::Value; use nodedb_types::columnar::StrictSchema; use crate::data::executor::doc_format; @@ -27,22 +28,19 @@ use crate::engine::document::store::RowIdentity; /// /// For callers that already hold the decoded document (the update paths /// re-project the image they just built rather than re-reading storage). -pub(in crate::data::executor) fn attach_row_id( - doc: &mut serde_json::Value, - identity: &RowIdentity, -) { - if let Some(obj) = doc.as_object_mut() +pub(in crate::data::executor) fn attach_row_id(doc: &mut Value, identity: &RowIdentity) { + if let Value::Object(obj) = doc && !obj.contains_key("id") { obj.insert( "id".to_string(), - serde_json::Value::String(identity.as_str().to_string()), + Value::String(identity.as_str().to_string()), ); } } /// Decode a STORED row body — a pre-image or a re-encoded post-image — into the -/// document a `RETURNING` projection reads. +/// typed document a `RETURNING` projection reads and the write gate judges. /// /// `strict_schema` is `Some` exactly when the collection stores Binary Tuples; /// passing `None` for a strict collection is the silent-misdecode failure this @@ -56,27 +54,58 @@ pub(in crate::data::executor) fn from_stored( body: &[u8], identity: &RowIdentity, strict_schema: Option<&StrictSchema>, -) -> crate::Result { +) -> crate::Result { let mut doc = match strict_schema { - Some(schema) => strict_format::binary_tuple_to_json(body, schema).ok_or_else(|| { - crate::Error::Serialization { - format: "binary_tuple".to_string(), - detail: format!( - "RETURNING row {identity}: stored body ({} bytes) is not a Binary Tuple \ - readable under the collection's strict schema", - body.len() - ), - } - })?, - // `inject_str_field` already honours an existing `id` and wraps a - // non-map body as `{id, value}`, which is the shape schemaless callers - // have always emitted for a body that is not a document map. - None => { - let with_id = - nodedb_query::msgpack_scan::inject_str_field(body, "id", identity.as_str()); - doc_format::decode_document(&with_id)? - } + Some(schema) => strict_format::binary_tuple_to_row_value(body, schema) + .ok_or_else(|| undecodable(identity, body.len()))?, + None => doc_format::decode_document_value(&with_identity(body, identity))?, }; attach_row_id(&mut doc, identity); Ok(doc) } + +/// [`from_stored`] for the JSON document the update pipeline patches and the +/// secondary-index diff reads. +/// +/// The index diff compares this image's rendered field values against the +/// entries the forward write path derived from the same JSON decoders, so a +/// pre-image must come from those decoders too — a binary field rendered by +/// a different converter would leave its index entry behind. +pub(in crate::data::executor) fn from_stored_json( + body: &[u8], + identity: &RowIdentity, + strict_schema: Option<&StrictSchema>, +) -> crate::Result { + let mut doc = match strict_schema { + Some(schema) => strict_format::binary_tuple_to_json(body, schema) + .ok_or_else(|| undecodable(identity, body.len()))?, + None => doc_format::decode_document(&with_identity(body, identity))?, + }; + if let serde_json::Value::Object(obj) = &mut doc + && !obj.contains_key("id") + { + obj.insert( + "id".to_string(), + serde_json::Value::String(identity.as_str().to_string()), + ); + } + Ok(doc) +} + +/// The schemaless body with the storage identity injected as `id`. +/// `inject_str_field` honours an existing `id` and wraps a non-map body as +/// `{id, value}`, which is the shape schemaless callers have always emitted +/// for a body that is not a document map. +fn with_identity(body: &[u8], identity: &RowIdentity) -> Vec { + nodedb_query::msgpack_scan::inject_str_field(body, "id", identity.as_str()) +} + +fn undecodable(identity: &RowIdentity, body_len: usize) -> crate::Error { + crate::Error::Serialization { + format: "binary_tuple".to_string(), + detail: format!( + "RETURNING row {identity}: stored body ({body_len} bytes) is not a Binary Tuple \ + readable under the collection's strict schema" + ), + } +} diff --git a/nodedb/src/data/executor/handlers/returning_rows.rs b/nodedb/src/data/executor/handlers/returning_rows.rs index 83aa32145..c3d9e1441 100644 --- a/nodedb/src/data/executor/handlers/returning_rows.rs +++ b/nodedb/src/data/executor/handlers/returning_rows.rs @@ -15,8 +15,10 @@ use crate::data::executor::scan_normalize::{kv_row_to_doc, sparse_row_to_doc}; use crate::data::executor::sparse_body_format::SparseBodyFormatRef; use crate::data::executor::task::ExecutionTask; use crate::engine::document::store::RowIdentity; +use crate::util::rmpv_value::rmpv_to_value; use nodedb_physical::physical_plan::{ReturningColumns, ReturningSpec}; use nodedb_types::columnar::StrictSchema; +use nodedb_types::{NativeCell, Value}; /// Rows a write path hands back to a `RETURNING` projection: the row's /// client-visible identity paired with the exact bytes stored for it. @@ -78,11 +80,11 @@ pub(in crate::data::executor) fn kv_stored_rows_payload( rls_filters: &[u8], rows: &[KvStoredRow<'_>], ) -> crate::Result> { - let docs: Vec = rows + let docs: Vec = rows .iter() .map(|(key, value)| { let (_key_str, body) = kv_row_to_doc(key, value); - doc_format::decode_document(&body) + doc_format::decode_document_value(&body) }) .collect::>>()?; build_rows_payload(spec, rls_filters, &docs).map_err(|e| crate::Error::Internal { @@ -103,11 +105,9 @@ impl CoreLoop { schema: &nodedb_types::columnar::ColumnarSchema, rows: &[Vec], ) -> Response { - let docs: Vec = rows + let docs: Vec = rows .iter() - .map(|row| { - serde_json::Value::from(super::columnar_write::row_values_to_object(schema, row)) - }) + .map(|row| super::columnar_write::row_values_to_object(schema, row)) .collect(); match build_rows_payload(spec, rls_filters, &docs) { Ok(payload) => self.response_with_payload(task, payload), @@ -133,7 +133,7 @@ impl CoreLoop { rls_filters: &[u8], rows: &[rmpv::Value], ) -> Response { - let docs: Vec = rows.iter().map(rmpv_row_to_json).collect(); + let docs: Vec = rows.iter().map(rmpv_to_value).collect(); match build_rows_payload(spec, rls_filters, &docs) { Ok(payload) => self.response_with_payload(task, payload), Err(e) => self.response_error( @@ -164,7 +164,7 @@ impl CoreLoop { let (_id, mp) = sparse_row_to_doc(row_key, sidecar, SparseBodyFormatRef::VectorSidecar); // An empty row set here would report "the write affected nothing" for a // write that did land, so an unreadable sidecar fails the statement. - let docs: Vec = match doc_format::decode_document(&mp) { + let docs: Vec = match doc_format::decode_document_value(&mp) { Ok(doc) => vec![doc], Err(e) => return self.response_error(task, e), }; @@ -180,39 +180,6 @@ impl CoreLoop { } } -/// Re-key one scan-projected timeseries row into JSON for -/// `build_rows_payload`. A straight transcode: msgpack nil stays SQL NULL. -fn rmpv_row_to_json(row: &rmpv::Value) -> serde_json::Value { - let rmpv::Value::Map(fields) = row else { - return serde_json::Value::Null; - }; - let mut obj = serde_json::Map::with_capacity(fields.len()); - for (key, value) in fields { - let Some(name) = key.as_str() else { continue }; - let cell = match value { - rmpv::Value::Nil => serde_json::Value::Null, - rmpv::Value::Boolean(b) => serde_json::Value::Bool(*b), - rmpv::Value::Integer(n) => n - .as_i64() - .map(|i| serde_json::Value::Number(i.into())) - .unwrap_or(serde_json::Value::Null), - rmpv::Value::F64(f) => serde_json::Number::from_f64(*f) - .map(serde_json::Value::Number) - .unwrap_or(serde_json::Value::Null), - rmpv::Value::F32(f) => serde_json::Number::from_f64(f64::from(*f)) - .map(serde_json::Value::Number) - .unwrap_or(serde_json::Value::Null), - rmpv::Value::String(s) => s - .as_str() - .map(|text| serde_json::Value::String(text.to_string())) - .unwrap_or(serde_json::Value::Null), - _ => serde_json::Value::Null, - }; - obj.insert(name.to_string(), cell); - } - serde_json::Value::Object(obj) -} - /// Project the STORED post-images of freshly written rows into a /// `RowsPayload` — the write paths hold the exact bytes handed to storage, /// so `RETURNING` reports what landed rather than echoing the submitted @@ -224,27 +191,27 @@ pub(in crate::data::executor) fn build_stored_rows_payload( strict_schema: Option<&StrictSchema>, rows: &[StoredRow<'_>], ) -> crate::Result> { - let docs: Vec = rows + let docs: Vec = rows .iter() .map(|(doc_id, body)| returning_doc::from_stored(body, doc_id, strict_schema)) .collect::>>()?; build_rows_payload(spec, rls_filters, &docs) } -/// Project documents per `spec` (Star = insertion order, Named = spec order, -/// missing/null → `None`) into a `RowsPayload` msgpack blob. `rls_filters` -/// (the compiled read policy) is applied BEFORE projection — a predicate -/// often references a column `RETURNING` omits, so a post-projection check -/// would leak the row. Only the visible row set shrinks; the affected count -/// already counted the write. +/// Project documents per `spec` (Star = field name order, Named = spec +/// order, missing → `Value::Null`) into a `RowsPayload` msgpack blob. +/// `rls_filters` (the compiled read policy) is applied BEFORE projection — a +/// predicate often references a column `RETURNING` omits, so a +/// post-projection check would leak the row. Only the visible row set +/// shrinks; the affected count already counted the write. pub(super) fn build_rows_payload( spec: &ReturningSpec, rls_filters: &[u8], - docs: &[serde_json::Value], + docs: &[Value], ) -> crate::Result> { - let visible: Vec<&serde_json::Value> = docs + let visible: Vec<&Value> = docs .iter() - .filter(|doc| rls_eval::rls_check_document(rls_filters, doc)) + .filter(|doc| rls_eval::rls_check_value(rls_filters, doc)) .collect(); let (columns, source_names) = match &spec.columns { @@ -252,13 +219,14 @@ pub(super) fn build_rows_payload( if visible.is_empty() { return encode_empty(Vec::new()); } - // Derive column names from the first doc's keys; both output and - // source names are identical for `RETURNING *`. - let cols: Vec = visible - .first() - .and_then(|d| d.as_object()) - .map(|obj| obj.keys().cloned().collect()) - .unwrap_or_default(); + // Derive column names from the first doc's keys, sorted so the + // shape is deterministic; both output and source names are + // identical for `RETURNING *`. + let mut cols: Vec = match visible.first() { + Some(Value::Object(obj)) => obj.keys().cloned().collect(), + Some(_) | None => Vec::new(), + }; + cols.sort_unstable(); (cols.clone(), cols) } ReturningColumns::Named(items) => { @@ -271,7 +239,7 @@ pub(super) fn build_rows_payload( } }; - let rows: Vec>> = visible + let rows: Vec> = visible .iter() .map(|doc| project_row(doc, &source_names)) .collect(); @@ -292,27 +260,27 @@ fn encode_empty(columns: Vec) -> crate::Result> { }) } -/// Project a single document into one cell per source name. +/// Project a single document into one typed cell per source name. /// -/// Returns `None` for missing fields or JSON null, `Some(text)` otherwise. -fn project_row(doc: &serde_json::Value, source_names: &[String]) -> Vec> { - let obj = doc.as_object(); +/// A missing field is `Value::Null`; a document that is not an object has +/// every cell `Value::Null`. +fn project_row(doc: &Value, source_names: &[String]) -> Vec { + let fields = match doc { + Value::Object(obj) => Some(obj), + _ => None, + }; source_names .iter() - .map(|name| obj.and_then(|o| o.get(name)).and_then(value_to_text)) + .map(|name| { + let cell = match fields.and_then(|obj| obj.get(name)) { + Some(value) => value.clone(), + None => Value::Null, + }; + NativeCell(cell) + }) .collect() } -/// Convert a JSON value to its TEXT representation for pgwire. `None` for -/// JSON null (real SQL NULL); strings are as-is, other types use JSON text. -fn value_to_text(val: &serde_json::Value) -> Option { - match val { - serde_json::Value::Null => None, - serde_json::Value::String(s) => Some(s.clone()), - other => Some(other.to_string()), - } -} - #[cfg(test)] mod tests { use super::*; @@ -349,10 +317,14 @@ mod tests { zerompk::from_msgpack(payload).expect("decode RowsPayload") } - fn docs() -> Vec { + fn cells(row: &[NativeCell]) -> Vec { + row.iter().map(|c| c.0.clone()).collect() + } + + fn docs() -> Vec { vec![ - json!({"id": "r1", "owner": "alice", "note": "hidden"}), - json!({"id": "r2", "owner": "bob", "note": "shown"}), + Value::from(json!({"id": "r1", "owner": "alice", "note": "hidden"})), + Value::from(json!({"id": "r2", "owner": "bob", "note": "shown"})), ] } @@ -368,7 +340,8 @@ mod tests { let payload = build_rows_payload(&named(&["id"]), &owner_policy("bob"), &docs()).expect("build"); let decoded = decode(&payload); - assert_eq!(decoded.rows, vec![vec![Some("r2".to_string())]]); + assert_eq!(decoded.rows.len(), 1); + assert_eq!(cells(&decoded.rows[0]), vec![Value::String("r2".into())]); } /// The predicate names a column the projection omits — the filter must @@ -379,7 +352,8 @@ mod tests { build_rows_payload(&named(&["note"]), &owner_policy("bob"), &docs()).expect("build"); let decoded = decode(&payload); assert_eq!(decoded.columns, vec!["note".to_string()]); - assert_eq!(decoded.rows, vec![vec![Some("shown".to_string())]]); + assert_eq!(decoded.rows.len(), 1); + assert_eq!(cells(&decoded.rows[0]), vec![Value::String("shown".into())]); } /// `RETURNING *` derives its column list from the first VISIBLE row, so a @@ -387,8 +361,8 @@ mod tests { #[test] fn star_columns_come_from_the_first_visible_row() { let docs = vec![ - json!({"id": "r1", "owner": "alice", "secret": "hidden"}), - json!({"id": "r2", "owner": "bob"}), + Value::from(json!({"id": "r1", "owner": "alice", "secret": "hidden"})), + Value::from(json!({"id": "r2", "owner": "bob"})), ]; let spec = ReturningSpec { columns: ReturningColumns::Star, @@ -405,4 +379,52 @@ mod tests { let payload = build_rows_payload(&named(&["id"]), &[0xFF, 0xFE], &docs()).expect("build"); assert!(decode(&payload).rows.is_empty()); } + + /// Cells keep their stored type: an instant stays an instant, a missing + /// field is SQL NULL, a nested object stays nested. + #[test] + fn cells_keep_their_stored_type() { + let at = nodedb_types::NdbDateTime::from_micros(1_583_402_400_000_000); + let mut doc = std::collections::HashMap::new(); + doc.insert("at".to_string(), Value::NaiveDateTime(at)); + doc.insert("n".to_string(), Value::Integer(7)); + doc.insert( + "meta".to_string(), + Value::Object(std::collections::HashMap::from([( + "k".to_string(), + Value::Bool(true), + )])), + ); + let payload = build_rows_payload( + &named(&["at", "n", "meta", "absent"]), + &[], + &[Value::Object(doc)], + ) + .expect("build"); + let decoded = decode(&payload); + assert_eq!( + cells(&decoded.rows[0]), + vec![ + Value::NaiveDateTime(at), + Value::Integer(7), + Value::Object(std::collections::HashMap::from([( + "k".to_string(), + Value::Bool(true), + )])), + Value::Null, + ] + ); + } + + /// A timeseries row's instant ext crosses into the payload typed. + #[test] + fn a_timeseries_instant_cell_stays_typed() { + let at = nodedb_types::NdbDateTime::from_micros(42); + let row = crate::util::rmpv_value::value_to_rmpv(&Value::Object( + std::collections::HashMap::from([("t".to_string(), Value::DateTime(at))]), + )); + let docs = vec![rmpv_to_value(&row)]; + let payload = build_rows_payload(&named(&["t"]), &[], &docs).expect("build"); + assert_eq!(cells(&decode(&payload).rows[0]), vec![Value::DateTime(at)]); + } } diff --git a/nodedb/src/data/executor/handlers/rls_eval.rs b/nodedb/src/data/executor/handlers/rls_eval.rs index 87dc2e136..844624e4a 100644 --- a/nodedb/src/data/executor/handlers/rls_eval.rs +++ b/nodedb/src/data/executor/handlers/rls_eval.rs @@ -10,38 +10,36 @@ //! not pass all filters, the handler MUST return `NOT_FOUND` (no info leak). //! Empty `rls_filters` means no RLS policies apply — allow unconditionally. +use nodedb_types::Value; + use crate::bridge::scan_filter::ScanFilter; -/// Evaluate RLS filters against a document. +/// Evaluate RLS filters against a JSON document. /// /// Returns `true` if the document passes all RLS filters (or if no filters). /// Returns `false` if any filter rejects the document (caller must deny). /// -/// Used by point-get and key-get handlers after fetching the raw document. +/// Used by the write gate over the JSON post-images the update paths build. pub fn rls_check_document(rls_filters: &[u8], doc: &serde_json::Value) -> bool { if rls_filters.is_empty() { return true; } + check_encoded(rls_filters, &nodedb_types::json_to_msgpack_or_empty(doc)) +} - let filters: Vec = match zerompk::from_msgpack(rls_filters) { - Ok(f) => f, - Err(_) => { - // Deserialization failure → deny (fail-closed). - tracing::warn!("RLS filter deserialization failed — denying access"); - return false; - } - }; - - let msgpack = nodedb_types::json_to_msgpack_or_empty(doc); - // RLS is a security boundary: fail closed on a division/modulo-by-zero - // in a filter, exactly like the deserialization failure above — deny - // rather than propagate a query error, so a - // malformed/adversarial RLS predicate can never be used to distinguish - // "row exists but errors" from "row doesn't exist". - match ScanFilter::all_match_binary(&filters, &msgpack) { - Ok(pass) => pass, +/// Evaluate RLS filters against a typed `Value` document. +/// +/// Same contract as [`rls_check_document`]. A document that does not encode +/// is denied: a row the policy could not be evaluated against is not a row +/// the policy admitted. +pub fn rls_check_value(rls_filters: &[u8], doc: &Value) -> bool { + if rls_filters.is_empty() { + return true; + } + match nodedb_types::value_to_msgpack(doc) { + Ok(msgpack) => check_encoded(rls_filters, &msgpack), Err(e) => { - tracing::warn!(error = %e, "RLS filter evaluation failed — denying access"); + tracing::warn!(error = %e, "RLS document encode failed — denying access"); false } } @@ -55,7 +53,17 @@ pub fn rls_check_msgpack_bytes(rls_filters: &[u8], doc_bytes: &[u8]) -> bool { if rls_filters.is_empty() { return true; } + // Ensure bytes are standard msgpack for matches_binary. + let mp = super::super::doc_format::json_to_msgpack(doc_bytes); + check_encoded(rls_filters, &mp) +} +/// Decode the compiled filters and match them against one standard msgpack +/// document. RLS is a security boundary, so it fails closed: an undecodable +/// filter payload or a division/modulo-by-zero inside a filter denies rather +/// than propagating a query error, so a malformed or adversarial predicate +/// can never distinguish "row exists but errors" from "row doesn't exist". +fn check_encoded(rls_filters: &[u8], msgpack: &[u8]) -> bool { let filters: Vec = match zerompk::from_msgpack(rls_filters) { Ok(f) => f, Err(_) => { @@ -63,11 +71,7 @@ pub fn rls_check_msgpack_bytes(rls_filters: &[u8], doc_bytes: &[u8]) -> bool { return false; } }; - - // Ensure bytes are standard msgpack for matches_binary. - let mp = super::super::doc_format::json_to_msgpack(doc_bytes); - // RLS is a security boundary: fail closed — see `rls_check_document`. - match ScanFilter::all_match_binary(&filters, &mp) { + match ScanFilter::all_match_binary(&filters, msgpack) { Ok(pass) => pass, Err(e) => { tracing::warn!(error = %e, "RLS filter evaluation failed — denying access"); @@ -119,6 +123,16 @@ mod tests { assert!(!rls_check_document(&rls, &doc)); } + #[test] + fn typed_documents_evaluate_the_same_filters() { + let rls = make_rls_bytes("user_id", "eq", nodedb_types::Value::String("42".into())); + let ok = Value::from(json!({"user_id": "42"})); + let bad = Value::from(json!({"user_id": "99"})); + assert!(rls_check_value(&rls, &ok)); + assert!(!rls_check_value(&rls, &bad)); + assert!(rls_check_value(&[], &bad)); + } + #[test] fn corrupt_filters_deny() { let corrupt = vec![0xFF, 0xFE, 0xFD]; diff --git a/nodedb/src/data/executor/handlers/rls_write_gate.rs b/nodedb/src/data/executor/handlers/rls_write_gate.rs index bba77abdd..3e985a81d 100644 --- a/nodedb/src/data/executor/handlers/rls_write_gate.rs +++ b/nodedb/src/data/executor/handlers/rls_write_gate.rs @@ -37,7 +37,8 @@ use super::columnar_write::row_values_to_object; use super::returning_doc; use super::rls_eval; -/// Decide one already-decoded row image against the compiled write policy. +/// Decide one already-decoded JSON row image against the compiled write +/// policy. /// /// Fails closed: an undecodable filter payload or an evaluation error denies, /// so an adversarial predicate cannot be turned into an admitted write. @@ -46,11 +47,42 @@ pub(in crate::data::executor) fn admit_row( image: &serde_json::Value, tid: u64, collection: &str, +) -> crate::Result<()> { + decide(rls_write_check, tid, collection, |bytes| { + rls_eval::rls_check_document(bytes, image) + }) +} + +/// Decide one already-decoded typed DOCUMENT row image against the compiled +/// write policy. +/// +/// The document read side tests the stored msgpack body, so the image is +/// encoded to msgpack and tested by the same evaluator — one compiled +/// predicate means the same thing on both sides. Columnar rows go through +/// [`admit_value_row`], whose evaluator is the columnar read side's. +pub(in crate::data::executor) fn admit_document_value( + rls_write_check: &RlsWriteCheck, + image: &nodedb_types::Value, + tid: u64, + collection: &str, +) -> crate::Result<()> { + decide(rls_write_check, tid, collection, |bytes| { + rls_eval::rls_check_value(bytes, image) + }) +} + +/// Map the gate's decision to a result, running `passes` on the compiled +/// predicate only when there is one to evaluate. +fn decide( + rls_write_check: &RlsWriteCheck, + tid: u64, + collection: &str, + passes: impl FnOnce(&[u8]) -> bool, ) -> crate::Result<()> { match rls_write_check.decision() { WriteGateDecision::AdmitAll => Ok(()), WriteGateDecision::Evaluate(bytes) => { - if rls_eval::rls_check_document(bytes, image) { + if passes(bytes) { return Ok(()); } Err(crate::Error::RejectedAuthz { @@ -100,7 +132,7 @@ pub(in crate::data::executor) fn admit_stored_row( }), WriteGateDecision::Evaluate(_) => { match returning_doc::from_stored(body, identity, strict_schema) { - Ok(image) => admit_row(rls_write_check, &image, tid, collection), + Ok(image) => admit_document_value(rls_write_check, &image, tid, collection), Err(e) => Err(crate::Error::RejectedAuthz { tenant_id: crate::types::TenantId::new(tid), resource: format!( @@ -175,30 +207,15 @@ pub(in crate::data::executor) fn admit_value_row( tid: u64, collection: &str, ) -> crate::Result<()> { - let bytes = match rls_write_check.decision() { - WriteGateDecision::AdmitAll => return Ok(()), - WriteGateDecision::DenyNotInjected => { - return Err(crate::Error::Internal { - detail: format!( - "write plan for '{collection}' reached the Data Plane RLS write gate before \ - RLS injection ran; this is an internal invariant break, not a policy \ - rejection" - ), - }); - } - WriteGateDecision::Evaluate(bytes) => bytes, - }; - let admitted = match zerompk::from_msgpack::>(bytes) { - Ok(filters) => value_matches_filters(image, &filters).unwrap_or(false), - Err(_) => false, - }; - if admitted { - return Ok(()); - } - Err(crate::Error::RejectedAuthz { - tenant_id: crate::types::TenantId::new(tid), - resource: format!("RLS write policy on '{collection}' rejected the row"), - }) + decide( + rls_write_check, + tid, + collection, + |bytes| match zerompk::from_msgpack::>(bytes) { + Ok(filters) => value_matches_filters(image, &filters).unwrap_or(false), + Err(_) => false, + }, + ) } /// Decide one schema-ordered columnar row — the values about to be written, or diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs index 63180c16b..c482c8ee0 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Row emission helpers — build `rmpv::Value` directly, plus value conversions. +//! Row emission helpers — build `rmpv::Value` directly. use std::collections::HashMap; use nodedb_types::columnar::schema::TS_SYSTEM; +use crate::util::rmpv_value::{rmpv_to_value, value_to_rmpv}; + use crate::engine::timeseries::columnar_memtable::{ColumnData, ColumnType}; /// Extract the `_ts_system` value from an rmpv-encoded row for audit-log @@ -154,7 +156,7 @@ pub(super) fn apply_computed_columns_rmpv( row: rmpv::Value, computed_cols: &[crate::bridge::expr_eval::ComputedColumn], ) -> crate::Result { - let doc = rmpv_to_nodedb_value(&row); + let doc = rmpv_to_value(&row); let mut fields: Vec<(rmpv::Value, rmpv::Value)> = Vec::with_capacity(computed_cols.len()); for cc in computed_cols { // A computed column is projection-shaped: a division/modulo-by-zero @@ -163,54 +165,12 @@ pub(super) fn apply_computed_columns_rmpv( let result = cc.expr.eval(&doc)?; fields.push(( rmpv::Value::String(cc.alias.as_str().into()), - nodedb_value_to_rmpv(&result), + value_to_rmpv(&result), )); } Ok(rmpv::Value::Map(fields)) } -/// Convert rmpv row to nodedb_types::Value for expression evaluation. -pub(super) fn rmpv_to_nodedb_value(row: &rmpv::Value) -> nodedb_types::Value { - match row { - rmpv::Value::Map(fields) => { - let mut map = std::collections::HashMap::new(); - for (k, v) in fields { - let key = match k { - rmpv::Value::String(s) => s.as_str().unwrap_or("").to_string(), - _ => continue, - }; - let val = match v { - rmpv::Value::Integer(n) => { - nodedb_types::Value::Integer(n.as_i64().unwrap_or(0)) - } - rmpv::Value::F64(f) => nodedb_types::Value::Float(*f), - rmpv::Value::String(s) => { - nodedb_types::Value::String(s.as_str().unwrap_or("").to_string()) - } - rmpv::Value::Nil => nodedb_types::Value::Null, - rmpv::Value::Boolean(b) => nodedb_types::Value::Bool(*b), - _ => nodedb_types::Value::Null, - }; - map.insert(key, val); - } - nodedb_types::Value::Object(map) - } - _ => nodedb_types::Value::Null, - } -} - -/// Convert nodedb_types::Value back to rmpv::Value for response encoding. -pub(super) fn nodedb_value_to_rmpv(v: &nodedb_types::Value) -> rmpv::Value { - match v { - nodedb_types::Value::Integer(n) => rmpv::Value::Integer((*n).into()), - nodedb_types::Value::Float(f) => rmpv::Value::F64(*f), - nodedb_types::Value::String(s) => rmpv::Value::String(s.as_str().into()), - nodedb_types::Value::Bool(b) => rmpv::Value::Boolean(*b), - nodedb_types::Value::Null => rmpv::Value::Nil, - _ => rmpv::Value::Nil, - } -} - /// Rescale every declared-instant cell of `rows` from the milliseconds the /// memtable and the partitions store to the epoch microseconds a `TIMESTAMP` /// cell carries on the wire. diff --git a/nodedb/src/data/executor/handlers/timeseries/sort.rs b/nodedb/src/data/executor/handlers/timeseries/sort.rs index 8e8f650cd..6aaeeb13b 100644 --- a/nodedb/src/data/executor/handlers/timeseries/sort.rs +++ b/nodedb/src/data/executor/handlers/timeseries/sort.rs @@ -11,6 +11,9 @@ use std::cmp::Ordering; use nodedb_physical::physical_plan::SortKeySpec; +use nodedb_types::json_msgpack::instant_from_ext; + +use crate::util::rmpv_value::{rmpv_to_value, value_to_rmpv}; /// Sort materialized result rows by the planner's ORDER BY terms. /// @@ -70,54 +73,6 @@ fn eval_row_keys(row: &rmpv::Value, sort_keys: &[SortKeySpec]) -> crate::Result< Ok(out) } -fn rmpv_to_value(row: &rmpv::Value) -> nodedb_types::Value { - let rmpv::Value::Map(entries) = row else { - return nodedb_types::Value::Null; - }; - let mut map = std::collections::HashMap::with_capacity(entries.len()); - for (key, value) in entries { - if let rmpv::Value::String(name) = key - && let Some(name) = name.as_str() - { - map.insert(name.to_string(), rmpv_value_to_value(value)); - } - } - nodedb_types::Value::Object(map) -} - -fn rmpv_value_to_value(value: &rmpv::Value) -> nodedb_types::Value { - match value { - rmpv::Value::Nil => nodedb_types::Value::Null, - rmpv::Value::Boolean(b) => nodedb_types::Value::Bool(*b), - rmpv::Value::Integer(n) => n - .as_i64() - .map(nodedb_types::Value::Integer) - .or_else(|| n.as_f64().map(nodedb_types::Value::Float)) - .unwrap_or(nodedb_types::Value::Null), - rmpv::Value::F32(f) => nodedb_types::Value::Float(*f as f64), - rmpv::Value::F64(f) => nodedb_types::Value::Float(*f), - rmpv::Value::String(s) => s - .as_str() - .map(|s| nodedb_types::Value::String(s.to_string())) - .unwrap_or(nodedb_types::Value::Null), - rmpv::Value::Array(items) => { - nodedb_types::Value::Array(items.iter().map(rmpv_value_to_value).collect()) - } - _ => nodedb_types::Value::Null, - } -} - -fn value_to_rmpv(value: &nodedb_types::Value) -> rmpv::Value { - match value { - nodedb_types::Value::Null => rmpv::Value::Nil, - nodedb_types::Value::Bool(b) => rmpv::Value::Boolean(*b), - nodedb_types::Value::Integer(n) => rmpv::Value::Integer((*n).into()), - nodedb_types::Value::Float(f) => rmpv::Value::F64(*f), - nodedb_types::Value::String(s) => rmpv::Value::String(s.clone().into()), - other => rmpv::Value::String(format!("{other:?}").into()), - } -} - fn compare_key_rows(a: &[rmpv::Value], b: &[rmpv::Value], sort_keys: &[SortKeySpec]) -> Ordering { for (idx, key) in sort_keys.iter().enumerate() { let av = a.get(idx).filter(|v| !matches!(v, rmpv::Value::Nil)); @@ -167,6 +122,14 @@ fn compare_values(a: Option<&rmpv::Value>, b: Option<&rmpv::Value>) -> Ordering x.as_str().unwrap_or("").cmp(y.as_str().unwrap_or("")) } (rmpv::Value::Boolean(x), rmpv::Value::Boolean(y)) => x.cmp(y), + // A computed key that evaluates to an instant arrives as the instant + // ext; two instants order by their epoch microseconds. + (rmpv::Value::Ext(tx, px), rmpv::Value::Ext(ty, py)) => { + match (instant_from_ext(*tx, px), instant_from_ext(*ty, py)) { + (Some((_, x)), Some((_, y))) => x.cmp(&y), + _ => Ordering::Equal, + } + } // Exotic shapes never appear in a timeseries result row; keep the // order stable rather than inventing one. _ => Ordering::Equal, diff --git a/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs b/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs index 3d725774d..3450409fb 100644 --- a/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs +++ b/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs @@ -26,6 +26,7 @@ use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::handlers::transaction::overlay::Staged; use crate::types::{DatabaseId, TenantId, TxnId}; +use crate::util::rmpv_value::value_to_rmpv; /// Inputs for [`CoreLoop::merge_overlay_into_timeseries_scan`]. pub(in crate::data::executor) struct TimeseriesOverlayMergeParams<'a> { @@ -72,24 +73,8 @@ fn row_timestamp_ms(row: &Value, time_column: &str) -> Option { /// row shape the base raw scan emits, so a merged staged row is /// indistinguishable from a base row downstream (computed columns, encoding). fn staged_row_to_rmpv(row: &Value) -> rmpv::Value { - let Value::Object(map) = row else { - return rmpv::Value::Nil; - }; - let fields: Vec<(rmpv::Value, rmpv::Value)> = map - .iter() - .map(|(k, v)| (rmpv::Value::String(k.as_str().into()), scalar_to_rmpv(v))) - .collect(); - rmpv::Value::Map(fields) -} - -/// Scalar `nodedb_types::Value` → `rmpv::Value`, matching the raw scan's own -/// value emission (`row_emit::nodedb_value_to_rmpv`). -fn scalar_to_rmpv(v: &Value) -> rmpv::Value { - match v { - Value::Integer(n) => rmpv::Value::Integer((*n).into()), - Value::Float(f) => rmpv::Value::F64(*f), - Value::String(s) => rmpv::Value::String(s.as_str().into()), - Value::Bool(b) => rmpv::Value::Boolean(*b), + match row { + Value::Object(_) => value_to_rmpv(row), _ => rmpv::Value::Nil, } } diff --git a/nodedb/src/data/executor/handlers/update_from_join_write.rs b/nodedb/src/data/executor/handlers/update_from_join_write.rs index 01fe76e60..cd5f57afd 100644 --- a/nodedb/src/data/executor/handlers/update_from_join_write.rs +++ b/nodedb/src/data/executor/handlers/update_from_join_write.rs @@ -27,7 +27,7 @@ pub(in crate::data::executor) struct UpdateFromJoinWriteOutcome { pub write_set: Vec, /// Post-image JSON per affected row, populated only when the caller asked /// for `RETURNING`. - pub returned_docs: Vec, + pub returned_docs: Vec, } /// Everything the write pass needs about the statement, gathered once by the @@ -73,7 +73,7 @@ impl CoreLoop { ); let mut affected = 0u64; let mut write_set: Vec = Vec::new(); - let mut returned_docs: Vec = if want_returning { + let mut returned_docs: Vec = if want_returning { Vec::with_capacity(rows.len()) } else { Vec::new() @@ -84,7 +84,7 @@ impl CoreLoop { key: storage_key, body: updated_bytes, old_body, - mut doc, + doc, } = row; // Period lock, both images — matching `execute_point_update`: a @@ -239,8 +239,9 @@ impl CoreLoop { // `row_identity` only stands in as `id` for a row that // declares no primary key of its own — overwriting a // declared key would return a value the client never wrote. - returning_doc::attach_row_id(&mut doc, &row_identity); - returned_docs.push(doc); + let mut row = nodedb_types::Value::from(doc); + returning_doc::attach_row_id(&mut row, &row_identity); + returned_docs.push(row); } } } diff --git a/nodedb/src/data/executor/response_codec/encode.rs b/nodedb/src/data/executor/response_codec/encode.rs index d563cbf42..0f384112c 100644 --- a/nodedb/src/data/executor/response_codec/encode.rs +++ b/nodedb/src/data/executor/response_codec/encode.rs @@ -140,26 +140,29 @@ pub fn decode_payload(payload: &[u8]) -> crate::R /// to JSON text via streaming transcoder (no intermediate `serde_json::Value`). /// If already JSON (starts with `[` or `{`), returns as-is. pub fn decode_payload_to_json(payload: &[u8]) -> String { - if payload.is_empty() { + let Some(&first) = payload.first() else { return String::new(); + }; + + if looks_like_json(first) { + return String::from_utf8_lossy(payload).into_owned(); } - let first = payload[0]; + nodedb_types::msgpack_to_json_string(payload) + .unwrap_or_else(|_| String::from_utf8_lossy(payload).into_owned()) +} - let is_likely_json = first == b'[' +/// True when `first` can open JSON text: an array, object, string, number or +/// one of the `true` / `false` / `null` literals. No msgpack marker for a map +/// or array shares these bytes. +fn looks_like_json(first: u8) -> bool { + first == b'[' || first == b'{' || first == b'"' || first.is_ascii_digit() || first == b't' || first == b'f' - || first == b'n'; - - if is_likely_json { - return String::from_utf8_lossy(payload).into_owned(); - } - - nodedb_types::msgpack_to_json_string(payload) - .unwrap_or_else(|_| String::from_utf8_lossy(payload).into_owned()) + || first == b'n' } #[cfg(test)] diff --git a/nodedb/src/data/executor/response_codec/hits.rs b/nodedb/src/data/executor/response_codec/hits.rs index a93312f83..77891681d 100644 --- a/nodedb/src/data/executor/response_codec/hits.rs +++ b/nodedb/src/data/executor/response_codec/hits.rs @@ -9,6 +9,7 @@ //! score under a dynamic key. Without this the alias would be lost and the //! response would always carry `rrf_score` regardless of what the SQL named it. +use nodedb_types::NativeCell; use serde::Serialize; use crate::data::executor::handlers::hybrid_key::HybridFusionKey; @@ -86,15 +87,19 @@ pub(in crate::data::executor) struct GraphRagResult { /// Carries one entry per affected row, with the projected column values. /// The Control Plane decodes this to build a multi-column pgwire QueryResponse /// (one pgwire field per entry in `columns`). -#[derive(Serialize, serde::Deserialize, zerompk::ToMessagePack, zerompk::FromMessagePack)] +/// +/// Cells are typed `Value`s in plain msgpack (`NativeCell`), so an instant +/// crosses as the instant ext and the JSON transcoder renders the payload as +/// `{"columns": [...], "rows": [[cell, ...], ...]}` with plain cells. +#[derive(zerompk::ToMessagePack, zerompk::FromMessagePack)] #[msgpack(map)] pub(crate) struct RowsPayload { /// Projected column names (output names, respecting AS aliases). pub columns: Vec, /// One inner Vec per affected row; each inner Vec has one cell per - /// column in the same order as `columns`. `None` denotes SQL NULL - /// (missing field or JSON null); `Some` carries the TEXT representation. - pub rows: Vec>>, + /// column in the same order as `columns`. `Value::Null` denotes SQL NULL + /// (missing field or a stored null). + pub rows: Vec>, } /// Carries the row payload alongside a flag that signals whether the @@ -215,4 +220,33 @@ mod tests { assert!(json.contains("\"id\"")); assert!(json.contains("\"distance\"")); } + + /// A `RowsPayload` transcodes to JSON with plain cells: a typed instant + /// renders as ISO-8601, an integer as a number, SQL NULL as `null`. + #[test] + fn rows_payload_cells_transcode_plain() { + use nodedb_types::{NdbDateTime, Value}; + + let payload = RowsPayload { + columns: vec!["id".into(), "n".into(), "at".into(), "gone".into()], + rows: vec![vec![ + NativeCell(Value::String("r1".into())), + NativeCell(Value::Integer(7)), + NativeCell(Value::NaiveDateTime(NdbDateTime::from_micros( + 1_583_402_400_000_000, + ))), + NativeCell(Value::Null), + ]], + }; + let bytes = zerompk::to_msgpack_vec(&payload).unwrap(); + let json = decode_payload_to_json(&bytes); + assert_eq!( + json, + "{\"columns\":[\"id\",\"n\",\"at\",\"gone\"],\"rows\":[[\"r1\",7,\"2020-03-05T10:00:00.000000Z\",null]]}" + ); + + let back: RowsPayload = zerompk::from_msgpack(&bytes).unwrap(); + assert_eq!(back.columns, payload.columns); + assert_eq!(back.rows, payload.rows); + } } diff --git a/nodedb/src/data/executor/strict_format/coerce.rs b/nodedb/src/data/executor/strict_format/coerce.rs index d2ec8eec1..4ebba44fd 100644 --- a/nodedb/src/data/executor/strict_format/coerce.rs +++ b/nodedb/src/data/executor/strict_format/coerce.rs @@ -268,8 +268,10 @@ pub fn value_to_json(val: &Value) -> serde_json::Value { &base64::engine::general_purpose::STANDARD, b, )), - Value::DateTime(dt) | Value::NaiveDateTime(dt) => serde_json::json!(dt.micros / 1000), - Value::Duration(d) => serde_json::json!(d.as_millis()), + Value::DateTime(dt) | Value::NaiveDateTime(dt) => { + serde_json::Value::String(dt.to_iso8601()) + } + Value::Duration(d) => serde_json::Value::String(d.to_string()), Value::Decimal(d) => serde_json::Value::String(d.to_string()), Value::Array(arr) => serde_json::Value::Array(arr.iter().map(value_to_json).collect()), Value::Object(map) => { diff --git a/nodedb/src/data/executor/strict_format/decode.rs b/nodedb/src/data/executor/strict_format/decode.rs index 3dfa174e5..848a37b61 100644 --- a/nodedb/src/data/executor/strict_format/decode.rs +++ b/nodedb/src/data/executor/strict_format/decode.rs @@ -81,26 +81,46 @@ pub fn undecodable_strict_row(collection: &str, identity: &str) -> crate::Error } } +/// Decode a Binary Tuple to the row a projection reads: every declared +/// column, in schema order, with the reserved bitemporal bookkeeping columns +/// dropped. A column the tuple does not carry is `Value::Null`. +/// +/// The walk is shared by [`binary_tuple_to_json`], so the JSON and `Value` +/// forms of a row can never disagree about which columns it has. +pub fn binary_tuple_to_row_value(tuple_bytes: &[u8], schema: &StrictSchema) -> Option { + let Value::Object(mut map) = binary_tuple_to_value(tuple_bytes, schema)? else { + return None; + }; + let mut row = std::collections::HashMap::with_capacity(map.len()); + for col in projected_columns(schema) { + let v = map.remove(&col.name).unwrap_or(Value::Null); + row.insert(col.name.clone(), v); + } + Some(Value::Object(row)) +} + /// Decode a Binary Tuple to a JSON object using the schema (for pgwire output). pub fn binary_tuple_to_json( tuple_bytes: &[u8], schema: &StrictSchema, ) -> Option { - // Delegate to binary_tuple_to_value (which handles version-aware decoding) - // then convert Value → JSON. - let val = binary_tuple_to_value(tuple_bytes, schema)?; - match val { - Value::Object(map) => { - let mut obj = serde_json::Map::with_capacity(map.len()); - for col in &schema.columns { - if is_reserved_bitemporal_column(&col.name) { - continue; - } - let v = map.get(&col.name).unwrap_or(&Value::Null); - obj.insert(col.name.clone(), value_to_json(v)); - } - Some(serde_json::Value::Object(obj)) - } - _ => None, + let Value::Object(map) = binary_tuple_to_row_value(tuple_bytes, schema)? else { + return None; + }; + let mut obj = serde_json::Map::with_capacity(map.len()); + for col in projected_columns(schema) { + let v = map.get(&col.name).unwrap_or(&Value::Null); + obj.insert(col.name.clone(), value_to_json(v)); } + Some(serde_json::Value::Object(obj)) +} + +/// The schema's columns a projection sees, in declaration order. +fn projected_columns( + schema: &StrictSchema, +) -> impl Iterator { + schema + .columns + .iter() + .filter(|col| !is_reserved_bitemporal_column(&col.name)) } diff --git a/nodedb/src/data/executor/strict_format/mod.rs b/nodedb/src/data/executor/strict_format/mod.rs index f2dc6e30a..2b4625d8d 100644 --- a/nodedb/src/data/executor/strict_format/mod.rs +++ b/nodedb/src/data/executor/strict_format/mod.rs @@ -10,7 +10,8 @@ mod decode; mod encode; pub(crate) use decode::{ - binary_tuple_to_json, binary_tuple_to_msgpack, binary_tuple_to_value, undecodable_strict_row, + binary_tuple_to_json, binary_tuple_to_msgpack, binary_tuple_to_row_value, + binary_tuple_to_value, undecodable_strict_row, }; pub(super) use encode::{ bytes_to_binary_tuple, bytes_to_binary_tuple_bitemporal, value_to_binary_tuple, diff --git a/nodedb/src/util.rs b/nodedb/src/util.rs index da33524e5..652c63432 100644 --- a/nodedb/src/util.rs +++ b/nodedb/src/util.rs @@ -4,6 +4,7 @@ pub mod bounded_json; pub mod bounded_msgpack; +pub mod rmpv_value; /// FNV-1a 64-bit hash of a byte slice. /// diff --git a/nodedb/src/util/rmpv_value.rs b/nodedb/src/util/rmpv_value.rs new file mode 100644 index 000000000..f79e5ca26 --- /dev/null +++ b/nodedb/src/util/rmpv_value.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The one converter pair between `rmpv::Value` and `nodedb_types::Value`. +//! +//! Timeseries rows, trigger batches and `RETURNING` projections all move +//! between the two dynamic forms. Both directions live here so an instant, +//! a binary blob or a nested map renders the same way on every path. +//! +//! Instants cross as the msgpack ext `nodedb_types::write_instant` produces: +//! `rmpv::Value::Ext(1 | 2, )`. Every other ext type reads as +//! `Value::Null`, the rule `nodedb_types::value_from_msgpack` applies. + +use std::collections::HashMap; + +use nodedb_types::json_msgpack::instant_from_ext; +use nodedb_types::{InstantKind, Value}; + +/// Convert an `rmpv::Value` to a `nodedb_types::Value`. +/// +/// - `Integer` outside `i64` reads as its `u64` bits cast to `i64`. +/// - A `String` that is not UTF-8 reads as `Value::Bytes` of its raw bytes. +/// - A map key that is not a string renders through rmpv's `Display`. +/// - `Ext` reads as an instant for ext types 1 and 2, else `Value::Null`. +pub fn rmpv_to_value(v: &rmpv::Value) -> Value { + match v { + rmpv::Value::Nil => Value::Null, + rmpv::Value::Boolean(b) => Value::Bool(*b), + rmpv::Value::Integer(n) => match (n.as_i64(), n.as_u64()) { + (Some(i), _) => Value::Integer(i), + (None, Some(u)) => Value::Integer(u as i64), + (None, None) => Value::Null, + }, + rmpv::Value::F32(f) => Value::Float(f64::from(*f)), + rmpv::Value::F64(f) => Value::Float(*f), + rmpv::Value::String(s) => match s.as_str() { + Some(text) => Value::String(text.to_string()), + None => Value::Bytes(s.as_bytes().to_vec()), + }, + rmpv::Value::Binary(b) => Value::Bytes(b.clone()), + rmpv::Value::Array(items) => Value::Array(items.iter().map(rmpv_to_value).collect()), + rmpv::Value::Map(entries) => { + let mut map = HashMap::with_capacity(entries.len()); + for (key, value) in entries { + let name = match key { + rmpv::Value::String(s) => match s.as_str() { + Some(text) => text.to_string(), + None => key.to_string(), + }, + other => other.to_string(), + }; + map.insert(name, rmpv_to_value(value)); + } + Value::Object(map) + } + rmpv::Value::Ext(ext_type, payload) => match instant_from_ext(*ext_type, payload) { + Some((kind, micros)) => kind.from_micros(micros), + None => Value::Null, + }, + } +} + +/// Convert a `nodedb_types::Value` to an `rmpv::Value`. +/// +/// Mirrors `nodedb_types::value_to_msgpack` variant for variant, so an rmpv +/// row encodes to the same bytes a `Value` row does: +/// +/// - `Uuid`, `Ulid`, `Regex`, `Duration`, `Decimal` and `Geometry` are strings. +/// - `Set` is an array. `Vector` is an array of `F64`. +/// - `ArrayCell` is a `{coords, attrs}` map, plus `_ts_system` when versioned. +/// - `Range` and `Record` have no rmpv form and become `Nil`. +pub fn value_to_rmpv(v: &Value) -> rmpv::Value { + match v { + Value::Null => rmpv::Value::Nil, + Value::Bool(b) => rmpv::Value::Boolean(*b), + Value::Integer(n) => rmpv::Value::Integer((*n).into()), + Value::Float(f) => rmpv::Value::F64(*f), + Value::String(s) | Value::Uuid(s) | Value::Ulid(s) | Value::Regex(s) => { + rmpv::Value::String(s.as_str().into()) + } + Value::Bytes(b) => rmpv::Value::Binary(b.clone()), + Value::Array(items) | Value::Set(items) => { + rmpv::Value::Array(items.iter().map(value_to_rmpv).collect()) + } + Value::Object(map) => rmpv::Value::Map( + map.iter() + .map(|(k, v)| (rmpv::Value::String(k.as_str().into()), value_to_rmpv(v))) + .collect(), + ), + Value::DateTime(dt) => instant_ext(InstantKind::Utc, dt.micros), + Value::NaiveDateTime(dt) => instant_ext(InstantKind::Naive, dt.micros), + Value::Duration(d) => rmpv::Value::String(d.to_string().into()), + Value::Decimal(d) => rmpv::Value::String(d.to_string().into()), + Value::Geometry(g) => match sonic_rs::to_string(g) { + Ok(text) => rmpv::Value::String(text.into()), + Err(_) => rmpv::Value::Nil, + }, + Value::Range { .. } | Value::Record { .. } => rmpv::Value::Nil, + Value::Vector(floats) => rmpv::Value::Array( + floats + .iter() + .map(|f| rmpv::Value::F64(f64::from(*f))) + .collect(), + ), + Value::ArrayCell(cell) => { + let mut fields = vec![ + ( + rmpv::Value::String("coords".into()), + rmpv::Value::Array(cell.coords.iter().map(value_to_rmpv).collect()), + ), + ( + rmpv::Value::String("attrs".into()), + rmpv::Value::Array(cell.attrs.iter().map(value_to_rmpv).collect()), + ), + ]; + if let Some(ts) = cell.system_time { + fields.push(( + rmpv::Value::String("_ts_system".into()), + rmpv::Value::Integer(ts.into()), + )); + } + rmpv::Value::Map(fields) + } + // `Value` is `#[non_exhaustive]`: a variant this crate cannot name + // has no rmpv form. + _ => rmpv::Value::Nil, + } +} + +fn instant_ext(kind: InstantKind, micros: i64) -> rmpv::Value { + rmpv::Value::Ext(kind.ext_type(), micros.to_be_bytes().to_vec()) +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_types::NdbDateTime; + + #[test] + fn instants_round_trip_both_kinds() { + for v in [ + Value::DateTime(NdbDateTime::from_micros(1_583_402_400_000_000)), + Value::NaiveDateTime(NdbDateTime::from_micros(1_583_402_400_000_000)), + ] { + let ext = value_to_rmpv(&v); + assert!(matches!(ext, rmpv::Value::Ext(1 | 2, ref b) if b.len() == 8)); + assert_eq!(rmpv_to_value(&ext), v); + } + } + + #[test] + fn negative_micros_round_trip() { + let v = Value::NaiveDateTime(NdbDateTime::from_micros(-86_400_000_000)); + assert_eq!(rmpv_to_value(&value_to_rmpv(&v)), v); + } + + #[test] + fn instant_ext_matches_the_native_encoding() { + let v = Value::DateTime(NdbDateTime::from_micros(42)); + let mut via_rmpv = Vec::new(); + rmpv::encode::write_value(&mut via_rmpv, &value_to_rmpv(&v)).expect("encode"); + let native = nodedb_types::value_to_msgpack(&v).expect("native"); + assert_eq!(via_rmpv, native); + } + + #[test] + fn nested_object_and_array_round_trip() { + let mut inner = HashMap::new(); + inner.insert("n".to_string(), Value::Integer(-7)); + inner.insert("b".to_string(), Value::Bytes(vec![9, 8])); + let mut outer = HashMap::new(); + outer.insert( + "arr".to_string(), + Value::Array(vec![ + Value::Null, + Value::Bool(true), + Value::Float(1.5), + Value::String("x".into()), + Value::Object(inner), + ]), + ); + let v = Value::Object(outer); + assert_eq!(rmpv_to_value(&value_to_rmpv(&v)), v); + } + + #[test] + fn unknown_ext_reads_as_null() { + assert_eq!(rmpv_to_value(&rmpv::Value::Ext(7, vec![0; 8])), Value::Null); + } + + #[test] + fn u64_beyond_i64_reads_as_its_bits() { + let v = rmpv::Value::Integer(rmpv::Integer::from(u64::MAX)); + assert_eq!(rmpv_to_value(&v), Value::Integer(-1)); + } +} From 61483837864ac367e26c52c4443f7e36197dbbd7 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 16 Sep 2026 14:48:46 +0800 Subject: [PATCH 05/21] fix(timeseries): distinguish instant kind and millis in time columns Replace the unit-less TsColumnType::Timestamp with Timestamp(TimeKind), where TimeKind::Instant carries InstantKind::Naive/Utc and TimeKind::Millis marks a declared integer time column. The columnar bridge, segment codec, schema evolution, sparse index, grouped scan, ILP ingest, and detection paths now key off this kind instead of assuming every timestamp column is the same representation, so a UTC instant maps to Timestamptz and a millis column keeps its Int64 storage instead of being coerced to a bare timestamp. Split columnar_memtable/memtable.rs and grouped_scan/strategies.rs into per-concern submodule directories. --- .../backup/restore/timeseries_reissue.rs | 4 +- .../src/control/server/ilp_batch/preflight.rs | 12 +- .../executor/core_loop/ts_declared_schema.rs | 218 ++-- .../data/executor/handlers/columnar_agg.rs | 4 +- .../executor/handlers/columnar_agg_support.rs | 2 +- .../executor/handlers/columnar_filter/eval.rs | 6 +- .../columnar_filter/memtable_source.rs | 4 +- .../columnar_filter/partition_source.rs | 7 +- .../handlers/columnar_read/convert.rs | 2 +- .../columnar_read/materialize_scan_ts.rs | 4 +- .../executor/handlers/timeseries/encode.rs | 2 +- .../executor/handlers/timeseries/ingest.rs | 14 +- .../handlers/timeseries/raw_scan/row_emit.rs | 8 +- .../handlers/transaction/undo/apply.rs | 4 +- .../src/engine/timeseries/columnar_bridge.rs | 51 +- .../timeseries/columnar_memtable/memtable.rs | 952 ------------------ .../columnar_memtable/memtable/drain.rs | 180 ++++ .../columnar_memtable/memtable/evolve.rs | 36 + .../columnar_memtable/memtable/ingest.rs | 407 ++++++++ .../columnar_memtable/memtable/mod.rs | 9 + .../columnar_memtable/memtable/snapshot_io.rs | 293 ++++++ .../columnar_memtable/memtable/table.rs | 137 +++ .../timeseries/columnar_memtable/mod.rs | 2 +- .../timeseries/columnar_memtable/snapshot.rs | 4 +- .../timeseries/columnar_memtable/types.rs | 49 +- .../timeseries/columnar_segment/codec.rs | 6 +- .../timeseries/columnar_segment/reader.rs | 21 +- .../timeseries/columnar_segment/schema.rs | 71 +- .../timeseries/columnar_segment/writer.rs | 29 +- .../timeseries/continuous_agg/manager.rs | 4 +- .../src/engine/timeseries/grouped_filter.rs | 2 +- .../timeseries/grouped_scan/strategies.rs | 537 ---------- .../grouped_scan/strategies/bucket.rs | 74 ++ .../grouped_scan/strategies/dispatch.rs | 87 ++ .../grouped_scan/strategies/hashed.rs | 214 ++++ .../grouped_scan/strategies/keys.rs | 153 +++ .../timeseries/grouped_scan/strategies/mod.rs | 8 + .../engine/timeseries/grouped_scan/types.rs | 26 +- nodedb/src/engine/timeseries/ilp_ingest.rs | 10 +- nodedb/src/engine/timeseries/ilp_schema.rs | 9 +- nodedb/src/engine/timeseries/merge/o3.rs | 18 +- .../src/engine/timeseries/merge/partitions.rs | 26 +- nodedb/src/engine/timeseries/projection.rs | 10 +- .../src/engine/timeseries/schema_evolution.rs | 38 +- .../engine/timeseries/sparse_index/index.rs | 14 +- .../src/engine/timeseries/tag_autocomplete.rs | 4 +- nodedb/src/engine/timeseries/ts_detect.rs | 9 +- nodedb/src/engine/timeseries/verification.rs | 19 +- 48 files changed, 2026 insertions(+), 1774 deletions(-) delete mode 100644 nodedb/src/engine/timeseries/columnar_memtable/memtable.rs create mode 100644 nodedb/src/engine/timeseries/columnar_memtable/memtable/drain.rs create mode 100644 nodedb/src/engine/timeseries/columnar_memtable/memtable/evolve.rs create mode 100644 nodedb/src/engine/timeseries/columnar_memtable/memtable/ingest.rs create mode 100644 nodedb/src/engine/timeseries/columnar_memtable/memtable/mod.rs create mode 100644 nodedb/src/engine/timeseries/columnar_memtable/memtable/snapshot_io.rs create mode 100644 nodedb/src/engine/timeseries/columnar_memtable/memtable/table.rs delete mode 100644 nodedb/src/engine/timeseries/grouped_scan/strategies.rs create mode 100644 nodedb/src/engine/timeseries/grouped_scan/strategies/bucket.rs create mode 100644 nodedb/src/engine/timeseries/grouped_scan/strategies/dispatch.rs create mode 100644 nodedb/src/engine/timeseries/grouped_scan/strategies/hashed.rs create mode 100644 nodedb/src/engine/timeseries/grouped_scan/strategies/keys.rs create mode 100644 nodedb/src/engine/timeseries/grouped_scan/strategies/mod.rs diff --git a/nodedb/src/control/backup/restore/timeseries_reissue.rs b/nodedb/src/control/backup/restore/timeseries_reissue.rs index 64c9f3b34..11015bd90 100644 --- a/nodedb/src/control/backup/restore/timeseries_reissue.rs +++ b/nodedb/src/control/backup/restore/timeseries_reissue.rs @@ -99,7 +99,7 @@ fn decode_memtable_rows( /// Extract one cell from a memtable column as a `Value`. fn memtable_cell(mt: &ColumnarMemtable, col_idx: usize, ty: ColumnType, idx: usize) -> Value { match ty { - ColumnType::Timestamp => Value::Integer(mt.column(col_idx).as_timestamps()[idx]), + ColumnType::Timestamp(_) => Value::Integer(mt.column(col_idx).as_timestamps()[idx]), ColumnType::Int64 => Value::Integer(mt.column(col_idx).as_i64()[idx]), ColumnType::Float64 => { let v = mt.column(col_idx).as_f64()[idx]; @@ -207,7 +207,7 @@ fn partition_cell( idx: usize, ) -> Value { match ty { - ColumnType::Timestamp => Value::Integer(data.as_timestamps()[idx]), + ColumnType::Timestamp(_) => Value::Integer(data.as_timestamps()[idx]), ColumnType::Int64 => Value::Integer(data.as_i64()[idx]), ColumnType::Float64 => { let v = data.as_f64()[idx]; diff --git a/nodedb/src/control/server/ilp_batch/preflight.rs b/nodedb/src/control/server/ilp_batch/preflight.rs index 95e7756fc..a8822c760 100644 --- a/nodedb/src/control/server/ilp_batch/preflight.rs +++ b/nodedb/src/control/server/ilp_batch/preflight.rs @@ -78,17 +78,7 @@ pub(super) fn preflight_ilp_batch( let catalog_fields = schema .columns .iter() - .map(|(name, ty)| { - let sql_type = match ty { - crate::engine::timeseries::columnar_memtable::ColumnType::Timestamp => { - "TIMESTAMP" - } - crate::engine::timeseries::columnar_memtable::ColumnType::Float64 => "FLOAT", - crate::engine::timeseries::columnar_memtable::ColumnType::Int64 => "BIGINT", - crate::engine::timeseries::columnar_memtable::ColumnType::Symbol => "VARCHAR", - }; - (name.clone(), sql_type.to_owned()) - }) + .map(|(name, ty)| (name.clone(), ty.ddl_type_name().to_owned())) .collect(); groups.push(IlpMeasurementBatch { measurement, diff --git a/nodedb/src/data/executor/core_loop/ts_declared_schema.rs b/nodedb/src/data/executor/core_loop/ts_declared_schema.rs index acad6049d..1ccf3b6fd 100644 --- a/nodedb/src/data/executor/core_loop/ts_declared_schema.rs +++ b/nodedb/src/data/executor/core_loop/ts_declared_schema.rs @@ -18,8 +18,9 @@ //! line's own timestamp as the time column. use nodedb_physical::physical_plan::TimeseriesSchema; +use nodedb_types::InstantKind; -use crate::engine::timeseries::columnar_memtable::{ColumnType, ColumnarSchema}; +use crate::engine::timeseries::columnar_memtable::{ColumnType, ColumnarSchema, TimeKind}; use crate::types::{DatabaseId, TenantId}; use super::state::CoreLoop; @@ -117,13 +118,33 @@ impl CoreLoop { }) } + /// Run `f` against the memtable schema that types this collection's + /// columns: the resident memtable's when one exists, else the declared + /// one. A memtable is created on first ingest, so a node serving only + /// flushed partitions after a restart has none, and the declared shape + /// is the same schema that memtable would have been built from. `None` + /// when the collection has neither. + fn with_ts_schema( + &self, + database_id: DatabaseId, + tid: TenantId, + collection: &str, + f: impl FnOnce(&ColumnarSchema) -> R, + ) -> Option { + let key = (database_id, tid, collection.to_string()); + if let Some(memtable) = self.columnar_memtables.get(&key) { + return Some(f(memtable.schema())); + } + self.declared_ts_memtable_schema(database_id, tid, collection) + .map(|schema| f(&schema)) + } + /// How each named GROUP BY column of a timeseries collection renders. /// /// A grouped column must carry the type it carries ungrouped, so this - /// resolves the same declared shape row emission reads. A declared - /// collection answers from its DDL; a measurement ingested over the raw - /// ILP protocol has no DDL, so its resident memtable schema answers. - /// A column present in neither renders as text. + /// answers from the memtable schema, whose column types carry the time + /// kind. A column absent from the schema, or a collection with no schema + /// at all, renders as text. pub(in crate::data::executor) fn ts_group_key_kinds( &self, database_id: DatabaseId, @@ -131,73 +152,46 @@ impl CoreLoop { collection: &str, group_by: &[String], ) -> Vec { - if let Some(declared) = self.declared_timeseries(database_id, tid, collection) { - let time_key_index = declared.time_key_index(); - return group_by + self.with_ts_schema(database_id, tid, collection, |schema| { + group_by .iter() .map(|name| { - let Some(index) = declared.columns.iter().position(|(c, _)| c == name) else { - return TsGroupKeyKind::Text; - }; - let declared_type = declared.columns[index].1.as_str(); - if declared_type_is_instant(declared_type) { - return TsGroupKeyKind::Instant; - } - kind_of_storage(memtable_column_type( - declared_type, - Some(index) == time_key_index, - )) + schema + .columns + .iter() + .find(|(c, _)| c == name) + .map(|(_, ty)| kind_of_storage(*ty)) + .unwrap_or(TsGroupKeyKind::Text) }) - .collect(); - } - - let key = (database_id, tid, collection.to_string()); - let Some(memtable) = self.columnar_memtables.get(&key) else { - return vec![TsGroupKeyKind::Text; group_by.len()]; - }; - let schema = memtable.schema(); - group_by - .iter() - .map(|name| { - schema - .columns - .iter() - .find(|(c, _)| c == name) - .map(|(_, ty)| kind_of_storage(*ty)) - .unwrap_or(TsGroupKeyKind::Text) - }) - .collect() + .collect() + }) + .unwrap_or_else(|| vec![TsGroupKeyKind::Text; group_by.len()]) } - /// Declared columns of a timeseries collection that carry an instant. + /// Columns of a timeseries collection whose memtable type is an instant. /// - /// A column declared `TIMESTAMP` or `TIMESTAMPTZ` is one. The memtable - /// keeps every timestamp column in epoch milliseconds, while a client - /// reads a `TIMESTAMP` cell as epoch microseconds, so row emission scales - /// exactly these columns. + /// The memtable keeps every time column in epoch milliseconds, while a + /// client reads a `TIMESTAMP` cell as epoch microseconds, so row emission + /// scales exactly these columns. /// - /// A `BIGINT TIME_KEY` shares the same millisecond column and is absent - /// from this list: its declared type is an integer, so it hands back the - /// number that was inserted. - /// - /// An undeclared measurement (raw ILP protocol ingest) has no entry and - /// yields an empty list — the planner types its columns as text, so no - /// cell of it is read as an instant. + /// A `BIGINT TIME_KEY` shares the same millisecond storage but its kind + /// is `Millis`, so it is absent from this list and hands back the number + /// that was inserted. A collection with no schema yields an empty list. pub(in crate::data::executor) fn ts_instant_columns( &self, database_id: DatabaseId, tid: TenantId, collection: &str, ) -> Vec { - let Some(declared) = self.declared_timeseries(database_id, tid, collection) else { - return Vec::new(); - }; - declared - .columns - .iter() - .filter(|(_, type_str)| declared_type_is_instant(type_str)) - .map(|(name, _)| name.clone()) - .collect() + self.with_ts_schema(database_id, tid, collection, |schema| { + schema + .columns + .iter() + .filter(|(_, ty)| matches!(ty, ColumnType::Timestamp(TimeKind::Instant(_)))) + .map(|(name, _)| name.clone()) + .collect() + }) + .unwrap_or_default() } } @@ -209,7 +203,7 @@ impl CoreLoop { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(in crate::data::executor) enum TsGroupKeyKind { /// A declared `TIMESTAMP` / `TIMESTAMPTZ` column: epoch microseconds. - Instant, + Instant(InstantKind), /// An integer column, including a `BIGINT TIME_KEY`, in its stored unit. Integer, /// A floating-point column. @@ -220,43 +214,36 @@ pub(in crate::data::executor) enum TsGroupKeyKind { /// The wire shape a memtable storage type renders as. /// -/// `Timestamp` maps to `Integer` here: the instant case is decided from the -/// declared DDL type before this runs, so what reaches it is a `BIGINT` -/// time key or a system-time column, both of which render as the number -/// storage holds. +/// A time column answers from its own kind: an instant renders as one, a +/// `Millis` column (a `BIGINT` time key or the system-time column) renders +/// as the number storage holds. fn kind_of_storage(storage: ColumnType) -> TsGroupKeyKind { match storage { - ColumnType::Int64 | ColumnType::Timestamp => TsGroupKeyKind::Integer, + ColumnType::Timestamp(TimeKind::Instant(kind)) => TsGroupKeyKind::Instant(kind), + ColumnType::Int64 | ColumnType::Timestamp(TimeKind::Millis) => TsGroupKeyKind::Integer, ColumnType::Float64 => TsGroupKeyKind::Float, ColumnType::Symbol => TsGroupKeyKind::Text, } } -/// Whether a declared DDL type makes a column an instant on the wire. -/// -/// Both planes answer this from `nodedb_types::columnar::ColumnType`: the -/// Control Plane decides how a cell is READ, this decides the unit it is -/// WRITTEN in, and one classifier keeps them from naming different sets. -fn declared_type_is_instant(declared_type: &str) -> bool { - nodedb_types::columnar::ColumnType::from_declared_type(declared_type) - .is_some_and(|declared| declared.is_instant()) -} - /// Map a declared SQL type onto the memtable's storage type. /// -/// The designated time key is always the memtable's `Timestamp` column -/// regardless of how it was spelled — `TIMESTAMP`, `TIMESTAMPTZ`, and -/// `BIGINT` time keys all store epoch milliseconds. +/// The designated time key is always a memtable `Timestamp` column. Its +/// kind comes from the declared type: `TIMESTAMP` and `TIMESTAMPTZ` are +/// instants, while a `BIGINT` time key (or any other spelling) is `Millis`. +/// The engine-assigned system-time column is `Millis` too. fn memtable_column_type(declared_type: &str, is_time_key: bool) -> ColumnType { use nodedb_types::columnar::ColumnType as DeclaredType; - if is_time_key { - return ColumnType::Timestamp; - } match DeclaredType::from_declared_type(declared_type) { - Some( - DeclaredType::Timestamp | DeclaredType::Timestamptz | DeclaredType::SystemTimestamp, - ) => ColumnType::Timestamp, + Some(DeclaredType::Timestamp) => { + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive)) + } + Some(DeclaredType::Timestamptz) => { + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)) + } + Some(DeclaredType::SystemTimestamp) => ColumnType::Timestamp(TimeKind::Millis), + _ if is_time_key => ColumnType::Timestamp(TimeKind::Millis), Some(DeclaredType::Int64) => ColumnType::Int64, // The memtable has no boolean column; ILP ingest widens booleans to // f64, so a declared BOOLEAN lands in the same place. @@ -273,20 +260,47 @@ fn memtable_column_type(declared_type: &str, is_time_key: bool) -> ColumnType { mod tests { use super::*; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + const NAIVE: ColumnType = ColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive)); + const UTC: ColumnType = ColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)); + #[test] fn time_key_is_the_timestamp_column_whatever_its_declared_type() { + assert!(memtable_column_type("BIGINT TIME_KEY", true).is_time()); + assert!(memtable_column_type("TIMESTAMP TIME_KEY", true).is_time()); + assert!(memtable_column_type("TIMESTAMPTZ", true).is_time()); + assert!(memtable_column_type("TEXT", true).is_time()); + } + + /// The kind follows the declared type: a `BIGINT` time key reads back as + /// the integer stored, a `TIMESTAMP` as a naive instant, a `TIMESTAMPTZ` + /// as a UTC instant. The system-time column is engine-assigned and reads + /// back as milliseconds. + #[test] + fn time_kind_follows_the_declared_type() { + assert_eq!(memtable_column_type("BIGINT TIME_KEY", true), MILLIS); + assert_eq!(memtable_column_type("TIMESTAMP TIME_KEY", true), NAIVE); + assert_eq!(memtable_column_type("timestamp", true), NAIVE); + assert_eq!(memtable_column_type("TIMESTAMPTZ", true), UTC); + assert_eq!(memtable_column_type("TIMESTAMPTZ", false), UTC); + assert_eq!(memtable_column_type("SYSTEM_TIMESTAMP", false), MILLIS); + assert_eq!(memtable_column_type("SYSTEM_TIMESTAMP", true), MILLIS); + } + + #[test] + fn group_key_kind_comes_from_the_column_kind() { assert_eq!( - memtable_column_type("BIGINT TIME_KEY", true), - ColumnType::Timestamp - ); - assert_eq!( - memtable_column_type("TIMESTAMP TIME_KEY", true), - ColumnType::Timestamp + kind_of_storage(NAIVE), + TsGroupKeyKind::Instant(InstantKind::Naive) ); assert_eq!( - memtable_column_type("TIMESTAMPTZ", true), - ColumnType::Timestamp + kind_of_storage(UTC), + TsGroupKeyKind::Instant(InstantKind::Utc) ); + assert_eq!(kind_of_storage(MILLIS), TsGroupKeyKind::Integer); + assert_eq!(kind_of_storage(ColumnType::Int64), TsGroupKeyKind::Integer); + assert_eq!(kind_of_storage(ColumnType::Float64), TsGroupKeyKind::Float); + assert_eq!(kind_of_storage(ColumnType::Symbol), TsGroupKeyKind::Text); } #[test] @@ -308,25 +322,7 @@ mod tests { // Only the designated key drives partitioning, but a non-key // timestamp column keeps timestamp storage — its value comes from the // row, not from the ingest line's clock. - assert_eq!( - memtable_column_type("TIMESTAMP", false), - ColumnType::Timestamp - ); - } - - /// The declared types the planner reads back as instants are exactly the - /// ones emission scales. A `BIGINT` time key is stored in the same - /// millisecond column and must NOT be scaled. - #[test] - fn only_declared_timestamp_types_are_instants() { - assert!(declared_type_is_instant("TIMESTAMP TIME_KEY")); - assert!(declared_type_is_instant("TIMESTAMPTZ")); - assert!(declared_type_is_instant("timestamp")); - assert!(!declared_type_is_instant("BIGINT TIME_KEY")); - assert!(!declared_type_is_instant("INT")); - assert!(!declared_type_is_instant("TEXT")); - assert!(!declared_type_is_instant("SYSTEM_TIMESTAMP")); - assert!(!declared_type_is_instant("")); + assert_eq!(memtable_column_type("TIMESTAMP", false), NAIVE); } #[test] diff --git a/nodedb/src/data/executor/handlers/columnar_agg.rs b/nodedb/src/data/executor/handlers/columnar_agg.rs index 346dc2f48..c2e5c8a8b 100644 --- a/nodedb/src/data/executor/handlers/columnar_agg.rs +++ b/nodedb/src/data/executor/handlers/columnar_agg.rs @@ -436,14 +436,14 @@ fn build_results_from_groups( mod tests { use super::*; use crate::engine::timeseries::columnar_memtable::{ - ColumnType, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, + ColumnType, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; use nodedb_types::timeseries::SeriesId; fn make_test_memtable() -> ColumnarMemtable { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), ("value".into(), ColumnType::Float64), ("qname".into(), ColumnType::Symbol), ("qtype".into(), ColumnType::Symbol), diff --git a/nodedb/src/data/executor/handlers/columnar_agg_support.rs b/nodedb/src/data/executor/handlers/columnar_agg_support.rs index 7cf012d66..ec71cb21d 100644 --- a/nodedb/src/data/executor/handlers/columnar_agg_support.rs +++ b/nodedb/src/data/executor/handlers/columnar_agg_support.rs @@ -108,7 +108,7 @@ pub(in crate::data::executor::handlers) fn extract_group_key_part( GroupKeyPart::Null } } - ColumnType::Timestamp => { + ColumnType::Timestamp(_) => { if let ColumnData::Timestamp(vals) = col_data { GroupKeyPart::Int64(vals[row_idx]) } else { diff --git a/nodedb/src/data/executor/handlers/columnar_filter/eval.rs b/nodedb/src/data/executor/handlers/columnar_filter/eval.rs index 6c9c06cc9..4d02773e0 100644 --- a/nodedb/src/data/executor/handlers/columnar_filter/eval.rs +++ b/nodedb/src/data/executor/handlers/columnar_filter/eval.rs @@ -68,7 +68,7 @@ pub(crate) fn eval_filters_sparse( } } } - ColumnType::Timestamp => { + ColumnType::Timestamp(_) => { let fv = nodedb_query::scan_filter::value_as_timestamp_ms(&f.value)?; if let ColumnData::Timestamp(vals) = col_data { for (mi, &idx) in indices.iter().enumerate() { @@ -142,7 +142,7 @@ pub(crate) fn eval_filters_dense( } } } - ColumnType::Timestamp => { + ColumnType::Timestamp(_) => { let fv = nodedb_query::scan_filter::value_as_timestamp_ms(&f.value)?; if let ColumnData::Timestamp(vals) = col_data { for i in 0..row_count { @@ -273,7 +273,7 @@ pub(crate) fn eval_filters_bitmask( _ => return None, } } - ColumnType::Timestamp => { + ColumnType::Timestamp(_) => { let fv = nodedb_query::scan_filter::value_as_timestamp_ms(&f.value)?; let ColumnData::Timestamp(vals) = col_data else { return None; diff --git a/nodedb/src/data/executor/handlers/columnar_filter/memtable_source.rs b/nodedb/src/data/executor/handlers/columnar_filter/memtable_source.rs index 707a47a86..6f6019007 100644 --- a/nodedb/src/data/executor/handlers/columnar_filter/memtable_source.rs +++ b/nodedb/src/data/executor/handlers/columnar_filter/memtable_source.rs @@ -24,7 +24,7 @@ mod tests { use super::*; use crate::bridge::scan_filter::ScanFilter; use crate::engine::timeseries::columnar_memtable::{ - ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, + ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; use nodedb_types::timeseries::SeriesId; @@ -33,7 +33,7 @@ mod tests { fn make_test_mt() -> ColumnarMemtable { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), ("value".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], diff --git a/nodedb/src/data/executor/handlers/columnar_filter/partition_source.rs b/nodedb/src/data/executor/handlers/columnar_filter/partition_source.rs index 0dc91dc15..67f5969d5 100644 --- a/nodedb/src/data/executor/handlers/columnar_filter/partition_source.rs +++ b/nodedb/src/data/executor/handlers/columnar_filter/partition_source.rs @@ -248,7 +248,12 @@ mod tests { fn partition_columns_adapter() { // Simulate sealed partition data. let schema = vec![ - ("timestamp".into(), ColumnType::Timestamp), + ( + "timestamp".into(), + ColumnType::Timestamp( + crate::engine::timeseries::columnar_memtable::TimeKind::Millis, + ), + ), ("value".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ]; diff --git a/nodedb/src/data/executor/handlers/columnar_read/convert.rs b/nodedb/src/data/executor/handlers/columnar_read/convert.rs index dd49e7298..9cea0c69f 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/convert.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/convert.rs @@ -104,7 +104,7 @@ pub(in crate::data::executor) fn emit_column_value( ColumnData as TsColumnData, ColumnType as TsColumnType, }; match col_type { - TsColumnType::Timestamp => { + TsColumnType::Timestamp(_) => { nodedb_query::msgpack_scan::write_i64(buf, col_data.as_timestamps()[row_idx]); } TsColumnType::Float64 => { diff --git a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs index 862054d24..1aedecb25 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs @@ -370,7 +370,7 @@ fn memtable_col_to_value( row_idx: usize, ) -> Value { match col_type { - ColumnType::Timestamp => Value::Integer(col_data.as_timestamps()[row_idx]), + ColumnType::Timestamp(_) => Value::Integer(col_data.as_timestamps()[row_idx]), ColumnType::Float64 => { let v = col_data.as_f64()[row_idx]; if v.is_nan() { @@ -399,7 +399,7 @@ fn partition_col_to_value( row_idx: usize, ) -> Value { match col_type { - ColumnType::Timestamp => Value::Integer(data.as_timestamps()[row_idx]), + ColumnType::Timestamp(_) => Value::Integer(data.as_timestamps()[row_idx]), ColumnType::Float64 => { let v = data.as_f64()[row_idx]; if v.is_nan() { diff --git a/nodedb/src/data/executor/handlers/timeseries/encode.rs b/nodedb/src/data/executor/handlers/timeseries/encode.rs index 922b4dfdc..0990839ff 100644 --- a/nodedb/src/data/executor/handlers/timeseries/encode.rs +++ b/nodedb/src/data/executor/handlers/timeseries/encode.rs @@ -21,7 +21,7 @@ fn group_key_value(part: Option<&&str>, kind: TsGroupKeyKind) -> crate::Result match text.parse::() { + TsGroupKeyKind::Instant(_) => match text.parse::() { Ok(millis) => { let micros = nodedb_types::NdbDateTime::from_millis(millis) .map_err(|e| crate::Error::Internal { diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest.rs b/nodedb/src/data/executor/handlers/timeseries/ingest.rs index 1135c32a6..c2a02fdd5 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest.rs @@ -15,9 +15,7 @@ use crate::bridge::envelope::{ErrorCode, Payload, Response, Status}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::response_codec; -use crate::engine::timeseries::columnar_memtable::{ - ColumnType, ColumnarMemtable, ColumnarMemtableConfig, -}; +use crate::engine::timeseries::columnar_memtable::{ColumnarMemtable, ColumnarMemtableConfig}; use crate::engine::timeseries::ilp; use crate::engine::timeseries::ilp_ingest; @@ -402,15 +400,7 @@ impl CoreLoop { .schema() .columns .iter() - .map(|(name, col_type)| { - let type_str = match col_type { - ColumnType::Timestamp => "TIMESTAMP", - ColumnType::Float64 => "FLOAT", - ColumnType::Int64 => "BIGINT", - ColumnType::Symbol => "VARCHAR", - }; - serde_json::json!([name, type_str]) - }) + .map(|(name, col_type)| serde_json::json!([name, col_type.ddl_type_name()])) .collect(); serde_json::json!({ "accepted": accepted, diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs index c482c8ee0..919525d2a 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs @@ -101,7 +101,7 @@ pub(super) fn emit_partition_row( continue; }; let val = match col_type { - ColumnType::Timestamp => rmpv::Value::Integer(data.as_timestamps()[idx].into()), + ColumnType::Timestamp(_) => rmpv::Value::Integer(data.as_timestamps()[idx].into()), ColumnType::Float64 => { let v = data.as_f64()[idx]; if v.is_nan() { @@ -181,9 +181,9 @@ pub(super) fn apply_computed_columns_rmpv( /// columns — so nothing inside the engine sees the wire unit. /// /// `instant_columns` comes from `CoreLoop::ts_instant_columns`, which lists -/// the columns declared `TIMESTAMP` or `TIMESTAMPTZ`. A `BIGINT TIME_KEY` -/// lives in the same millisecond column and is not in that list, so it keeps -/// the integer the client inserted. +/// the columns whose memtable time kind is an instant. A `BIGINT TIME_KEY` +/// lives in the same millisecond storage with kind `Millis` and is not in +/// that list, so it keeps the integer the client inserted. /// /// SQL NULL cells pass through untouched. A stored value that cannot be /// expressed in microseconds fails the read rather than wrapping. diff --git a/nodedb/src/data/executor/handlers/transaction/undo/apply.rs b/nodedb/src/data/executor/handlers/transaction/undo/apply.rs index 6c756e434..5425640a9 100644 --- a/nodedb/src/data/executor/handlers/transaction/undo/apply.rs +++ b/nodedb/src/data/executor/handlers/transaction/undo/apply.rs @@ -465,7 +465,7 @@ mod tests { use super::*; use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; use crate::engine::timeseries::columnar_memtable::{ - ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, + ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; use crate::engine::timeseries::last_value_cache::LastValueCache; use crate::types::{DatabaseId, TenantId}; @@ -486,7 +486,7 @@ mod tests { ColumnarMemtable::new( ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), ("value".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], diff --git a/nodedb/src/engine/timeseries/columnar_bridge.rs b/nodedb/src/engine/timeseries/columnar_bridge.rs index 567b160d3..d8ad1c782 100644 --- a/nodedb/src/engine/timeseries/columnar_bridge.rs +++ b/nodedb/src/engine/timeseries/columnar_bridge.rs @@ -16,10 +16,11 @@ use nodedb_columnar::predicate::ScanPredicate; use nodedb_columnar::reader::{DecodedColumn, SegmentReader}; use nodedb_columnar::writer::SegmentWriter; use nodedb_mem::ScopedMemory; +use nodedb_types::InstantKind; use nodedb_types::columnar::{ColumnDef, ColumnType as SharedColumnType, ColumnarSchema}; use super::columnar_memtable::{ - ColumnData as TsColumnData, ColumnType as TsColumnType, ColumnarDrainResult, + ColumnData as TsColumnData, ColumnType as TsColumnType, ColumnarDrainResult, TimeKind, }; /// Convert a timeseries ColumnarSchema to a shared ColumnarSchema. @@ -46,9 +47,19 @@ pub fn ts_schema_to_shared(ts_schema: &super::columnar_memtable::ColumnarSchema) } /// Map a timeseries ColumnType to a shared ColumnType. +/// +/// A time column maps by its kind: a naive instant is `Timestamp`, a UTC +/// instant is `Timestamptz`, and a `Millis` column is the `Int64` its +/// declaration named. All three share `i64` storage on both sides. fn ts_column_type_to_shared(ts_type: TsColumnType) -> SharedColumnType { match ts_type { - TsColumnType::Timestamp => SharedColumnType::Timestamp, + TsColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive)) => { + SharedColumnType::Timestamp + } + TsColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)) => { + SharedColumnType::Timestamptz + } + TsColumnType::Timestamp(TimeKind::Millis) => SharedColumnType::Int64, TsColumnType::Float64 => SharedColumnType::Float64, TsColumnType::Int64 => SharedColumnType::Int64, // Symbol columns store u32 IDs — represented as Int64 in shared format. @@ -79,10 +90,18 @@ fn ts_column_to_shared( row_count: usize, ) -> SharedColumnData { match (ts_col, ts_type) { - (TsColumnData::Timestamp(values), TsColumnType::Timestamp) => SharedColumnData::Timestamp { - values: values.clone(), - valid: None, // Timeseries columns are non-nullable. - }, + (TsColumnData::Timestamp(values), TsColumnType::Timestamp(TimeKind::Instant(_))) => { + SharedColumnData::Timestamp { + values: values.clone(), + valid: None, // Timeseries columns are non-nullable. + } + } + (TsColumnData::Timestamp(values), TsColumnType::Timestamp(TimeKind::Millis)) => { + SharedColumnData::Int64 { + values: values.clone(), + valid: None, + } + } (TsColumnData::Float64(values), TsColumnType::Float64) => SharedColumnData::Float64 { values: values.clone(), valid: None, @@ -297,20 +316,34 @@ mod tests { fn schema_conversion() { let ts_schema = super::super::columnar_memtable::ColumnarSchema { columns: vec![ - ("timestamp".into(), TsColumnType::Timestamp), + ( + "timestamp".into(), + TsColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive)), + ), ("value".into(), TsColumnType::Float64), ("host".into(), TsColumnType::Symbol), + ( + "seen_at".into(), + TsColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)), + ), + ( + "_ts_system".into(), + TsColumnType::Timestamp(TimeKind::Millis), + ), ], timestamp_idx: 0, - codecs: vec![nodedb_codec::ColumnCodec::Auto; 3], + codecs: vec![nodedb_codec::ColumnCodec::Auto; 5], }; let shared = ts_schema_to_shared(&ts_schema); - assert_eq!(shared.columns.len(), 3); + assert_eq!(shared.columns.len(), 5); assert_eq!(shared.columns[0].column_type, SharedColumnType::Timestamp); assert_eq!(shared.columns[1].column_type, SharedColumnType::Float64); // Symbol → Int64 in shared format. assert_eq!(shared.columns[2].column_type, SharedColumnType::Int64); + assert_eq!(shared.columns[3].column_type, SharedColumnType::Timestamptz); + // A millisecond integer time column is the integer it was declared as. + assert_eq!(shared.columns[4].column_type, SharedColumnType::Int64); } #[test] diff --git a/nodedb/src/engine/timeseries/columnar_memtable/memtable.rs b/nodedb/src/engine/timeseries/columnar_memtable/memtable.rs deleted file mode 100644 index 3307a91a2..000000000 --- a/nodedb/src/engine/timeseries/columnar_memtable/memtable.rs +++ /dev/null @@ -1,952 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! `ColumnarMemtable` — per-column ingest buffer for timeseries data. -//! -//! NOT thread-safe — lives on a single Data Plane core (!Send by design). - -use std::collections::HashMap; - -use nodedb_types::timeseries::{IngestResult, MetricSample, SeriesId, SymbolDictionary}; - -use super::snapshot::{MemtableSnapshot, column_to_snapshot, rebuild_columns}; -use super::types::{ - ColumnData, ColumnType, ColumnValue, ColumnarDrainResult, ColumnarFlushView, - ColumnarMemtableConfig, ColumnarSchema, max_system_ts_of, -}; - -/// Columnar memtable: per-column vectors instead of per-series hash maps. -/// -/// Each row is a flat tuple of (timestamp, value, tag1, tag2, ...). -/// Series identity is derived from the tag columns at query time. -/// This layout is SIMD-friendly: aggregation functions operate on -/// contiguous `&[f64]` or `&[i64]` slices. -pub struct ColumnarMemtable { - schema: ColumnarSchema, - columns: Vec, - /// Per-series row count for quick cardinality checks. - series_row_counts: HashMap, - /// Per-tag-column symbol dictionary. - symbol_dicts: HashMap, - row_count: u64, - memory_bytes: usize, - config: ColumnarMemtableConfig, - min_ts: i64, - max_ts: i64, -} - -impl ColumnarMemtable { - /// Create a new columnar memtable with the given schema. - pub fn new(schema: ColumnarSchema, config: ColumnarMemtableConfig) -> Self { - let columns: Vec = schema - .columns - .iter() - .map(|(_, ty)| ColumnData::new(*ty)) - .collect(); - - // Initialize symbol dicts for tag columns. - let mut symbol_dicts = HashMap::new(); - for (i, (_, ty)) in schema.columns.iter().enumerate() { - if *ty == ColumnType::Symbol { - symbol_dicts.insert(i, SymbolDictionary::new()); - } - } - - Self { - schema, - columns, - series_row_counts: HashMap::new(), - symbol_dicts, - row_count: 0, - memory_bytes: 0, - config, - min_ts: i64::MAX, - max_ts: i64::MIN, - } - } - - /// Create a simple metrics memtable (timestamp + f64 value, no tags). - pub fn new_metric(config: ColumnarMemtableConfig) -> Self { - Self::new(ColumnarSchema::metric_default(), config) - } - - /// Ingest a metric sample into the default (timestamp, value) layout. - /// - /// For the simple 2-column schema. For multi-column schemas with tags, - /// use `ingest_row()` instead. - /// - /// Does NOT enforce `hard_memory_limit`, for the same reason `ingest_row` - /// does not (see its doc): the sole production caller is WAL replay - /// (`replay_timeseries_payload`), and a sample reaching here belongs to a - /// record that has ALREADY COMMITTED, so refusing it is not backpressure — - /// it is silent loss of a durable write. Replay must take the record whole; - /// the ceiling lives at the record boundary in the live ingest handler's - /// admission gate, never here. NOTE: if a structured-`TimeseriesWalBatch` - /// ingest producer is ever added, its replay must gain that same - /// record-boundary flush, or a mid-record replay flush would stamp the - /// partition with an LSN covering rows it does not hold. - /// `replay_timeseries_wal` advances `ts_max_ingested_lsn` only AFTER a - /// record is applied, which is what keeps that stamp honest. - pub fn ingest_metric(&mut self, series_id: SeriesId, sample: MetricSample) -> IngestResult { - // Push to timestamp column. - if let ColumnData::Timestamp(ref mut v) = self.columns[self.schema.timestamp_idx] { - v.push(sample.timestamp_ms); - } - - // Push to value column (assume index 1 for default schema). - if self.columns.len() > 1 - && let ColumnData::Float64(ref mut v) = self.columns[1] - { - v.push(sample.value); - } - - self.update_stats(series_id, sample.timestamp_ms, 16); - self.check_flush_state() - } - - /// Ingest a row with explicit column values. - /// - /// `values` must match the schema length. Tag string values are resolved - /// to symbol IDs via the per-column dictionary. - /// - /// ## Why this does NOT enforce `hard_memory_limit` - /// - /// It used to: a row arriving above the ceiling came back - /// `Ok(IngestResult::Rejected)`. But every row reaching here belongs to a - /// WAL record that has ALREADY COMMITTED, so refusing it is not - /// backpressure — it is silent loss of a durable write. The caller then - /// tried to rescue the refusal by flushing and re-ingesting mid-record, - /// which stamped the flushed partition with the PREVIOUS record's LSN - /// while it held part of the current one; replay, gated on that stamp, - /// re-appended the whole record on top. - /// - /// The ceiling now lives at the record boundary instead (the ingest - /// handler's admission gate flushes BEFORE a record when the memtable is - /// at or over it), so a flush always lands on a whole-record prefix and - /// the partition's stamp is true of every row in it. Errors returned here - /// are therefore genuine per-row data faults — bad arity, type mismatch, - /// exhausted tag dictionary — never "come back later". - pub fn ingest_row( - &mut self, - series_id: SeriesId, - values: &[ColumnValue], - ) -> crate::Result { - let col_types: Vec<(String, ColumnType)> = self.schema.columns.clone(); - - if values.len() != col_types.len() { - return Err(crate::Error::BadRequest { - detail: format!("expected {} columns, got {}", col_types.len(), values.len()), - }); - } - - let mut ts = 0i64; - let mut row_bytes = 0usize; - let max_card = self.config.max_tag_cardinality; - - for (i, (val, (col_name, col_type))) in values.iter().zip(col_types.iter()).enumerate() { - match (val, col_type) { - (ColumnValue::Timestamp(t), ColumnType::Timestamp) => { - if let ColumnData::Timestamp(ref mut v) = self.columns[i] { - v.push(*t); - } - ts = *t; - row_bytes += 8; - } - (ColumnValue::Float64(f), ColumnType::Float64) => { - if let ColumnData::Float64(ref mut v) = self.columns[i] { - v.push(*f); - } - row_bytes += 8; - } - (ColumnValue::Int64(n), ColumnType::Int64) => { - if let ColumnData::Int64(ref mut v) = self.columns[i] { - v.push(*n); - } - row_bytes += 8; - } - (ColumnValue::Symbol(s), ColumnType::Symbol) => { - let dict = - self.symbol_dicts - .get_mut(&i) - .ok_or_else(|| crate::Error::BadRequest { - detail: format!( - "internal error: symbol dict missing for column {i}" - ), - })?; - match dict.resolve(s, max_card) { - Some(sym_id) => { - if let ColumnData::Symbol(ref mut v) = self.columns[i] { - v.push(sym_id); - } - } - None => { - self.rollback_partial_row(i); - return Err(crate::Error::BadRequest { - detail: format!( - "tag cardinality limit ({max_card}) exceeded for column '{col_name}'" - ), - }); - } - } - row_bytes += 4; - } - _ => { - self.rollback_partial_row(i); - return Err(crate::Error::BadRequest { - detail: format!("type mismatch at column {i}: expected {col_type:?}"), - }); - } - } - } - - self.update_stats(series_id, ts, row_bytes); - Ok(self.check_flush_state()) - } - - /// Roll back a partially written row (called on error during `ingest_row`). - fn rollback_partial_row(&mut self, columns_written: usize) { - for col in self.columns.iter_mut().take(columns_written) { - match col { - ColumnData::Timestamp(v) => { - v.pop(); - } - ColumnData::Float64(v) => { - v.pop(); - } - ColumnData::Int64(v) => { - v.pop(); - } - ColumnData::Symbol(v) => { - v.pop(); - } - ColumnData::DictEncoded { ids, valid, .. } => { - ids.pop(); - valid.pop(); - } - } - } - } - - fn update_stats(&mut self, series_id: SeriesId, ts: i64, row_bytes: usize) { - *self.series_row_counts.entry(series_id).or_insert(0) += 1; - self.row_count += 1; - self.memory_bytes += row_bytes; - if ts < self.min_ts { - self.min_ts = ts; - } - if ts > self.max_ts { - self.max_ts = ts; - } - } - - fn check_flush_state(&self) -> IngestResult { - if self.memory_bytes >= self.config.max_memory_bytes { - IngestResult::FlushNeeded - } else { - IngestResult::Ok - } - } - - /// Borrow this memtable's live rows as the payload a flush would write. - /// - /// The read-only half of a flush: a segment can be encoded and landed from - /// this view, and only then does [`Self::drain`] take the rows out. Nothing - /// leaves memory before it is durable, so a failed segment write leaves the - /// memtable exactly as it was. - pub fn flush_view(&self) -> ColumnarFlushView<'_> { - ColumnarFlushView { - columns: &self.columns, - schema: &self.schema, - symbol_dicts: &self.symbol_dicts, - row_count: self.row_count, - min_ts: self.min_ts, - max_ts: self.max_ts, - max_system_ts: max_system_ts_of(&self.schema, &self.columns), - } - } - - /// Drain all data from the memtable, resetting it for reuse. - /// - /// Returns the column data, schema, symbol dicts, and stats. - /// - /// Callers that flush must write the segment from [`Self::flush_view`] FIRST - /// and drain only once that write has committed — the rows here have no - /// other copy but the WAL, and the checkpoint that calls the flush is what - /// authorises deleting it. - pub fn drain(&mut self) -> ColumnarDrainResult { - let mut drained_columns = Vec::with_capacity(self.columns.len()); - for col in &mut self.columns { - // DictEncoded columns are drained by swapping in a fresh Symbol - // placeholder. The flusher converts Symbol → DictEncoded during - // segment encoding once it has enough cardinality data. - let col_type = match col { - ColumnData::Timestamp(_) => ColumnType::Timestamp, - ColumnData::Float64(_) => ColumnType::Float64, - ColumnData::Int64(_) => ColumnType::Int64, - ColumnData::Symbol(_) => ColumnType::Symbol, - ColumnData::DictEncoded { .. } => ColumnType::Symbol, - }; - let mut empty = ColumnData::new(col_type); - std::mem::swap(col, &mut empty); - drained_columns.push(empty); - } - - let drained_dicts = std::mem::take(&mut self.symbol_dicts); - // Reinitialize symbol dicts. - for (i, (_, ty)) in self.schema.columns.iter().enumerate() { - if *ty == ColumnType::Symbol { - self.symbol_dicts.insert(i, SymbolDictionary::new()); - } - } - - // Scan `_ts_system` column (if present) for retention's system-time axis. - let max_system_ts = max_system_ts_of(&self.schema, &drained_columns); - - let result = ColumnarDrainResult { - columns: drained_columns, - schema: self.schema.clone(), - symbol_dicts: drained_dicts, - row_count: self.row_count, - min_ts: self.min_ts, - max_ts: self.max_ts, - max_system_ts, - series_row_counts: std::mem::take(&mut self.series_row_counts), - }; - - self.row_count = 0; - self.memory_bytes = 0; - self.min_ts = i64::MAX; - self.max_ts = i64::MIN; - - result - } - - // -- Mutators -- - - /// Truncate this memtable back to `n` rows. - /// - /// Used during transaction rollback to reverse a `TimeseriesIngest` operation. - /// All column vectors are truncated; aggregate stats are recomputed from the - /// surviving rows. `series_row_counts` is rebuilt from scratch so per-series - /// cardinality remains consistent. - pub fn truncate_to(&mut self, n: u64) { - if n >= self.row_count { - return; - } - let n_usize = n as usize; - let ts_idx = self.schema.timestamp_idx; - for col in &mut self.columns { - match col { - ColumnData::Timestamp(v) | ColumnData::Int64(v) => v.truncate(n_usize), - ColumnData::Float64(v) => v.truncate(n_usize), - ColumnData::Symbol(v) => v.truncate(n_usize), - ColumnData::DictEncoded { ids, valid, .. } => { - ids.truncate(n_usize); - valid.truncate(n_usize); - } - } - } - self.row_count = n; - // Recompute ts range from surviving timestamps. - if n == 0 { - self.min_ts = i64::MAX; - self.max_ts = i64::MIN; - self.series_row_counts.clear(); - } else if let ColumnData::Timestamp(ts) = &self.columns[ts_idx] { - self.min_ts = ts.iter().copied().min().unwrap_or(i64::MAX); - self.max_ts = ts.iter().copied().max().unwrap_or(i64::MIN); - } - // Recompute memory_bytes estimate by re-summing column capacities. - self.memory_bytes = self - .columns - .iter() - .map(|c| match c { - ColumnData::Timestamp(v) | ColumnData::Int64(v) => v.capacity() * 8, - ColumnData::Float64(v) => v.capacity() * 8, - ColumnData::Symbol(v) => v.capacity() * 4, - ColumnData::DictEncoded { - ids, - valid, - dictionary, - .. - } => ids.capacity() * 4 + valid.capacity() + dictionary.len() * 32, - }) - .sum(); - } - - // -- Accessors -- - - pub fn row_count(&self) -> u64 { - self.row_count - } - - /// Approximate memory usage. Uses incremental tracking with periodic - /// recomputation from column capacities for accuracy. - pub fn memory_bytes(&self) -> usize { - let col_bytes: usize = self.columns.iter().map(|c| c.memory_bytes()).sum(); - let dict_bytes: usize = self.symbol_dicts.len() * 256; // rough estimate - self.memory_bytes.max(col_bytes + dict_bytes) - } - - pub fn min_ts(&self) -> i64 { - self.min_ts - } - - pub fn max_ts(&self) -> i64 { - self.max_ts - } - - pub fn series_count(&self) -> usize { - self.series_row_counts.len() - } - - pub fn schema(&self) -> &ColumnarSchema { - &self.schema - } - - /// Return the immutable admission configuration used to construct this - /// memtable. Transaction undo uses this with a snapshot so restoration - /// preserves the original limits even if live operator tuning changed. - pub fn config(&self) -> ColumnarMemtableConfig { - self.config.clone() - } - - /// Restore the pre-transaction resident-byte accounting after replacing - /// the memtable from a logical snapshot. - /// - /// A snapshot preserves values and dictionaries, but rebuilding its vectors - /// intentionally does not preserve their spare capacity. Transaction undo - /// retains the original governor reservation, so it must also reinstate the - /// original reported footprint rather than silently undercounting it. - pub(crate) fn restore_memory_bytes_for_undo(&mut self, memory_bytes: usize) { - self.memory_bytes = memory_bytes; - } - - /// Export a lossless snapshot (carries column types + symbol dicts). INFALLIBLE. - pub fn export_snapshot(&self) -> MemtableSnapshot { - let columns: Vec<_> = self.columns.iter().map(column_to_snapshot).collect(); - - // Stable ordering for deterministic snapshots. - let mut symbol_dicts: Vec<(usize, SymbolDictionary)> = self - .symbol_dicts - .iter() - .map(|(&idx, dict)| (idx, dict.clone())) - .collect(); - symbol_dicts.sort_by_key(|(idx, _)| *idx); - - let mut series_row_counts: Vec<(SeriesId, u64)> = self - .series_row_counts - .iter() - .map(|(&k, &v)| (k, v)) - .collect(); - series_row_counts.sort_by_key(|(id, _)| *id); - - MemtableSnapshot { - schema_columns: self.schema.columns.clone(), - timestamp_idx: self.schema.timestamp_idx, - columns, - symbol_dicts, - series_row_counts, - row_count: self.row_count, - min_ts: self.min_ts, - max_ts: self.max_ts, - } - } - - /// Reconstruct a memtable from a snapshot. Returns a typed error on any - /// schema/row-count mismatch. - /// - /// `config` is not carried in the snapshot — it is operator tuning, not - /// data — so callers pass the live `ColumnarMemtableConfig::from_tuning` - /// value. A memtable keeps its limits for its whole life, so restoring with - /// compiled defaults would silently ignore the operator's configuration. - pub fn from_snapshot( - snap: MemtableSnapshot, - config: ColumnarMemtableConfig, - ) -> crate::Result { - if snap.timestamp_idx >= snap.schema_columns.len() { - return Err(crate::Error::BadRequest { - detail: format!( - "snapshot timestamp_idx {} out of range for {} columns", - snap.timestamp_idx, - snap.schema_columns.len(), - ), - }); - } - - let columns = rebuild_columns(snap.columns, &snap.schema_columns, snap.row_count)?; - let n = snap.schema_columns.len(); - // Codecs are ephemeral — not serialized; rebuild as Auto. - let schema = ColumnarSchema { - columns: snap.schema_columns, - timestamp_idx: snap.timestamp_idx, - codecs: vec![nodedb_codec::ColumnCodec::Auto; n], - }; - - let symbol_dicts: HashMap = - snap.symbol_dicts.into_iter().collect(); - let series_row_counts: HashMap = - snap.series_row_counts.into_iter().collect(); - let memory_bytes: usize = - columns.iter().map(|c| c.memory_bytes()).sum::() + symbol_dicts.len() * 256; - - Ok(Self { - schema, - columns, - series_row_counts, - symbol_dicts, - row_count: snap.row_count, - memory_bytes, - config, - min_ts: snap.min_ts, - max_ts: snap.max_ts, - }) - } - - pub fn column(&self, idx: usize) -> &ColumnData { - &self.columns[idx] - } - - pub fn symbol_dict(&self, col_idx: usize) -> Option<&SymbolDictionary> { - self.symbol_dicts.get(&col_idx) - } - - pub fn is_empty(&self) -> bool { - self.row_count == 0 - } - - /// Add a new column to the memtable schema, backfilling existing rows - /// with NULL-equivalent values. - /// - /// Used for ILP schema evolution: when a new field appears in a later - /// batch, the column is added and old rows get NaN/0/null-symbol. - pub fn add_column(&mut self, name: String, col_type: ColumnType) { - // Don't add duplicates. - if self.schema.columns.iter().any(|(n, _)| n == &name) { - return; - } - let existing_rows = self.row_count as usize; - let col = match col_type { - ColumnType::Float64 => ColumnData::Float64(vec![f64::NAN; existing_rows]), - ColumnType::Int64 => ColumnData::Int64(vec![0; existing_rows]), - ColumnType::Symbol => ColumnData::Symbol(vec![u32::MAX; existing_rows]), - ColumnType::Timestamp => return, // never add a second timestamp - }; - let idx = self.columns.len(); - self.columns.push(col); - self.schema.columns.push((name, col_type)); - self.schema.codecs.push(nodedb_codec::ColumnCodec::Auto); - if col_type == ColumnType::Symbol { - self.symbol_dicts.insert(idx, SymbolDictionary::new()); - } - } -} - -#[cfg(test)] -mod tests { - use nodedb_types::timeseries::MetricSample; - - use super::super::snapshot::ColumnSnapshot; - use super::*; - - fn default_config() -> ColumnarMemtableConfig { - ColumnarMemtableConfig { - max_memory_bytes: 1024 * 1024, - hard_memory_limit: 2 * 1024 * 1024, - max_tag_cardinality: 1000, - } - } - - #[test] - fn empty_memtable() { - let mt = ColumnarMemtable::new_metric(default_config()); - assert_eq!(mt.row_count(), 0); - assert!(mt.is_empty()); - assert_eq!(mt.series_count(), 0); - } - - #[test] - fn ingest_simple_metric() { - let mut mt = ColumnarMemtable::new_metric(default_config()); - let result = mt.ingest_metric( - 1, - MetricSample { - timestamp_ms: 1000, - value: 42.5, - }, - ); - assert_eq!(result, IngestResult::Ok); - assert_eq!(mt.row_count(), 1); - assert_eq!(mt.min_ts(), 1000); - assert_eq!(mt.max_ts(), 1000); - - let ts_col = mt.column(0).as_timestamps(); - assert_eq!(ts_col, &[1000]); - let val_col = mt.column(1).as_f64(); - assert!((val_col[0] - 42.5).abs() < f64::EPSILON); - } - - #[test] - fn ingest_multiple_metrics() { - let mut mt = ColumnarMemtable::new_metric(default_config()); - for i in 0..100 { - mt.ingest_metric( - i % 10, - MetricSample { - timestamp_ms: 1000 + i as i64, - value: i as f64, - }, - ); - } - assert_eq!(mt.row_count(), 100); - assert_eq!(mt.series_count(), 10); - assert_eq!(mt.min_ts(), 1000); - assert_eq!(mt.max_ts(), 1099); - } - - #[test] - fn ingest_row_with_tags() { - let schema = ColumnarSchema { - columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), - ("value".into(), ColumnType::Float64), - ("host".into(), ColumnType::Symbol), - ("dc".into(), ColumnType::Symbol), - ], - timestamp_idx: 0, - codecs: vec![nodedb_codec::ColumnCodec::Auto; 4], - }; - let mut mt = ColumnarMemtable::new(schema, default_config()); - - let result = mt.ingest_row( - 1, - &[ - ColumnValue::Timestamp(5000), - ColumnValue::Float64(99.9), - ColumnValue::Symbol("prod-1".to_string()), - ColumnValue::Symbol("us-east".to_string()), - ], - ); - assert!(result.is_ok()); - assert_eq!(mt.row_count(), 1); - - // Verify symbol dictionaries were populated. - let host_dict = mt.symbol_dict(2).unwrap(); - assert_eq!(host_dict.len(), 1); - assert_eq!(host_dict.get(0), Some("prod-1")); - - let dc_dict = mt.symbol_dict(3).unwrap(); - assert_eq!(dc_dict.get(0), Some("us-east")); - } - - #[test] - fn tag_cardinality_breaker() { - let schema = ColumnarSchema { - columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), - ("value".into(), ColumnType::Float64), - ("tag".into(), ColumnType::Symbol), - ], - timestamp_idx: 0, - codecs: vec![nodedb_codec::ColumnCodec::Auto; 3], - }; - let config = ColumnarMemtableConfig { - max_tag_cardinality: 5, - ..default_config() - }; - let mut mt = ColumnarMemtable::new(schema, config); - - // First 5 unique tags work. - for i in 0..5 { - let tag = format!("val-{i}"); - let r = mt.ingest_row( - i as u64, - &[ - ColumnValue::Timestamp(1000 + i as i64), - ColumnValue::Float64(1.0), - ColumnValue::Symbol(tag.clone()), - ], - ); - assert!(r.is_ok()); - } - assert_eq!(mt.row_count(), 5); - - // 6th unique tag is rejected. - let r = mt.ingest_row( - 99, - &[ - ColumnValue::Timestamp(2000), - ColumnValue::Float64(1.0), - ColumnValue::Symbol("one-too-many".to_string()), - ], - ); - assert!(r.is_err()); - // Row count didn't increase (rolled back). - assert_eq!(mt.row_count(), 5); - } - - #[test] - fn drain_returns_data_and_resets() { - let mut mt = ColumnarMemtable::new_metric(default_config()); - for i in 0..50 { - mt.ingest_metric( - 1, - MetricSample { - timestamp_ms: 1000 + i, - value: i as f64, - }, - ); - } - assert_eq!(mt.row_count(), 50); - - let result = mt.drain(); - assert_eq!(result.row_count, 50); - assert_eq!(result.min_ts, 1000); - assert_eq!(result.max_ts, 1049); - assert_eq!(result.columns.len(), 2); - assert_eq!(result.columns[0].len(), 50); - assert_eq!(result.columns[1].len(), 50); - - // Memtable is reset. - assert_eq!(mt.row_count(), 0); - assert!(mt.is_empty()); - } - - #[test] - fn ingest_metric_accepts_past_hard_limit() { - // A sample reaching the memtable belongs to an already-committed WAL - // record; refusing it would silently drop a durable write on replay. - // So `ingest_metric` accepts every sample regardless of the ceiling — - // it never returns `Rejected`, and the resident footprint overshoots - // the hard limit rather than losing data. - let config = ColumnarMemtableConfig { - max_memory_bytes: 100, - hard_memory_limit: 200, - max_tag_cardinality: 1000, - }; - let mut mt = ColumnarMemtable::new_metric(config); - - for i in 0..1000 { - let r = mt.ingest_metric( - 1, - MetricSample { - timestamp_ms: i, - value: 1.0, - }, - ); - assert_ne!( - r, - IngestResult::Rejected, - "sample {i} must not be rejected — that would drop a durable record on replay" - ); - } - assert_eq!( - mt.row_count(), - 1000, - "every sample past the limit is retained" - ); - assert!( - mt.memory_bytes() >= 200, - "the footprint is allowed to overshoot the hard limit" - ); - } - - #[test] - fn flush_needed_signal() { - let config = ColumnarMemtableConfig { - max_memory_bytes: 100, - hard_memory_limit: 200, - max_tag_cardinality: 1000, - }; - let mut mt = ColumnarMemtable::new_metric(config); - - let mut flush_signaled = false; - for i in 0..100 { - let r = mt.ingest_metric( - 1, - MetricSample { - timestamp_ms: i, - value: 1.0, - }, - ); - if r == IngestResult::FlushNeeded { - flush_signaled = true; - break; - } - } - assert!(flush_signaled); - } - - #[test] - fn type_mismatch_rejected() { - let schema = ColumnarSchema { - columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), - ("value".into(), ColumnType::Float64), - ], - timestamp_idx: 0, - codecs: vec![nodedb_codec::ColumnCodec::Auto; 2], - }; - let mut mt = ColumnarMemtable::new(schema, default_config()); - - let r = mt.ingest_row( - 1, - &[ - ColumnValue::Timestamp(1000), - ColumnValue::Int64(42), // Wrong: schema says Float64 - ], - ); - assert!(r.is_err()); - assert_eq!(mt.row_count(), 0); // Rolled back. - } - - // ----------------------------------------------------------------------- - // Snapshot round-trip tests - // ----------------------------------------------------------------------- - - #[test] - fn snapshot_roundtrip_empty_memtable() { - let mt = ColumnarMemtable::new_metric(default_config()); - let snap = mt.export_snapshot(); - let bytes = zerompk::to_msgpack_vec(&snap).expect("serialize"); - let snap2: MemtableSnapshot = zerompk::from_msgpack(&bytes).expect("deserialize"); - let mt2 = ColumnarMemtable::from_snapshot(snap2, default_config()).expect("from_snapshot"); - assert_eq!(mt2.row_count(), 0); - assert!(mt2.is_empty()); - assert_eq!(mt2.schema().columns.len(), 2); - assert_eq!(mt2.schema().timestamp_idx, 0); - } - - #[test] - fn snapshot_roundtrip_multi_column_with_symbol_tags() { - let schema = ColumnarSchema { - columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), - ("value".into(), ColumnType::Float64), - ("host".into(), ColumnType::Symbol), - ], - timestamp_idx: 0, - codecs: vec![nodedb_codec::ColumnCodec::Auto; 3], - }; - let mut mt = ColumnarMemtable::new(schema, default_config()); - - for i in 0..5u64 { - mt.ingest_row( - i, - &[ - ColumnValue::Timestamp(1000 + i as i64), - ColumnValue::Float64(i as f64 * 1.5), - ColumnValue::Symbol(format!("host-{i}")), - ], - ) - .expect("ingest"); - } - // Also re-use an existing host to verify symbol dict cardinality. - mt.ingest_row( - 99, - &[ - ColumnValue::Timestamp(2000), - ColumnValue::Float64(99.0), - ColumnValue::Symbol("host-0".to_string()), - ], - ) - .expect("ingest existing host"); - - let expected_row_count = mt.row_count(); - let expected_min_ts = mt.min_ts(); - let expected_max_ts = mt.max_ts(); - let expected_schema_cols: Vec<(String, ColumnType)> = mt.schema().columns.clone(); - let expected_ts_idx = mt.schema().timestamp_idx; - let expected_dict_len = mt.symbol_dict(2).map(|d| d.len()).unwrap_or(0); - - let snap = mt.export_snapshot(); - let bytes = zerompk::to_msgpack_vec(&snap).expect("serialize"); - let snap2: MemtableSnapshot = zerompk::from_msgpack(&bytes).expect("deserialize"); - let mt2 = ColumnarMemtable::from_snapshot(snap2, default_config()).expect("from_snapshot"); - - assert_eq!(mt2.row_count(), expected_row_count); - assert_eq!(mt2.min_ts(), expected_min_ts); - assert_eq!(mt2.max_ts(), expected_max_ts); - assert_eq!(mt2.schema().columns, expected_schema_cols); - assert_eq!(mt2.schema().timestamp_idx, expected_ts_idx); - - // Verify symbol dict was faithfully restored. - let dict2 = mt2.symbol_dict(2).expect("host dict present"); - assert_eq!(dict2.len(), expected_dict_len); - assert_eq!(dict2.get(0), Some("host-0")); - - // Verify column data lengths match. - for i in 0..3 { - assert_eq!(mt2.column(i).len(), expected_row_count as usize); - } - } - - #[test] - fn snapshot_roundtrip_dict_encoded_column_rebuilds_reverse() { - // Construct a DictEncoded snapshot directly (bypass ingest path). - let snap = MemtableSnapshot { - schema_columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), - ("tag".into(), ColumnType::Symbol), - ], - timestamp_idx: 0, - columns: vec![ - ColumnSnapshot::Timestamp(vec![1000, 2000, 3000]), - ColumnSnapshot::DictEncoded { - ids: vec![0, 1, 0], - dictionary: vec!["alpha".to_string(), "beta".to_string()], - valid: vec![true, true, true], - }, - ], - symbol_dicts: vec![], - series_row_counts: vec![], - row_count: 3, - min_ts: 1000, - max_ts: 3000, - }; - - let mt = ColumnarMemtable::from_snapshot(snap, default_config()).expect("from_snapshot"); - assert_eq!(mt.row_count(), 3); - // Verify the DictEncoded column has correct data. - match mt.column(1) { - ColumnData::DictEncoded { - ids, - dictionary, - reverse, - valid, - } => { - assert_eq!(ids, &[0u32, 1, 0]); - assert_eq!(dictionary, &["alpha", "beta"]); - assert_eq!(reverse.get("alpha"), Some(&0u32)); - assert_eq!(reverse.get("beta"), Some(&1u32)); - assert_eq!(valid, &[true, true, true]); - } - other => panic!("expected DictEncoded, got {:?}", other), - } - } - - #[test] - fn snapshot_from_invalid_column_lengths_returns_error() { - // row_count says 3 but timestamp column has only 2 rows → mismatch. - let snap = MemtableSnapshot { - schema_columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), - ("value".into(), ColumnType::Float64), - ], - timestamp_idx: 0, - columns: vec![ - ColumnSnapshot::Timestamp(vec![1000, 2000]), // 2 rows - ColumnSnapshot::Float64(vec![1.0, 2.0, 3.0]), // 3 rows — mismatch - ], - symbol_dicts: vec![], - series_row_counts: vec![], - row_count: 3, - min_ts: 1000, - max_ts: 2000, - }; - let result = ColumnarMemtable::from_snapshot(snap, default_config()); - assert!( - matches!(result, Err(crate::Error::BadRequest { .. })), - "expected BadRequest error on length mismatch" - ); - } -} diff --git a/nodedb/src/engine/timeseries/columnar_memtable/memtable/drain.rs b/nodedb/src/engine/timeseries/columnar_memtable/memtable/drain.rs new file mode 100644 index 000000000..129f942a1 --- /dev/null +++ b/nodedb/src/engine/timeseries/columnar_memtable/memtable/drain.rs @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Taking rows out of a [`ColumnarMemtable`]: flush view, drain, truncate. + +use nodedb_types::timeseries::SymbolDictionary; + +use super::super::types::{ + ColumnData, ColumnType, ColumnarDrainResult, ColumnarFlushView, max_system_ts_of, +}; +use super::table::ColumnarMemtable; + +impl ColumnarMemtable { + /// Borrow this memtable's live rows as the payload a flush would write. + /// + /// The read-only half of a flush: a segment can be encoded and landed from + /// this view, and only then does [`Self::drain`] take the rows out. Nothing + /// leaves memory before it is durable, so a failed segment write leaves the + /// memtable exactly as it was. + pub fn flush_view(&self) -> ColumnarFlushView<'_> { + ColumnarFlushView { + columns: &self.columns, + schema: &self.schema, + symbol_dicts: &self.symbol_dicts, + row_count: self.row_count, + min_ts: self.min_ts, + max_ts: self.max_ts, + max_system_ts: max_system_ts_of(&self.schema, &self.columns), + } + } + + /// Drain all data from the memtable, resetting it for reuse. + /// + /// Returns the column data, schema, symbol dicts, and stats. + /// + /// Callers that flush must write the segment from [`Self::flush_view`] FIRST + /// and drain only once that write has committed — the rows here have no + /// other copy but the WAL, and the checkpoint that calls the flush is what + /// authorises deleting it. + pub fn drain(&mut self) -> ColumnarDrainResult { + let mut drained_columns = Vec::with_capacity(self.columns.len()); + for (col, (_, schema_type)) in self.columns.iter_mut().zip(self.schema.columns.iter()) { + // DictEncoded columns are drained by swapping in a fresh Symbol + // placeholder. The flusher converts Symbol → DictEncoded during + // segment encoding once it has enough cardinality data. + let col_type = match col { + ColumnData::Timestamp(_) => *schema_type, + ColumnData::Float64(_) => ColumnType::Float64, + ColumnData::Int64(_) => ColumnType::Int64, + ColumnData::Symbol(_) => ColumnType::Symbol, + ColumnData::DictEncoded { .. } => ColumnType::Symbol, + }; + let mut empty = ColumnData::new(col_type); + std::mem::swap(col, &mut empty); + drained_columns.push(empty); + } + + let drained_dicts = std::mem::take(&mut self.symbol_dicts); + // Reinitialize symbol dicts. + for (i, (_, ty)) in self.schema.columns.iter().enumerate() { + if *ty == ColumnType::Symbol { + self.symbol_dicts.insert(i, SymbolDictionary::new()); + } + } + + // Scan `_ts_system` column (if present) for retention's system-time axis. + let max_system_ts = max_system_ts_of(&self.schema, &drained_columns); + + let result = ColumnarDrainResult { + columns: drained_columns, + schema: self.schema.clone(), + symbol_dicts: drained_dicts, + row_count: self.row_count, + min_ts: self.min_ts, + max_ts: self.max_ts, + max_system_ts, + series_row_counts: std::mem::take(&mut self.series_row_counts), + }; + + self.row_count = 0; + self.memory_bytes = 0; + self.min_ts = i64::MAX; + self.max_ts = i64::MIN; + + result + } + + /// Truncate this memtable back to `n` rows. + /// + /// Used during transaction rollback to reverse a `TimeseriesIngest` operation. + /// All column vectors are truncated; aggregate stats are recomputed from the + /// surviving rows. `series_row_counts` is rebuilt from scratch so per-series + /// cardinality remains consistent. + pub fn truncate_to(&mut self, n: u64) { + if n >= self.row_count { + return; + } + let n_usize = n as usize; + let ts_idx = self.schema.timestamp_idx; + for col in &mut self.columns { + match col { + ColumnData::Timestamp(v) | ColumnData::Int64(v) => v.truncate(n_usize), + ColumnData::Float64(v) => v.truncate(n_usize), + ColumnData::Symbol(v) => v.truncate(n_usize), + ColumnData::DictEncoded { ids, valid, .. } => { + ids.truncate(n_usize); + valid.truncate(n_usize); + } + } + } + self.row_count = n; + // Recompute ts range from surviving timestamps. + if n == 0 { + self.min_ts = i64::MAX; + self.max_ts = i64::MIN; + self.series_row_counts.clear(); + } else if let ColumnData::Timestamp(ts) = &self.columns[ts_idx] { + self.min_ts = ts.iter().copied().min().unwrap_or(i64::MAX); + self.max_ts = ts.iter().copied().max().unwrap_or(i64::MIN); + } + // Recompute memory_bytes estimate by re-summing column capacities. + self.memory_bytes = self + .columns + .iter() + .map(|c| match c { + ColumnData::Timestamp(v) | ColumnData::Int64(v) => v.capacity() * 8, + ColumnData::Float64(v) => v.capacity() * 8, + ColumnData::Symbol(v) => v.capacity() * 4, + ColumnData::DictEncoded { + ids, + valid, + dictionary, + .. + } => ids.capacity() * 4 + valid.capacity() + dictionary.len() * 32, + }) + .sum(); + } +} + +#[cfg(test)] +mod tests { + use nodedb_types::timeseries::MetricSample; + + use super::super::super::types::ColumnarMemtableConfig; + use super::*; + + fn default_config() -> ColumnarMemtableConfig { + ColumnarMemtableConfig { + max_memory_bytes: 1024 * 1024, + hard_memory_limit: 2 * 1024 * 1024, + max_tag_cardinality: 1000, + } + } + + #[test] + fn drain_returns_data_and_resets() { + let mut mt = ColumnarMemtable::new_metric(default_config()); + for i in 0..50 { + mt.ingest_metric( + 1, + MetricSample { + timestamp_ms: 1000 + i, + value: i as f64, + }, + ); + } + assert_eq!(mt.row_count(), 50); + + let result = mt.drain(); + assert_eq!(result.row_count, 50); + assert_eq!(result.min_ts, 1000); + assert_eq!(result.max_ts, 1049); + assert_eq!(result.columns.len(), 2); + assert_eq!(result.columns[0].len(), 50); + assert_eq!(result.columns[1].len(), 50); + + // Memtable is reset. + assert_eq!(mt.row_count(), 0); + assert!(mt.is_empty()); + } +} diff --git a/nodedb/src/engine/timeseries/columnar_memtable/memtable/evolve.rs b/nodedb/src/engine/timeseries/columnar_memtable/memtable/evolve.rs new file mode 100644 index 000000000..e712b46a2 --- /dev/null +++ b/nodedb/src/engine/timeseries/columnar_memtable/memtable/evolve.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Schema evolution on a live [`ColumnarMemtable`]. + +use nodedb_types::timeseries::SymbolDictionary; + +use super::super::types::{ColumnData, ColumnType}; +use super::table::ColumnarMemtable; + +impl ColumnarMemtable { + /// Add a new column to the memtable schema, backfilling existing rows + /// with NULL-equivalent values. + /// + /// Used for ILP schema evolution: when a new field appears in a later + /// batch, the column is added and old rows get NaN/0/null-symbol. + pub fn add_column(&mut self, name: String, col_type: ColumnType) { + // Don't add duplicates. + if self.schema.columns.iter().any(|(n, _)| n == &name) { + return; + } + let existing_rows = self.row_count as usize; + let col = match col_type { + ColumnType::Float64 => ColumnData::Float64(vec![f64::NAN; existing_rows]), + ColumnType::Int64 => ColumnData::Int64(vec![0; existing_rows]), + ColumnType::Symbol => ColumnData::Symbol(vec![u32::MAX; existing_rows]), + ColumnType::Timestamp(_) => return, // never add a second timestamp + }; + let idx = self.columns.len(); + self.columns.push(col); + self.schema.columns.push((name, col_type)); + self.schema.codecs.push(nodedb_codec::ColumnCodec::Auto); + if col_type == ColumnType::Symbol { + self.symbol_dicts.insert(idx, SymbolDictionary::new()); + } + } +} diff --git a/nodedb/src/engine/timeseries/columnar_memtable/memtable/ingest.rs b/nodedb/src/engine/timeseries/columnar_memtable/memtable/ingest.rs new file mode 100644 index 000000000..ab45c7e2d --- /dev/null +++ b/nodedb/src/engine/timeseries/columnar_memtable/memtable/ingest.rs @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Row and sample ingest into a [`ColumnarMemtable`]. + +use nodedb_types::timeseries::{IngestResult, MetricSample, SeriesId}; + +use super::super::types::{ColumnData, ColumnType, ColumnValue}; +use super::table::ColumnarMemtable; + +impl ColumnarMemtable { + /// Ingest a metric sample into the default (timestamp, value) layout. + /// + /// For the simple 2-column schema. For multi-column schemas with tags, + /// use `ingest_row()` instead. + /// + /// Does NOT enforce `hard_memory_limit`, for the same reason `ingest_row` + /// does not (see its doc): the sole production caller is WAL replay + /// (`replay_timeseries_payload`), and a sample reaching here belongs to a + /// record that has ALREADY COMMITTED, so refusing it is not backpressure — + /// it is silent loss of a durable write. Replay must take the record whole; + /// the ceiling lives at the record boundary in the live ingest handler's + /// admission gate, never here. NOTE: if a structured-`TimeseriesWalBatch` + /// ingest producer is ever added, its replay must gain that same + /// record-boundary flush, or a mid-record replay flush would stamp the + /// partition with an LSN covering rows it does not hold. + /// `replay_timeseries_wal` advances `ts_max_ingested_lsn` only AFTER a + /// record is applied, which is what keeps that stamp honest. + pub fn ingest_metric(&mut self, series_id: SeriesId, sample: MetricSample) -> IngestResult { + // Push to timestamp column. + if let ColumnData::Timestamp(ref mut v) = self.columns[self.schema.timestamp_idx] { + v.push(sample.timestamp_ms); + } + + // Push to value column (assume index 1 for default schema). + if self.columns.len() > 1 + && let ColumnData::Float64(ref mut v) = self.columns[1] + { + v.push(sample.value); + } + + self.update_stats(series_id, sample.timestamp_ms, 16); + self.check_flush_state() + } + + /// Ingest a row with explicit column values. + /// + /// `values` must match the schema length. Tag string values are resolved + /// to symbol IDs via the per-column dictionary. + /// + /// ## Why this does NOT enforce `hard_memory_limit` + /// + /// Every row reaching here belongs to a WAL record that has ALREADY + /// COMMITTED, so refusing it is not backpressure — it is silent loss of a + /// durable write. A refusal rescued by flushing and re-ingesting + /// mid-record stamps the flushed partition with the PREVIOUS record's LSN + /// while it holds part of the current one; replay, gated on that stamp, + /// re-appends the whole record on top. + /// + /// The ceiling lives at the record boundary instead (the ingest handler's + /// admission gate flushes BEFORE a record when the memtable is at or over + /// it), so a flush always lands on a whole-record prefix and the + /// partition's stamp is true of every row in it. Errors returned here are + /// therefore genuine per-row data faults — bad arity, type mismatch, + /// exhausted tag dictionary — never "come back later". + pub fn ingest_row( + &mut self, + series_id: SeriesId, + values: &[ColumnValue], + ) -> crate::Result { + let col_types: Vec<(String, ColumnType)> = self.schema.columns.clone(); + + if values.len() != col_types.len() { + return Err(crate::Error::BadRequest { + detail: format!("expected {} columns, got {}", col_types.len(), values.len()), + }); + } + + let mut ts = 0i64; + let mut row_bytes = 0usize; + let max_card = self.config.max_tag_cardinality; + + for (i, (val, (col_name, col_type))) in values.iter().zip(col_types.iter()).enumerate() { + match (val, col_type) { + (ColumnValue::Timestamp(t), ColumnType::Timestamp(_)) => { + if let ColumnData::Timestamp(ref mut v) = self.columns[i] { + v.push(*t); + } + ts = *t; + row_bytes += 8; + } + (ColumnValue::Float64(f), ColumnType::Float64) => { + if let ColumnData::Float64(ref mut v) = self.columns[i] { + v.push(*f); + } + row_bytes += 8; + } + (ColumnValue::Int64(n), ColumnType::Int64) => { + if let ColumnData::Int64(ref mut v) = self.columns[i] { + v.push(*n); + } + row_bytes += 8; + } + (ColumnValue::Symbol(s), ColumnType::Symbol) => { + let dict = + self.symbol_dicts + .get_mut(&i) + .ok_or_else(|| crate::Error::BadRequest { + detail: format!( + "internal error: symbol dict missing for column {i}" + ), + })?; + match dict.resolve(s, max_card) { + Some(sym_id) => { + if let ColumnData::Symbol(ref mut v) = self.columns[i] { + v.push(sym_id); + } + } + None => { + self.rollback_partial_row(i); + return Err(crate::Error::BadRequest { + detail: format!( + "tag cardinality limit ({max_card}) exceeded for column '{col_name}'" + ), + }); + } + } + row_bytes += 4; + } + _ => { + self.rollback_partial_row(i); + return Err(crate::Error::BadRequest { + detail: format!("type mismatch at column {i}: expected {col_type:?}"), + }); + } + } + } + + self.update_stats(series_id, ts, row_bytes); + Ok(self.check_flush_state()) + } + + /// Roll back a partially written row (called on error during `ingest_row`). + fn rollback_partial_row(&mut self, columns_written: usize) { + for col in self.columns.iter_mut().take(columns_written) { + match col { + ColumnData::Timestamp(v) => { + v.pop(); + } + ColumnData::Float64(v) => { + v.pop(); + } + ColumnData::Int64(v) => { + v.pop(); + } + ColumnData::Symbol(v) => { + v.pop(); + } + ColumnData::DictEncoded { ids, valid, .. } => { + ids.pop(); + valid.pop(); + } + } + } + } + + fn update_stats(&mut self, series_id: SeriesId, ts: i64, row_bytes: usize) { + *self.series_row_counts.entry(series_id).or_insert(0) += 1; + self.row_count += 1; + self.memory_bytes += row_bytes; + if ts < self.min_ts { + self.min_ts = ts; + } + if ts > self.max_ts { + self.max_ts = ts; + } + } + + fn check_flush_state(&self) -> IngestResult { + if self.memory_bytes >= self.config.max_memory_bytes { + IngestResult::FlushNeeded + } else { + IngestResult::Ok + } + } +} + +#[cfg(test)] +mod tests { + use super::super::super::types::{ColumnarMemtableConfig, ColumnarSchema, TimeKind}; + use super::*; + + fn default_config() -> ColumnarMemtableConfig { + ColumnarMemtableConfig { + max_memory_bytes: 1024 * 1024, + hard_memory_limit: 2 * 1024 * 1024, + max_tag_cardinality: 1000, + } + } + + #[test] + fn ingest_simple_metric() { + let mut mt = ColumnarMemtable::new_metric(default_config()); + let result = mt.ingest_metric( + 1, + MetricSample { + timestamp_ms: 1000, + value: 42.5, + }, + ); + assert_eq!(result, IngestResult::Ok); + assert_eq!(mt.row_count(), 1); + assert_eq!(mt.min_ts(), 1000); + assert_eq!(mt.max_ts(), 1000); + + let ts_col = mt.column(0).as_timestamps(); + assert_eq!(ts_col, &[1000]); + let val_col = mt.column(1).as_f64(); + assert!((val_col[0] - 42.5).abs() < f64::EPSILON); + } + + #[test] + fn ingest_multiple_metrics() { + let mut mt = ColumnarMemtable::new_metric(default_config()); + for i in 0..100 { + mt.ingest_metric( + i % 10, + MetricSample { + timestamp_ms: 1000 + i as i64, + value: i as f64, + }, + ); + } + assert_eq!(mt.row_count(), 100); + assert_eq!(mt.series_count(), 10); + assert_eq!(mt.min_ts(), 1000); + assert_eq!(mt.max_ts(), 1099); + } + + #[test] + fn ingest_row_with_tags() { + let schema = ColumnarSchema { + columns: vec![ + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), + ("value".into(), ColumnType::Float64), + ("host".into(), ColumnType::Symbol), + ("dc".into(), ColumnType::Symbol), + ], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 4], + }; + let mut mt = ColumnarMemtable::new(schema, default_config()); + + let result = mt.ingest_row( + 1, + &[ + ColumnValue::Timestamp(5000), + ColumnValue::Float64(99.9), + ColumnValue::Symbol("prod-1".to_string()), + ColumnValue::Symbol("us-east".to_string()), + ], + ); + assert!(result.is_ok()); + assert_eq!(mt.row_count(), 1); + + // Verify symbol dictionaries were populated. + let host_dict = mt.symbol_dict(2).unwrap(); + assert_eq!(host_dict.len(), 1); + assert_eq!(host_dict.get(0), Some("prod-1")); + + let dc_dict = mt.symbol_dict(3).unwrap(); + assert_eq!(dc_dict.get(0), Some("us-east")); + } + + #[test] + fn tag_cardinality_breaker() { + let schema = ColumnarSchema { + columns: vec![ + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), + ("value".into(), ColumnType::Float64), + ("tag".into(), ColumnType::Symbol), + ], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 3], + }; + let config = ColumnarMemtableConfig { + max_tag_cardinality: 5, + ..default_config() + }; + let mut mt = ColumnarMemtable::new(schema, config); + + // First 5 unique tags work. + for i in 0..5 { + let tag = format!("val-{i}"); + let r = mt.ingest_row( + i as u64, + &[ + ColumnValue::Timestamp(1000 + i as i64), + ColumnValue::Float64(1.0), + ColumnValue::Symbol(tag.clone()), + ], + ); + assert!(r.is_ok()); + } + assert_eq!(mt.row_count(), 5); + + // 6th unique tag is rejected. + let r = mt.ingest_row( + 99, + &[ + ColumnValue::Timestamp(2000), + ColumnValue::Float64(1.0), + ColumnValue::Symbol("one-too-many".to_string()), + ], + ); + assert!(r.is_err()); + // Row count didn't increase (rolled back). + assert_eq!(mt.row_count(), 5); + } + + #[test] + fn ingest_metric_accepts_past_hard_limit() { + // A sample reaching the memtable belongs to an already-committed WAL + // record; refusing it would silently drop a durable write on replay. + // So `ingest_metric` accepts every sample regardless of the ceiling — + // it never returns `Rejected`, and the resident footprint overshoots + // the hard limit rather than losing data. + let config = ColumnarMemtableConfig { + max_memory_bytes: 100, + hard_memory_limit: 200, + max_tag_cardinality: 1000, + }; + let mut mt = ColumnarMemtable::new_metric(config); + + for i in 0..1000 { + let r = mt.ingest_metric( + 1, + MetricSample { + timestamp_ms: i, + value: 1.0, + }, + ); + assert_ne!( + r, + IngestResult::Rejected, + "sample {i} must not be rejected — that would drop a durable record on replay" + ); + } + assert_eq!( + mt.row_count(), + 1000, + "every sample past the limit is retained" + ); + assert!( + mt.memory_bytes() >= 200, + "the footprint is allowed to overshoot the hard limit" + ); + } + + #[test] + fn flush_needed_signal() { + let config = ColumnarMemtableConfig { + max_memory_bytes: 100, + hard_memory_limit: 200, + max_tag_cardinality: 1000, + }; + let mut mt = ColumnarMemtable::new_metric(config); + + let mut flush_signaled = false; + for i in 0..100 { + let r = mt.ingest_metric( + 1, + MetricSample { + timestamp_ms: i, + value: 1.0, + }, + ); + if r == IngestResult::FlushNeeded { + flush_signaled = true; + break; + } + } + assert!(flush_signaled); + } + + #[test] + fn type_mismatch_rejected() { + let schema = ColumnarSchema { + columns: vec![ + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), + ("value".into(), ColumnType::Float64), + ], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 2], + }; + let mut mt = ColumnarMemtable::new(schema, default_config()); + + let r = mt.ingest_row( + 1, + &[ + ColumnValue::Timestamp(1000), + ColumnValue::Int64(42), // Wrong: schema says Float64 + ], + ); + assert!(r.is_err()); + assert_eq!(mt.row_count(), 0); // Rolled back. + } +} diff --git a/nodedb/src/engine/timeseries/columnar_memtable/memtable/mod.rs b/nodedb/src/engine/timeseries/columnar_memtable/memtable/mod.rs new file mode 100644 index 000000000..462d96b53 --- /dev/null +++ b/nodedb/src/engine/timeseries/columnar_memtable/memtable/mod.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BUSL-1.1 + +mod drain; +mod evolve; +mod ingest; +mod snapshot_io; +mod table; + +pub use table::ColumnarMemtable; diff --git a/nodedb/src/engine/timeseries/columnar_memtable/memtable/snapshot_io.rs b/nodedb/src/engine/timeseries/columnar_memtable/memtable/snapshot_io.rs new file mode 100644 index 000000000..8557b6370 --- /dev/null +++ b/nodedb/src/engine/timeseries/columnar_memtable/memtable/snapshot_io.rs @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Lossless snapshot export and import for a [`ColumnarMemtable`]. + +use std::collections::HashMap; + +use nodedb_types::timeseries::{SeriesId, SymbolDictionary}; + +use super::super::snapshot::{MemtableSnapshot, column_to_snapshot, rebuild_columns}; +use super::super::types::{ColumnarMemtableConfig, ColumnarSchema}; +use super::table::ColumnarMemtable; + +impl ColumnarMemtable { + /// Restore the pre-transaction resident-byte accounting after replacing + /// the memtable from a logical snapshot. + /// + /// A snapshot preserves values and dictionaries, but rebuilding its vectors + /// intentionally does not preserve their spare capacity. Transaction undo + /// retains the original governor reservation, so it must also reinstate the + /// original reported footprint rather than silently undercounting it. + pub(crate) fn restore_memory_bytes_for_undo(&mut self, memory_bytes: usize) { + self.memory_bytes = memory_bytes; + } + + /// Export a lossless snapshot (carries column types + symbol dicts). INFALLIBLE. + pub fn export_snapshot(&self) -> MemtableSnapshot { + let columns: Vec<_> = self.columns.iter().map(column_to_snapshot).collect(); + + // Stable ordering for deterministic snapshots. + let mut symbol_dicts: Vec<(usize, SymbolDictionary)> = self + .symbol_dicts + .iter() + .map(|(&idx, dict)| (idx, dict.clone())) + .collect(); + symbol_dicts.sort_by_key(|(idx, _)| *idx); + + let mut series_row_counts: Vec<(SeriesId, u64)> = self + .series_row_counts + .iter() + .map(|(&k, &v)| (k, v)) + .collect(); + series_row_counts.sort_by_key(|(id, _)| *id); + + MemtableSnapshot { + schema_columns: self.schema.columns.clone(), + timestamp_idx: self.schema.timestamp_idx, + columns, + symbol_dicts, + series_row_counts, + row_count: self.row_count, + min_ts: self.min_ts, + max_ts: self.max_ts, + } + } + + /// Reconstruct a memtable from a snapshot. Returns a typed error on any + /// schema/row-count mismatch. + /// + /// `config` is not carried in the snapshot — it is operator tuning, not + /// data — so callers pass the live `ColumnarMemtableConfig::from_tuning` + /// value. A memtable keeps its limits for its whole life, so restoring with + /// compiled defaults would silently ignore the operator's configuration. + pub fn from_snapshot( + snap: MemtableSnapshot, + config: ColumnarMemtableConfig, + ) -> crate::Result { + if snap.timestamp_idx >= snap.schema_columns.len() { + return Err(crate::Error::BadRequest { + detail: format!( + "snapshot timestamp_idx {} out of range for {} columns", + snap.timestamp_idx, + snap.schema_columns.len(), + ), + }); + } + + let columns = rebuild_columns(snap.columns, &snap.schema_columns, snap.row_count)?; + let n = snap.schema_columns.len(); + // Codecs are ephemeral — not serialized; rebuild as Auto. + let schema = ColumnarSchema { + columns: snap.schema_columns, + timestamp_idx: snap.timestamp_idx, + codecs: vec![nodedb_codec::ColumnCodec::Auto; n], + }; + + let symbol_dicts: HashMap = + snap.symbol_dicts.into_iter().collect(); + let series_row_counts: HashMap = + snap.series_row_counts.into_iter().collect(); + let memory_bytes: usize = + columns.iter().map(|c| c.memory_bytes()).sum::() + symbol_dicts.len() * 256; + + Ok(Self { + schema, + columns, + series_row_counts, + symbol_dicts, + row_count: snap.row_count, + memory_bytes, + config, + min_ts: snap.min_ts, + max_ts: snap.max_ts, + }) + } +} + +#[cfg(test)] +mod tests { + use super::super::super::snapshot::ColumnSnapshot; + use super::super::super::types::{ColumnData, ColumnType, ColumnValue, TimeKind}; + use super::*; + + fn default_config() -> ColumnarMemtableConfig { + ColumnarMemtableConfig { + max_memory_bytes: 1024 * 1024, + hard_memory_limit: 2 * 1024 * 1024, + max_tag_cardinality: 1000, + } + } + + #[test] + fn snapshot_roundtrip_empty_memtable() { + let mt = ColumnarMemtable::new_metric(default_config()); + let snap = mt.export_snapshot(); + let bytes = zerompk::to_msgpack_vec(&snap).expect("serialize"); + let snap2: MemtableSnapshot = zerompk::from_msgpack(&bytes).expect("deserialize"); + let mt2 = ColumnarMemtable::from_snapshot(snap2, default_config()).expect("from_snapshot"); + assert_eq!(mt2.row_count(), 0); + assert!(mt2.is_empty()); + assert_eq!(mt2.schema().columns.len(), 2); + assert_eq!(mt2.schema().timestamp_idx, 0); + } + + #[test] + fn snapshot_roundtrip_multi_column_with_symbol_tags() { + let schema = ColumnarSchema { + columns: vec![ + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), + ("value".into(), ColumnType::Float64), + ("host".into(), ColumnType::Symbol), + ], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 3], + }; + let mut mt = ColumnarMemtable::new(schema, default_config()); + + for i in 0..5u64 { + mt.ingest_row( + i, + &[ + ColumnValue::Timestamp(1000 + i as i64), + ColumnValue::Float64(i as f64 * 1.5), + ColumnValue::Symbol(format!("host-{i}")), + ], + ) + .expect("ingest"); + } + // Also re-use an existing host to verify symbol dict cardinality. + mt.ingest_row( + 99, + &[ + ColumnValue::Timestamp(2000), + ColumnValue::Float64(99.0), + ColumnValue::Symbol("host-0".to_string()), + ], + ) + .expect("ingest existing host"); + + let expected_row_count = mt.row_count(); + let expected_min_ts = mt.min_ts(); + let expected_max_ts = mt.max_ts(); + let expected_schema_cols: Vec<(String, ColumnType)> = mt.schema().columns.clone(); + let expected_ts_idx = mt.schema().timestamp_idx; + let expected_dict_len = mt.symbol_dict(2).map(|d| d.len()).unwrap_or(0); + + let snap = mt.export_snapshot(); + let bytes = zerompk::to_msgpack_vec(&snap).expect("serialize"); + let snap2: MemtableSnapshot = zerompk::from_msgpack(&bytes).expect("deserialize"); + let mt2 = ColumnarMemtable::from_snapshot(snap2, default_config()).expect("from_snapshot"); + + assert_eq!(mt2.row_count(), expected_row_count); + assert_eq!(mt2.min_ts(), expected_min_ts); + assert_eq!(mt2.max_ts(), expected_max_ts); + assert_eq!(mt2.schema().columns, expected_schema_cols); + assert_eq!(mt2.schema().timestamp_idx, expected_ts_idx); + + // Verify symbol dict was faithfully restored. + let dict2 = mt2.symbol_dict(2).expect("host dict present"); + assert_eq!(dict2.len(), expected_dict_len); + assert_eq!(dict2.get(0), Some("host-0")); + + // Verify column data lengths match. + for i in 0..3 { + assert_eq!(mt2.column(i).len(), expected_row_count as usize); + } + } + + /// The time kind survives the snapshot round trip for every kind. + #[test] + fn snapshot_roundtrip_keeps_time_kind() { + for kind in [ + TimeKind::Millis, + TimeKind::Instant(nodedb_types::InstantKind::Naive), + TimeKind::Instant(nodedb_types::InstantKind::Utc), + ] { + let schema = ColumnarSchema { + columns: vec![ + ("ts".into(), ColumnType::Timestamp(kind)), + ("value".into(), ColumnType::Float64), + ], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 2], + }; + let mt = ColumnarMemtable::new(schema, default_config()); + let snap = mt.export_snapshot(); + let bytes = zerompk::to_msgpack_vec(&snap).expect("serialize"); + let snap2: MemtableSnapshot = zerompk::from_msgpack(&bytes).expect("deserialize"); + let mt2 = + ColumnarMemtable::from_snapshot(snap2, default_config()).expect("from_snapshot"); + assert_eq!(mt2.schema().columns[0].1, ColumnType::Timestamp(kind)); + } + } + + #[test] + fn snapshot_roundtrip_dict_encoded_column_rebuilds_reverse() { + // Construct a DictEncoded snapshot directly (bypass ingest path). + let snap = MemtableSnapshot { + schema_columns: vec![ + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), + ("tag".into(), ColumnType::Symbol), + ], + timestamp_idx: 0, + columns: vec![ + ColumnSnapshot::Timestamp(vec![1000, 2000, 3000]), + ColumnSnapshot::DictEncoded { + ids: vec![0, 1, 0], + dictionary: vec!["alpha".to_string(), "beta".to_string()], + valid: vec![true, true, true], + }, + ], + symbol_dicts: vec![], + series_row_counts: vec![], + row_count: 3, + min_ts: 1000, + max_ts: 3000, + }; + + let mt = ColumnarMemtable::from_snapshot(snap, default_config()).expect("from_snapshot"); + assert_eq!(mt.row_count(), 3); + // Verify the DictEncoded column has correct data. + match mt.column(1) { + ColumnData::DictEncoded { + ids, + dictionary, + reverse, + valid, + } => { + assert_eq!(ids, &[0u32, 1, 0]); + assert_eq!(dictionary, &["alpha", "beta"]); + assert_eq!(reverse.get("alpha"), Some(&0u32)); + assert_eq!(reverse.get("beta"), Some(&1u32)); + assert_eq!(valid, &[true, true, true]); + } + other => panic!("expected DictEncoded, got {:?}", other), + } + } + + #[test] + fn snapshot_from_invalid_column_lengths_returns_error() { + // row_count says 3 but timestamp column has only 2 rows → mismatch. + let snap = MemtableSnapshot { + schema_columns: vec![ + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), + ("value".into(), ColumnType::Float64), + ], + timestamp_idx: 0, + columns: vec![ + ColumnSnapshot::Timestamp(vec![1000, 2000]), // 2 rows + ColumnSnapshot::Float64(vec![1.0, 2.0, 3.0]), // 3 rows — mismatch + ], + symbol_dicts: vec![], + series_row_counts: vec![], + row_count: 3, + min_ts: 1000, + max_ts: 2000, + }; + let result = ColumnarMemtable::from_snapshot(snap, default_config()); + assert!( + matches!(result, Err(crate::Error::BadRequest { .. })), + "expected BadRequest error on length mismatch" + ); + } +} diff --git a/nodedb/src/engine/timeseries/columnar_memtable/memtable/table.rs b/nodedb/src/engine/timeseries/columnar_memtable/memtable/table.rs new file mode 100644 index 000000000..7eac30bdb --- /dev/null +++ b/nodedb/src/engine/timeseries/columnar_memtable/memtable/table.rs @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `ColumnarMemtable` — per-column ingest buffer for timeseries data. +//! +//! NOT thread-safe — lives on a single Data Plane core (!Send by design). + +use std::collections::HashMap; + +use nodedb_types::timeseries::{SeriesId, SymbolDictionary}; + +use super::super::types::{ColumnData, ColumnType, ColumnarMemtableConfig, ColumnarSchema}; + +/// Columnar memtable: per-column vectors instead of per-series hash maps. +/// +/// Each row is a flat tuple of (timestamp, value, tag1, tag2, ...). +/// Series identity is derived from the tag columns at query time. +/// This layout is SIMD-friendly: aggregation functions operate on +/// contiguous `&[f64]` or `&[i64]` slices. +pub struct ColumnarMemtable { + pub(super) schema: ColumnarSchema, + pub(super) columns: Vec, + /// Per-series row count for quick cardinality checks. + pub(super) series_row_counts: HashMap, + /// Per-tag-column symbol dictionary. + pub(super) symbol_dicts: HashMap, + pub(super) row_count: u64, + pub(super) memory_bytes: usize, + pub(super) config: ColumnarMemtableConfig, + pub(super) min_ts: i64, + pub(super) max_ts: i64, +} + +impl ColumnarMemtable { + /// Create a new columnar memtable with the given schema. + pub fn new(schema: ColumnarSchema, config: ColumnarMemtableConfig) -> Self { + let columns: Vec = schema + .columns + .iter() + .map(|(_, ty)| ColumnData::new(*ty)) + .collect(); + + // Initialize symbol dicts for tag columns. + let mut symbol_dicts = HashMap::new(); + for (i, (_, ty)) in schema.columns.iter().enumerate() { + if *ty == ColumnType::Symbol { + symbol_dicts.insert(i, SymbolDictionary::new()); + } + } + + Self { + schema, + columns, + series_row_counts: HashMap::new(), + symbol_dicts, + row_count: 0, + memory_bytes: 0, + config, + min_ts: i64::MAX, + max_ts: i64::MIN, + } + } + + /// Create a simple metrics memtable (timestamp + f64 value, no tags). + pub fn new_metric(config: ColumnarMemtableConfig) -> Self { + Self::new(ColumnarSchema::metric_default(), config) + } + + // -- Accessors -- + + pub fn row_count(&self) -> u64 { + self.row_count + } + + /// Approximate memory usage. Uses incremental tracking with periodic + /// recomputation from column capacities for accuracy. + pub fn memory_bytes(&self) -> usize { + let col_bytes: usize = self.columns.iter().map(|c| c.memory_bytes()).sum(); + let dict_bytes: usize = self.symbol_dicts.len() * 256; // rough estimate + self.memory_bytes.max(col_bytes + dict_bytes) + } + + pub fn min_ts(&self) -> i64 { + self.min_ts + } + + pub fn max_ts(&self) -> i64 { + self.max_ts + } + + pub fn series_count(&self) -> usize { + self.series_row_counts.len() + } + + pub fn schema(&self) -> &ColumnarSchema { + &self.schema + } + + /// Return the immutable admission configuration used to construct this + /// memtable. Transaction undo uses this with a snapshot so restoration + /// preserves the original limits even if live operator tuning changed. + pub fn config(&self) -> ColumnarMemtableConfig { + self.config.clone() + } + + pub fn column(&self, idx: usize) -> &ColumnData { + &self.columns[idx] + } + + pub fn symbol_dict(&self, col_idx: usize) -> Option<&SymbolDictionary> { + self.symbol_dicts.get(&col_idx) + } + + pub fn is_empty(&self) -> bool { + self.row_count == 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn default_config() -> ColumnarMemtableConfig { + ColumnarMemtableConfig { + max_memory_bytes: 1024 * 1024, + hard_memory_limit: 2 * 1024 * 1024, + max_tag_cardinality: 1000, + } + } + + #[test] + fn empty_memtable() { + let mt = ColumnarMemtable::new_metric(default_config()); + assert_eq!(mt.row_count(), 0); + assert!(mt.is_empty()); + assert_eq!(mt.series_count(), 0); + } +} diff --git a/nodedb/src/engine/timeseries/columnar_memtable/mod.rs b/nodedb/src/engine/timeseries/columnar_memtable/mod.rs index f7d36f55e..e990d80ba 100644 --- a/nodedb/src/engine/timeseries/columnar_memtable/mod.rs +++ b/nodedb/src/engine/timeseries/columnar_memtable/mod.rs @@ -8,5 +8,5 @@ pub use memtable::ColumnarMemtable; pub use snapshot::{ColumnSnapshot, MemtableSnapshot}; pub use types::{ ColumnData, ColumnType, ColumnValue, ColumnarDrainResult, ColumnarFlushView, - ColumnarMemtableConfig, ColumnarSchema, + ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; diff --git a/nodedb/src/engine/timeseries/columnar_memtable/snapshot.rs b/nodedb/src/engine/timeseries/columnar_memtable/snapshot.rs index cb2321e67..35cf3c0e2 100644 --- a/nodedb/src/engine/timeseries/columnar_memtable/snapshot.rs +++ b/nodedb/src/engine/timeseries/columnar_memtable/snapshot.rs @@ -66,7 +66,7 @@ pub enum ColumnSnapshot { } // --------------------------------------------------------------------------- -// Column conversion helpers (called from memtable.rs) +// Column conversion helpers (called from `export_snapshot`) // --------------------------------------------------------------------------- /// Convert a single [`ColumnData`] value into its [`ColumnSnapshot`] wire form. @@ -94,7 +94,7 @@ pub(super) fn column_to_snapshot(col: &ColumnData) -> ColumnSnapshot { } // --------------------------------------------------------------------------- -// Column rebuild helper (called from memtable.rs `from_snapshot`) +// Column rebuild helper (called from `from_snapshot`) // --------------------------------------------------------------------------- /// Rebuild a `Vec` from the snapshot columns and validate diff --git a/nodedb/src/engine/timeseries/columnar_memtable/types.rs b/nodedb/src/engine/timeseries/columnar_memtable/types.rs index 3955fda9a..14a74397d 100644 --- a/nodedb/src/engine/timeseries/columnar_memtable/types.rs +++ b/nodedb/src/engine/timeseries/columnar_memtable/types.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; +use nodedb_types::InstantKind; use nodedb_types::columnar::schema::TS_SYSTEM; use nodedb_types::timeseries::{SeriesId, SymbolDictionary}; use serde::{Deserialize, Serialize}; @@ -12,6 +13,25 @@ use serde::{Deserialize, Serialize}; // Schema // --------------------------------------------------------------------------- +/// What a millisecond time column denotes when its cells are read. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum TimeKind { + /// An integer count of milliseconds, read back as the integer stored. + Millis, + /// A declared TIMESTAMP (naive) or TIMESTAMPTZ (utc) instant. + Instant(InstantKind), +} + /// Column data type in a columnar memtable. #[derive( Debug, @@ -25,8 +45,8 @@ use serde::{Deserialize, Serialize}; zerompk::FromMessagePack, )] pub enum ColumnType { - /// Designated timestamp column (i64 millis). - Timestamp, + /// Time column (i64 millis). The kind says how a cell is read back. + Timestamp(TimeKind), /// Floating-point metric value. Float64, /// Integer metric value. @@ -35,6 +55,27 @@ pub enum ColumnType { Symbol, } +impl ColumnType { + /// Whether this is a time column of any kind. + pub fn is_time(&self) -> bool { + matches!(self, ColumnType::Timestamp(_)) + } + + /// The SQL DDL type name a client sees for this column (schema-preview + /// responses, catalog field lists). + pub fn ddl_type_name(&self) -> &'static str { + match self { + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)) => "TIMESTAMPTZ", + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive) | TimeKind::Millis) => { + "TIMESTAMP" + } + ColumnType::Float64 => "FLOAT", + ColumnType::Int64 => "BIGINT", + ColumnType::Symbol => "VARCHAR", + } + } +} + /// Schema for a columnar memtable (column names + types, in order). #[derive(Debug, Clone)] pub struct ColumnarSchema { @@ -51,7 +92,7 @@ impl ColumnarSchema { pub fn metric_default() -> Self { Self { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), ("value".into(), ColumnType::Float64), ], timestamp_idx: 0, @@ -106,7 +147,7 @@ pub enum ColumnData { impl ColumnData { pub(super) fn new(ty: ColumnType) -> Self { match ty { - ColumnType::Timestamp => Self::Timestamp(Vec::with_capacity(4096)), + ColumnType::Timestamp(_) => Self::Timestamp(Vec::with_capacity(4096)), ColumnType::Float64 => Self::Float64(Vec::with_capacity(4096)), ColumnType::Int64 => Self::Int64(Vec::with_capacity(4096)), ColumnType::Symbol => Self::Symbol(Vec::with_capacity(4096)), diff --git a/nodedb/src/engine/timeseries/columnar_segment/codec.rs b/nodedb/src/engine/timeseries/columnar_segment/codec.rs index b4fc0d936..d942856bc 100644 --- a/nodedb/src/engine/timeseries/columnar_segment/codec.rs +++ b/nodedb/src/engine/timeseries/columnar_segment/codec.rs @@ -10,7 +10,7 @@ use super::error::SegmentError; /// Legacy default codecs for partitions written before V2 codec metadata. pub(super) fn legacy_default_codec(col_type: ColumnType) -> ResolvedColumnCodec { match col_type { - ColumnType::Timestamp => ResolvedColumnCodec::Gorilla, + ColumnType::Timestamp(_) => ResolvedColumnCodec::Gorilla, ColumnType::Float64 => ResolvedColumnCodec::Gorilla, ColumnType::Int64 => ResolvedColumnCodec::Raw, ColumnType::Symbol => ResolvedColumnCodec::Raw, @@ -28,7 +28,7 @@ pub(super) fn encode_column( requested_codec: ColumnCodec, ) -> Result<(Vec, ResolvedColumnCodec, ColumnStatistics), SegmentError> { match col_type { - ColumnType::Timestamp => { + ColumnType::Timestamp(_) => { let values = col_data.as_timestamps(); let codec = if requested_codec == ColumnCodec::Auto { nodedb_codec::detect::detect_i64_codec(values) @@ -99,7 +99,7 @@ pub(super) fn decode_column( let map_err = |e: nodedb_codec::CodecError| SegmentError::Corrupt(format!("{codec}: {e}")); match col_type { - ColumnType::Timestamp => { + ColumnType::Timestamp(_) => { let values = nodedb_codec::decode_i64_pipeline(data, codec.into_column_codec()) .map_err(map_err)?; Ok(ColumnData::Timestamp(values)) diff --git a/nodedb/src/engine/timeseries/columnar_segment/reader.rs b/nodedb/src/engine/timeseries/columnar_segment/reader.rs index 36f0ecd04..e200c59cb 100644 --- a/nodedb/src/engine/timeseries/columnar_segment/reader.rs +++ b/nodedb/src/engine/timeseries/columnar_segment/reader.rs @@ -229,7 +229,7 @@ impl ColumnarSegmentReader { } let data = match col_type { - ColumnType::Timestamp => ColumnData::Timestamp(all_values), + ColumnType::Timestamp(_) => ColumnData::Timestamp(all_values), ColumnType::Int64 => ColumnData::Int64(all_values), ColumnType::Float64 => { let f64_vals: Vec = all_values @@ -417,11 +417,13 @@ mod tests { use tempfile::TempDir; use super::super::super::columnar_memtable::{ - ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, + ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, TimeKind, }; use super::super::writer::ColumnarSegmentWriter; use super::*; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + fn test_config() -> ColumnarMemtableConfig { ColumnarMemtableConfig { max_memory_bytes: 10 * 1024 * 1024, @@ -459,7 +461,7 @@ mod tests { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("value".into(), ColumnType::Float64), ("extra".into(), ColumnType::Int64), ], @@ -492,7 +494,7 @@ mod tests { let projected = ColumnarSegmentReader::read_columns( &part_dir, &[ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("value".into(), ColumnType::Float64), ], None, @@ -544,7 +546,7 @@ mod tests { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ], timestamp_idx: 0, @@ -720,13 +722,8 @@ mod tests { bytes[nodedb_wal::crypto::SEGMENT_ENVELOPE_PREAMBLE_SIZE + 2] ^= 0xFF; std::fs::write(&col_path, &bytes).unwrap(); - let err = ColumnarSegmentReader::read_column( - &part_dir, - "timestamp", - ColumnType::Timestamp, - Some(&kek), - ) - .unwrap_err(); + let err = ColumnarSegmentReader::read_column(&part_dir, "timestamp", MILLIS, Some(&kek)) + .unwrap_err(); assert!( matches!(err, SegmentError::DecryptionFailed(_)), "expected DecryptionFailed, got {err:?}" diff --git a/nodedb/src/engine/timeseries/columnar_segment/schema.rs b/nodedb/src/engine/timeseries/columnar_segment/schema.rs index d8324c8e5..d629b41eb 100644 --- a/nodedb/src/engine/timeseries/columnar_segment/schema.rs +++ b/nodedb/src/engine/timeseries/columnar_segment/schema.rs @@ -3,10 +3,18 @@ //! Schema serialization (V2: includes codec per column). use nodedb_codec::ColumnCodec; +use nodedb_types::InstantKind; -use super::super::columnar_memtable::{ColumnType, ColumnarSchema}; +use super::super::columnar_memtable::{ColumnType, ColumnarSchema, TimeKind}; use super::error::SegmentError; +/// On-disk type name of a millisecond time column read back as an integer. +const TYPE_TIMESTAMP: &str = "timestamp"; +/// On-disk type name of a time column read back as a naive instant. +const TYPE_INSTANT_NAIVE: &str = "instant_naive"; +/// On-disk type name of a time column read back as a UTC instant. +const TYPE_INSTANT_UTC: &str = "instant_utc"; + /// Schema entry for JSON serialization. #[derive(serde::Serialize, serde::Deserialize)] pub(super) struct SchemaEntry { @@ -46,7 +54,9 @@ pub(super) fn schema_to_json(schema: &ColumnarSchema) -> Vec { .enumerate() .map(|(i, (name, ty))| { let ty_str = match ty { - ColumnType::Timestamp => "timestamp", + ColumnType::Timestamp(TimeKind::Millis) => TYPE_TIMESTAMP, + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive)) => TYPE_INSTANT_NAIVE, + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)) => TYPE_INSTANT_UTC, ColumnType::Float64 => "float64", ColumnType::Int64 => "int64", ColumnType::Symbol => "symbol", @@ -66,10 +76,7 @@ pub(super) fn schema_to_json(schema: &ColumnarSchema) -> Vec { /// (pre-marker) schema carries implicitly, since those were written by /// inference, which always emits the designated column first. fn first_timestamp_idx(columns: &[(String, ColumnType)]) -> usize { - columns - .iter() - .position(|(_, ty)| *ty == ColumnType::Timestamp) - .unwrap_or(0) + columns.iter().position(|(_, ty)| ty.is_time()).unwrap_or(0) } pub(super) fn schema_from_parsed(json: &SchemaJson) -> Result { @@ -81,7 +88,7 @@ pub(super) fn schema_from_parsed(json: &SchemaJson) -> Result Result Result { match ty_str { - "timestamp" => Ok(ColumnType::Timestamp), + TYPE_TIMESTAMP => Ok(ColumnType::Timestamp(TimeKind::Millis)), + TYPE_INSTANT_NAIVE => Ok(ColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive))), + TYPE_INSTANT_UTC => Ok(ColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc))), "float64" => Ok(ColumnType::Float64), "int64" => Ok(ColumnType::Int64), "symbol" => Ok(ColumnType::Symbol), @@ -133,7 +142,7 @@ mod tests { fn schema_v2_roundtrip() { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), ("cpu".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], @@ -152,4 +161,48 @@ mod tests { assert_eq!(recovered.timestamp_idx, 0); assert_eq!(recovered.codecs, schema.codecs); } + + /// Every time kind has its own on-disk type name and reads back as itself. + #[test] + fn schema_keeps_the_time_kind_of_every_time_column() { + let schema = ColumnarSchema { + columns: vec![ + ( + "ts".into(), + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive)), + ), + ( + "seen_at".into(), + ColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)), + ), + ("_ts_system".into(), ColumnType::Timestamp(TimeKind::Millis)), + ], + timestamp_idx: 0, + codecs: vec![ColumnCodec::Auto; 3], + }; + let entries = schema_to_json(&schema); + assert_eq!(entries[0].col_type, TYPE_INSTANT_NAIVE); + assert_eq!(entries[1].col_type, TYPE_INSTANT_UTC); + assert_eq!(entries[2].col_type, TYPE_TIMESTAMP); + + let json = sonic_rs::to_vec(&entries).unwrap(); + let parsed: SchemaJson = sonic_rs::from_slice(&json).unwrap(); + let recovered = schema_from_parsed(&parsed).unwrap(); + assert_eq!(recovered.columns, schema.columns); + assert_eq!(recovered.timestamp_idx, 0); + } + + /// A V1 tuple schema names its time column `timestamp` and reads back as + /// `Millis`. + #[test] + fn v1_schema_time_column_is_millis() { + let json = br#"[["timestamp","timestamp"],["value","float64"]]"#; + let parsed: SchemaJson = sonic_rs::from_slice(json).unwrap(); + let recovered = schema_from_parsed(&parsed).unwrap(); + assert_eq!( + recovered.columns[0].1, + ColumnType::Timestamp(TimeKind::Millis) + ); + assert_eq!(recovered.timestamp_idx, 0); + } } diff --git a/nodedb/src/engine/timeseries/columnar_segment/writer.rs b/nodedb/src/engine/timeseries/columnar_segment/writer.rs index e93961fb2..83533ae8c 100644 --- a/nodedb/src/engine/timeseries/columnar_segment/writer.rs +++ b/nodedb/src/engine/timeseries/columnar_segment/writer.rs @@ -74,10 +74,10 @@ impl ColumnarSegmentWriter { // declared collection to reach the inference fallback and flush a // partition under inferred names — `timestamp.col` instead of the // declared TIME_KEY — the seed would have to arrive empty, which needs - // one of: the catalog unreadable at boot (now an error rather than a - // silent empty seed, see `CatalogForRead::open`), a core spawned - // without `doc_config_seed`, or the collection missing from the - // catalog. All three are boot-integrity failures, not steady state. + // one of: the catalog unreadable at boot (an error, see + // `CatalogForRead::open`), a core spawned without `doc_config_seed`, + // or the collection missing from the catalog. All three are + // boot-integrity failures, not steady state. // // If that ever regresses, the damage is durable and silent: those // partitions keep projecting under the inferred name after the @@ -198,11 +198,13 @@ mod tests { use tempfile::TempDir; use super::super::super::columnar_memtable::{ - ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, + ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, TimeKind, }; use super::super::reader::ColumnarSegmentReader; use super::*; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + fn test_config() -> ColumnarMemtableConfig { ColumnarMemtableConfig { max_memory_bytes: 10 * 1024 * 1024, @@ -281,7 +283,7 @@ mod tests { let ts_col = ColumnarSegmentReader::read_column_with_codec( &part_dir, "timestamp", - ColumnType::Timestamp, + MILLIS, Some(ResolvedColumnCodec::DoubleDelta), None, ) @@ -312,7 +314,7 @@ mod tests { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], @@ -367,7 +369,7 @@ mod tests { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("value".into(), ColumnType::Float64), ], timestamp_idx: 0, @@ -401,7 +403,7 @@ mod tests { let ts_col = ColumnarSegmentReader::read_column_with_codec( &part_dir, "timestamp", - ColumnType::Timestamp, + MILLIS, Some(ResolvedColumnCodec::Gorilla), None, ) @@ -507,13 +509,8 @@ mod tests { let meta = ColumnarSegmentReader::read_meta(&part_dir, Some(&kek)).unwrap(); assert_eq!(meta.row_count, 100); - let ts_col = ColumnarSegmentReader::read_column( - &part_dir, - "timestamp", - ColumnType::Timestamp, - Some(&kek), - ) - .unwrap(); + let ts_col = + ColumnarSegmentReader::read_column(&part_dir, "timestamp", MILLIS, Some(&kek)).unwrap(); assert_eq!(ts_col.as_timestamps().len(), 100); let sparse = ColumnarSegmentReader::read_sparse_index(&part_dir, Some(&kek)) diff --git a/nodedb/src/engine/timeseries/continuous_agg/manager.rs b/nodedb/src/engine/timeseries/continuous_agg/manager.rs index ac8eea5eb..f89ad2911 100644 --- a/nodedb/src/engine/timeseries/continuous_agg/manager.rs +++ b/nodedb/src/engine/timeseries/continuous_agg/manager.rs @@ -331,7 +331,7 @@ pub struct AggregateInfo { mod tests { use super::*; use crate::engine::timeseries::columnar_memtable::{ - ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, + ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; use crate::engine::timeseries::continuous_agg::definition::{ AggFunction, AggregateExpr, RefreshPolicy, @@ -451,7 +451,7 @@ mod tests { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), ("value".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], diff --git a/nodedb/src/engine/timeseries/grouped_filter.rs b/nodedb/src/engine/timeseries/grouped_filter.rs index 04f17dbab..0b8503b7b 100644 --- a/nodedb/src/engine/timeseries/grouped_filter.rs +++ b/nodedb/src/engine/timeseries/grouped_filter.rs @@ -60,7 +60,7 @@ pub fn eval_filters_to_bitmask<'a>( _ => return None, } } - ColumnType::Int64 | ColumnType::Timestamp => { + ColumnType::Int64 | ColumnType::Timestamp(_) => { let fv = f.value.as_i64()?; let vals = if *col_type == ColumnType::Int64 { col_data.as_i64() diff --git a/nodedb/src/engine/timeseries/grouped_scan/strategies.rs b/nodedb/src/engine/timeseries/grouped_scan/strategies.rs deleted file mode 100644 index 0389af52a..000000000 --- a/nodedb/src/engine/timeseries/grouped_scan/strategies.rs +++ /dev/null @@ -1,537 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Tiered grouping strategies: dispatch, direct-index, FxHash, generic, -//! time-bucket, and integer-to-string key resolution. - -use rustc_hash::FxHashMap; - -use super::super::columnar_agg::AggAccum; -use super::super::columnar_memtable::{ColumnData, ColumnType}; -use super::types::{GroupedAggResult, ResolvedSchema, accumulate_row, for_each_set_bit}; - -const DIRECT_INDEX_MAX_CARDINALITY: u32 = 65536; - -/// Integer group key for local (per-source) grouping. -#[derive(Clone, PartialEq, Eq, Hash)] -pub(super) enum IntGroupKey { - None, - SingleU32(u32), - Multi(Vec), -} - -/// Row-level scan inputs shared by every grouping strategy: the resolved -/// schema, per-column data, the row-selection bitmask, row count, and -/// aggregate count. Bundled so the top-level dispatch/bucket entry points -/// stay within clippy's argument budget. -#[derive(Clone, Copy)] -pub(super) struct GroupedScanInputs<'a> { - pub resolved: &'a ResolvedSchema, - pub columns: &'a [Option<&'a ColumnData>], - pub mask: &'a [u64], - pub row_count: usize, - pub num_aggs: usize, -} - -pub(super) fn dispatch_grouping<'a>( - inputs: GroupedScanInputs<'a>, - group_by: &[String], - sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, - timestamps: Option<&[i64]>, - bucket_interval_ms: i64, -) -> GroupedAggResult { - let GroupedScanInputs { - resolved, - columns, - mask, - row_count, - num_aggs, - } = inputs; - let has_bucket = bucket_interval_ms > 0 && timestamps.is_some(); - - if has_bucket && let Some(ts) = timestamps { - return aggregate_with_bucket(inputs, ts, bucket_interval_ms, sym_lookup); - } - - let local_groups = if group_by.is_empty() { - aggregate_no_group(resolved, columns, mask, row_count, num_aggs) - } else if group_by.len() == 1 && resolved.group_cols[0].1 == ColumnType::Symbol { - let (col_idx, _) = resolved.group_cols[0]; - if let Some(data) = columns[col_idx] { - let sym_ids = data.as_symbols(); - let cardinality = sym_lookup(col_idx).map(|d| d.len() as u32).unwrap_or(0); - if cardinality <= DIRECT_INDEX_MAX_CARDINALITY && cardinality > 0 { - aggregate_direct_index( - resolved, - columns, - mask, - row_count, - num_aggs, - sym_ids, - cardinality, - ) - } else { - aggregate_hash_u32(resolved, columns, mask, row_count, num_aggs, sym_ids) - } - } else { - aggregate_hash_generic(resolved, columns, mask, row_count, num_aggs) - } - } else if row_count > 100_000 { - aggregate_two_level(resolved, columns, mask, row_count, num_aggs) - } else { - aggregate_hash_generic(resolved, columns, mask, row_count, num_aggs) - }; - - // Resolve integer keys to strings. - let mut result = GroupedAggResult::new(num_aggs); - for (int_key, accums) in local_groups { - let str_key = resolve_group_key(&int_key, resolved, sym_lookup); - let entry = result - .groups - .entry(str_key) - .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); - for (i, a) in accums.iter().enumerate() { - entry[i].merge(a); - } - } - result -} - -// --------------------------------------------------------------------------- -// Grouping strategies -// --------------------------------------------------------------------------- - -fn aggregate_no_group( - resolved: &ResolvedSchema, - columns: &[Option<&ColumnData>], - mask: &[u64], - row_count: usize, - num_aggs: usize, -) -> Vec<(IntGroupKey, Vec)> { - let mut accums: Vec = (0..num_aggs).map(|_| AggAccum::default()).collect(); - for_each_set_bit(mask, row_count, |row_idx| { - accumulate_row(&mut accums, resolved, columns, row_idx); - }); - vec![(IntGroupKey::None, accums)] -} - -fn aggregate_direct_index( - resolved: &ResolvedSchema, - columns: &[Option<&ColumnData>], - mask: &[u64], - row_count: usize, - num_aggs: usize, - sym_ids: &[u32], - cardinality: u32, -) -> Vec<(IntGroupKey, Vec)> { - let card = cardinality as usize; - let mut table: Vec> = (0..card) - .map(|_| (0..num_aggs).map(|_| AggAccum::default()).collect()) - .collect(); - - for_each_set_bit(mask, row_count, |row_idx| { - let id = sym_ids[row_idx] as usize; - if id < card { - accumulate_row(&mut table[id], resolved, columns, row_idx); - } - }); - - table - .into_iter() - .enumerate() - .filter(|(_, accums)| accums.iter().any(|a| a.count > 0)) - .map(|(id, accums)| (IntGroupKey::SingleU32(id as u32), accums)) - .collect() -} - -fn aggregate_hash_u32( - resolved: &ResolvedSchema, - columns: &[Option<&ColumnData>], - mask: &[u64], - row_count: usize, - num_aggs: usize, - sym_ids: &[u32], -) -> Vec<(IntGroupKey, Vec)> { - let mut groups: FxHashMap> = FxHashMap::default(); - - for_each_set_bit(mask, row_count, |row_idx| { - let id = sym_ids[row_idx]; - let accums = groups - .entry(id) - .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); - accumulate_row(accums, resolved, columns, row_idx); - }); - - groups - .into_iter() - .map(|(id, accums)| (IntGroupKey::SingleU32(id), accums)) - .collect() -} - -fn aggregate_hash_generic( - resolved: &ResolvedSchema, - columns: &[Option<&ColumnData>], - mask: &[u64], - row_count: usize, - num_aggs: usize, -) -> Vec<(IntGroupKey, Vec)> { - let mut groups: FxHashMap, Vec> = FxHashMap::default(); - - for_each_set_bit(mask, row_count, |row_idx| { - let key = build_generic_key(resolved, columns, row_idx); - let accums = groups - .entry(key) - .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); - accumulate_row(accums, resolved, columns, row_idx); - }); - - groups - .into_iter() - .map(|(key, accums)| (IntGroupKey::Multi(key), accums)) - .collect() -} - -/// Two-level aggregation for high-cardinality GROUP BY (2M+ keys). -/// -/// Phase 1: Partition rows into buckets by hash prefix (top 8 bits → 256 buckets). -/// Phase 2: Aggregate each bucket independently (small HashMap, cache-friendly). -/// Phase 3: Flatten all buckets into the final result. -fn aggregate_two_level( - resolved: &ResolvedSchema, - columns: &[Option<&ColumnData>], - mask: &[u64], - row_count: usize, - num_aggs: usize, -) -> Vec<(IntGroupKey, Vec)> { - const NUM_BUCKETS: usize = 256; - - // Phase 1: Build key for each row, hash it, and partition into buckets by top 8 bits. - // The key is stored alongside the row index so Phase 2 does not recompute it. - let mut buckets: Vec)>> = (0..NUM_BUCKETS).map(|_| Vec::new()).collect(); - for_each_set_bit(mask, row_count, |row_idx| { - let key = build_generic_key(resolved, columns, row_idx); - let hash = fx_hash_key(&key); - let bucket = (hash >> 56) as usize; - buckets[bucket].push((row_idx, key)); - }); - - // Phase 2 + 3: Aggregate each bucket independently and flatten. - let mut all_results: Vec<(IntGroupKey, Vec)> = Vec::new(); - for bucket_rows in buckets { - if bucket_rows.is_empty() { - continue; - } - let mut groups: FxHashMap, Vec> = FxHashMap::default(); - for (row_idx, key) in bucket_rows { - let accums = groups - .entry(key) - .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); - accumulate_row(accums, resolved, columns, row_idx); - } - all_results.extend(groups.into_iter().map(|(k, a)| (IntGroupKey::Multi(k), a))); - } - - all_results -} - -/// Time-bucket aggregation with integer keys. -/// -/// Packs (bucket_ts, group_key_parts...) into a `Vec`: -/// - `parts[0]` = bucket_ts as u64 -/// - `parts[1..]` = group column values (symbol IDs, i64, f64 bits) -/// -/// Resolves to string keys with "bucket_ts\0group1\0group2" format -/// so `emit_grouped_results` can parse them. -fn aggregate_with_bucket<'a>( - inputs: GroupedScanInputs<'a>, - timestamps: &[i64], - bucket_interval_ms: i64, - sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, -) -> GroupedAggResult { - let GroupedScanInputs { - resolved, - columns, - mask, - row_count, - num_aggs, - } = inputs; - let key_len = 1 + resolved.group_cols.len(); // bucket + group columns - - let mut groups: FxHashMap, Vec> = FxHashMap::default(); - - for_each_set_bit(mask, row_count, |row_idx| { - let bucket = - super::super::time_bucket::time_bucket(bucket_interval_ms, timestamps[row_idx]); - - let mut key = Vec::with_capacity(key_len); - key.push(bucket as u64); - - // Pack group-by columns as integers. - for &(col_idx, ty) in &resolved.group_cols { - let part = columns[col_idx] - .map(|data| match ty { - ColumnType::Symbol => { - if let ColumnData::Symbol(ids) = data { - ids[row_idx] as u64 - } else { - u64::MAX - } - } - ColumnType::Int64 => { - if let ColumnData::Int64(v) = data { - v[row_idx] as u64 - } else { - u64::MAX - } - } - ColumnType::Float64 => { - if let ColumnData::Float64(v) = data { - v[row_idx].to_bits() - } else { - u64::MAX - } - } - ColumnType::Timestamp => { - if let ColumnData::Timestamp(v) = data { - v[row_idx] as u64 - } else { - u64::MAX - } - } - }) - .unwrap_or(u64::MAX); - key.push(part); - } - - let accums = groups - .entry(key) - .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); - accumulate_row(accums, resolved, columns, row_idx); - }); - - // Resolve integer keys to string keys: "bucket_ts\0group1\0group2" - let mut result = GroupedAggResult::new(num_aggs); - for (key_parts, accums) in groups { - let bucket_ts = key_parts[0] as i64; - let mut str_key = bucket_ts.to_string(); - - for (i, &part) in key_parts[1..].iter().enumerate() { - str_key.push('\0'); - if i < resolved.group_cols.len() { - let (col_idx, ty) = resolved.group_cols[i]; - match ty { - ColumnType::Symbol => { - if let Some(dict) = sym_lookup(col_idx) - && let Some(name) = dict.get(part as u32) - { - str_key.push_str(name); - } - } - ColumnType::Int64 | ColumnType::Timestamp => { - use std::fmt::Write; - let _ = write!(str_key, "{}", part as i64); - } - ColumnType::Float64 => { - use std::fmt::Write; - let _ = write!(str_key, "{}", f64::from_bits(part)); - } - } - } - } - - let entry = result - .groups - .entry(str_key) - .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); - for (i, a) in accums.iter().enumerate() { - entry[i].merge(a); - } - } - - result -} - -// --------------------------------------------------------------------------- -// Key helpers -// --------------------------------------------------------------------------- - -/// FxHash-style multiplicative hash over a slice of u64 values. -/// -/// Used by `aggregate_two_level` to assign rows to buckets in Phase 1. -#[inline] -fn fx_hash_key(key: &[u64]) -> u64 { - let mut hash: u64 = 0; - for &v in key { - hash = hash.wrapping_mul(0x517cc1b727220a95).wrapping_add(v); - } - hash -} - -fn build_generic_key( - resolved: &ResolvedSchema, - columns: &[Option<&ColumnData>], - row_idx: usize, -) -> Vec { - resolved - .group_cols - .iter() - .map(|&(idx, ty)| { - columns[idx] - .map(|data| match ty { - ColumnType::Symbol => { - if let ColumnData::Symbol(ids) = data { - ids[row_idx] as u64 - } else { - u64::MAX - } - } - ColumnType::Int64 => { - if let ColumnData::Int64(vals) = data { - vals[row_idx] as u64 - } else { - u64::MAX - } - } - ColumnType::Float64 => { - if let ColumnData::Float64(vals) = data { - vals[row_idx].to_bits() - } else { - u64::MAX - } - } - ColumnType::Timestamp => { - if let ColumnData::Timestamp(vals) = data { - vals[row_idx] as u64 - } else { - u64::MAX - } - } - }) - .unwrap_or(u64::MAX) - }) - .collect() -} - -fn resolve_group_key<'a>( - key: &IntGroupKey, - resolved: &ResolvedSchema, - sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, -) -> String { - match key { - IntGroupKey::None => String::new(), - IntGroupKey::SingleU32(id) => { - let col_idx = resolved.group_cols[0].0; - if resolved.group_cols[0].1 == ColumnType::Symbol { - sym_lookup(col_idx) - .and_then(|d: &nodedb_types::timeseries::SymbolDictionary| d.get(*id)) - .unwrap_or("") - .to_string() - } else { - id.to_string() - } - } - IntGroupKey::Multi(parts) => { - let mut s = String::with_capacity(parts.len() * 16); - for (i, &part) in parts.iter().enumerate() { - if i > 0 { - s.push('\0'); - } - let (col_idx, ty) = resolved.group_cols[i]; - match ty { - ColumnType::Symbol => { - if let Some(dict) = sym_lookup(col_idx) - && let Some(name) = dict.get(part as u32) - { - s.push_str(name); - } - } - ColumnType::Int64 | ColumnType::Timestamp => { - use std::fmt::Write; - let _ = write!(s, "{}", part as i64); - } - ColumnType::Float64 => { - use std::fmt::Write; - let _ = write!(s, "{}", f64::from_bits(part)); - } - } - } - s - } - } -} - -#[cfg(test)] -mod tests { - use super::super::super::columnar_memtable::ColumnType; - use super::super::types::AggColInfo; - use super::super::types::ResolvedSchema; - use super::*; - - fn make_resolved_count(col_types: &[(usize, ColumnType)]) -> ResolvedSchema { - ResolvedSchema { - group_cols: col_types.to_vec(), - agg_cols: vec![AggColInfo::CountStar], - ts_idx: 0, - } - } - - /// Build a full-set mask for `row_count` rows. - fn full_mask(row_count: usize) -> Vec { - let words = row_count.div_ceil(64); - let mut mask = vec![u64::MAX; words]; - let rem = row_count % 64; - if rem != 0 { - *mask.last_mut().unwrap() = (1u64 << rem) - 1; - } - mask - } - - /// Collect aggregation results into a sorted Vec of (key, count) for comparison. - fn collect_counts(results: Vec<(IntGroupKey, Vec)>) -> Vec<(Vec, u64)> { - let mut out: Vec<(Vec, u64)> = results - .into_iter() - .map(|(key, accums)| { - let k = match key { - IntGroupKey::Multi(v) => v, - IntGroupKey::SingleU32(v) => vec![v as u64], - IntGroupKey::None => vec![], - }; - let count = accums.first().map(|a| a.count).unwrap_or(0); - (k, count) - }) - .collect(); - out.sort_by(|a, b| a.0.cmp(&b.0)); - out - } - - #[test] - fn two_level_matches_generic_multi_column() { - // Build two Int64 columns with 8 distinct combinations repeated many times. - let n = 200_000usize; - let col_a: Vec = (0..n).map(|i| (i % 4) as i64).collect(); - let col_b: Vec = (0..n).map(|i| (i % 2) as i64).collect(); - - let data_a = ColumnData::Int64(col_a); - let data_b = ColumnData::Int64(col_b); - let columns: Vec> = vec![Some(&data_a), Some(&data_b)]; - - let resolved = make_resolved_count(&[(0, ColumnType::Int64), (1, ColumnType::Int64)]); - let mask = full_mask(n); - - // num_aggs = 1 so AggAccum::count tracks row membership. - let generic = aggregate_hash_generic(&resolved, &columns, &mask, n, 1); - let two_level = aggregate_two_level(&resolved, &columns, &mask, n, 1); - - assert_eq!( - collect_counts(generic), - collect_counts(two_level), - "two-level and generic must produce identical group counts" - ); - } - - #[test] - fn fx_hash_key_deterministic() { - let key = vec![1u64, 2, 3, 4]; - assert_eq!(fx_hash_key(&key), fx_hash_key(&key)); - assert_ne!(fx_hash_key(&[1, 2]), fx_hash_key(&[2, 1])); - } -} diff --git a/nodedb/src/engine/timeseries/grouped_scan/strategies/bucket.rs b/nodedb/src/engine/timeseries/grouped_scan/strategies/bucket.rs new file mode 100644 index 000000000..0afa20f65 --- /dev/null +++ b/nodedb/src/engine/timeseries/grouped_scan/strategies/bucket.rs @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Time-bucket grouping: rows keyed by `time_bucket(ts)` plus the group +//! columns, resolved to `"bucket_ts\0group1\0group2"` string keys. + +use rustc_hash::FxHashMap; + +use super::super::super::columnar_agg::AggAccum; +use super::super::types::{GroupedAggResult, accumulate_row, for_each_set_bit}; +use super::dispatch::GroupedScanInputs; +use super::keys::{key_part, push_key_part}; + +/// Time-bucket aggregation with integer keys. +/// +/// Packs (bucket_ts, group_key_parts...) into a `Vec`: +/// - `parts[0]` = bucket_ts as u64 +/// - `parts[1..]` = group column values (symbol IDs, i64, f64 bits) +/// +/// Resolves to string keys with "bucket_ts\0group1\0group2" format +/// so `emit_grouped_results` can parse them. +pub(super) fn aggregate_with_bucket<'a>( + inputs: GroupedScanInputs<'a>, + timestamps: &[i64], + bucket_interval_ms: i64, + sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, +) -> GroupedAggResult { + let GroupedScanInputs { + resolved, + columns, + mask, + row_count, + num_aggs, + } = inputs; + let key_len = 1 + resolved.group_cols.len(); // bucket + group columns + + let mut groups: FxHashMap, Vec> = FxHashMap::default(); + + for_each_set_bit(mask, row_count, |row_idx| { + let bucket = + super::super::super::time_bucket::time_bucket(bucket_interval_ms, timestamps[row_idx]); + + let mut key = Vec::with_capacity(key_len); + key.push(bucket as u64); + + // Pack group-by columns as integers. + for &(col_idx, ty) in &resolved.group_cols { + key.push(key_part(ty, columns[col_idx], row_idx)); + } + + let accums = groups + .entry(key) + .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); + accumulate_row(accums, resolved, columns, row_idx); + }); + + // Resolve integer keys to string keys: "bucket_ts\0group1\0group2" + let mut result = GroupedAggResult::new(num_aggs); + for (key_parts, accums) in groups { + let bucket_ts = key_parts[0] as i64; + let mut str_key = bucket_ts.to_string(); + + for (i, &part) in key_parts[1..].iter().enumerate() { + str_key.push('\0'); + if i < resolved.group_cols.len() { + let (col_idx, ty) = resolved.group_cols[i]; + push_key_part(&mut str_key, col_idx, ty, part, sym_lookup); + } + } + + result.merge_group(str_key, &accums); + } + + result +} diff --git a/nodedb/src/engine/timeseries/grouped_scan/strategies/dispatch.rs b/nodedb/src/engine/timeseries/grouped_scan/strategies/dispatch.rs new file mode 100644 index 000000000..7ee21c3c2 --- /dev/null +++ b/nodedb/src/engine/timeseries/grouped_scan/strategies/dispatch.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Strategy selection: picks the grouping strategy for a scan from its +//! GROUP BY shape, symbol cardinality, and row count, then resolves the +//! integer keys to strings. + +use super::super::super::columnar_memtable::{ColumnData, ColumnType}; +use super::super::types::{GroupedAggResult, ResolvedSchema}; +use super::bucket::aggregate_with_bucket; +use super::hashed::{ + aggregate_direct_index, aggregate_hash_generic, aggregate_hash_u32, aggregate_no_group, + aggregate_two_level, +}; +use super::keys::resolve_group_key; + +const DIRECT_INDEX_MAX_CARDINALITY: u32 = 65536; + +/// Row-level scan inputs shared by every grouping strategy: the resolved +/// schema, per-column data, the row-selection bitmask, row count, and +/// aggregate count. Bundled so the top-level dispatch/bucket entry points +/// stay within clippy's argument budget. +#[derive(Clone, Copy)] +pub(in crate::engine::timeseries::grouped_scan) struct GroupedScanInputs<'a> { + pub resolved: &'a ResolvedSchema, + pub columns: &'a [Option<&'a ColumnData>], + pub mask: &'a [u64], + pub row_count: usize, + pub num_aggs: usize, +} + +pub(in crate::engine::timeseries::grouped_scan) fn dispatch_grouping<'a>( + inputs: GroupedScanInputs<'a>, + group_by: &[String], + sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, + timestamps: Option<&[i64]>, + bucket_interval_ms: i64, +) -> GroupedAggResult { + let GroupedScanInputs { + resolved, + columns, + mask, + row_count, + num_aggs, + } = inputs; + let has_bucket = bucket_interval_ms > 0 && timestamps.is_some(); + + if has_bucket && let Some(ts) = timestamps { + return aggregate_with_bucket(inputs, ts, bucket_interval_ms, sym_lookup); + } + + let local_groups = if group_by.is_empty() { + aggregate_no_group(resolved, columns, mask, row_count, num_aggs) + } else if group_by.len() == 1 && resolved.group_cols[0].1 == ColumnType::Symbol { + let (col_idx, _) = resolved.group_cols[0]; + if let Some(data) = columns[col_idx] { + let sym_ids = data.as_symbols(); + let cardinality = sym_lookup(col_idx).map(|d| d.len() as u32).unwrap_or(0); + if cardinality <= DIRECT_INDEX_MAX_CARDINALITY && cardinality > 0 { + aggregate_direct_index( + resolved, + columns, + mask, + row_count, + num_aggs, + sym_ids, + cardinality, + ) + } else { + aggregate_hash_u32(resolved, columns, mask, row_count, num_aggs, sym_ids) + } + } else { + aggregate_hash_generic(resolved, columns, mask, row_count, num_aggs) + } + } else if row_count > 100_000 { + aggregate_two_level(resolved, columns, mask, row_count, num_aggs) + } else { + aggregate_hash_generic(resolved, columns, mask, row_count, num_aggs) + }; + + // Resolve integer keys to strings. + let mut result = GroupedAggResult::new(num_aggs); + for (int_key, accums) in local_groups { + let str_key = resolve_group_key(&int_key, resolved, sym_lookup); + result.merge_group(str_key, &accums); + } + result +} diff --git a/nodedb/src/engine/timeseries/grouped_scan/strategies/hashed.rs b/nodedb/src/engine/timeseries/grouped_scan/strategies/hashed.rs new file mode 100644 index 000000000..429153ac8 --- /dev/null +++ b/nodedb/src/engine/timeseries/grouped_scan/strategies/hashed.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Grouping strategies over integer keys: no-group, direct-index, FxHash +//! on a single symbol column, generic multi-column hash, and two-level +//! bucketed hash for high cardinality. + +use rustc_hash::FxHashMap; + +use super::super::super::columnar_agg::AggAccum; +use super::super::super::columnar_memtable::ColumnData; +use super::super::types::{ResolvedSchema, accumulate_row, for_each_set_bit}; +use super::keys::{IntGroupKey, build_generic_key, fx_hash_key}; + +pub(super) fn aggregate_no_group( + resolved: &ResolvedSchema, + columns: &[Option<&ColumnData>], + mask: &[u64], + row_count: usize, + num_aggs: usize, +) -> Vec<(IntGroupKey, Vec)> { + let mut accums: Vec = (0..num_aggs).map(|_| AggAccum::default()).collect(); + for_each_set_bit(mask, row_count, |row_idx| { + accumulate_row(&mut accums, resolved, columns, row_idx); + }); + vec![(IntGroupKey::None, accums)] +} + +pub(super) fn aggregate_direct_index( + resolved: &ResolvedSchema, + columns: &[Option<&ColumnData>], + mask: &[u64], + row_count: usize, + num_aggs: usize, + sym_ids: &[u32], + cardinality: u32, +) -> Vec<(IntGroupKey, Vec)> { + let card = cardinality as usize; + let mut table: Vec> = (0..card) + .map(|_| (0..num_aggs).map(|_| AggAccum::default()).collect()) + .collect(); + + for_each_set_bit(mask, row_count, |row_idx| { + let id = sym_ids[row_idx] as usize; + if id < card { + accumulate_row(&mut table[id], resolved, columns, row_idx); + } + }); + + table + .into_iter() + .enumerate() + .filter(|(_, accums)| accums.iter().any(|a| a.count > 0)) + .map(|(id, accums)| (IntGroupKey::SingleU32(id as u32), accums)) + .collect() +} + +pub(super) fn aggregate_hash_u32( + resolved: &ResolvedSchema, + columns: &[Option<&ColumnData>], + mask: &[u64], + row_count: usize, + num_aggs: usize, + sym_ids: &[u32], +) -> Vec<(IntGroupKey, Vec)> { + let mut groups: FxHashMap> = FxHashMap::default(); + + for_each_set_bit(mask, row_count, |row_idx| { + let id = sym_ids[row_idx]; + let accums = groups + .entry(id) + .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); + accumulate_row(accums, resolved, columns, row_idx); + }); + + groups + .into_iter() + .map(|(id, accums)| (IntGroupKey::SingleU32(id), accums)) + .collect() +} + +pub(super) fn aggregate_hash_generic( + resolved: &ResolvedSchema, + columns: &[Option<&ColumnData>], + mask: &[u64], + row_count: usize, + num_aggs: usize, +) -> Vec<(IntGroupKey, Vec)> { + let mut groups: FxHashMap, Vec> = FxHashMap::default(); + + for_each_set_bit(mask, row_count, |row_idx| { + let key = build_generic_key(resolved, columns, row_idx); + let accums = groups + .entry(key) + .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); + accumulate_row(accums, resolved, columns, row_idx); + }); + + groups + .into_iter() + .map(|(key, accums)| (IntGroupKey::Multi(key), accums)) + .collect() +} + +/// Two-level aggregation for high-cardinality GROUP BY (2M+ keys). +/// +/// Phase 1: Partition rows into buckets by hash prefix (top 8 bits → 256 buckets). +/// Phase 2: Aggregate each bucket independently (small HashMap, cache-friendly). +/// Phase 3: Flatten all buckets into the final result. +pub(super) fn aggregate_two_level( + resolved: &ResolvedSchema, + columns: &[Option<&ColumnData>], + mask: &[u64], + row_count: usize, + num_aggs: usize, +) -> Vec<(IntGroupKey, Vec)> { + const NUM_BUCKETS: usize = 256; + + // Phase 1: Build key for each row, hash it, and partition into buckets by top 8 bits. + // The key is stored alongside the row index so Phase 2 does not recompute it. + let mut buckets: Vec)>> = (0..NUM_BUCKETS).map(|_| Vec::new()).collect(); + for_each_set_bit(mask, row_count, |row_idx| { + let key = build_generic_key(resolved, columns, row_idx); + let hash = fx_hash_key(&key); + let bucket = (hash >> 56) as usize; + buckets[bucket].push((row_idx, key)); + }); + + // Phase 2 + 3: Aggregate each bucket independently and flatten. + let mut all_results: Vec<(IntGroupKey, Vec)> = Vec::new(); + for bucket_rows in buckets { + if bucket_rows.is_empty() { + continue; + } + let mut groups: FxHashMap, Vec> = FxHashMap::default(); + for (row_idx, key) in bucket_rows { + let accums = groups + .entry(key) + .or_insert_with(|| (0..num_aggs).map(|_| AggAccum::default()).collect()); + accumulate_row(accums, resolved, columns, row_idx); + } + all_results.extend(groups.into_iter().map(|(k, a)| (IntGroupKey::Multi(k), a))); + } + + all_results +} + +#[cfg(test)] +mod tests { + use super::super::super::super::columnar_memtable::ColumnType; + use super::super::super::types::AggColInfo; + use super::*; + + fn make_resolved_count(col_types: &[(usize, ColumnType)]) -> ResolvedSchema { + ResolvedSchema { + group_cols: col_types.to_vec(), + agg_cols: vec![AggColInfo::CountStar], + ts_idx: 0, + } + } + + /// Build a full-set mask for `row_count` rows. + fn full_mask(row_count: usize) -> Vec { + let words = row_count.div_ceil(64); + let mut mask = vec![u64::MAX; words]; + let rem = row_count % 64; + if rem != 0 { + *mask.last_mut().unwrap() = (1u64 << rem) - 1; + } + mask + } + + /// Collect aggregation results into a sorted Vec of (key, count) for comparison. + fn collect_counts(results: Vec<(IntGroupKey, Vec)>) -> Vec<(Vec, u64)> { + let mut out: Vec<(Vec, u64)> = results + .into_iter() + .map(|(key, accums)| { + let k = match key { + IntGroupKey::Multi(v) => v, + IntGroupKey::SingleU32(v) => vec![v as u64], + IntGroupKey::None => vec![], + }; + let count = accums.first().map(|a| a.count).unwrap_or(0); + (k, count) + }) + .collect(); + out.sort_by(|a, b| a.0.cmp(&b.0)); + out + } + + #[test] + fn two_level_matches_generic_multi_column() { + // Build two Int64 columns with 8 distinct combinations repeated many times. + let n = 200_000usize; + let col_a: Vec = (0..n).map(|i| (i % 4) as i64).collect(); + let col_b: Vec = (0..n).map(|i| (i % 2) as i64).collect(); + + let data_a = ColumnData::Int64(col_a); + let data_b = ColumnData::Int64(col_b); + let columns: Vec> = vec![Some(&data_a), Some(&data_b)]; + + let resolved = make_resolved_count(&[(0, ColumnType::Int64), (1, ColumnType::Int64)]); + let mask = full_mask(n); + + // num_aggs = 1 so AggAccum::count tracks row membership. + let generic = aggregate_hash_generic(&resolved, &columns, &mask, n, 1); + let two_level = aggregate_two_level(&resolved, &columns, &mask, n, 1); + + assert_eq!( + collect_counts(generic), + collect_counts(two_level), + "two-level and generic must produce identical group counts" + ); + } +} diff --git a/nodedb/src/engine/timeseries/grouped_scan/strategies/keys.rs b/nodedb/src/engine/timeseries/grouped_scan/strategies/keys.rs new file mode 100644 index 000000000..c7b86b21d --- /dev/null +++ b/nodedb/src/engine/timeseries/grouped_scan/strategies/keys.rs @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Integer group keys: packing a row's group columns into `u64` parts, +//! hashing them, and resolving them back to the string key emission reads. + +use super::super::super::columnar_memtable::{ColumnData, ColumnType}; +use super::super::types::ResolvedSchema; + +/// Integer group key for local (per-source) grouping. +#[derive(Clone, PartialEq, Eq, Hash)] +pub(super) enum IntGroupKey { + None, + SingleU32(u32), + Multi(Vec), +} + +/// FxHash-style multiplicative hash over a slice of u64 values. +/// +/// Used by `aggregate_two_level` to assign rows to buckets in Phase 1. +#[inline] +pub(super) fn fx_hash_key(key: &[u64]) -> u64 { + let mut hash: u64 = 0; + for &v in key { + hash = hash.wrapping_mul(0x517cc1b727220a95).wrapping_add(v); + } + hash +} + +/// Pack one group column's cell at `row_idx` into a `u64` key part. +/// +/// Symbols and integers pack as their value, floats as their bit pattern, +/// and a time column as its millisecond count. A column whose data is absent +/// or of the wrong shape packs as `u64::MAX`. +#[inline] +pub(super) fn key_part(ty: ColumnType, data: Option<&ColumnData>, row_idx: usize) -> u64 { + data.map(|data| match ty { + ColumnType::Symbol => { + if let ColumnData::Symbol(ids) = data { + ids[row_idx] as u64 + } else { + u64::MAX + } + } + ColumnType::Int64 => { + if let ColumnData::Int64(vals) = data { + vals[row_idx] as u64 + } else { + u64::MAX + } + } + ColumnType::Float64 => { + if let ColumnData::Float64(vals) = data { + vals[row_idx].to_bits() + } else { + u64::MAX + } + } + ColumnType::Timestamp(_) => { + if let ColumnData::Timestamp(vals) = data { + vals[row_idx] as u64 + } else { + u64::MAX + } + } + }) + .unwrap_or(u64::MAX) +} + +pub(super) fn build_generic_key( + resolved: &ResolvedSchema, + columns: &[Option<&ColumnData>], + row_idx: usize, +) -> Vec { + resolved + .group_cols + .iter() + .map(|&(idx, ty)| key_part(ty, columns[idx], row_idx)) + .collect() +} + +/// Append the string form of one packed key part for a column of type `ty`. +/// +/// A symbol resolves through its dictionary and appends nothing when the id +/// is unknown. Integers and time columns render as the signed integer packed, +/// floats from their bit pattern. +pub(super) fn push_key_part<'a>( + out: &mut String, + col_idx: usize, + ty: ColumnType, + part: u64, + sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, +) { + use std::fmt::Write; + match ty { + ColumnType::Symbol => { + if let Some(dict) = sym_lookup(col_idx) + && let Some(name) = dict.get(part as u32) + { + out.push_str(name); + } + } + ColumnType::Int64 | ColumnType::Timestamp(_) => { + let _ = write!(out, "{}", part as i64); + } + ColumnType::Float64 => { + let _ = write!(out, "{}", f64::from_bits(part)); + } + } +} + +pub(super) fn resolve_group_key<'a>( + key: &IntGroupKey, + resolved: &ResolvedSchema, + sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, +) -> String { + match key { + IntGroupKey::None => String::new(), + IntGroupKey::SingleU32(id) => { + let col_idx = resolved.group_cols[0].0; + if resolved.group_cols[0].1 == ColumnType::Symbol { + sym_lookup(col_idx) + .and_then(|d: &nodedb_types::timeseries::SymbolDictionary| d.get(*id)) + .unwrap_or("") + .to_string() + } else { + id.to_string() + } + } + IntGroupKey::Multi(parts) => { + let mut s = String::with_capacity(parts.len() * 16); + for (i, &part) in parts.iter().enumerate() { + if i > 0 { + s.push('\0'); + } + let (col_idx, ty) = resolved.group_cols[i]; + push_key_part(&mut s, col_idx, ty, part, sym_lookup); + } + s + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fx_hash_key_deterministic() { + let key = vec![1u64, 2, 3, 4]; + assert_eq!(fx_hash_key(&key), fx_hash_key(&key)); + assert_ne!(fx_hash_key(&[1, 2]), fx_hash_key(&[2, 1])); + } +} diff --git a/nodedb/src/engine/timeseries/grouped_scan/strategies/mod.rs b/nodedb/src/engine/timeseries/grouped_scan/strategies/mod.rs new file mode 100644 index 000000000..19512c80a --- /dev/null +++ b/nodedb/src/engine/timeseries/grouped_scan/strategies/mod.rs @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: BUSL-1.1 + +mod bucket; +mod dispatch; +mod hashed; +mod keys; + +pub(super) use dispatch::{GroupedScanInputs, dispatch_grouping}; diff --git a/nodedb/src/engine/timeseries/grouped_scan/types.rs b/nodedb/src/engine/timeseries/grouped_scan/types.rs index 461c2af1f..a033ae539 100644 --- a/nodedb/src/engine/timeseries/grouped_scan/types.rs +++ b/nodedb/src/engine/timeseries/grouped_scan/types.rs @@ -25,14 +25,20 @@ impl GroupedAggResult { pub fn merge(&mut self, other: &GroupedAggResult) { for (key, other_accums) in &other.groups { - let accums = self - .groups - .entry(key.clone()) - .or_insert_with(|| (0..self.num_aggs).map(|_| AggAccum::default()).collect()); - for (i, a) in other_accums.iter().enumerate() { - if i < accums.len() { - accums[i].merge(a); - } + self.merge_group(key.clone(), other_accums); + } + } + + /// Merge one group's accumulators into this result, creating the group's + /// entry if it is not already present. + pub(super) fn merge_group(&mut self, key: String, accums: &[AggAccum]) { + let entry = self + .groups + .entry(key) + .or_insert_with(|| (0..self.num_aggs).map(|_| AggAccum::default()).collect()); + for (i, a) in accums.iter().enumerate() { + if i < entry.len() { + entry[i].merge(a); } } } @@ -91,10 +97,10 @@ pub(super) fn resolve_schema( AggColInfo::CountField } else { match ty { - ColumnType::Float64 | ColumnType::Int64 | ColumnType::Timestamp => { + ColumnType::Float64 | ColumnType::Int64 | ColumnType::Timestamp(_) => { AggColInfo::Numeric(idx) } - _ => AggColInfo::Skip, + ColumnType::Symbol => AggColInfo::Skip, } } } else if op == "count" { diff --git a/nodedb/src/engine/timeseries/ilp_ingest.rs b/nodedb/src/engine/timeseries/ilp_ingest.rs index e7b932423..106983d0d 100644 --- a/nodedb/src/engine/timeseries/ilp_ingest.rs +++ b/nodedb/src/engine/timeseries/ilp_ingest.rs @@ -140,10 +140,10 @@ pub fn ingest_batch_with_lvc(args: IngestBatchArgs<'_, '_>) -> IngestBatchOutcom // Only the designated time column takes the line's timestamp. // Any other timestamp column is an ordinary column and carries // whatever the row itself supplied. - ColumnType::Timestamp if col_idx == schema.timestamp_idx => { + ColumnType::Timestamp(_) if col_idx == schema.timestamp_idx => { values.push(ColumnValue::Timestamp(ts_ms)); } - ColumnType::Timestamp => { + ColumnType::Timestamp(_) => { values.push(ColumnValue::Timestamp(find_field_timestamp_ms( &line.fields, col_name, @@ -325,7 +325,9 @@ fn find_field_i64_opt<'a>(fields: &[(Cow<'a, str>, FieldValue<'a>)], name: &str) #[cfg(test)] mod tests { use super::*; - use crate::engine::timeseries::columnar_memtable::{ColumnData, ColumnarMemtableConfig}; + use crate::engine::timeseries::columnar_memtable::{ + ColumnData, ColumnarMemtableConfig, TimeKind, + }; use crate::engine::timeseries::ilp::parse_batch; fn default_config() -> ColumnarMemtableConfig { @@ -347,7 +349,7 @@ mod tests { assert_eq!(schema.columns.len(), 5); assert_eq!( schema.columns[0], - ("timestamp".into(), ColumnType::Timestamp) + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)) ); assert_eq!(schema.columns[1].1, ColumnType::Symbol); // host assert_eq!(schema.columns[2].1, ColumnType::Symbol); // dc diff --git a/nodedb/src/engine/timeseries/ilp_schema.rs b/nodedb/src/engine/timeseries/ilp_schema.rs index 0efe4a229..94d025f1e 100644 --- a/nodedb/src/engine/timeseries/ilp_schema.rs +++ b/nodedb/src/engine/timeseries/ilp_schema.rs @@ -13,13 +13,15 @@ use nodedb_types::columnar::schema::{TS_SYSTEM, TS_VALID_FROM, TS_VALID_UNTIL}; -use super::columnar_memtable::{ColumnType, ColumnarMemtable, ColumnarSchema}; +use super::columnar_memtable::{ColumnType, ColumnarMemtable, ColumnarSchema, TimeKind}; use super::ilp::{FieldValue, IlpLine}; /// Infers a columnar schema from a batch of ILP lines. /// /// Scans all lines to discover tag keys and field keys, then builds /// a schema: timestamp + tag columns (Symbol) + field columns (typed). +/// The inferred time column is `Millis`: no declaration names it an +/// instant, so it reads back as the integer the line carried. pub fn infer_schema(lines: &[IlpLine<'_>]) -> ColumnarSchema { let mut tag_keys: Vec = Vec::new(); let mut field_keys: Vec<(String, ColumnType)> = Vec::new(); @@ -46,7 +48,10 @@ pub fn infer_schema(lines: &[IlpLine<'_>]) -> ColumnarSchema { } let mut columns = Vec::with_capacity(1 + tag_keys.len() + field_keys.len()); - columns.push(("timestamp".to_string(), ColumnType::Timestamp)); + columns.push(( + "timestamp".to_string(), + ColumnType::Timestamp(TimeKind::Millis), + )); for tag in &tag_keys { columns.push((tag.clone(), ColumnType::Symbol)); } diff --git a/nodedb/src/engine/timeseries/merge/o3.rs b/nodedb/src/engine/timeseries/merge/o3.rs index fec4d97f9..98eab09c3 100644 --- a/nodedb/src/engine/timeseries/merge/o3.rs +++ b/nodedb/src/engine/timeseries/merge/o3.rs @@ -38,13 +38,17 @@ pub fn merge_o3_into_partition( // Read existing partition data. let existing_meta = ColumnarSegmentReader::read_meta(&partition_dir, None)?; let existing_schema = ColumnarSegmentReader::read_schema(&partition_dir, None)?; - - let ts_col = ColumnarSegmentReader::read_column( - &partition_dir, - "timestamp", - ColumnType::Timestamp, - None, - )?; + let ts_type = existing_schema + .columns + .get(existing_schema.timestamp_idx) + .map(|(_, ty)| *ty) + .ok_or_else(|| { + SegmentError::Corrupt(format!( + "partition {partition_dir_name} schema has no designated time column" + )) + })?; + + let ts_col = ColumnarSegmentReader::read_column(&partition_dir, "timestamp", ts_type, None)?; let val_col = ColumnarSegmentReader::read_column(&partition_dir, "value", ColumnType::Float64, None)?; diff --git a/nodedb/src/engine/timeseries/merge/partitions.rs b/nodedb/src/engine/timeseries/merge/partitions.rs index 0e08ec248..d095bd58e 100644 --- a/nodedb/src/engine/timeseries/merge/partitions.rs +++ b/nodedb/src/engine/timeseries/merge/partitions.rs @@ -194,7 +194,7 @@ pub struct MergeResult { impl ColumnData { pub(super) fn new_empty(ty: ColumnType) -> Self { match ty { - ColumnType::Timestamp => Self::Timestamp(Vec::new()), + ColumnType::Timestamp(_) => Self::Timestamp(Vec::new()), ColumnType::Float64 => Self::Float64(Vec::new()), ColumnType::Int64 => Self::Int64(Vec::new()), ColumnType::Symbol => Self::Symbol(Vec::new()), @@ -258,11 +258,13 @@ mod tests { use tempfile::TempDir; use crate::engine::timeseries::columnar_memtable::{ - ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, + ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; use super::*; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + fn test_config() -> ColumnarMemtableConfig { ColumnarMemtableConfig { max_memory_bytes: 10 * 1024 * 1024, @@ -305,13 +307,8 @@ mod tests { // Read back merged data. let merged_dir = tmp.path().join("ts-merged"); - let ts_col = ColumnarSegmentReader::read_column( - &merged_dir, - "timestamp", - ColumnType::Timestamp, - None, - ) - .unwrap(); + let ts_col = + ColumnarSegmentReader::read_column(&merged_dir, "timestamp", MILLIS, None).unwrap(); let timestamps = ts_col.as_timestamps(); assert_eq!(timestamps.len(), 100); // Should be sorted. @@ -325,7 +322,7 @@ mod tests { let tmp = TempDir::new().unwrap(); let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("value".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], @@ -384,13 +381,8 @@ mod tests { ); let merged_dir = tmp.path().join("ts-sorted"); - let ts_col = ColumnarSegmentReader::read_column( - &merged_dir, - "timestamp", - ColumnType::Timestamp, - None, - ) - .unwrap(); + let ts_col = + ColumnarSegmentReader::read_column(&merged_dir, "timestamp", MILLIS, None).unwrap(); let timestamps = ts_col.as_timestamps(); // All timestamps from partition 2 (1000-1019) should come before partition 1 (5000-5019). assert_eq!(timestamps[0], 1000); diff --git a/nodedb/src/engine/timeseries/projection.rs b/nodedb/src/engine/timeseries/projection.rs index 20fd9e4f3..1b854fa30 100644 --- a/nodedb/src/engine/timeseries/projection.rs +++ b/nodedb/src/engine/timeseries/projection.rs @@ -104,9 +104,11 @@ pub fn should_use_tag_projection( mod tests { use super::*; use crate::engine::timeseries::columnar_memtable::{ - ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, + ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, TimeKind, }; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + fn test_config() -> ColumnarMemtableConfig { ColumnarMemtableConfig { max_memory_bytes: 10 * 1024 * 1024, @@ -119,7 +121,7 @@ mod tests { fn sort_by_tag_then_timestamp() { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("value".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], @@ -158,7 +160,7 @@ mod tests { fn identity_permutation_for_sorted_data() { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("value".into(), ColumnType::Float64), ], timestamp_idx: 0, @@ -182,7 +184,7 @@ mod tests { fn tag_projection_heuristic() { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("host".into(), ColumnType::Symbol), ], timestamp_idx: 0, diff --git a/nodedb/src/engine/timeseries/schema_evolution.rs b/nodedb/src/engine/timeseries/schema_evolution.rs index aa7251e98..ce222b159 100644 --- a/nodedb/src/engine/timeseries/schema_evolution.rs +++ b/nodedb/src/engine/timeseries/schema_evolution.rs @@ -46,7 +46,9 @@ pub fn build_column_mappings( { Some(idx) => { let (_, p_type) = &partition_schema.columns[idx]; - if p_type == q_type { + // Every time kind shares one millisecond storage, so a + // time column is present whatever kind either side names. + if p_type == q_type || (p_type.is_time() && q_type.is_time()) { ColumnMapping::Present(idx) } else if can_widen(*p_type, *q_type) { ColumnMapping::Widen { @@ -98,7 +100,7 @@ pub fn apply_mappings( /// Create a NULL-equivalent column of the given type and length. fn null_column(ty: ColumnType, rows: usize) -> ColumnData { match ty { - ColumnType::Timestamp => ColumnData::Timestamp(vec![0; rows]), + ColumnType::Timestamp(_) => ColumnData::Timestamp(vec![0; rows]), ColumnType::Float64 => ColumnData::Float64(vec![f64::NAN; rows]), ColumnType::Int64 => ColumnData::Int64(vec![0; rows]), ColumnType::Symbol => ColumnData::Symbol(vec![u32::MAX; rows]), // sentinel @@ -188,12 +190,15 @@ pub fn apply_schema_changes( #[cfg(test)] mod tests { + use super::super::columnar_memtable::TimeKind; use super::*; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + fn schema_v1() -> ColumnarSchema { ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ], @@ -205,7 +210,7 @@ mod tests { fn schema_v2_added_column() -> ColumnarSchema { ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ("mem".into(), ColumnType::Float64), @@ -258,7 +263,7 @@ mod tests { fn widen_int64_to_float64() { let query = ColumnarSchema { columns: vec![ - ("ts".into(), ColumnType::Timestamp), + ("ts".into(), MILLIS), ("val".into(), ColumnType::Float64), // query expects f64 ], timestamp_idx: 0, @@ -266,7 +271,7 @@ mod tests { }; let partition = ColumnarSchema { columns: vec![ - ("ts".into(), ColumnType::Timestamp), + ("ts".into(), MILLIS), ("val".into(), ColumnType::Int64), // partition has i64 ], timestamp_idx: 0, @@ -284,6 +289,27 @@ mod tests { assert!((val[0] - 42.0).abs() < f64::EPSILON); } + /// A partition whose time column carries another kind still maps as + /// present: the kind is how a cell is read, not what is stored. + #[test] + fn time_column_maps_present_across_kinds() { + let query = ColumnarSchema { + columns: vec![( + "ts".into(), + ColumnType::Timestamp(TimeKind::Instant(nodedb_types::InstantKind::Naive)), + )], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 1], + }; + let partition = ColumnarSchema { + columns: vec![("ts".into(), MILLIS)], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 1], + }; + let mappings = build_column_mappings(&query, &partition); + assert!(matches!(mappings[0], ColumnMapping::Present(0))); + } + #[test] fn add_column_schema_change() { let s = schema_v1(); diff --git a/nodedb/src/engine/timeseries/sparse_index/index.rs b/nodedb/src/engine/timeseries/sparse_index/index.rs index 0b2e344d5..b1abe762b 100644 --- a/nodedb/src/engine/timeseries/sparse_index/index.rs +++ b/nodedb/src/engine/timeseries/sparse_index/index.rs @@ -356,7 +356,7 @@ fn compute_block_stats( row_end: usize, ) -> BlockColumnStats { match (col, col_type) { - (ColumnData::Timestamp(v), ColumnType::Timestamp) => { + (ColumnData::Timestamp(v), ColumnType::Timestamp(_)) => { let slice = &v[row_start..row_end]; if slice.is_empty() { return BlockColumnStats::none(); @@ -424,9 +424,13 @@ fn compute_block_stats( #[cfg(test)] mod tests { - use super::super::super::columnar_memtable::{ColumnData, ColumnType, ColumnarSchema}; + use super::super::super::columnar_memtable::{ + ColumnData, ColumnType, ColumnarSchema, TimeKind, + }; use super::*; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + fn make_test_columns(row_count: usize) -> (Vec, ColumnarSchema) { let timestamps: Vec = (0..row_count as i64) .map(|i| 1_700_000_000_000 + i * 10_000) @@ -439,7 +443,7 @@ mod tests { ]; let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ], timestamp_idx: 0, @@ -475,7 +479,7 @@ mod tests { let columns = vec![ColumnData::Timestamp(vec![]), ColumnData::Float64(vec![])]; let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ], timestamp_idx: 0, @@ -570,7 +574,7 @@ mod tests { ]; let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ], timestamp_idx: 0, diff --git a/nodedb/src/engine/timeseries/tag_autocomplete.rs b/nodedb/src/engine/timeseries/tag_autocomplete.rs index e0d4bf3d9..7bd35d080 100644 --- a/nodedb/src/engine/timeseries/tag_autocomplete.rs +++ b/nodedb/src/engine/timeseries/tag_autocomplete.rs @@ -109,7 +109,7 @@ pub enum TagAutoError { mod tests { use super::*; use crate::engine::timeseries::columnar_memtable::{ - ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, + ColumnType, ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; use crate::engine::timeseries::columnar_segment::ColumnarSegmentWriter; use tempfile::TempDir; @@ -125,7 +125,7 @@ mod tests { fn make_tagged_memtable() -> ColumnarMemtable { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), ColumnType::Timestamp(TimeKind::Millis)), ("value".into(), ColumnType::Float64), ("host".into(), ColumnType::Symbol), ("region".into(), ColumnType::Symbol), diff --git a/nodedb/src/engine/timeseries/ts_detect.rs b/nodedb/src/engine/timeseries/ts_detect.rs index 824422b3c..a624ffa91 100644 --- a/nodedb/src/engine/timeseries/ts_detect.rs +++ b/nodedb/src/engine/timeseries/ts_detect.rs @@ -70,7 +70,7 @@ pub fn detect_timestamp( ) -> TsDetection { // Tier 1: Column type is Timestamp. for (i, (name, ty)) in columns.iter().enumerate() { - if *ty == ColumnType::Timestamp { + if ty.is_time() { return TsDetection::ByType { column_index: i, column_name: name.clone(), @@ -133,13 +133,16 @@ fn looks_epoch_like(values: &[i64]) -> bool { #[cfg(test)] mod tests { + use super::super::columnar_memtable::TimeKind; use super::*; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + #[test] fn detect_by_type() { let cols = vec![ ("id".into(), ColumnType::Int64), - ("ts".into(), ColumnType::Timestamp), + ("ts".into(), MILLIS), ("value".into(), ColumnType::Float64), ]; let result = detect_timestamp(&cols, None); @@ -238,7 +241,7 @@ mod tests { fn type_takes_priority_over_name() { let cols = vec![ ("timestamp".into(), ColumnType::Int64), // name match - ("t".into(), ColumnType::Timestamp), // type match + ("t".into(), MILLIS), // type match ]; let result = detect_timestamp(&cols, None); // Type should win. diff --git a/nodedb/src/engine/timeseries/verification.rs b/nodedb/src/engine/timeseries/verification.rs index a41a00207..a6780cff9 100644 --- a/nodedb/src/engine/timeseries/verification.rs +++ b/nodedb/src/engine/timeseries/verification.rs @@ -15,6 +15,8 @@ mod tests { use nodedb_types::timeseries::*; use tempfile::TempDir; + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + fn test_memtable_config() -> ColumnarMemtableConfig { ColumnarMemtableConfig { max_memory_bytes: 10 * 1024 * 1024, @@ -188,13 +190,8 @@ mod tests { let mut total_rows = 0; for entry in &matching { let part_dir = tmp.path().join(&entry.dir_name); - let ts_col = ColumnarSegmentReader::read_column( - &part_dir, - "timestamp", - ColumnType::Timestamp, - None, - ) - .unwrap(); + let ts_col = + ColumnarSegmentReader::read_column(&part_dir, "timestamp", MILLIS, None).unwrap(); total_rows += ts_col.len(); } assert_eq!(total_rows, 300, "all 300 rows should be readable"); @@ -240,7 +237,7 @@ mod tests { // V1 schema: timestamp + cpu let schema_v1 = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ], timestamp_idx: 0, @@ -262,7 +259,7 @@ mod tests { // V2 schema: timestamp + cpu + mem (added) let schema_v2 = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("cpu".into(), ColumnType::Float64), ("mem".into(), ColumnType::Float64), ], @@ -295,7 +292,7 @@ mod tests { ColumnarSegmentReader::read_column( &tmp.path().join("ts-v1"), "timestamp", - ColumnType::Timestamp, + MILLIS, None, ) .unwrap(), @@ -332,7 +329,7 @@ mod tests { fn symbol_cardinality_breaker_rejects_with_message() { let schema = ColumnarSchema { columns: vec![ - ("timestamp".into(), ColumnType::Timestamp), + ("timestamp".into(), MILLIS), ("value".into(), ColumnType::Float64), ("uuid_tag".into(), ColumnType::Symbol), ], From 5db8fbfb9de65dad8fff4567ad9db34b841e091f Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 16 Sep 2026 16:24:39 +0800 Subject: [PATCH 06/21] fix(timeseries): type time cells at the scan source, not join rescale Time columns now render as their declared kind (typed instant or raw milliseconds) at the point a scan emits them, using the column's own TimeKind rather than the ts_instant_columns() name list. Columnar memtable and partition row emission, aggregate/sort/ingest paths, and msgpack filter/group-key/value-ops helpers thread this typed rendering through. This removes the join-side instant rescale pass entirely: a join no longer needs to know which locally-scanned columns are instants and rewrite them after emission, since every row already carries correctly typed time cells by the time a join reads it. instant_scale.rs and its threading through join params/dispatch are deleted. --- .../cases/shuffle_join_end_to_end.rs | 14 +- .../join_cross_node.rs | 17 +- nodedb-query/src/msgpack_scan/filter.rs | 46 +++++ nodedb-query/src/msgpack_scan/group_key.rs | 37 +++- nodedb-query/src/value_ops.rs | 43 ++++- .../executor/core_loop/ts_declared_schema.rs | 33 ++-- nodedb/src/data/executor/dispatch/query.rs | 118 +++--------- .../handlers/columnar_read/convert.rs | 117 +++++++++++- .../columnar_read/materialize_scan_ts.rs | 102 ++++++----- .../executor/handlers/columnar_read/mod.rs | 2 +- .../executor/handlers/join/instant_scale.rs | 86 --------- nodedb/src/data/executor/handlers/join/mod.rs | 1 - .../executor/handlers/join/nested_loop.rs | 13 -- .../src/data/executor/handlers/join/params.rs | 63 ------- .../executor/handlers/join/shuffle_join.rs | 1 - .../data/executor/handlers/join/sort_merge.rs | 13 -- .../executor/handlers/timeseries/aggregate.rs | 6 +- .../executor/handlers/timeseries/encode.rs | 118 ++++++++++-- .../executor/handlers/timeseries/ingest.rs | 15 +- .../handlers/timeseries/raw_scan/mod.rs | 2 +- .../timeseries/raw_scan/partition_scan.rs | 39 ++-- .../handlers/timeseries/raw_scan/row_emit.rs | 171 ++++-------------- .../handlers/timeseries/raw_scan/scan.rs | 21 +-- .../data/executor/handlers/timeseries/sort.rs | 5 +- nodedb/src/data/executor/scan_normalize.rs | 17 +- .../cases/timeseries_declared_time_key.rs | 19 +- .../cases/timeseries_join_time_rendering.rs | 61 +++---- 27 files changed, 566 insertions(+), 614 deletions(-) delete mode 100644 nodedb/src/data/executor/handlers/join/instant_scale.rs diff --git a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_end_to_end.rs b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_end_to_end.rs index 2ce02e070..9939abf38 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_end_to_end.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_end_to_end.rs @@ -28,12 +28,9 @@ use crate::common::cluster_harness::{TestCluster, wait_for}; /// stores in its timeseries collection. const EARLY: &str = "2020-03-05 10:00:00"; /// `EARLY` as a declared `TIMESTAMP` time key renders it. The engine stores -/// 1583402400000 epoch milliseconds; a `TIMESTAMP` cell carries epoch -/// microseconds, which the pgwire encoder writes as ISO-8601 UTC. +/// 1583402400000 epoch milliseconds and emits the cell as a typed instant, +/// which the pgwire encoder writes as ISO-8601 UTC. const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; -/// `EARLY` as epoch microseconds — 1583402400000 milliseconds times 1000. -/// A projection that announces no catalog type leaves its cells this number. -const EARLY_MICROS: &str = "1583402400000000"; /// Run `sql` and collect the `id` column of every returned data row, sorted, so /// the result is order-independent for equality assertions. @@ -344,10 +341,9 @@ async fn a_shuffle_join_renders_a_time_key_as_the_stored_instant() { .collect(); assert_eq!(values.len(), 1, "one event matches one host: {values:?}"); - assert!( - values[0] == EARLY_ISO || values[0] == EARLY_MICROS, - "a shuffle-joined time key must denote {EARLY}: expected {EARLY_ISO} \ - or {EARLY_MICROS}, got {values:?}" + assert_eq!( + values[0], EARLY_ISO, + "a shuffle-joined time key must denote {EARLY}: got {values:?}" ); cluster.shutdown().await; diff --git a/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/join_cross_node.rs b/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/join_cross_node.rs index 59beca7e7..452708b45 100644 --- a/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/join_cross_node.rs +++ b/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/join_cross_node.rs @@ -281,14 +281,10 @@ async fn cross_node_join_compares_time_keys_in_one_unit() { ); } - // The joined TIME_KEY itself must denote the stored instant, whichever - // unit the coordinator's gather step renders it in: ISO-8601 UTC (a - // typed TIMESTAMP cell) or epoch microseconds (an untyped cell). Epoch - // MILLISECONDS denote 1970-01-19 under either reading, so a millisecond - // value fails both arms and reveals the gather step decoded the wrong - // unit. + // The joined TIME_KEY is a typed instant on every node, so the + // coordinator's gather step renders it as ISO-8601 UTC. The stored + // 1583402400000 milliseconds read as a number would denote 1970-01-19. const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; - const EARLY_MICROS: &str = "1583402400000000"; let time_key_sql = format!( "SELECT {EVENTS}.captured_at FROM {EVENTS} INNER JOIN {FEATURES} \ ON {EVENTS}.host = {FEATURES}.host" @@ -300,10 +296,9 @@ async fn cross_node_join_compares_time_keys_in_one_unit() { 1, "node {idx}: the join must yield one row: {rows:?}" ); - assert!( - rows[0] == EARLY_ISO || rows[0] == EARLY_MICROS, - "node {idx}: joined time key must denote 2020-03-05T10:00:00Z: \ - expected {EARLY_ISO} or {EARLY_MICROS}, got {rows:?}" + assert_eq!( + rows[0], EARLY_ISO, + "node {idx}: joined time key must denote 2020-03-05T10:00:00Z: got {rows:?}" ); } diff --git a/nodedb-query/src/msgpack_scan/filter.rs b/nodedb-query/src/msgpack_scan/filter.rs index f05f93381..990a2cc57 100644 --- a/nodedb-query/src/msgpack_scan/filter.rs +++ b/nodedb-query/src/msgpack_scan/filter.rs @@ -341,6 +341,52 @@ mod tests { ); } + /// An instant cell compares by epoch microseconds against a typed + /// instant literal and against an ISO-8601 string literal. + #[test] + fn instant_cell_compares_by_micros() { + use nodedb_types::{InstantKind, NdbDateTime, Value}; + const MICROS: i64 = 1_583_402_400_000_000; + let mut doc = Vec::new(); + crate::msgpack_scan::write_map_header(&mut doc, 1); + crate::msgpack_scan::write_kv_instant(&mut doc, "ts", InstantKind::Naive, MICROS); + + let same = Value::NaiveDateTime(NdbDateTime::from_micros(MICROS)); + let later = Value::NaiveDateTime(NdbDateTime::from_micros(MICROS + 1)); + assert!( + filter("ts", "eq", same.clone()) + .matches_binary(&doc) + .unwrap() + ); + assert!( + !filter("ts", "eq", later.clone()) + .matches_binary(&doc) + .unwrap() + ); + assert!(filter("ts", "lt", later).matches_binary(&doc).unwrap()); + assert!(filter("ts", "gte", same).matches_binary(&doc).unwrap()); + assert!( + filter("ts", "eq", Value::String("2020-03-05T10:00:00Z".into())) + .matches_binary(&doc) + .unwrap() + ); + assert!( + filter("ts", "gt", Value::String("2020-03-05 09:00:00".into())) + .matches_binary(&doc) + .unwrap() + ); + let idx = FieldIndex::build(&doc, 0).unwrap_or_else(FieldIndex::empty); + assert!( + filter( + "ts", + "eq", + Value::NaiveDateTime(NdbDateTime::from_micros(MICROS)) + ) + .matches_binary_indexed(&doc, &idx) + .unwrap() + ); + } + #[test] fn eq_coerces_string_to_integer() { let doc = encode(&json!({"age": 25})); diff --git a/nodedb-query/src/msgpack_scan/group_key.rs b/nodedb-query/src/msgpack_scan/group_key.rs index e3af3408c..916d22a0d 100644 --- a/nodedb-query/src/msgpack_scan/group_key.rs +++ b/nodedb-query/src/msgpack_scan/group_key.rs @@ -5,6 +5,8 @@ //! Builds a deterministic string key from field values extracted directly //! from msgpack bytes, avoiding full document decode. +use nodedb_types::{NdbDateTime, read_instant}; + use crate::expr::{EvalError, GroupKeySpec, SqlExpr}; use crate::msgpack_scan::field::extract_field; use crate::msgpack_scan::index::FieldIndex; @@ -94,7 +96,12 @@ pub fn build_group_key_indexed( } /// Append the msgpack value at `doc[start..end]` to the key buffer as a JSON -/// literal (string quoted, numbers/null verbatim, complex values hex-encoded). +/// literal (string quoted, numbers/null verbatim, an instant as its quoted +/// ISO-8601 form, complex values hex-encoded). +/// +/// An instant keys by its epoch microseconds rendered as ISO-8601: the same +/// text the JSON transcoder gives the cell, so a group column parsed back +/// from the key reads as the cell would. fn append_value_at(buf: &mut String, doc: &[u8], start: usize, end: usize) { if read_null(doc, start) { buf.push_str("null"); @@ -102,6 +109,10 @@ fn append_value_at(buf: &mut String, doc: &[u8], start: usize, end: usize) { buf.push('"'); buf.push_str(s); buf.push('"'); + } else if let Some((_, micros)) = read_instant(doc, start) { + buf.push('"'); + buf.push_str(&NdbDateTime::from_micros(micros).to_iso8601()); + buf.push('"'); } else if let Some(n) = read_i64(doc, start) { use std::fmt::Write; let _ = write!(buf, "{n}"); @@ -219,6 +230,30 @@ mod tests { assert_eq!(key, "[36.6]"); } + /// An instant cell keys by its ISO-8601 form, a scalar JSON string, and + /// the indexed builder produces the same key. + #[test] + fn instant_field_keys_as_iso8601() { + use nodedb_types::InstantKind; + let mut doc = Vec::new(); + crate::msgpack_scan::write_map_header(&mut doc, 2); + crate::msgpack_scan::write_kv_instant( + &mut doc, + "ts", + InstantKind::Naive, + 1_583_402_400_000_000, + ); + crate::msgpack_scan::write_kv_i64(&mut doc, "n", 1); + + let key = build_group_key(&doc, &keys(&["ts", "n"])).unwrap(); + assert_eq!(key, r#"["2020-03-05T10:00:00.000000Z",1]"#); + let idx = FieldIndex::build(&doc, 0).unwrap_or_else(FieldIndex::empty); + assert_eq!( + build_group_key_indexed(&doc, &keys(&["ts", "n"]), &idx).unwrap(), + key + ); + } + /// A computed group key evaluates its expression and folds the result into /// the key slot; `alpha` and `ALPHA` collapse under `UPPER(label)`. fn upper_label_spec() -> GroupKeySpec { diff --git a/nodedb-query/src/value_ops.rs b/nodedb-query/src/value_ops.rs index a2e13622d..3ab6950ca 100644 --- a/nodedb-query/src/value_ops.rs +++ b/nodedb-query/src/value_ops.rs @@ -30,11 +30,22 @@ pub fn value_to_f64(v: &Value, coerce_bool: bool) -> Option { } } +/// Whether either side is a typed instant, so the pair compares by epoch +/// microseconds through `Value::cmp_coerced` (an ISO string on the other +/// side is parsed). +fn involves_instant(a: &Value, b: &Value) -> bool { + a.as_instant().is_some() || b.as_instant().is_some() +} + /// Compare two Values with type coercion. /// -/// Tries numeric comparison first (with bool coercion), then falls -/// back to string comparison. +/// A typed instant compares by epoch microseconds against another instant +/// or an ISO-8601 string. Otherwise numeric comparison first (with bool +/// coercion), then string comparison. pub fn compare_values(a: &Value, b: &Value) -> Ordering { + if involves_instant(a, b) { + return a.cmp_coerced(b); + } if let (Some(na), Some(nb)) = (value_to_f64(a, true), value_to_f64(b, true)) { return na.partial_cmp(&nb).unwrap_or(Ordering::Equal); } @@ -45,12 +56,16 @@ pub fn compare_values(a: &Value, b: &Value) -> Ordering { /// Check equality with type coercion. /// -/// Handles `"5" == 5` by coercing both sides to f64 when one is a -/// number and the other is a numeric string. +/// A typed instant equals another instant or an ISO-8601 string with the +/// same epoch microseconds. Handles `"5" == 5` by coercing both sides to +/// f64 when one is a number and the other is a numeric string. pub fn coerced_eq(a: &Value, b: &Value) -> bool { if a == b { return true; } + if involves_instant(a, b) { + return a.eq_coerced(b); + } if let (Some(af), Some(bf)) = (value_to_f64(a, true), value_to_f64(b, true)) { return (af - bf).abs() < f64::EPSILON; } @@ -138,6 +153,26 @@ mod tests { ); } + #[test] + fn instants_compare_by_micros_against_instants_and_iso_strings() { + let earlier = Value::NaiveDateTime(nodedb_types::NdbDateTime::from_micros( + 1_583_402_400_000_000, + )); + let later = Value::DateTime(nodedb_types::NdbDateTime::from_micros( + 1_583_406_000_000_000, + )); + assert_eq!(compare_values(&earlier, &later), Ordering::Less); + assert_eq!( + compare_values(&later, &Value::String("2020-03-05 10:00:00".into())), + Ordering::Greater + ); + assert!(coerced_eq( + &earlier, + &Value::String("2020-03-05T10:00:00Z".into()) + )); + assert!(!coerced_eq(&earlier, &later)); + } + #[test] fn truthiness() { assert!(is_truthy(&Value::Bool(true))); diff --git a/nodedb/src/data/executor/core_loop/ts_declared_schema.rs b/nodedb/src/data/executor/core_loop/ts_declared_schema.rs index 1ccf3b6fd..1b377db87 100644 --- a/nodedb/src/data/executor/core_loop/ts_declared_schema.rs +++ b/nodedb/src/data/executor/core_loop/ts_declared_schema.rs @@ -168,30 +168,27 @@ impl CoreLoop { .unwrap_or_else(|| vec![TsGroupKeyKind::Text; group_by.len()]) } - /// Columns of a timeseries collection whose memtable type is an instant. + /// The time kind of a timeseries collection's designated time column. /// - /// The memtable keeps every time column in epoch milliseconds, while a - /// client reads a `TIMESTAMP` cell as epoch microseconds, so row emission - /// scales exactly these columns. - /// - /// A `BIGINT TIME_KEY` shares the same millisecond storage but its kind - /// is `Millis`, so it is absent from this list and hands back the number - /// that was inserted. A collection with no schema yields an empty list. - pub(in crate::data::executor) fn ts_instant_columns( + /// A `time_bucket` result is derived from that column and renders with + /// its kind: a declared `TIMESTAMP` key yields an instant, a `BIGINT` + /// key yields the integer stored. A collection with no schema, or a + /// schema whose time index is out of range, renders as `Millis`. + pub(in crate::data::executor) fn ts_time_key_kind( &self, database_id: DatabaseId, tid: TenantId, collection: &str, - ) -> Vec { + ) -> TimeKind { self.with_ts_schema(database_id, tid, collection, |schema| { - schema - .columns - .iter() - .filter(|(_, ty)| matches!(ty, ColumnType::Timestamp(TimeKind::Instant(_)))) - .map(|(name, _)| name.clone()) - .collect() + match schema.columns.get(schema.timestamp_idx) { + Some((_, ColumnType::Timestamp(kind))) => *kind, + Some((_, ColumnType::Int64 | ColumnType::Float64 | ColumnType::Symbol)) | None => { + TimeKind::Millis + } + } }) - .unwrap_or_default() + .unwrap_or(TimeKind::Millis) } } @@ -202,7 +199,7 @@ impl CoreLoop { /// timeseries column can take on the wire. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(in crate::data::executor) enum TsGroupKeyKind { - /// A declared `TIMESTAMP` / `TIMESTAMPTZ` column: epoch microseconds. + /// A declared `TIMESTAMP` / `TIMESTAMPTZ` column: a typed instant. Instant(InstantKind), /// An integer column, including a `BIGINT TIME_KEY`, in its stored unit. Integer, diff --git a/nodedb/src/data/executor/dispatch/query.rs b/nodedb/src/data/executor/dispatch/query.rs index cbe383a73..180a112d0 100644 --- a/nodedb/src/data/executor/dispatch/query.rs +++ b/nodedb/src/data/executor/dispatch/query.rs @@ -8,7 +8,6 @@ use nodedb_physical::physical_plan::{GroupKeySpec, QueryOp}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::handlers::join::{ HashJoinParams, JoinParams, NestedLoopJoinParams, ShuffleJoinInputs, SortMergeJoinParams, - instant_scale::JoinInstantSide, lateral::{LateralLoopParams, LateralTopKParams}, }; use crate::data::executor::task::ExecutionTask; @@ -111,34 +110,7 @@ impl CoreLoop { left_scan_filters, right_scan_filters, .. - } => { - // Only a side this join scans itself carries stored - // milliseconds. A side supplied by a sub-plan was already - // rescaled by the handler that read it, so listing it here - // would scale one instant twice. - let mut local_sides = Vec::with_capacity(2); - if left_input.is_none() { - local_sides.push(JoinInstantSide { - collection: left_collection.as_str(), - qualifier: left_alias - .as_deref() - .unwrap_or_else(|| left_collection.as_str()), - }); - } - if right_input.is_none() { - local_sides.push(JoinInstantSide { - collection: right_collection.as_str(), - qualifier: right_alias - .as_deref() - .unwrap_or_else(|| right_collection.as_str()), - }); - } - let instant_columns = self.join_instant_columns( - task.request.database_id, - crate::types::TenantId::new(tid), - &local_sides, - ); - self.execute_hash_join(HashJoinParams { + } => self.execute_hash_join(HashJoinParams { join: JoinParams { task, on, @@ -148,7 +120,6 @@ impl CoreLoop { computed_projection_bytes: computed_projection, join_filter_bytes: join_filters, post_filter_bytes: post_filters, - instant_columns: &instant_columns, }, tid, left_collection: left_collection.as_str(), @@ -163,8 +134,7 @@ impl CoreLoop { right_rls_filters, left_scan_filters, right_scan_filters, - }) - } + }), QueryOp::ShuffleJoinConsume { build_path, @@ -191,10 +161,6 @@ impl CoreLoop { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &[], - // A shuffle consumer's rows are gathered by the - // coordinator, not sent to a client, and its staged - // frames were produced by the sides' own scans. - instant_columns: &[], }; let inputs = ShuffleJoinInputs { build_path: std::path::PathBuf::from(build_path), @@ -216,34 +182,17 @@ impl CoreLoop { limit, left_rls_filters, right_rls_filters, - } => { - let instant_columns = self.join_instant_columns( - task.request.database_id, - crate::types::TenantId::new(tid), - &[ - JoinInstantSide { - collection: left_collection.as_str(), - qualifier: left_collection.as_str(), - }, - JoinInstantSide { - collection: right_collection.as_str(), - qualifier: right_collection.as_str(), - }, - ], - ); - self.execute_nested_loop_join(NestedLoopJoinParams { - task, - tid, - left_collection: left_collection.as_str(), - right_collection: right_collection.as_str(), - condition, - join_type, - limit: *limit, - left_rls_filters, - right_rls_filters, - instant_columns: &instant_columns, - }) - } + } => self.execute_nested_loop_join(NestedLoopJoinParams { + task, + tid, + left_collection: left_collection.as_str(), + right_collection: right_collection.as_str(), + condition, + join_type, + limit: *limit, + left_rls_filters, + right_rls_filters, + }), QueryOp::SortMergeJoin { left_collection, @@ -254,35 +203,18 @@ impl CoreLoop { pre_sorted, left_rls_filters, right_rls_filters, - } => { - let instant_columns = self.join_instant_columns( - task.request.database_id, - crate::types::TenantId::new(tid), - &[ - JoinInstantSide { - collection: left_collection.as_str(), - qualifier: left_collection.as_str(), - }, - JoinInstantSide { - collection: right_collection.as_str(), - qualifier: right_collection.as_str(), - }, - ], - ); - self.execute_sort_merge_join(SortMergeJoinParams { - task, - tid, - left_collection: left_collection.as_str(), - right_collection: right_collection.as_str(), - on, - join_type, - limit: *limit, - pre_sorted: *pre_sorted, - left_rls_filters, - right_rls_filters, - instant_columns: &instant_columns, - }) - } + } => self.execute_sort_merge_join(SortMergeJoinParams { + task, + tid, + left_collection: left_collection.as_str(), + right_collection: right_collection.as_str(), + on, + join_type, + limit: *limit, + pre_sorted: *pre_sorted, + left_rls_filters, + right_rls_filters, + }), QueryOp::RecursiveScan { collection, diff --git a/nodedb/src/data/executor/handlers/columnar_read/convert.rs b/nodedb/src/data/executor/handlers/columnar_read/convert.rs index 9cea0c69f..076a72b07 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/convert.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/convert.rs @@ -92,6 +92,11 @@ pub(in crate::data::executor) fn row_to_projected_json( /// Encodes the column value at the given row index directly into `buf` /// without intermediate decoding. Used by timeseries raw_scan and aggregate /// handlers that still use the internal `ColumnarMemtable`. +/// +/// A time cell is written as its column's kind says: an instant column +/// yields a typed instant ext, a `Millis` column the integer stored. `Err` +/// when a stored millisecond count overflows the microsecond range an +/// instant carries; the cell is never written wrapped. pub(in crate::data::executor) fn emit_column_value( buf: &mut Vec, mt: &crate::engine::timeseries::columnar_memtable::ColumnarMemtable, @@ -99,13 +104,14 @@ pub(in crate::data::executor) fn emit_column_value( col_type: &crate::engine::timeseries::columnar_memtable::ColumnType, col_data: &crate::engine::timeseries::columnar_memtable::ColumnData, row_idx: usize, -) { +) -> crate::Result<()> { use crate::engine::timeseries::columnar_memtable::{ ColumnData as TsColumnData, ColumnType as TsColumnType, }; match col_type { - TsColumnType::Timestamp(_) => { - nodedb_query::msgpack_scan::write_i64(buf, col_data.as_timestamps()[row_idx]); + TsColumnType::Timestamp(kind) => { + let millis = col_data.as_timestamps()[row_idx]; + write_time_cell(buf, *kind, millis)?; } TsColumnType::Float64 => { let v = col_data.as_f64()[row_idx]; @@ -135,4 +141,109 @@ pub(in crate::data::executor) fn emit_column_value( } } } + Ok(()) +} + +/// Epoch microseconds for a stored millisecond count. +/// +/// `Err` when `millis * 1000` overflows `i64`: the stored value cannot be +/// expressed as an instant, and wrapping it would hand back a different one. +fn instant_micros(millis: i64) -> crate::Result { + nodedb_types::NdbDateTime::from_millis(millis) + .map(|dt| dt.micros) + .map_err(|e| crate::Error::Internal { + detail: format!("timeseries time cell at {millis} ms: {e}"), + }) +} + +/// Write a stored millisecond time cell as the value its kind denotes. +pub(in crate::data::executor) fn write_time_cell( + buf: &mut Vec, + kind: crate::engine::timeseries::columnar_memtable::TimeKind, + millis: i64, +) -> crate::Result<()> { + use crate::engine::timeseries::columnar_memtable::TimeKind; + match kind { + TimeKind::Instant(k) => nodedb_types::write_instant(buf, k, instant_micros(millis)?), + TimeKind::Millis => nodedb_query::msgpack_scan::write_i64(buf, millis), + } + Ok(()) +} + +/// The rmpv cell for a stored millisecond time value, typed by its kind. +/// +/// An instant column yields the ten-byte instant ext, a `Millis` column the +/// integer stored. +pub(in crate::data::executor) fn rmpv_time_cell( + kind: crate::engine::timeseries::columnar_memtable::TimeKind, + millis: i64, +) -> crate::Result { + use crate::engine::timeseries::columnar_memtable::TimeKind; + Ok(match kind { + TimeKind::Instant(k) => { + rmpv::Value::Ext(k.ext_type(), instant_micros(millis)?.to_be_bytes().to_vec()) + } + TimeKind::Millis => rmpv::Value::Integer(millis.into()), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::engine::timeseries::columnar_memtable::{ + ColumnData, ColumnType, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, + }; + use nodedb_types::InstantKind; + + const MS: i64 = 1_583_402_400_000; + + fn memtable(kind: TimeKind) -> ColumnarMemtable { + let schema = ColumnarSchema { + columns: vec![ + ("ts".into(), ColumnType::Timestamp(kind)), + ("v".into(), ColumnType::Float64), + ], + timestamp_idx: 0, + codecs: vec![nodedb_codec::ColumnCodec::Auto; 2], + }; + ColumnarMemtable::new(schema, ColumnarMemtableConfig::default()) + } + + fn emit(kind: TimeKind, millis: i64) -> crate::Result> { + let mt = memtable(kind); + let data = ColumnData::Timestamp(vec![millis]); + let mut buf = Vec::new(); + emit_column_value(&mut buf, &mt, 0, &ColumnType::Timestamp(kind), &data, 0)?; + Ok(buf) + } + + #[test] + fn an_instant_column_emits_a_typed_instant_ext() { + let buf = emit(TimeKind::Instant(InstantKind::Naive), MS).expect("emit"); + assert_eq!( + nodedb_types::read_instant(&buf, 0), + Some((InstantKind::Naive, MS * 1000)) + ); + let buf = emit(TimeKind::Instant(InstantKind::Utc), MS).expect("emit"); + assert_eq!( + nodedb_types::read_instant(&buf, 0), + Some((InstantKind::Utc, MS * 1000)) + ); + } + + #[test] + fn a_millis_column_emits_the_integer_stored() { + let buf = emit(TimeKind::Millis, MS).expect("emit"); + let mut expected = Vec::new(); + nodedb_query::msgpack_scan::write_i64(&mut expected, MS); + assert_eq!(buf, expected); + } + + #[test] + fn a_millisecond_count_past_the_microsecond_range_is_an_error() { + let err = emit(TimeKind::Instant(InstantKind::Naive), i64::MAX) + .expect_err("i64::MAX ms cannot be expressed in microseconds"); + assert!(err.to_string().contains("time cell"), "{err}"); + assert!(emit(TimeKind::Millis, i64::MAX).is_ok()); + } } diff --git a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs index 1aedecb25..7829aa2d9 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs @@ -51,7 +51,7 @@ use nodedb_types::value::Value; use super::materialize_scan::{build_response, encode_cursor}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; -use crate::engine::timeseries::columnar_memtable::{ColumnData, ColumnType}; +use crate::engine::timeseries::columnar_memtable::{ColumnData, ColumnType, TimeKind}; use crate::engine::timeseries::columnar_segment::ColumnarSegmentReader; impl CoreLoop { @@ -98,9 +98,9 @@ impl CoreLoop { } } - let value_bytes = match encode_ts_memtable_row(mt, col_count, row_idx, collection) { - Some(b) => b, - None => continue, + let value_bytes = match encode_ts_memtable_row(mt, col_count, row_idx) { + Ok(b) => b, + Err(e) => return self.response_error(task, e), }; // Surrogate: bit 31 set (memtable) | lower 31 bits = row_idx. @@ -223,10 +223,9 @@ impl CoreLoop { &col_data, &sym_dicts, row_idx, - collection, ) { - Some(b) => b, - None => continue, + Ok(b) => b, + Err(e) => return self.response_error(task, e), }; // Surrogate: (part_id & 0xFFFF) << 16 | (row_idx & 0xFFFF). @@ -293,35 +292,24 @@ pub(super) fn encode_ts_part_surrogate(part_id_1based: u32, row_idx: u32) -> u32 /// Encode a memtable row as msgpack `Value::Object` bytes. /// -/// Returns `None` if serialization fails (logged by caller). +/// A cell that cannot be read as its column's type, or a row that cannot +/// be encoded, fails the scan: a skipped row is a silently short result. fn encode_ts_memtable_row( mt: &crate::engine::timeseries::columnar_memtable::ColumnarMemtable, col_count: usize, row_idx: usize, - collection: &str, -) -> Option> { +) -> crate::Result> { let mut map: HashMap = HashMap::with_capacity(col_count); let schema = mt.schema(); for (col_idx, (col_name, col_type)) in schema.columns.iter().enumerate() { let col_data = mt.column(col_idx); - let val = memtable_col_to_value(col_data, col_type, col_idx, mt, row_idx); + let val = memtable_col_to_value(col_data, col_type, col_idx, mt, row_idx) + .map_err(|e| instant_read_error(col_name, e))?; map.insert(col_name.clone(), val); } - let ndb_val = Value::Object(map); - match nodedb_types::value_to_msgpack(&ndb_val) { - Ok(b) => Some(b), - Err(e) => { - tracing::warn!( - collection, - row_idx, - error = %e, - "ts_materialize_scan: memtable row encode failed; skipping" - ); - None - } - } + encode_row_map(map, row_idx) } /// Encode a partition row as msgpack `Value::Object` bytes. @@ -330,30 +318,33 @@ fn encode_ts_partition_row( col_data: &[Option], sym_dicts: &HashMap, row_idx: usize, - collection: &str, -) -> Option> { +) -> crate::Result> { let mut map: HashMap = HashMap::with_capacity(schema_columns.len()); for (col_i, (col_name, col_type)) in schema_columns.iter().enumerate() { let Some(data) = &col_data[col_i] else { continue; }; - let val = partition_col_to_value(data, col_type, col_i, sym_dicts, row_idx); + let val = partition_col_to_value(data, col_type, col_i, sym_dicts, row_idx) + .map_err(|e| instant_read_error(col_name, e))?; map.insert(col_name.clone(), val); } - let ndb_val = Value::Object(map); - match nodedb_types::value_to_msgpack(&ndb_val) { - Ok(b) => Some(b), - Err(e) => { - tracing::warn!( - collection, - row_idx, - error = %e, - "ts_materialize_scan: partition row encode failed; skipping" - ); - None - } + encode_row_map(map, row_idx) +} + +/// Serialize one row map to msgpack bytes. +fn encode_row_map(map: HashMap, row_idx: usize) -> crate::Result> { + nodedb_types::value_to_msgpack(&Value::Object(map)).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("timeseries materialize scan row {row_idx}: {e}"), + }) +} + +/// The error for a stored millisecond count that no instant can carry. +fn instant_read_error(column: &str, e: nodedb_types::NdbDateTimeError) -> crate::Error { + crate::Error::Internal { + detail: format!("timeseries column {column}: {e}"), } } @@ -361,6 +352,17 @@ fn encode_ts_partition_row( // Column-to-Value converters // --------------------------------------------------------------------------- +/// Read a stored millisecond time cell as the value its kind denotes. +/// +/// An instant column yields a typed instant, a `Millis` column the integer +/// stored. `Err` when the milliseconds overflow the microsecond range. +fn time_cell_value(kind: TimeKind, millis: i64) -> Result { + match kind { + TimeKind::Instant(k) => k.from_millis(millis), + TimeKind::Millis => Ok(Value::Integer(millis)), + } +} + /// Convert a memtable column entry to `nodedb_types::Value`. fn memtable_col_to_value( col_data: &ColumnData, @@ -368,9 +370,11 @@ fn memtable_col_to_value( col_idx: usize, mt: &crate::engine::timeseries::columnar_memtable::ColumnarMemtable, row_idx: usize, -) -> Value { - match col_type { - ColumnType::Timestamp(_) => Value::Integer(col_data.as_timestamps()[row_idx]), +) -> Result { + let value = match col_type { + ColumnType::Timestamp(kind) => { + return time_cell_value(*kind, col_data.as_timestamps()[row_idx]); + } ColumnType::Float64 => { let v = col_data.as_f64()[row_idx]; if v.is_nan() { @@ -387,7 +391,8 @@ fn memtable_col_to_value( .map(|s| Value::String(s.to_string())) .unwrap_or(Value::Null) } - } + }; + Ok(value) } /// Convert a partition column entry to `nodedb_types::Value`. @@ -397,9 +402,11 @@ fn partition_col_to_value( col_i: usize, sym_dicts: &HashMap, row_idx: usize, -) -> Value { - match col_type { - ColumnType::Timestamp(_) => Value::Integer(data.as_timestamps()[row_idx]), +) -> Result { + let value = match col_type { + ColumnType::Timestamp(kind) => { + return time_cell_value(*kind, data.as_timestamps()[row_idx]); + } ColumnType::Float64 => { let v = data.as_f64()[row_idx]; if v.is_nan() { @@ -426,7 +433,8 @@ fn partition_col_to_value( Value::Null } } - } + }; + Ok(value) } /// Extract a timestamp value from a column (for `_ts_system` filtering). diff --git a/nodedb/src/data/executor/handlers/columnar_read/mod.rs b/nodedb/src/data/executor/handlers/columnar_read/mod.rs index bb289151c..5fde1a0de 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/mod.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/mod.rs @@ -15,5 +15,5 @@ pub mod scan; pub mod scan_flushed; pub mod sort; -pub(in crate::data::executor) use convert::emit_column_value; +pub(in crate::data::executor) use convert::{emit_column_value, rmpv_time_cell}; pub(in crate::data::executor) use scan::ColumnarScanParams; diff --git a/nodedb/src/data/executor/handlers/join/instant_scale.rs b/nodedb/src/data/executor/handlers/join/instant_scale.rs deleted file mode 100644 index 4f2e41620..000000000 --- a/nodedb/src/data/executor/handlers/join/instant_scale.rs +++ /dev/null @@ -1,86 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Rescale the declared-instant cells a join materialized itself. -//! -//! A timeseries collection stores every timestamp column in epoch -//! milliseconds, and a client reads a `TIMESTAMP` cell as epoch microseconds. -//! A join scans its local sides through `scan_collection`, which hands back -//! the stored millisecond value, so the join rescales those cells as it emits -//! them — after every predicate has run, so the join itself still compares -//! milliseconds against milliseconds. -//! -//! Exactly once: a handler rescales ONLY the cells it read from a local -//! collection scan. Rows that arrive from a sub-plan response were already -//! rescaled by the handler that read them, and are never touched again. -//! -//! The rescale runs on the merged row, before any projection. A cell is named -//! by the key the join itself wrote, so a rename cannot hide it: the rename -//! happens strictly downstream, and a projected cell copies bytes the rescale -//! has already corrected. - -use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::handlers::timeseries::raw_scan::scale_instant_cells; -use crate::types::{DatabaseId, TenantId}; - -/// One locally-scanned join side: the collection its rows come from, and the -/// qualifier the merged row prefixes that side's keys with. -pub(in crate::data::executor) struct JoinInstantSide<'a> { - pub(in crate::data::executor) collection: &'a str, - pub(in crate::data::executor) qualifier: &'a str, -} - -impl CoreLoop { - /// The merged-row keys of the declared-instant cells `sides` contribute. - /// - /// A merged join row keys every cell `.`, so the - /// returned names are what the row carries before any projection runs. A - /// side that is not a timeseries collection contributes none, so a join - /// over ordinary collections returns an empty list and pays nothing. - pub(in crate::data::executor) fn join_instant_columns( - &self, - database_id: DatabaseId, - tid: TenantId, - sides: &[JoinInstantSide<'_>], - ) -> Vec { - let mut merged_keys = Vec::new(); - for side in sides { - for column in self.ts_instant_columns(database_id, tid, side.collection) { - merged_keys.push(format!("{}.{column}", side.qualifier)); - } - } - merged_keys - } -} - -/// Rescale the named cells of merged join rows, in place. -/// -/// Each row is a msgpack map. An empty `instant_columns` is a no-op, so a -/// join that touches no timeseries collection never decodes a row. -pub(in crate::data::executor) fn scale_join_instant_rows( - rows: &mut [Vec], - instant_columns: &[String], -) -> crate::Result<()> { - if instant_columns.is_empty() { - return Ok(()); - } - for row in rows.iter_mut() { - let mut decoded = [ - crate::util::bounded_msgpack::read_value(row.as_slice()).map_err(|e| { - crate::Error::Serialization { - format: "msgpack".into(), - detail: format!("decode join row for instant rescale: {e}"), - } - })?, - ]; - scale_instant_cells(&mut decoded, instant_columns)?; - let mut buf = Vec::with_capacity(row.len()); - rmpv::encode::write_value(&mut buf, &decoded[0]).map_err(|e| { - crate::Error::Serialization { - format: "msgpack".into(), - detail: format!("encode join row after instant rescale: {e}"), - } - })?; - *row = buf; - } - Ok(()) -} diff --git a/nodedb/src/data/executor/handlers/join/mod.rs b/nodedb/src/data/executor/handlers/join/mod.rs index a6dc9bff6..f9d4f8ad7 100644 --- a/nodedb/src/data/executor/handlers/join/mod.rs +++ b/nodedb/src/data/executor/handlers/join/mod.rs @@ -10,7 +10,6 @@ mod grace_repartition; mod grace_spill; pub mod hash; mod hash_handlers; -pub(in crate::data::executor) mod instant_scale; pub mod lateral; pub mod nested_loop; pub mod params; diff --git a/nodedb/src/data/executor/handlers/join/nested_loop.rs b/nodedb/src/data/executor/handlers/join/nested_loop.rs index d961bbc6a..e0e6db89a 100644 --- a/nodedb/src/data/executor/handlers/join/nested_loop.rs +++ b/nodedb/src/data/executor/handlers/join/nested_loop.rs @@ -30,7 +30,6 @@ impl CoreLoop { limit, left_rls_filters, right_rls_filters, - instant_columns, } = p; debug!( core = self.core_id, @@ -225,18 +224,6 @@ impl CoreLoop { return self.response_error(task, ErrorCode::ResourcesExhausted); } - // Last step before emission: the scans above compared the - // milliseconds storage holds, and the client reads microseconds. - if let Err(e) = super::instant_scale::scale_join_instant_rows(&mut results, instant_columns) - { - return self.response_error( - task, - ErrorCode::Internal { - detail: e.to_string(), - }, - ); - } - let payload = super::super::super::response_codec::encode_binary_rows(&results); self.response_with_payload(task, payload) } diff --git a/nodedb/src/data/executor/handlers/join/params.rs b/nodedb/src/data/executor/handlers/join/params.rs index 8b14df27f..c5620115e 100644 --- a/nodedb/src/data/executor/handlers/join/params.rs +++ b/nodedb/src/data/executor/handlers/join/params.rs @@ -17,11 +17,6 @@ pub(crate) struct JoinParams<'a> { pub computed_projection_bytes: &'a [u8], pub join_filter_bytes: &'a [u8], pub post_filter_bytes: &'a [u8], - /// Merged-row keys of the declared-instant cells this join reads from its - /// own local scans. Empty when no timeseries collection is scanned - /// locally — including for every side supplied by a sub-plan, whose rows - /// were already rescaled by the handler that read them. - pub instant_columns: &'a [String], } /// Hash join: scans both sides from storage or executes resolved child sub-plans. @@ -81,9 +76,6 @@ pub(crate) struct NestedLoopJoinParams<'a> { pub left_rls_filters: &'a [u8], /// Row-level-security filters for the locally-scanned right side. pub right_rls_filters: &'a [u8], - /// Merged-row keys of the declared-instant cells the two local scans - /// contribute. Empty when neither side is a timeseries collection. - pub instant_columns: &'a [String], } /// Sort-merge join: O((N+M)·log N) equi-join with optional pre-sorted inputs. @@ -103,9 +95,6 @@ pub(crate) struct SortMergeJoinParams<'a> { pub left_rls_filters: &'a [u8], /// Row-level-security filters for the locally-scanned right side. pub right_rls_filters: &'a [u8], - /// Merged-row keys of the declared-instant cells the two local scans - /// contribute. Empty when neither side is a timeseries collection. - pub instant_columns: &'a [String], } // ── Test helpers ───────────────────────────────────────────────────────────── @@ -202,11 +191,6 @@ impl JoinParams<'_> { } } - // Every predicate above compared the milliseconds storage holds, and - // the client reads microseconds. Scale here, on the keys the join - // wrote, so a projection that renames a cell copies a corrected value. - super::instant_scale::scale_join_instant_rows(results, self.instant_columns)?; - if !self.computed_projection_bytes.is_empty() { let computed: Vec = zerompk::from_msgpack(self.computed_projection_bytes).map_err(|e| { @@ -274,7 +258,6 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &[], - instant_columns: &[], }; let mut results = vec![vec![1u8, 2, 3], vec![4u8, 5, 6]]; assert!(params.filter_and_project(&mut results).is_ok()); @@ -297,7 +280,6 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: corrupt, - instant_columns: &[], }; let mut results = vec![vec![0u8; 8]]; // would be "leaked" under the old code let err = params.filter_and_project(&mut results); @@ -313,50 +295,6 @@ mod tests { ); } - /// A renaming projection cannot hide an instant cell from the rescale. - /// - /// The rescale names the merged key the join wrote, and the projection - /// renames the cell afterwards, so the emitted cell carries microseconds - /// under its output name. - #[test] - fn projection_rename_still_rescales_the_instant_cell() { - let task = make_dummy_task(); - let projection = vec![JoinProjection { - source: "e.captured_at".into(), - output: "ts".into(), - }]; - let params = JoinParams { - task: &task, - on: &[], - join_type: "inner", - limit: usize::MAX, - projection: &projection, - computed_projection_bytes: &[], - join_filter_bytes: &[], - post_filter_bytes: &[], - instant_columns: &["e.captured_at".to_string()], - }; - let mut results = vec![ - nodedb_types::json_to_msgpack( - &serde_json::json!({"e.captured_at": 1_583_402_400_000i64}), - ) - .expect("encode test row"), - ]; - params - .filter_and_project(&mut results) - .expect("filter_and_project"); - - let decoded = nodedb_types::value_from_msgpack(&results[0]).expect("decode emitted row"); - let nodedb_types::Value::Object(fields) = decoded else { - panic!("emitted row must be a map, got {decoded:?}"); - }; - assert_eq!( - fields.get("ts"), - Some(&nodedb_types::Value::Integer(1_583_402_400_000_000)), - "the renamed cell must carry epoch microseconds: {fields:?}" - ); - } - /// Valid encoded filters → matching rows retained, non-matching rows dropped /// (happy path unchanged). #[test] @@ -372,7 +310,6 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &filter_bytes, - instant_columns: &[], }; let mut results = vec![ row_with_score(99), // should be kept diff --git a/nodedb/src/data/executor/handlers/join/shuffle_join.rs b/nodedb/src/data/executor/handlers/join/shuffle_join.rs index 2cbe7abfd..d1fceefad 100644 --- a/nodedb/src/data/executor/handlers/join/shuffle_join.rs +++ b/nodedb/src/data/executor/handlers/join/shuffle_join.rs @@ -439,7 +439,6 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &[], - instant_columns: &[], }; let inputs = ShuffleJoinInputs { build_path, diff --git a/nodedb/src/data/executor/handlers/join/sort_merge.rs b/nodedb/src/data/executor/handlers/join/sort_merge.rs index c4803a8a5..286de26f6 100644 --- a/nodedb/src/data/executor/handlers/join/sort_merge.rs +++ b/nodedb/src/data/executor/handlers/join/sort_merge.rs @@ -51,7 +51,6 @@ impl CoreLoop { pre_sorted, left_rls_filters, right_rls_filters, - instant_columns, } = p; debug!( core = self.core_id, @@ -300,18 +299,6 @@ impl CoreLoop { return self.response_error(task, ErrorCode::ResourcesExhausted); } - // Last step before emission: the scans above compared the - // milliseconds storage holds, and the client reads microseconds. - if let Err(e) = super::instant_scale::scale_join_instant_rows(&mut results, instant_columns) - { - return self.response_error( - task, - ErrorCode::Internal { - detail: e.to_string(), - }, - ); - } - let payload = super::super::super::response_codec::encode_binary_rows(&results); self.response_with_payload(task, payload) } diff --git a/nodedb/src/data/executor/handlers/timeseries/aggregate.rs b/nodedb/src/data/executor/handlers/timeseries/aggregate.rs index 7ee65f367..b762d0c49 100644 --- a/nodedb/src/data/executor/handlers/timeseries/aggregate.rs +++ b/nodedb/src/data/executor/handlers/timeseries/aggregate.rs @@ -188,6 +188,7 @@ impl CoreLoop { // encoder is told each key column's declared shape. let group_key_kinds = self.ts_group_key_kinds(task.request.database_id, tid, collection, group_by); + let bucket_kind = self.ts_time_key_kind(task.request.database_id, tid, collection); let payload = match super::encode::encode_grouped_results( &merged, group_by, @@ -195,7 +196,10 @@ impl CoreLoop { limit, bucket_interval_ms, sort_keys, - &group_key_kinds, + super::encode::GroupedKeyTypes { + group_key_kinds: &group_key_kinds, + bucket_kind, + }, ) { Ok(p) => p, Err(e) => return self.response_error(task, e), diff --git a/nodedb/src/data/executor/handlers/timeseries/encode.rs b/nodedb/src/data/executor/handlers/timeseries/encode.rs index 0990839ff..5d0d9a269 100644 --- a/nodedb/src/data/executor/handlers/timeseries/encode.rs +++ b/nodedb/src/data/executor/handlers/timeseries/encode.rs @@ -5,13 +5,24 @@ use nodedb_query::agg_key::canonical_agg_key; use crate::data::executor::core_loop::TsGroupKeyKind; +use crate::data::executor::handlers::columnar_read::rmpv_time_cell; +use crate::engine::timeseries::columnar_memtable::TimeKind; + +/// The wire types of a grouped result's key columns. +pub(in crate::data::executor) struct GroupedKeyTypes<'a> { + /// One kind per GROUP BY column, in `group_by` order. + pub group_key_kinds: &'a [TsGroupKeyKind], + /// The kind of the collection's time key, which the `bucket` column + /// derives from. + pub bucket_kind: TimeKind, +} /// Render one GROUP BY key part with the type its column carries ungrouped. /// /// The grouped scan reduces every key to a string, so the column's own type /// is put back here. An empty part is SQL NULL. A declared instant is stored -/// in milliseconds and read in microseconds, exactly as row emission reads -/// it, so the two routes to one stored instant render it identically. +/// in milliseconds and rendered as a typed instant, exactly as row emission +/// renders it, so the two routes to one stored instant render it identically. /// /// A part that does not parse as its column's type falls back to the text it /// holds: the key is data the scan produced, and dropping the group would @@ -21,15 +32,8 @@ fn group_key_value(part: Option<&&str>, kind: TsGroupKeyKind) -> crate::Result match text.parse::() { - Ok(millis) => { - let micros = nodedb_types::NdbDateTime::from_millis(millis) - .map_err(|e| crate::Error::Internal { - detail: format!("grouped timeseries key at {millis} ms: {e}"), - })? - .micros; - rmpv::Value::Integer(micros.into()) - } + TsGroupKeyKind::Instant(k) => match text.parse::() { + Ok(millis) => rmpv_time_cell(TimeKind::Instant(k), millis)?, Err(_) => rmpv::Value::String((*text).into()), }, TsGroupKeyKind::Integer => match text.parse::() { @@ -54,6 +58,9 @@ fn group_key_value(part: Option<&&str>, kind: TsGroupKeyKind) -> crate::Result, ) -> crate::Result> { + let GroupedKeyTypes { + group_key_kinds, + bucket_kind, + } = key_types; let has_bucket = bucket_interval_ms > 0; // An ordered query has to see every group before cutting to `limit`: // groups arrive in hash-map order, so the first `limit` of them are an @@ -100,7 +111,7 @@ pub(in crate::data::executor) fn encode_grouped_results( .unwrap_or(0); fields.push(( rmpv::Value::String("bucket".into()), - rmpv::Value::Integer(bucket_ts.into()), + rmpv_time_cell(bucket_kind, bucket_ts)?, )); for (i, field) in group_by.iter().enumerate() { @@ -152,3 +163,84 @@ pub(in crate::data::executor) fn encode_grouped_results( rmpv::encode::write_value(&mut buf, &array).unwrap_or(()); Ok(buf) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::engine::timeseries::grouped_scan::GroupedAggResult; + use nodedb_types::InstantKind; + + const BUCKET_MS: i64 = 1_583_402_400_000; + + fn one_bucket() -> GroupedAggResult { + let mut result = GroupedAggResult::new(1); + result + .groups + .insert(BUCKET_MS.to_string(), vec![Default::default()]); + result + } + + fn bucket_cell(bucket_kind: TimeKind) -> rmpv::Value { + let bytes = encode_grouped_results( + &one_bucket(), + &[], + &[("count".to_string(), "v".to_string())], + usize::MAX, + 60_000, + &[], + GroupedKeyTypes { + group_key_kinds: &[], + bucket_kind, + }, + ) + .expect("encode"); + let rmpv::Value::Array(rows) = + crate::util::bounded_msgpack::read_value(&bytes).expect("decode") + else { + panic!("not an array"); + }; + let rmpv::Value::Map(fields) = &rows[0] else { + panic!("not a map"); + }; + fields + .iter() + .find(|(k, _)| k.as_str() == Some("bucket")) + .map(|(_, v)| v.clone()) + .expect("bucket column") + } + + #[test] + fn bucket_over_an_instant_time_key_is_a_typed_instant() { + assert_eq!( + bucket_cell(TimeKind::Instant(InstantKind::Naive)), + rmpv::Value::Ext( + InstantKind::Naive.ext_type(), + (BUCKET_MS * 1000).to_be_bytes().to_vec() + ) + ); + } + + #[test] + fn bucket_over_a_millis_time_key_is_the_integer_stored() { + assert_eq!( + bucket_cell(TimeKind::Millis), + rmpv::Value::Integer(BUCKET_MS.into()) + ); + } + + #[test] + fn an_instant_group_key_is_a_typed_instant() { + let cell = group_key_value( + Some(&BUCKET_MS.to_string().as_str()), + TsGroupKeyKind::Instant(InstantKind::Utc), + ) + .expect("group key"); + assert_eq!( + cell, + rmpv::Value::Ext( + InstantKind::Utc.ext_type(), + (BUCKET_MS * 1000).to_be_bytes().to_vec() + ) + ); + } +} diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest.rs b/nodedb/src/data/executor/handlers/timeseries/ingest.rs index c2a02fdd5..39f825d98 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest.rs @@ -325,22 +325,19 @@ impl CoreLoop { // missing float field is stored as NaN and both paths render it as SQL // NULL, which a hand-written projection over the ingest values would // have printed as "NaN". - let mut returned_rows: Vec = match returning { + let returned_rows: Vec = match returning { Some(_) => match self.columnar_memtables.get(&key) { Some(mt) => { - super::raw_scan::emit_memtable_rows_at(mt, &outcome.accepted_row_indices) + match super::raw_scan::emit_memtable_rows_at(mt, &outcome.accepted_row_indices) + { + Ok(rows) => rows, + Err(e) => return self.response_error(task, e), + } } None => Vec::new(), }, None => Vec::new(), }; - // Same scan-unit rule `SELECT` applies: a declared `TIMESTAMP` cell - // leaves the engine as epoch microseconds, not the milliseconds - // storage holds. - let instant_columns = self.ts_instant_columns(task.request.database_id, tid, collection); - if let Err(e) = super::raw_scan::scale_instant_cells(&mut returned_rows, &instant_columns) { - return self.response_error(task, e); - } if accepted > 0 && let Some(lsn) = wal_lsn diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs index 9b346ff12..081da03b5 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs @@ -8,5 +8,5 @@ pub mod partition_scan; pub mod row_emit; pub mod scan; -pub(in crate::data::executor) use row_emit::{emit_memtable_rows_at, scale_instant_cells}; +pub(in crate::data::executor) use row_emit::emit_memtable_rows_at; pub(in crate::data::executor) use scan::RawScanParams; diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/partition_scan.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/partition_scan.rs index 4f3eeabc1..375c80609 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/partition_scan.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/partition_scan.rs @@ -11,18 +11,20 @@ use crate::engine::timeseries::columnar_segment::ColumnarSegmentReader; use super::row_emit::{emit_partition_row, extract_timestamp}; /// Scan disk partitions in parallel, returning rmpv rows sorted by timestamp. +/// +/// `Err` when a stored time cell cannot be read as its column's instant. pub(super) fn scan_partitions_parallel( partition_dirs: &[std::path::PathBuf], time_range: (i64, i64), limit: usize, filter_predicates: &[crate::bridge::scan_filter::ScanFilter], has_filters: bool, -) -> Vec { +) -> crate::Result> { if partition_dirs.len() <= 1 { - return partition_dirs - .first() - .map(|dir| scan_one_partition(dir, time_range, limit, filter_predicates, has_filters)) - .unwrap_or_default(); + return match partition_dirs.first() { + Some(dir) => scan_one_partition(dir, time_range, limit, filter_predicates, has_filters), + None => Ok(Vec::new()), + }; } #[cfg(not(target_arch = "wasm32"))] @@ -61,8 +63,11 @@ pub(super) fn scan_partitions_parallel( }) .collect(); - handles.into_iter().filter_map(|h| h.join().ok()).collect() - }); + handles + .into_iter() + .filter_map(|h| h.join().ok()) + .collect::>>() + })?; // Merge: each thread's results are already time-sorted (partitions are // time-ordered). Flatten, sort globally, truncate to limit. @@ -74,7 +79,7 @@ pub(super) fn scan_partitions_parallel( // Sort by timestamp (first field in each row map). merged.sort_by_key(extract_timestamp); merged.truncate(limit); - merged + Ok(merged) } #[cfg(target_arch = "wasm32")] @@ -95,31 +100,33 @@ pub(super) fn scan_partitions_sequential( limit: usize, filter_predicates: &[crate::bridge::scan_filter::ScanFilter], has_filters: bool, -) -> Vec { +) -> crate::Result> { let mut results = Vec::new(); for dir in partition_dirs { if results.len() >= limit { break; } let remaining = limit - results.len(); - let rows = scan_one_partition(dir, time_range, remaining, filter_predicates, has_filters); + let rows = scan_one_partition(dir, time_range, remaining, filter_predicates, has_filters)?; results.extend(rows); } results.truncate(limit); - results + Ok(results) } /// Scan a single disk partition, returning rmpv rows. +/// +/// `Err` when a stored time cell cannot be read as its column's instant. pub(super) fn scan_one_partition( part_dir: &std::path::Path, time_range: (i64, i64), limit: usize, filter_predicates: &[crate::bridge::scan_filter::ScanFilter], has_filters: bool, -) -> Vec { +) -> crate::Result> { let schema = match ColumnarSegmentReader::read_schema(part_dir, None) { Ok(s) => s, - Err(_) => return Vec::new(), + Err(_) => return Ok(Vec::new()), }; // Prefetch all column files into page cache before reading. @@ -146,7 +153,7 @@ pub(super) fn scan_one_partition( let ts_col = col_data.get(schema.timestamp_idx).and_then(|d| d.as_ref()); let Some(ts_col) = ts_col else { - return Vec::new(); + return Ok(Vec::new()); }; let timestamps = ts_col.as_timestamps(); let indices = timestamp_range_filter(timestamps, time_range.0, time_range.1); @@ -189,12 +196,12 @@ pub(super) fn scan_one_partition( if rows.len() >= limit { break; } - let row = emit_partition_row(&schema_vec, &col_data, &sym_dicts, idx as usize); + let row = emit_partition_row(&schema_vec, &col_data, &sym_dicts, idx as usize)?; rows.push(row); } // Release page cache for this partition. crate::data::io::fadvise::release_partition_columns(part_dir, &all_col_names); - rows + Ok(rows) } diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs index 919525d2a..4e72d1a48 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs @@ -6,9 +6,9 @@ use std::collections::HashMap; use nodedb_types::columnar::schema::TS_SYSTEM; -use crate::util::rmpv_value::{rmpv_to_value, value_to_rmpv}; - +use crate::data::executor::handlers::columnar_read::{emit_column_value, rmpv_time_cell}; use crate::engine::timeseries::columnar_memtable::{ColumnData, ColumnType}; +use crate::util::rmpv_value::{rmpv_to_value, value_to_rmpv}; /// Extract the `_ts_system` value from an rmpv-encoded row for audit-log /// ordering. Rows without the column sort first (treated as `i64::MIN`). @@ -41,10 +41,12 @@ pub(super) fn rmpv_system_time(row: &rmpv::Value) -> i64 { /// An index past the memtable's row count is skipped rather than panicking: /// the caller reads indices recorded before a flush, and a flush in between /// would invalidate them. Callers must project before flushing. +/// +/// `Err` when a stored time cell cannot be read as its column's instant. pub(in crate::data::executor) fn emit_memtable_rows_at( mt: &crate::engine::timeseries::columnar_memtable::ColumnarMemtable, row_indices: &[usize], -) -> Vec { +) -> crate::Result> { let schema = mt.schema().clone(); let columns: Vec<_> = schema .columns @@ -61,30 +63,34 @@ pub(in crate::data::executor) fn emit_memtable_rows_at( } /// Emit a single row from the memtable as rmpv::Value::Map. +/// +/// Every cell is written by [`emit_column_value`], so a time cell carries +/// the type its column kind declares. `Err` when a stored time cell cannot +/// be read as its column's instant. pub(super) fn emit_memtable_row( mt: &crate::engine::timeseries::columnar_memtable::ColumnarMemtable, columns: &[(usize, &String, &ColumnType, &ColumnData)], idx: usize, -) -> rmpv::Value { +) -> crate::Result { // Build raw msgpack bytes, then decode to rmpv::Value. let mut buf = Vec::with_capacity(columns.len() * 32); nodedb_query::msgpack_scan::write_map_header(&mut buf, columns.len()); for (col_idx, col_name, col_type, col_data) in columns { nodedb_query::msgpack_scan::write_str(&mut buf, col_name); - crate::data::executor::handlers::columnar_read::emit_column_value( - &mut buf, mt, *col_idx, col_type, col_data, idx, - ); + emit_column_value(&mut buf, mt, *col_idx, col_type, col_data, idx)?; } - crate::util::bounded_msgpack::read_value(&buf).unwrap_or(rmpv::Value::Nil) + Ok(crate::util::bounded_msgpack::read_value(&buf).unwrap_or(rmpv::Value::Nil)) } /// Emit a single row from a disk partition as rmpv::Value::Map. +/// +/// `Err` when a stored time cell cannot be read as its column's instant. pub(super) fn emit_partition_row( schema: &[(String, ColumnType)], col_data: &[Option], sym_dicts: &HashMap, idx: usize, -) -> rmpv::Value { +) -> crate::Result { let mut fields: Vec<(rmpv::Value, rmpv::Value)> = Vec::with_capacity(schema.len()); for (col_i, (col_name, col_type)) in schema.iter().enumerate() { // A column whose file could not be read is emitted as NULL, never @@ -101,7 +107,7 @@ pub(super) fn emit_partition_row( continue; }; let val = match col_type { - ColumnType::Timestamp(_) => rmpv::Value::Integer(data.as_timestamps()[idx].into()), + ColumnType::Timestamp(kind) => rmpv_time_cell(*kind, data.as_timestamps()[idx])?, ColumnType::Float64 => { let v = data.as_f64()[idx]; if v.is_nan() { @@ -131,15 +137,27 @@ pub(super) fn emit_partition_row( }; fields.push((rmpv::Value::String(col_name.as_str().into()), val)); } - rmpv::Value::Map(fields) + Ok(rmpv::Value::Map(fields)) } -/// Extract timestamp from a row (first integer field) for sort-merge. +/// Extract the sort key of a partition row for the merge across partitions: +/// the first time-shaped field, an integer or an instant ext, as `i64`. +/// +/// Every row of one collection carries the same kind in that field, so the +/// key is comparable across the rows being merged. pub(super) fn extract_timestamp(row: &rmpv::Value) -> i64 { if let rmpv::Value::Map(fields) = row { for (_, v) in fields { - if let rmpv::Value::Integer(n) = v { - return n.as_i64().unwrap_or(0); + match v { + rmpv::Value::Integer(n) => return n.as_i64().unwrap_or(0), + rmpv::Value::Ext(ext_type, payload) => { + if let Some((_, micros)) = + nodedb_types::json_msgpack::instant_from_ext(*ext_type, payload) + { + return micros; + } + } + _ => {} } } } @@ -170,128 +188,3 @@ pub(super) fn apply_computed_columns_rmpv( } Ok(rmpv::Value::Map(fields)) } - -/// Rescale every declared-instant cell of `rows` from the milliseconds the -/// memtable and the partitions store to the epoch microseconds a `TIMESTAMP` -/// cell carries on the wire. -/// -/// The engine's own unit stays milliseconds: partition ranges, retention, -/// `time_bucket` and every scan predicate read it. The scale therefore runs -/// once, as rows leave the scan — after filtering, sorting and computed -/// columns — so nothing inside the engine sees the wire unit. -/// -/// `instant_columns` comes from `CoreLoop::ts_instant_columns`, which lists -/// the columns whose memtable time kind is an instant. A `BIGINT TIME_KEY` -/// lives in the same millisecond storage with kind `Millis` and is not in -/// that list, so it keeps the integer the client inserted. -/// -/// SQL NULL cells pass through untouched. A stored value that cannot be -/// expressed in microseconds fails the read rather than wrapping. -pub(in crate::data::executor) fn scale_instant_cells( - rows: &mut [rmpv::Value], - instant_columns: &[String], -) -> crate::Result<()> { - if instant_columns.is_empty() { - return Ok(()); - } - for row in rows.iter_mut() { - let rmpv::Value::Map(fields) = row else { - continue; - }; - for (key, value) in fields.iter_mut() { - let Some(name) = key.as_str() else { continue }; - if !instant_columns.iter().any(|c| c == name) { - continue; - } - let rmpv::Value::Integer(stored) = value else { - continue; - }; - let millis = stored.as_i64().ok_or_else(|| crate::Error::Internal { - detail: format!( - "timeseries column {name} holds {stored}, which is not a millisecond \ - count an instant can be read from" - ), - })?; - let micros = nodedb_types::NdbDateTime::from_millis(millis) - .map_err(|e| crate::Error::Internal { - detail: format!("timeseries column {name} at {millis} ms: {e}"), - })? - .micros; - *value = rmpv::Value::Integer(micros.into()); - } - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::scale_instant_cells; - - fn row(cells: &[(&str, i64)]) -> rmpv::Value { - rmpv::Value::Map( - cells - .iter() - .map(|(k, v)| { - ( - rmpv::Value::String((*k).into()), - rmpv::Value::Integer((*v).into()), - ) - }) - .collect(), - ) - } - - fn cell(row: &rmpv::Value, name: &str) -> Option { - let rmpv::Value::Map(fields) = row else { - return None; - }; - fields - .iter() - .find(|(k, _)| k.as_str() == Some(name)) - .and_then(|(_, v)| v.as_i64()) - } - - /// A declared `TIMESTAMP` column is read as epoch microseconds, so the - /// millisecond value storage holds is scaled on the way out. 2020-03-05 - /// stays 2020-03-05 instead of landing 50 years earlier. - #[test] - fn a_declared_instant_column_leaves_the_scan_in_microseconds() { - let mut rows = vec![row(&[("captured_at", 1_583_402_400_000)])]; - scale_instant_cells(&mut rows, &["captured_at".to_string()]).expect("scale"); - assert_eq!(cell(&rows[0], "captured_at"), Some(1_583_402_400_000_000)); - } - - /// A `BIGINT TIME_KEY` shares the millisecond storage column but is not a - /// declared instant, so its value is handed back exactly as inserted. - #[test] - fn a_column_that_is_not_a_declared_instant_keeps_its_value() { - let mut rows = vec![row(&[("ts", 1000), ("n", 7)])]; - scale_instant_cells(&mut rows, &["other".to_string()]).expect("scale"); - assert_eq!(cell(&rows[0], "ts"), Some(1000)); - assert_eq!(cell(&rows[0], "n"), Some(7)); - } - - /// A NULL instant cell stays NULL — there is no instant to scale. - #[test] - fn a_null_instant_cell_passes_through() { - let mut rows = vec![rmpv::Value::Map(vec![( - rmpv::Value::String("captured_at".into()), - rmpv::Value::Nil, - )])]; - scale_instant_cells(&mut rows, &["captured_at".to_string()]).expect("scale"); - assert_eq!(cell(&rows[0], "captured_at"), None); - } - - /// A stored millisecond count past the microsecond range fails the read. - /// Wrapping it would hand back an instant that is not the stored one. - #[test] - fn a_millisecond_value_beyond_the_microsecond_range_fails_the_read() { - let mut rows = vec![row(&[("captured_at", i64::MAX)])]; - let err = scale_instant_cells(&mut rows, &["captured_at".to_string()]) - .expect_err("i64::MAX ms cannot be expressed in microseconds"); - assert!( - err.to_string().contains("captured_at"), - "the error must name the column: {err}" - ); - } -} diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs index a51828b8f..48249acf0 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs @@ -144,7 +144,10 @@ impl CoreLoop { if results.len() >= gather_limit { break; } - let row = emit_memtable_row(mt, &columns, idx as usize); + let row = match emit_memtable_row(mt, &columns, idx as usize) { + Ok(row) => row, + Err(e) => return self.response_error(task, e), + }; if need_json_filter { // Encode rmpv row to msgpack bytes for binary filter eval. let mut buf = Vec::new(); @@ -186,13 +189,16 @@ impl CoreLoop { let remaining = gather_limit.saturating_sub(results.len()); if remaining > 0 && !partition_dirs.is_empty() { - let partition_rows = scan_partitions_parallel( + let partition_rows = match scan_partitions_parallel( &partition_dirs, time_range, remaining, filter_predicates, has_filters, - ); + ) { + Ok(rows) => rows, + Err(e) => return self.response_error(task, e), + }; results.extend(partition_rows); results.truncate(gather_limit); } @@ -271,15 +277,6 @@ impl CoreLoop { } results.truncate(limit); - // The engine stores its timestamp columns in milliseconds; a client - // reads a `TIMESTAMP` cell as epoch microseconds. Scale here, once - // every predicate, sort and computed column has run against the - // engine's own unit. - let instant_columns = self.ts_instant_columns(task.request.database_id, tid, collection); - if let Err(e) = super::row_emit::scale_instant_cells(&mut results, &instant_columns) { - return self.response_error(task, e); - } - let array = rmpv::Value::Array(results); let mut buf = Vec::new(); rmpv::encode::write_value(&mut buf, &array).unwrap_or(()); diff --git a/nodedb/src/data/executor/handlers/timeseries/sort.rs b/nodedb/src/data/executor/handlers/timeseries/sort.rs index 6aaeeb13b..3284d1268 100644 --- a/nodedb/src/data/executor/handlers/timeseries/sort.rs +++ b/nodedb/src/data/executor/handlers/timeseries/sort.rs @@ -122,8 +122,9 @@ fn compare_values(a: Option<&rmpv::Value>, b: Option<&rmpv::Value>) -> Ordering x.as_str().unwrap_or("").cmp(y.as_str().unwrap_or("")) } (rmpv::Value::Boolean(x), rmpv::Value::Boolean(y)) => x.cmp(y), - // A computed key that evaluates to an instant arrives as the instant - // ext; two instants order by their epoch microseconds. + // A declared instant column, and a computed key that evaluates to an + // instant, arrive as the instant ext; two instants order by their + // epoch microseconds. (rmpv::Value::Ext(tx, px), rmpv::Value::Ext(ty, py)) => { match (instant_from_ext(*tx, px), instant_from_ext(*ty, py)) { (Some((_, x)), Some((_, y))) => x.cmp(&y), diff --git a/nodedb/src/data/executor/scan_normalize.rs b/nodedb/src/data/executor/scan_normalize.rs index 4fc173ed9..fb6f09277 100644 --- a/nodedb/src/data/executor/scan_normalize.rs +++ b/nodedb/src/data/executor/scan_normalize.rs @@ -116,7 +116,7 @@ impl CoreLoop { } // 2. Columnar memtable - let col_docs = self.scan_columnar(did, tid, collection, limit); + let col_docs = self.scan_columnar(did, tid, collection, limit)?; if !col_docs.is_empty() { return Ok(col_docs); } @@ -182,7 +182,7 @@ impl CoreLoop { // 2. Columnar — materializes internally; iterate the batch per-row. // columnar stays materialized — per-row segment streaming is a separate // follow-up (flushed-segment decode). - let col_docs = self.scan_columnar(did, tid, collection, usize::MAX); + let col_docs = self.scan_columnar(did, tid, collection, usize::MAX)?; if !col_docs.is_empty() { for (id, bytes) in &col_docs { f(id, bytes)?; @@ -238,13 +238,16 @@ impl CoreLoop { } /// Scan columnar rows → standard msgpack. + /// + /// A timeseries memtable's time cells are typed by their column kind. + /// `Err` when a stored time cell cannot be read as its column's instant. fn scan_columnar( &self, database_id: u64, tid: u64, collection: &str, limit: usize, - ) -> Vec<(String, Vec)> { + ) -> crate::Result)>> { let columnar_key = ( nodedb_types::DatabaseId::new(database_id), crate::types::TenantId::new(tid), @@ -282,15 +285,15 @@ impl CoreLoop { } super::handlers::columnar_read::emit_column_value( &mut mp, mt, *col_idx, col_type, col_data, idx, - ); + )?; } results.push((id, mp)); } - return results; + return Ok(results); } let Some(engine) = self.columnar_engines.get(&columnar_key) else { - return Vec::new(); + return Ok(Vec::new()); }; let schema = engine.schema(); @@ -386,7 +389,7 @@ impl CoreLoop { } } - results + Ok(results) } /// Scan sparse/document engine → standard msgpack. diff --git a/nodedb/tests/wire/cases/timeseries_declared_time_key.rs b/nodedb/tests/wire/cases/timeseries_declared_time_key.rs index 464ecd0f6..9950f45f1 100644 --- a/nodedb/tests/wire/cases/timeseries_declared_time_key.rs +++ b/nodedb/tests/wire/cases/timeseries_declared_time_key.rs @@ -18,10 +18,10 @@ //! (`ts`, `timestamp`, `time`) is not special: it is the user's column. //! //! A declared `TIMESTAMP` time key reads back as the instant that was -//! inserted: the engine stores epoch milliseconds and a `TIMESTAMP` cell is -//! read as epoch microseconds, so the two units meet as the row leaves the -//! scan. A `BIGINT` time key shares that storage column and is not a -//! timestamp, so it reads back as the number that was inserted. +//! inserted: the engine stores epoch milliseconds and emits the cell as a +//! typed instant from the column's own kind. A `BIGINT` time key shares +//! that storage column and is not a timestamp, so it reads back as the +//! number that was inserted. use crate::harness::TestServer; @@ -34,13 +34,10 @@ const LATE: &str = "2020-03-05 13:00:00"; /// wall-clock time ever will. const AFTER_BOTH: &str = "2021-01-01 00:00:00"; /// `EARLY` as a declared `TIMESTAMP` column renders it. The engine stores -/// 1583402400000 epoch milliseconds; a `TIMESTAMP` cell carries epoch -/// microseconds, which the pgwire encoder writes as ISO-8601 UTC. +/// 1583402400000 epoch milliseconds and emits the cell as a typed instant, +/// which the pgwire encoder writes as ISO-8601 UTC. A star projection reads +/// the same typed cell, so it renders the same. const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; -/// `EARLY` as `SELECT *` renders it: a star projection announces no catalog -/// type, so its cells stay the raw stored number — here the same instant in -/// epoch microseconds. -const EARLY_MICROS: &str = "1583402400000000"; #[tokio::test] async fn time_key_named_ts_round_trips() { @@ -126,7 +123,7 @@ async fn select_star_projects_declared_columns_only() { ); assert_eq!( rows[0].get("ts").map(String::as_str), - Some(EARLY_MICROS), + Some(EARLY_ISO), "`SELECT *` must carry the inserted event time under `ts`: {rows:?}" ); } diff --git a/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs b/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs index 52a686157..ee390111a 100644 --- a/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs +++ b/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs @@ -20,12 +20,9 @@ use crate::harness::TestServer; /// ingest (wall-clock "now") is separable from the value the INSERT supplied. const EARLY: &str = "2020-03-05 10:00:00"; /// `EARLY` as a declared `TIMESTAMP` column renders it. The engine stores -/// 1583402400000 epoch milliseconds; a `TIMESTAMP` cell carries epoch -/// microseconds, which the pgwire encoder writes as ISO-8601 UTC. +/// 1583402400000 epoch milliseconds and emits the cell as a typed instant, +/// which the pgwire encoder writes as ISO-8601 UTC. const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; -/// `EARLY` as epoch microseconds — 1583402400000 milliseconds times 1000. -/// A projection that announces no catalog type leaves its cells this number. -const EARLY_MICROS: &str = "1583402400000000"; /// Create a timeseries collection and a document collection that join on the /// event's `host`, then insert exactly one row into each so the join yields @@ -142,17 +139,12 @@ async fn a_joined_time_key_denotes_the_stored_instant() { .expect("the time key projected through a JOIN must succeed"); assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); - // Two renderings denote 2020-03-05T10:00:00Z, and the expected value is - // whichever one the join announces a type for. EARLY_ISO is that instant - // written as ISO-8601 UTC, which a cell typed TIMESTAMP produces. - // EARLY_MICROS is the same instant in epoch microseconds — the unit a - // TIMESTAMP cell carries — which an untyped cell leaves as a number. - // Epoch MILLISECONDS denote 1970-01-19 read either way, so a millisecond - // value fails both arms. - assert!( - joined[0] == EARLY_ISO || joined[0] == EARLY_MICROS, - "a joined time key must denote {EARLY}: expected {EARLY_ISO} \ - or {EARLY_MICROS}, got {joined:?}" + // The cell is typed at emission, so the join carries the instant itself + // and renders it as ISO-8601 UTC. The stored 1583402400000 milliseconds + // read as a number would denote 1970-01-19. + assert_eq!( + joined[0], EARLY_ISO, + "a joined time key must denote {EARLY}: got {joined:?}" ); } @@ -202,23 +194,17 @@ async fn an_aliased_joined_time_key_denotes_the_stored_instant() { .expect("an aliased time key projected through a JOIN must succeed"); assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); - // The expected value is EARLY, the inserted instant, in whichever unit the - // alias announces a type for. EARLY_ISO is 2020-03-05T10:00:00Z as a cell - // typed TIMESTAMP renders it; EARLY_MICROS is the same instant in the epoch - // microseconds a TIMESTAMP cell carries, which an untyped cell leaves as a - // number. The stored 1583402400000 milliseconds denote 1970-01-19 read - // either way, so a millisecond value fails both arms. - assert!( - joined[0] == EARLY_ISO || joined[0] == EARLY_MICROS, - "an aliased joined time key must denote {EARLY}: expected {EARLY_ISO} \ - or {EARLY_MICROS}, got {joined:?}" + // The alias renames a cell that is already a typed instant, so it renders + // as ISO-8601 UTC under the new name. + assert_eq!( + joined[0], EARLY_ISO, + "an aliased joined time key must denote {EARLY}: got {joined:?}" ); } /// `EARLY` truncated to its hour, as `time_bucket('1 hour', ...)` denotes it. /// `EARLY` sits on the hour, so the bucket is the same instant. const EARLY_BUCKET_ISO: &str = EARLY_ISO; -const EARLY_BUCKET_MICROS: &str = EARLY_MICROS; /// A transforming computed projection of a time key renders the same through /// a JOIN as through a direct read. `time_bucket` arithmetic depends on the @@ -268,10 +254,9 @@ async fn a_time_bucket_of_a_time_key_denotes_the_stored_instant() { .expect("time_bucket over the time key must succeed"); assert_eq!(direct.len(), 1, "one stored point: {direct:?}"); - assert!( - direct[0] == EARLY_BUCKET_ISO || direct[0] == EARLY_BUCKET_MICROS, - "time_bucket of the time key must denote {EARLY}: expected {EARLY_BUCKET_ISO} \ - or {EARLY_BUCKET_MICROS}, got {direct:?}" + assert_eq!( + direct[0], EARLY_BUCKET_ISO, + "time_bucket of the time key must denote {EARLY}: got {direct:?}" ); } @@ -293,10 +278,9 @@ async fn a_joined_time_bucket_of_a_time_key_denotes_the_stored_instant() { .expect("time_bucket over the time key through a JOIN must succeed"); assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); - assert!( - joined[0] == EARLY_BUCKET_ISO || joined[0] == EARLY_BUCKET_MICROS, - "a joined time_bucket of the time key must denote {EARLY}: expected \ - {EARLY_BUCKET_ISO} or {EARLY_BUCKET_MICROS}, got {joined:?}" + assert_eq!( + joined[0], EARLY_BUCKET_ISO, + "a joined time_bucket of the time key must denote {EARLY}: got {joined:?}" ); } @@ -461,10 +445,9 @@ async fn a_grouped_time_bucket_denotes_the_stored_instant() { "one stored point falls in one group: {rows:?}" ); - assert!( - rows[0][0] == EARLY_ISO || rows[0][0] == EARLY_MICROS, - "a grouped time_bucket key must denote {EARLY}: expected {EARLY_ISO} \ - or {EARLY_MICROS}, got {rows:?}" + assert_eq!( + rows[0][0], EARLY_ISO, + "a grouped time_bucket key must denote {EARLY}: got {rows:?}" ); } From 34d5cbe7257cbec1a9746a0c1ef825e2da145eb1 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 16 Sep 2026 17:54:11 +0800 Subject: [PATCH 07/21] fix(columnar): type time cells by declared column type, not codec The segment reader infers a column's physical kind from its codec, so a declared TIMESTAMP/TIMESTAMPTZ column decodes as the same Int64 kind as any other eight-byte integer column, losing the instant/millis distinction the row emitter needs. ColumnType::instant_kind() and time_cell() give every reader (segment decode, memtable access, PK encoding) one place to type a stored time cell from the schema's declared type: Timestamp yields a naive instant, Timestamptz a UTC instant, and every other type (SystemTimestamp, Duration, the bitemporal audit columns) keeps the raw integer. decoded_col_to_value and ColumnData::get_value now take the column's declared ColumnType and merge the Int64/Timestamp decode arms, since both read the same physical storage. row_to_projected_json is renamed to row_to_projected_value and returns a nodedb_types::Value directly instead of routing through serde_json, since every caller already works with typed row values and now reuses the shared decoder for the DML predicate row read as well as the base scan. DecodedColumn drops #[non_exhaustive] now that every reader matches it exhaustively. columnar_read/scan.rs is split into a scan/ directory (execute, order, params, block_skip) to stay within the file size limit as its callers change. --- .../src/materialize_rows/extract.rs | 22 +- .../src/memtable/column_data/access.rs | 13 +- nodedb-columnar/src/memtable/iter.rs | 58 ++- nodedb-columnar/src/pk_index.rs | 2 +- nodedb-columnar/src/reader/types.rs | 5 +- nodedb-types/src/columnar/column_type.rs | 69 +++- .../columnar_checkpoint/geometry_restore.rs | 5 +- .../dispatch/meta_retention/columnar_plain.rs | 21 +- .../handlers/columnar_read/convert.rs | 123 +++--- .../columnar_read/materialize_scan.rs | 9 +- .../handlers/columnar_read/scan/block_skip.rs | 68 ++++ .../{scan.rs => scan/execute.rs} | 384 +++++++++--------- .../handlers/columnar_read/scan/mod.rs | 10 + .../handlers/columnar_read/scan/order.rs | 43 ++ .../handlers/columnar_read/scan/params.rs | 44 ++ .../handlers/columnar_read/scan_flushed.rs | 78 ++-- .../handlers/columnar_write/read_prior.rs | 3 +- .../transaction/overlay/columnar_merge.rs | 24 +- .../handlers/transaction/overlay/mod.rs | 4 +- .../transaction/overlay/spatial_merge.rs | 11 +- .../stage_write/stage_columnar_dml.rs | 14 +- nodedb/src/data/executor/row_shape.rs | 111 ++++- nodedb/src/data/executor/scan_normalize.rs | 6 +- .../tests/wire/cases/sql_division_by_zero.rs | 4 +- .../cases/strict_typed_column_rendering.rs | 91 +++++ 25 files changed, 841 insertions(+), 381 deletions(-) create mode 100644 nodedb/src/data/executor/handlers/columnar_read/scan/block_skip.rs rename nodedb/src/data/executor/handlers/columnar_read/{scan.rs => scan/execute.rs} (72%) create mode 100644 nodedb/src/data/executor/handlers/columnar_read/scan/mod.rs create mode 100644 nodedb/src/data/executor/handlers/columnar_read/scan/order.rs create mode 100644 nodedb/src/data/executor/handlers/columnar_read/scan/params.rs diff --git a/nodedb-columnar/src/materialize_rows/extract.rs b/nodedb-columnar/src/materialize_rows/extract.rs index cd8d7a3b7..faab06c22 100644 --- a/nodedb-columnar/src/materialize_rows/extract.rs +++ b/nodedb-columnar/src/materialize_rows/extract.rs @@ -22,11 +22,15 @@ pub(crate) fn extract_row_value( use nodedb_types::value::Value; let v = match col { - DecodedColumn::Int64 { values, valid } => { + // The reader infers a column's physical kind from its codec, so a + // time column decodes as `Int64`. The declared type decides the cell: + // an instant column yields the variant its declared type names; + // every other type yields the integer stored. + DecodedColumn::Int64 { values, valid } | DecodedColumn::Timestamp { values, valid } => { if !valid[row_idx] { Value::Null } else { - Value::Integer(values[row_idx]) + col_type.time_cell(values[row_idx]) } } DecodedColumn::Float64 { values, valid } => { @@ -36,20 +40,6 @@ pub(crate) fn extract_row_value( Value::Float(values[row_idx]) } } - DecodedColumn::Timestamp { values, valid } => { - if !valid[row_idx] { - Value::Null - } else { - let micros = values[row_idx]; - let dt = nodedb_types::datetime::NdbDateTime::from_micros(micros); - match col_type { - nodedb_types::columnar::ColumnType::Timestamptz - | nodedb_types::columnar::ColumnType::SystemTimestamp => Value::DateTime(dt), - // Timestamp (naive) and anything else that maps to i64 storage. - _ => Value::NaiveDateTime(dt), - } - } - } DecodedColumn::Bool { values, valid } => { if !valid[row_idx] { Value::Null diff --git a/nodedb-columnar/src/memtable/column_data/access.rs b/nodedb-columnar/src/memtable/column_data/access.rs index 1be4bb1c7..2d6a769ad 100644 --- a/nodedb-columnar/src/memtable/column_data/access.rs +++ b/nodedb-columnar/src/memtable/column_data/access.rs @@ -2,6 +2,7 @@ //! Read-only access methods on `ColumnData`: validity checks, value extraction. +use nodedb_types::columnar::ColumnType; use nodedb_types::value::Value; use nodedb_types::value_from_msgpack; @@ -57,7 +58,13 @@ impl ColumnData { } /// Extract a single row's value as `nodedb_types::Value`. - pub(crate) fn get_value(&self, row: usize) -> Value { + /// + /// A time cell is typed by the column's declared type: an instant column + /// yields `Value::NaiveDateTime` (`Timestamp`) or `Value::DateTime` + /// (`Timestamptz`) from the stored epoch microseconds, and every other + /// declared type backed by time storage (`SystemTimestamp`, `Duration`) + /// yields the integer stored. + pub(crate) fn get_value(&self, row: usize, declared: &ColumnType) -> Value { if self.is_null(row) { return Value::Null; } @@ -65,9 +72,7 @@ impl ColumnData { Self::Int64 { values, .. } => Value::Integer(values[row]), Self::Float64 { values, .. } => Value::Float(values[row]), Self::Bool { values, .. } => Value::Bool(values[row]), - Self::Timestamp { values, .. } => Value::DateTime( - nodedb_types::datetime::NdbDateTime::from_micros(values[row]), - ), + Self::Timestamp { values, .. } => declared.time_cell(values[row]), Self::Decimal { values, .. } => { Value::Decimal(rust_decimal::Decimal::deserialize(values[row])) } diff --git a/nodedb-columnar/src/memtable/iter.rs b/nodedb-columnar/src/memtable/iter.rs index a80236c40..d411a6b6b 100644 --- a/nodedb-columnar/src/memtable/iter.rs +++ b/nodedb-columnar/src/memtable/iter.rs @@ -2,6 +2,7 @@ //! Row-oriented iteration and single-row lookup over the memtable. +use nodedb_types::columnar::ColumnDef; use nodedb_types::value::Value; use super::column_data::ColumnData; @@ -9,9 +10,13 @@ use super::core::ColumnarMemtable; impl ColumnarMemtable { /// Iterate rows as `Vec`. For scan/read operations. + /// + /// Every cell is typed by its column's declared type, so a time cell + /// comes back as the instant variant the schema declares. pub fn iter_rows(&self) -> MemtableRowIter<'_> { MemtableRowIter { columns: &self.columns, + column_defs: &self.schema.columns, row_count: self.row_count, current: 0, } @@ -23,8 +28,8 @@ impl ColumnarMemtable { return None; } let mut row = Vec::with_capacity(self.columns.len()); - for col in &self.columns { - row.push(col.get_value(row_idx)); + for (col, def) in self.columns.iter().zip(&self.schema.columns) { + row.push(col.get_value(row_idx, &def.column_type)); } Some(row) } @@ -33,6 +38,7 @@ impl ColumnarMemtable { /// Row iterator over a columnar memtable. pub struct MemtableRowIter<'a> { columns: &'a [ColumnData], + column_defs: &'a [ColumnDef], row_count: usize, current: usize, } @@ -45,8 +51,8 @@ impl Iterator for MemtableRowIter<'_> { return None; } let mut row = Vec::with_capacity(self.columns.len()); - for col in self.columns { - row.push(col.get_value(self.current)); + for (col, def) in self.columns.iter().zip(self.column_defs) { + row.push(col.get_value(self.current, &def.column_type)); } self.current += 1; Some(row) @@ -59,3 +65,47 @@ impl Iterator for MemtableRowIter<'_> { } impl ExactSizeIterator for MemtableRowIter<'_> {} + +#[cfg(test)] +mod tests { + use nodedb_types::NdbDateTime; + use nodedb_types::columnar::{ColumnDef, ColumnType, ColumnarSchema}; + use nodedb_types::value::Value; + + use super::super::core::ColumnarMemtable; + + const MICROS: i64 = 1_583_402_400_000_000; + + /// A row read back from the memtable types each time cell by its declared + /// column: `Timestamp` is a naive instant, `Timestamptz` a UTC instant, + /// `SystemTimestamp` the integer stored. + #[test] + fn a_time_cell_reads_back_as_its_declared_type() { + let schema = ColumnarSchema::new(vec![ + ColumnDef::required("id", ColumnType::Int64).with_primary_key(), + ColumnDef::required("at", ColumnType::Timestamp), + ColumnDef::required("at_tz", ColumnType::Timestamptz), + ColumnDef::required("sys", ColumnType::SystemTimestamp), + ]) + .expect("valid schema"); + let mut mt = ColumnarMemtable::new(&schema); + let dt = NdbDateTime::from_micros(MICROS); + mt.append_row(&[ + Value::Integer(1), + Value::NaiveDateTime(dt), + Value::DateTime(dt), + Value::Integer(7), + ]) + .expect("append"); + + let expected = vec![ + Value::Integer(1), + Value::NaiveDateTime(dt), + Value::DateTime(dt), + Value::Integer(7), + ]; + assert_eq!(mt.get_row(0), Some(expected.clone())); + assert_eq!(mt.iter_rows().collect::>(), vec![expected]); + assert_eq!(mt.get_row(1), None); + } +} diff --git a/nodedb-columnar/src/pk_index.rs b/nodedb-columnar/src/pk_index.rs index f5014f58f..3bb691f32 100644 --- a/nodedb-columnar/src/pk_index.rs +++ b/nodedb-columnar/src/pk_index.rs @@ -178,7 +178,7 @@ pub fn encode_pk(value: &nodedb_types::value::Value) -> Vec { Value::String(s) => s.as_bytes().to_vec(), Value::Uuid(s) => s.as_bytes().to_vec(), Value::Decimal(d) => d.serialize().to_vec(), - Value::DateTime(dt) => { + Value::DateTime(dt) | Value::NaiveDateTime(dt) => { let sortable = (dt.micros as u64) ^ (1u64 << 63); sortable.to_be_bytes().to_vec() } diff --git a/nodedb-columnar/src/reader/types.rs b/nodedb-columnar/src/reader/types.rs index cf8ebd34f..8f0c605ed 100644 --- a/nodedb-columnar/src/reader/types.rs +++ b/nodedb-columnar/src/reader/types.rs @@ -1,8 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 /// Decoded column data from a segment scan. +/// +/// The variant set is the closed set of physical column encodings, so a +/// reader matches it exhaustively and the compiler names every site an added +/// encoding must handle. #[derive(Debug)] -#[non_exhaustive] pub enum DecodedColumn { Int64 { values: Vec, diff --git a/nodedb-types/src/columnar/column_type.rs b/nodedb-types/src/columnar/column_type.rs index 262ef622a..7eb45829c 100644 --- a/nodedb-types/src/columnar/column_type.rs +++ b/nodedb-types/src/columnar/column_type.rs @@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize}; +use crate::InstantKind; use crate::value::Value; /// Typed column definition for strict document and columnar collections. @@ -118,7 +119,56 @@ impl ColumnType { /// `SystemTimestamp` is not an instant. It is engine-assigned from HLC at /// commit and the planner types it as text. pub const fn is_instant(&self) -> bool { - matches!(self, Self::Timestamp | Self::Timestamptz) + self.instant_kind().is_some() + } + + /// The instant kind a column of this type carries: `Naive` for + /// `Timestamp`, `Utc` for `Timestamptz`, `None` for every other type. + /// + /// A stored time cell is wrapped as `Value::NaiveDateTime` or + /// `Value::DateTime` by this kind, so a reader never has to decide which + /// variant a column means. `SystemTimestamp` and `Duration` share the + /// eight-byte time storage but are not instants: their cells read back as + /// the integer stored. + pub const fn instant_kind(&self) -> Option { + match self { + Self::Timestamp => Some(InstantKind::Naive), + Self::Timestamptz => Some(InstantKind::Utc), + Self::Int64 + | Self::Float64 + | Self::String + | Self::Bool + | Self::Bytes + | Self::SystemTimestamp + | Self::Decimal { .. } + | Self::Geometry + | Self::Vector(_) + | Self::SparseVector + | Self::Uuid + | Self::Json + | Self::Ulid + | Self::Duration + | Self::Array + | Self::Set + | Self::Regex + | Self::Range + | Self::Record => None, + } + } + + /// Type a stored time cell by this declared type. + /// + /// An instant column (`Timestamp`, `Timestamptz`) wraps `micros` as its + /// `instant_kind()` variant; every other declared type backed by + /// eight-byte time storage (`SystemTimestamp`, `Duration`) returns the + /// integer stored. The single mapping every segment-cell and memtable-cell + /// reader uses, so a row reads identically whether the reader inferred the + /// column's physical kind as `Int64` or `Timestamp`. + pub fn time_cell(&self, micros: i64) -> Value { + match self.instant_kind() { + Some(kind) => kind.from_micros(micros), + None => Value::Integer(micros), + } } /// Return the canonical PostgreSQL type OID for this column type. @@ -223,6 +273,23 @@ mod tests { crate::datetime::NdbDateTime::from_micros(0) } + #[test] + fn instant_kind_names_the_variant_a_time_column_reads_back_as() { + assert_eq!( + ColumnType::Timestamp.instant_kind(), + Some(InstantKind::Naive) + ); + assert_eq!( + ColumnType::Timestamptz.instant_kind(), + Some(InstantKind::Utc) + ); + assert_eq!(ColumnType::SystemTimestamp.instant_kind(), None); + assert_eq!(ColumnType::Duration.instant_kind(), None); + assert_eq!(ColumnType::Int64.instant_kind(), None); + assert!(ColumnType::Timestamp.is_instant()); + assert!(!ColumnType::SystemTimestamp.is_instant()); + } + #[test] fn to_pg_oid_stable() { assert_eq!(ColumnType::Bool.to_pg_oid(), 16); diff --git a/nodedb/src/data/executor/columnar_checkpoint/geometry_restore.rs b/nodedb/src/data/executor/columnar_checkpoint/geometry_restore.rs index afd5ff1b0..86f592e92 100644 --- a/nodedb/src/data/executor/columnar_checkpoint/geometry_restore.rs +++ b/nodedb/src/data/executor/columnar_checkpoint/geometry_restore.rs @@ -199,7 +199,10 @@ impl CoreLoop { out.push( decoded_cols .iter() - .map(|dc| decoded_col_to_value(dc, row_idx)) + .zip(&schema.columns) + .map(|(dc, col_def)| { + decoded_col_to_value(dc, row_idx, &col_def.column_type) + }) .collect(), ); } diff --git a/nodedb/src/data/executor/dispatch/meta_retention/columnar_plain.rs b/nodedb/src/data/executor/dispatch/meta_retention/columnar_plain.rs index 73e3fb9e2..075ee38af 100644 --- a/nodedb/src/data/executor/dispatch/meta_retention/columnar_plain.rs +++ b/nodedb/src/data/executor/dispatch/meta_retention/columnar_plain.rs @@ -93,9 +93,16 @@ impl CoreLoop { engine: "columnar".into(), detail: format!("read _ts_system: {e}"), })?; - let pk_cols: Vec = pk_indices + let pk_cols: Vec<( + nodedb_columnar::reader::DecodedColumn, + &nodedb_types::columnar::ColumnType, + )> = pk_indices .iter() - .map(|&i| reader.read_column(i)) + .map(|&i| { + reader + .read_column(i) + .map(|col| (col, &schema.columns[i].column_type)) + }) .collect::>() .map_err(|e| crate::Error::Storage { engine: "columnar".into(), @@ -201,13 +208,19 @@ fn int64_from_decoded(col: &nodedb_columnar::reader::DecodedColumn, row_idx: usi } } +/// Encode the primary key of one flushed row. Each cell is typed by its +/// declared column type, so the key bytes match what the memtable path +/// encodes for the same row: a `TIMESTAMP` key cell is an instant on both. fn encode_pk_from_decoded_cols( - cols: &[nodedb_columnar::reader::DecodedColumn], + cols: &[( + nodedb_columnar::reader::DecodedColumn, + &nodedb_types::columnar::ColumnType, + )], row_idx: usize, ) -> Vec { let values: Vec = cols .iter() - .map(|c| decoded_col_to_value(c, row_idx)) + .map(|(c, declared)| decoded_col_to_value(c, row_idx, declared)) .collect(); if values.len() == 1 { nodedb_columnar::pk_index::encode_pk(&values[0]) diff --git a/nodedb/src/data/executor/handlers/columnar_read/convert.rs b/nodedb/src/data/executor/handlers/columnar_read/convert.rs index 076a72b07..6300f0bc5 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/convert.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/convert.rs @@ -1,54 +1,35 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Value conversions: engine Value → JSON for response encoding, and -//! timeseries columnar cell → raw msgpack for the timeseries scan path. - -/// Convert a `nodedb_types::Value` to `serde_json::Value` for response encoding. -pub(in crate::data::executor) fn value_to_json( - val: &nodedb_types::value::Value, -) -> serde_json::Value { - use nodedb_types::value::Value; - match val { - Value::Null => serde_json::Value::Null, - Value::Bool(b) => serde_json::Value::Bool(*b), - Value::Integer(i) => serde_json::Value::Number((*i).into()), - Value::Float(f) => serde_json::Number::from_f64(*f) - .map(serde_json::Value::Number) - .unwrap_or(serde_json::Value::Null), - Value::String(s) => serde_json::Value::String(s.clone()), - Value::DateTime(dt) | Value::NaiveDateTime(dt) => serde_json::Value::String(dt.to_string()), - Value::Decimal(d) => serde_json::Value::String(d.to_string()), - Value::Uuid(s) => serde_json::Value::String(s.clone()), - Value::Bytes(b) => { - use base64::Engine; - serde_json::Value::String(base64::engine::general_purpose::STANDARD.encode(b)) - } - Value::Array(arr) => serde_json::Value::Array(arr.iter().map(value_to_json).collect()), - Value::Geometry(g) => serde_json::to_value(g).unwrap_or(serde_json::Value::Null), - Value::Object(map) => { - let obj: serde_json::Map = map - .iter() - .map(|(k, v)| (k.clone(), value_to_json(v))) - .collect(); - serde_json::Value::Object(obj) - } - _ => serde_json::Value::Null, - } -} +//! Value conversions: a decoded columnar row → its projected response +//! object, and timeseries columnar cell → raw msgpack for the timeseries scan +//! path. + +use std::collections::HashMap; -/// Project a decoded columnar row into the scan's response JSON shape: -/// column projection, the forced `_ts_system` audit column, and computed -/// (scalar-expression) columns. Shared by the base memtable scan loop and -/// the in-transaction overlay merge (`merge_overlay_into_columnar_scan`) so -/// a staged row's JSON is built identically to a base row's. -pub(in crate::data::executor) fn row_to_projected_json( - row: &[nodedb_types::value::Value], +use nodedb_types::value::Value; + +/// Project a decoded columnar row into the scan's response object: column +/// projection, the forced `_ts_system` audit column, and computed +/// (scalar-expression) columns. Shared by the flushed-segment scan, the base +/// memtable scan loop, the in-transaction overlay merge +/// (`merge_overlay_into_columnar_scan`), and the predicate DML row read, so a +/// staged row's object is built identically to a base row's. +/// +/// Every cell is carried as the `Value` the row reader typed it as, so a +/// declared `TIMESTAMP` / `TIMESTAMPTZ` cell is an instant here and is +/// written as the instant ext by `value_to_msgpack`, from the live memtable +/// and from a flushed segment alike. The bitemporal `_ts_system`, +/// `_ts_valid_from`, `_ts_valid_until` columns are declared `Int64` and stay +/// integers: they hold epoch milliseconds with `i64::MIN` / `i64::MAX` as +/// the unbounded sentinels, which no instant can carry. +pub(in crate::data::executor) fn row_to_projected_value( + row: &[Value], schema: &nodedb_types::columnar::ColumnarSchema, projection: &[String], computed_cols: &[crate::bridge::expr_eval::ComputedColumn], all_versions: bool, -) -> crate::Result { - let mut obj = serde_json::Map::new(); +) -> crate::Result { + let mut obj: HashMap = HashMap::with_capacity(schema.columns.len()); for (i, col_def) in schema.columns.iter().enumerate() { let force_system_col = all_versions && col_def.name == nodedb_types::columnar::schema::TS_SYSTEM; @@ -59,22 +40,21 @@ pub(in crate::data::executor) fn row_to_projected_json( { continue; } - if i < row.len() { - obj.insert(col_def.name.clone(), value_to_json(&row[i])); + if let Some(cell) = row.get(i) { + obj.insert(col_def.name.clone(), cell.clone()); } } if !computed_cols.is_empty() { - let doc_val = nodedb_types::Value::from(serde_json::Value::Object(obj.clone())); + let doc_val = Value::Object(obj.clone()); for cc in computed_cols { - let existing = obj.get(&cc.alias); - if matches!(existing, Some(v) if !v.is_null()) { + if matches!(obj.get(&cc.alias), Some(v) if !matches!(v, Value::Null)) { continue; } // A computed column is projection-shaped: a division/modulo-by- // zero fails the whole scan rather than silently materializing // NULL into the response row. let v = cc.expr.eval(&doc_val)?; - obj.insert(cc.alias.clone(), serde_json::Value::from(v)); + obj.insert(cc.alias.clone(), v); } if !projection.is_empty() { obj.retain(|k, _| { @@ -84,7 +64,7 @@ pub(in crate::data::executor) fn row_to_projected_json( }); } } - Ok(serde_json::Value::Object(obj)) + Ok(Value::Object(obj)) } /// Write a timeseries columnar memtable cell value directly as msgpack bytes. @@ -239,6 +219,47 @@ mod tests { assert_eq!(buf, expected); } + /// A projected row carries a declared `TIMESTAMP` cell as the instant the + /// row reader typed it, and the `_ts_system` audit column as the integer + /// it is declared as. + #[test] + fn a_projected_row_keeps_instant_cells_and_integer_system_time() { + use nodedb_types::NdbDateTime; + use nodedb_types::columnar::schema::TS_SYSTEM; + use nodedb_types::columnar::{ColumnDef, ColumnType}; + + let schema = nodedb_types::columnar::ColumnarSchema::new(vec![ + ColumnDef::required("id", ColumnType::Int64).with_primary_key(), + ColumnDef::required("at", ColumnType::Timestamp), + ColumnDef::required(TS_SYSTEM, ColumnType::Int64), + ]) + .expect("valid schema"); + let at = Value::NaiveDateTime(NdbDateTime::from_micros(MS * 1000)); + let row = [Value::Integer(1), at.clone(), Value::Integer(MS)]; + + let Value::Object(all) = + row_to_projected_value(&row, &schema, &[], &[], false).expect("project") + else { + panic!("a projected row is an object"); + }; + assert_eq!(all.get("at"), Some(&at)); + assert_eq!(all.get(TS_SYSTEM), Some(&Value::Integer(MS))); + + let projection = ["at".to_string()]; + let Value::Object(audit) = + row_to_projected_value(&row, &schema, &projection, &[], true).expect("project") + else { + panic!("a projected row is an object"); + }; + assert_eq!(audit.get("at"), Some(&at)); + assert_eq!( + audit.get(TS_SYSTEM), + Some(&Value::Integer(MS)), + "an all-versions read forces the system-time column into the projection" + ); + assert!(!audit.contains_key("id")); + } + #[test] fn a_millisecond_count_past_the_microsecond_range_is_an_error() { let err = emit(TimeKind::Instant(InstantKind::Naive), i64::MAX) diff --git a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs index 9b5cdc96f..7220da011 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs @@ -166,7 +166,11 @@ impl CoreLoop { // Bitemporal system-time filter. if let (Some(ts_idx), Some(cutoff)) = (ts_system_idx, system_as_of_ms) { - let ts_val = decoded_col_to_value(&decoded_cols[ts_idx], row_idx); + let ts_val = decoded_col_to_value( + &decoded_cols[ts_idx], + row_idx, + &schema.columns[ts_idx].column_type, + ); if let Value::Integer(ts) = ts_val && ts > cutoff { @@ -177,7 +181,8 @@ impl CoreLoop { // Build a Value::Object for this row. let mut map = std::collections::HashMap::new(); for (col_idx, col_def) in schema.columns.iter().enumerate() { - let val = decoded_col_to_value(&decoded_cols[col_idx], row_idx); + let val = + decoded_col_to_value(&decoded_cols[col_idx], row_idx, &col_def.column_type); map.insert(col_def.name.clone(), val); } diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan/block_skip.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/block_skip.rs new file mode 100644 index 000000000..b54c7acc6 --- /dev/null +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/block_skip.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Block-boundary surrogate prefilter for the live memtable. + +use nodedb_types::Surrogate; +use nodedb_types::surrogate_bitmap::SurrogateBitmap; + +/// Whether the whole live memtable can be skipped before any row decoding. +/// +/// True when the prefilter is empty, or when none of the memtable's recorded +/// surrogates fall within the bitmap's `[min, max]` range. A memtable with no +/// recorded surrogate at all cannot be block-skipped: the row-boundary check +/// decides each of its rows. +pub(super) fn memtable_block_skipped( + bitmap: &SurrogateBitmap, + surrogates: &[Option], +) -> bool { + if bitmap.is_empty() { + return true; + } + let (mt_min, mt_max) = surrogates + .iter() + .flatten() + .fold((u32::MAX, u32::MIN), |(lo, hi), s| { + (lo.min(s.0), hi.max(s.0)) + }); + if mt_min > mt_max { + return false; + } + // The bitmap is non-empty, so both bounds are `Some`; the `None` arm is + // the type-level fallback and never skips a block. + match (bitmap.0.min(), bitmap.0.max()) { + (Some(bm_min), Some(bm_max)) => bm_max < mt_min || bm_min > mt_max, + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn bitmap(ids: &[u32]) -> SurrogateBitmap { + let mut b = SurrogateBitmap::new(); + for id in ids { + b.insert(Surrogate(*id)); + } + b + } + + #[test] + fn an_empty_prefilter_skips_the_block() { + assert!(memtable_block_skipped(&bitmap(&[]), &[Some(Surrogate(1))])); + } + + #[test] + fn a_disjoint_range_skips_the_block() { + let mt = [Some(Surrogate(10)), None, Some(Surrogate(20))]; + assert!(memtable_block_skipped(&bitmap(&[1, 5]), &mt)); + assert!(memtable_block_skipped(&bitmap(&[21, 30]), &mt)); + assert!(!memtable_block_skipped(&bitmap(&[15]), &mt)); + } + + #[test] + fn a_memtable_without_surrogates_is_never_block_skipped() { + assert!(!memtable_block_skipped(&bitmap(&[1]), &[None, None])); + assert!(!memtable_block_skipped(&bitmap(&[1]), &[])); + } +} diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs similarity index 72% rename from nodedb/src/data/executor/handlers/columnar_read/scan.rs rename to nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs index 1ad97ce26..2bafe1dc1 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/scan.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs @@ -1,59 +1,27 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Scan-params struct and the base scan entry point. +//! The columnar base scan entry point. use nodedb_types::columnar::schema::{TS_SYSTEM, TS_VALID_FROM, TS_VALID_UNTIL}; -use nodedb_types::surrogate_bitmap::SurrogateBitmap; +use nodedb_types::value::Value; use crate::bridge::envelope::{ErrorCode, Response}; use crate::bridge::expr_eval::ComputedColumn; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::columnar_read::bitemporal::bitemporal_row_visible; +use crate::data::executor::handlers::columnar_read::convert::row_to_projected_value; +use crate::data::executor::handlers::columnar_read::filter::row_matches_filters; +use crate::data::executor::handlers::columnar_read::scan_flushed::FlushedScanCtx; +use crate::data::executor::handlers::transaction::overlay::{ + ColumnarMatchedRow, ColumnarOverlayMergeParams, +}; use crate::data::executor::response_codec; use crate::data::executor::task::ExecutionTask; -use super::bitemporal::bitemporal_row_visible; -use super::filter::row_matches_filters; -use super::sort::{compare_sort_values, eval_row_sort_values}; - -/// Parameters for a columnar base scan. Bundled as a struct because the -/// raw parameter list exceeds the project's too-many-arguments bound. -pub(in crate::data::executor) struct ColumnarScanParams<'a> { - pub collection: &'a str, - pub projection: &'a [String], - pub limit: usize, - pub filters: &'a [u8], - /// RLS filter bytes — wiring is the responsibility of a separate - /// enforcement pass; the base scan handler itself does not consume - /// them (hence the `_` destructure). - #[allow(dead_code)] - pub rls_filters: &'a [u8], - pub sort_keys: &'a [nodedb_physical::physical_plan::SortKeySpec], - /// Bitemporal system-time selection. `Current` is a current-state read; - /// `AsOf(ms)` drops rows with `_ts_system > ms`; `AllVersions` emits every - /// `_ts_system` row ordered ascending (audit log), with the system-time - /// column projected. - pub system_time: nodedb_types::SystemTimeScope, - /// Bitemporal valid-time point: drop rows whose - /// `[_ts_valid_from, _ts_valid_until)` interval does not contain this - /// point. `None` skips valid-time filtering entirely. - pub valid_at_ms: Option, - /// Optional cross-engine surrogate prefilter. When `Some`, the scan - /// skips whole memtable blocks whose surrogate range does not intersect - /// the bitmap (block boundary) and skips individual rows whose surrogate - /// is absent from the bitmap (row boundary). `None` = no prefilter. - pub prefilter: Option<&'a SurrogateBitmap>, - /// MessagePack-serialized `Vec` for scalar projection - /// expressions such as JSON arrow operators. Empty slice means no - /// computed columns are requested. - pub computed_columns: &'a [u8], - /// The in-transaction identity of the caller, when the scan is issued - /// inside `BEGIN..COMMIT`. `Some` gates a post-scan overlay merge - /// (`merge_overlay_into_columnar_scan`) so the scan observes the - /// transaction's own staged, not-yet-durable `ColumnarOp::Insert` rows - /// (read-your-own-writes). `None` for autocommit reads. - pub txn_id: Option, -} +use super::block_skip::memtable_block_skipped; +use super::order::order_matched; +use super::params::ColumnarScanParams; impl CoreLoop { /// Execute a base columnar scan: flushed segments first, then the live @@ -64,6 +32,11 @@ impl CoreLoop { /// surrogate membership exactly like the live-memtable phase. See /// `scan_normalize::scan_columnar` for the parallel read path — keep both /// in sync on segment-iteration changes. + /// + /// Every result row is a `Value::Object` built by + /// `row_to_projected_value` and encoded by `encode_value_vec`, so a + /// declared `TIMESTAMP` / `TIMESTAMPTZ` cell reaches the client as a + /// typed instant from either phase. pub(in crate::data::executor) fn execute_columnar_scan( &mut self, task: &ExecutionTask, @@ -120,7 +93,11 @@ impl CoreLoop { // Bound the materialized row count to a ceiling derived from the // memory budget (+1 row to detect "more exist") so the scan does // not pull the whole memtable into the `matched` Vec. - super::super::scan_budget::fetch_limit_for(limit, 0, scan_budget_bytes) + crate::data::executor::handlers::scan_budget::fetch_limit_for( + limit, + 0, + scan_budget_bytes, + ) } else { limit }; @@ -142,7 +119,7 @@ impl CoreLoop { Some(e) => e, None => { // Empty result for missing collection. - return match response_codec::encode_json_vec_as_msgpack(&[]) { + return match response_codec::encode_value_vec(&[]) { Ok(payload) => self.response_with_payload(task, payload), Err(e) => self.response_error( task, @@ -172,16 +149,12 @@ impl CoreLoop { Vec::new() }; - // Collect matched rows as (row_values, json_object) pairs. We keep - // the raw `Vec` for sort-key comparison — the JSON form is - // emitted only after ORDER BY + limit are applied. When no sort - // is requested we short-circuit the limit enforcement inside the - // loop to avoid materialising the entire memtable. - let mut matched: Vec<( - Option, - Vec, - serde_json::Value, - )> = Vec::new(); + // Collect matched rows as (surrogate, row_values, projected object) + // triples. The raw `Vec` is kept for sort-key comparison — the + // projected object is emitted only after ORDER BY + limit are + // applied. When no sort is requested the limit is enforced inside the + // loop so the whole memtable is not materialised. + let mut matched: Vec = Vec::new(); let scan_budget = if sort_keys.is_empty() { limit.saturating_mul(10).max(limit) } else { @@ -194,44 +167,10 @@ impl CoreLoop { let ts_valid_from_idx = schema.columns.iter().position(|c| c.name == TS_VALID_FROM); let ts_valid_until_idx = schema.columns.iter().position(|c| c.name == TS_VALID_UNTIL); - // Block-boundary prefilter: if a prefilter is present and none of - // the memtable's surrogates fall within the bitmap's [min, max] - // range, the entire memtable block can be skipped before any row - // decoding takes place. - let block_skipped = if let Some(bitmap) = prefilter { - if bitmap.is_empty() { - true - } else { - let surrogates = engine.memtable_surrogates(); - // Compute the surrogate range of non-None entries in the memtable. - let (mt_min, mt_max) = surrogates - .iter() - .flatten() - .fold((u32::MAX, u32::MIN), |(lo, hi), s| { - (lo.min(s.0), hi.max(s.0)) - }); - // If no surrogate was found (mt_min > mt_max) or the bitmap's - // range lies entirely outside the memtable range, skip. - if mt_min > mt_max { - // No surrogates in memtable — cannot apply block skip. - false - } else { - // The bitmap is non-empty (guarded above); min/max are - // always `Some`. Pattern-match to avoid `unwrap` in - // non-test code while keeping the compiler honest. - match (bitmap.0.min(), bitmap.0.max()) { - (Some(bm_min), Some(bm_max)) => { - // Disjoint ranges: bitmap entirely before or after memtable. - bm_max < mt_min || bm_min > mt_max - } - // Unreachable: `is_empty()` was already checked above. - _ => false, - } - } - } - } else { - false - }; + // Block-boundary prefilter: skip the whole live memtable before any + // row decoding when no recorded surrogate can be in the bitmap. + let block_skipped = prefilter + .is_some_and(|bitmap| memtable_block_skipped(bitmap, engine.memtable_surrogates())); // Deadline safe points for this scan: the phase boundary below, every // 1024th memtable row, and the stage boundary after the sort. The @@ -250,7 +189,7 @@ impl CoreLoop { // apply a per-row membership test below, mirroring the live-memtable // phase. See the method-level doc comment for the full rationale. if let Err(e) = self.scan_flushed_columnar_segments( - super::scan_flushed::FlushedScanCtx { + FlushedScanCtx { collection, engine_key: &engine_key, schema, @@ -296,9 +235,9 @@ impl CoreLoop { { // Row-boundary prefilter: skip this row when its surrogate is // absent from the bitmap. Rows without a recorded surrogate - // (legacy / test paths) are always included when no prefilter - // is active; when a prefilter is active they are excluded - // because the surrogate identity is unknown. + // are always included when no prefilter is active; when a + // prefilter is active they are excluded because the surrogate + // identity is unknown. if let Some(bitmap) = prefilter { match row_surrogate { Some(s) if bitmap.contains(s) => {} @@ -331,7 +270,7 @@ impl CoreLoop { } } } - let obj = match super::convert::row_to_projected_json( + let obj = match row_to_projected_value( &row, schema, projection, @@ -339,7 +278,7 @@ impl CoreLoop { all_versions, ) { Ok(v) => v, - // `row_to_projected_json` returns `crate::Result<_>` + // `row_to_projected_value` returns `crate::Result<_>` // (unlike `row_matches_filters` above) — its only // fallible step is a computed-column expression eval, // and `computed_cols` here is real, not `&[]` like the @@ -383,7 +322,7 @@ impl CoreLoop { collection.to_string(), ); if let Err(e) = self.merge_overlay_into_columnar_scan( - crate::data::executor::handlers::transaction::overlay::ColumnarOverlayMergeParams { + ColumnarOverlayMergeParams { txn_id, coll_key: &coll_key, schema, @@ -405,43 +344,24 @@ impl CoreLoop { } } - if !sort_keys.is_empty() { - // Sort keys are expressions, so evaluating them can fail. Evaluate - // every row's keys first — `sort_by` has no way to report an error - // — then order by the results. - let mut keyed = Vec::with_capacity(matched.len()); - for (_, row, _) in &matched { - match eval_row_sort_values(row, schema, sort_keys) { - Ok(values) => keyed.push(values), - Err(e) => return self.response_error(task, e), - } - } - let mut order: Vec = (0..matched.len()).collect(); - order.sort_by(|&a, &b| compare_sort_values(&keyed[a], &keyed[b], sort_keys)); - let mut reordered: Vec<_> = order - .into_iter() - .map(|i| matched[i].clone()) - .collect::>(); - std::mem::swap(&mut matched, &mut reordered); - // Safe point: the sort is the one stage whose cost grows with the - // matched row count, and it has just finished. Nothing is emitted - // yet, so stopping here costs the client only the error. - if deadline.expired_now() { - return self.response_error(task, ErrorCode::DeadlineExceeded); - } - } else if all_versions { - // Audit-log order: ascending by system time. The hidden - // `_ts_system` column index was resolved above. - matched.sort_by(|(_, a, _), (_, b, _)| { - super::bitemporal::row_system_time(a, ts_system_idx) - .cmp(&super::bitemporal::row_system_time(b, ts_system_idx)) - }); + if let Err(e) = order_matched(&mut matched, schema, sort_keys, all_versions, ts_system_idx) + { + return self.response_error(task, e); + } + // Safe point: the sort is the one stage whose cost grows with the + // matched row count, and it has just finished. Nothing is emitted + // yet, so stopping here costs the client only the error. + if !sort_keys.is_empty() && deadline.expired_now() { + return self.response_error(task, ErrorCode::DeadlineExceeded); } - let results: Vec = - matched.into_iter().take(limit).map(|(_, _, j)| j).collect(); + let results: Vec = matched + .into_iter() + .take(limit) + .map(|(_, _, obj)| obj) + .collect(); - let payload = match response_codec::encode_json_vec_as_msgpack(&results) { + let payload = match response_codec::encode_value_vec(&results) { Ok(payload) => payload, Err(e) => { return self.response_error( @@ -458,7 +378,11 @@ impl CoreLoop { // surface a deterministic error if it exceeds the budget rather than // silently truncating. Spatial scans are bounded (finite limit) and so // skip this check. - if unbounded && super::super::scan_budget::budget_exceeded(payload.len(), scan_budget_bytes) + if unbounded + && crate::data::executor::handlers::scan_budget::budget_exceeded( + payload.len(), + scan_budget_bytes, + ) { return self.response_error(task, ErrorCode::ResourcesExhausted); } @@ -471,15 +395,14 @@ impl CoreLoop { mod tests { //! Cross-engine prefilter coverage for FLUSHED plain-columnar segments. //! - //! These tests prove the silent-wrong-results fix: rows that live only in a - //! flushed segment (drained out of the live memtable) are now visible to a - //! prefiltered scan and are filtered per-row by their cross-engine - //! surrogate, instead of being skipped wholesale. Each test forces a real - //! flush — drain the memtable, encode a `SegmentWriter` segment, push the - //! bytes + the captured surrogate sidecar in lockstep, then `on_memtable_flushed` - //! clears the memtable — so the rows truly exist only in the flushed segment - //! before the scan runs. This mirrors the production flush block in - //! `handlers/columnar_write/insert.rs`. + //! Rows that live only in a flushed segment (drained out of the live + //! memtable) are visible to a prefiltered scan and are filtered per-row by + //! their cross-engine surrogate, instead of being skipped wholesale. Each + //! test forces a real flush — drain the memtable, encode a `SegmentWriter` + //! segment, push the bytes + the captured surrogate sidecar in lockstep, + //! then `on_memtable_flushed` clears the memtable — so the rows truly exist + //! only in the flushed segment before the scan runs. This mirrors the + //! production flush block in `handlers/columnar_write/insert.rs`. use std::time::{Duration, Instant}; @@ -487,7 +410,7 @@ mod tests { use nodedb_columnar::MutationEngine; use nodedb_types::columnar::{ColumnDef, ColumnType, ColumnarSchema}; use nodedb_types::value::Value; - use nodedb_types::{Surrogate, SurrogateBitmap}; + use nodedb_types::{NdbDateTime, Surrogate, SurrogateBitmap}; use crate::bridge::dispatch::{BridgeRequest, BridgeResponse}; use crate::bridge::envelope::{PhysicalPlan, Priority, Request}; @@ -497,10 +420,13 @@ mod tests { use super::ColumnarScanParams; + const MICROS: i64 = 1_583_402_400_000_000; + fn schema() -> ColumnarSchema { ColumnarSchema::new(vec![ ColumnDef::required("id", ColumnType::Int64).with_primary_key(), ColumnDef::required("name", ColumnType::String), + ColumnDef::required("at", ColumnType::Timestamp), ]) .expect("valid schema") } @@ -548,27 +474,30 @@ mod tests { }) } - /// Insert `(id, name, surrogate)` rows into a fresh engine, then run the - /// EXACT production flush sequence so the rows end up only in a flushed - /// segment. Registers the engine + populates both lockstep maps on `core`. - /// Returns the engine key. - fn insert_and_flush( - core: &mut CoreLoop, - collection: &str, - rows: &[(i64, &str, Surrogate)], - ) -> (DatabaseId, TenantId, String) { - let key = ( + fn engine_key(collection: &str) -> (DatabaseId, TenantId, String) { + ( DatabaseId::DEFAULT, TenantId::new(1), collection.to_string(), - ); - let mut engine = MutationEngine::new(collection.to_string(), schema()); - for (id, name, surr) in rows { - engine - .insert_with_surrogate(&[Value::Integer(*id), Value::String((*name).into())], *surr) - .expect("insert_with_surrogate"); - } + ) + } + fn row(id: i64, name: &str) -> [Value; 3] { + [ + Value::Integer(id), + Value::String(name.into()), + Value::NaiveDateTime(NdbDateTime::from_micros(MICROS)), + ] + } + + /// Run the EXACT production flush sequence on `engine` so its memtable + /// rows end up only in a flushed segment, populating both lockstep maps + /// on `core`. + fn flush( + core: &mut CoreLoop, + key: &(DatabaseId, TenantId, String), + engine: &mut MutationEngine, + ) { // ── Mirror handlers/columnar_write/insert.rs flush block ──────────── let new_segment_id = engine.next_segment_id(); let (seg_schema, columns, row_count) = engine.memtable_mut().drain_optimized(); @@ -608,18 +537,33 @@ mod tests { 0, "memtable surrogates cleared after flush" ); - core.columnar_engines.insert(key.clone(), engine); - key } - /// Run a prefiltered scan and return the decoded result rows. - fn scan_with_prefilter( + /// Insert `(id, name, surrogate)` rows into a fresh engine, then flush so + /// the rows end up only in a flushed segment. Registers the engine on + /// `core`. Returns the engine key. + fn insert_and_flush( core: &mut CoreLoop, collection: &str, - prefilter: Option<&SurrogateBitmap>, - ) -> Vec { - let task = make_task(); - let params = ColumnarScanParams { + rows: &[(i64, &str, Surrogate)], + ) -> (DatabaseId, TenantId, String) { + let key = engine_key(collection); + let mut engine = MutationEngine::new(collection.to_string(), schema()); + for (id, name, surr) in rows { + engine + .insert_with_surrogate(&row(*id, name), *surr) + .expect("insert_with_surrogate"); + } + flush(core, &key, &mut engine); + core.columnar_engines.insert(key.clone(), engine); + key + } + + fn scan_params<'a>( + collection: &'a str, + prefilter: Option<&'a SurrogateBitmap>, + ) -> ColumnarScanParams<'a> { + ColumnarScanParams { collection, projection: &[], limit: 0, @@ -631,24 +575,48 @@ mod tests { prefilter, computed_columns: &[], txn_id: None, - }; - let resp = core.execute_columnar_scan(&task, params); - let decoded: Vec = - zerompk::from_msgpack(resp.payload.as_bytes()).expect("decode scan payload"); - decoded.into_iter().map(|j| j.0).collect() + } + } + + fn decode_rows(payload: &[u8]) -> Vec { + match nodedb_types::value_from_msgpack(payload).expect("decode scan payload") { + Value::Array(rows) => rows, + other => panic!("scan payload must be an array of rows: {other:?}"), + } } - fn ids(rows: &[serde_json::Value]) -> Vec { + /// Run a prefiltered scan and return the decoded result rows. + fn scan_with_prefilter( + core: &mut CoreLoop, + collection: &str, + prefilter: Option<&SurrogateBitmap>, + ) -> Vec { + let task = make_task(); + let resp = core.execute_columnar_scan(&task, scan_params(collection, prefilter)); + decode_rows(resp.payload.as_bytes()) + } + + fn field<'a>(row: &'a Value, name: &str) -> Option<&'a Value> { + match row { + Value::Object(map) => map.get(name), + _ => None, + } + } + + fn ids(rows: &[Value]) -> Vec { let mut v: Vec = rows .iter() - .filter_map(|r| r.get("id").and_then(|x| x.as_i64())) + .filter_map(|r| match field(r, "id") { + Some(Value::Integer(i)) => Some(*i), + _ => None, + }) .collect(); v.sort_unstable(); v } /// A prefilter that includes SOME flushed-row surrogates must return - /// exactly those rows — proving flushed rows are now prefiltered, not + /// exactly those rows — proving flushed rows are prefiltered, not /// skipped wholesale. #[test] fn flushed_rows_are_prefiltered_not_skipped() { @@ -698,8 +666,8 @@ mod tests { ); } - /// Sanity: with NO prefilter all flushed rows are returned (unchanged - /// behaviour — the gate only applies when a prefilter is present). + /// Sanity: with NO prefilter all flushed rows are returned (the gate only + /// applies when a prefilter is present). #[test] fn flushed_rows_no_prefilter_returns_all() { let (mut core, _dir) = make_core(); @@ -718,6 +686,39 @@ mod tests { assert_eq!(ids(&rows), vec![1, 2, 3]); } + /// A declared `TIMESTAMP` cell is the same naive instant whether the row + /// is read from the live memtable or from a flushed segment. + #[test] + fn a_timestamp_cell_is_the_same_instant_before_and_after_flush() { + let (mut core, _dir) = make_core(); + let coll = "cf_instant"; + let key = engine_key(coll); + let mut engine = MutationEngine::new(coll.to_string(), schema()); + engine + .insert_with_surrogate(&row(1, "live"), Surrogate(701)) + .expect("insert_with_surrogate"); + core.columnar_engines.insert(key.clone(), engine); + + let expected = Value::NaiveDateTime(NdbDateTime::from_micros(MICROS)); + let live = scan_with_prefilter(&mut core, coll, None); + assert_eq!(live.len(), 1); + assert_eq!(field(&live[0], "at"), Some(&expected), "live memtable cell"); + + let mut engine = core + .columnar_engines + .remove(&key) + .expect("engine registered"); + flush(&mut core, &key, &mut engine); + core.columnar_engines.insert(key.clone(), engine); + let flushed = scan_with_prefilter(&mut core, coll, None); + assert_eq!(flushed.len(), 1); + assert_eq!( + field(&flushed[0], "at"), + Some(&expected), + "flushed segment cell" + ); + } + /// Build a task whose deadline is already in the past. fn expired_task() -> ExecutionTask { let mut task = make_task(); @@ -730,22 +731,7 @@ mod tests { collection: &str, task: &ExecutionTask, ) -> crate::bridge::envelope::Response { - core.execute_columnar_scan( - task, - ColumnarScanParams { - collection, - projection: &[], - limit: 0, - filters: &[], - rls_filters: &[], - sort_keys: &[], - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - prefilter: None, - computed_columns: &[], - txn_id: None, - }, - ) + core.execute_columnar_scan(task, scan_params(collection, None)) } /// A statement over its deadline is stopped DURING execution, not merely @@ -796,9 +782,7 @@ mod tests { let resp = scan_response(&mut core, coll, &make_task()); assert_eq!(resp.status, crate::bridge::envelope::Status::Ok); - let decoded: Vec = - zerompk::from_msgpack(resp.payload.as_bytes()).expect("decode scan payload"); - let rows: Vec = decoded.into_iter().map(|j| j.0).collect(); + let rows = decode_rows(resp.payload.as_bytes()); assert_eq!(ids(&rows), vec![1, 2]); } diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan/mod.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/mod.rs new file mode 100644 index 000000000..1849f434c --- /dev/null +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Columnar base scan: parameters, the entry point, and its stages. + +mod block_skip; +mod execute; +mod order; +mod params; + +pub(in crate::data::executor) use params::ColumnarScanParams; diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan/order.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/order.rs new file mode 100644 index 000000000..023f22f5b --- /dev/null +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/order.rs @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Result ordering for a columnar base scan: `ORDER BY` keys, or audit-log +//! system-time order for an all-versions read. + +use nodedb_physical::physical_plan::SortKeySpec; +use nodedb_types::columnar::ColumnarSchema; + +use crate::data::executor::handlers::columnar_read::bitemporal::row_system_time; +use crate::data::executor::handlers::columnar_read::sort::{ + compare_sort_values, eval_row_sort_values, +}; +use crate::data::executor::handlers::transaction::overlay::ColumnarMatchedRow; + +/// Order `matched` in place. +/// +/// With sort keys, every row's keys are evaluated first — `sort_by` has no +/// way to report an error — then the rows are ordered by them. A computed +/// key that divides by zero fails the statement. Without sort keys, an +/// all-versions read orders ascending by `_ts_system` (audit-log order) and a +/// current read keeps scan order. +pub(super) fn order_matched( + matched: &mut Vec, + schema: &ColumnarSchema, + sort_keys: &[SortKeySpec], + all_versions: bool, + ts_system_idx: Option, +) -> crate::Result<()> { + if !sort_keys.is_empty() { + let mut keyed = Vec::with_capacity(matched.len()); + for (_, row, _) in matched.iter() { + keyed.push(eval_row_sort_values(row, schema, sort_keys)?); + } + let mut indexed: Vec<_> = keyed.into_iter().zip(matched.drain(..)).collect(); + indexed.sort_by(|a, b| compare_sort_values(&a.0, &b.0, sort_keys)); + matched.extend(indexed.into_iter().map(|(_, row)| row)); + } else if all_versions { + matched.sort_by(|(_, a, _), (_, b, _)| { + row_system_time(a, ts_system_idx).cmp(&row_system_time(b, ts_system_idx)) + }); + } + Ok(()) +} diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan/params.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/params.rs new file mode 100644 index 000000000..824df3613 --- /dev/null +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/params.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Parameters for a columnar base scan. + +use nodedb_types::surrogate_bitmap::SurrogateBitmap; + +/// Parameters for a columnar base scan. Bundled as a struct because the +/// raw parameter list exceeds the project's too-many-arguments bound. +pub(in crate::data::executor) struct ColumnarScanParams<'a> { + pub collection: &'a str, + pub projection: &'a [String], + pub limit: usize, + pub filters: &'a [u8], + /// RLS filter bytes — wiring is the responsibility of a separate + /// enforcement pass; the base scan handler itself does not consume + /// them (hence the `_` destructure). + #[allow(dead_code)] + pub rls_filters: &'a [u8], + pub sort_keys: &'a [nodedb_physical::physical_plan::SortKeySpec], + /// Bitemporal system-time selection. `Current` is a current-state read; + /// `AsOf(ms)` drops rows with `_ts_system > ms`; `AllVersions` emits every + /// `_ts_system` row ordered ascending (audit log), with the system-time + /// column projected. + pub system_time: nodedb_types::SystemTimeScope, + /// Bitemporal valid-time point: drop rows whose + /// `[_ts_valid_from, _ts_valid_until)` interval does not contain this + /// point. `None` skips valid-time filtering entirely. + pub valid_at_ms: Option, + /// Optional cross-engine surrogate prefilter. When `Some`, the scan + /// skips whole memtable blocks whose surrogate range does not intersect + /// the bitmap (block boundary) and skips individual rows whose surrogate + /// is absent from the bitmap (row boundary). `None` = no prefilter. + pub prefilter: Option<&'a SurrogateBitmap>, + /// MessagePack-serialized `Vec` for scalar projection + /// expressions such as JSON arrow operators. Empty slice means no + /// computed columns are requested. + pub computed_columns: &'a [u8], + /// The in-transaction identity of the caller, when the scan is issued + /// inside `BEGIN..COMMIT`. `Some` gates a post-scan overlay merge + /// (`merge_overlay_into_columnar_scan`) so the scan observes the + /// transaction's own staged, not-yet-durable `ColumnarOp::Insert` rows + /// (read-your-own-writes). `None` for autocommit reads. + pub txn_id: Option, +} diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan_flushed.rs b/nodedb/src/data/executor/handlers/columnar_read/scan_flushed.rs index aa559445b..38116a597 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/scan_flushed.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/scan_flushed.rs @@ -1,18 +1,15 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Phase-1 flushed-segment scanning, extracted from `scan.rs` to keep that -//! file within the 500-line non-test limit. Logic is verbatim from -//! `execute_columnar_scan`; only the parameterisation changes. - -use nodedb_types::columnar::schema::TS_SYSTEM; +//! The flushed-segment pass of the columnar base scan. use crate::bridge::expr_eval::ComputedColumn; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::transaction::overlay::ColumnarMatchedRow; use crate::data::executor::scan_normalize::decoded_col_to_value; use super::bitemporal::bitemporal_row_visible; -use super::convert::value_to_json; +use super::convert::row_to_projected_value; use super::filter::row_matches_filters; /// Read-only context for the flushed-segment scan phase. All fields are @@ -40,18 +37,14 @@ impl CoreLoop { /// rows to `matched`. This is Phase 1 of `execute_columnar_scan`; Phase 2 /// (live memtable) continues in the caller with the same `matched` Vec. /// - /// The body is verbatim from the `// ── Phase 1: flushed segments ──` block - /// in `scan.rs`. Only local variables that were defined earlier in - /// `execute_columnar_scan` are now method parameters (borrowed read-only via - /// `FlushedScanCtx`) or the mutable `matched` accumulator. + /// Each cell is decoded by `decoded_col_to_value` with its declared column + /// type, and each row is projected by `row_to_projected_value` — the same + /// two steps the live-memtable phase applies — so a row reads identically + /// from a segment and from the memtable. pub(in crate::data::executor) fn scan_flushed_columnar_segments( &self, ctx: FlushedScanCtx<'_>, - matched: &mut Vec<( - Option, - Vec, - serde_json::Value, - )>, + matched: &mut Vec, ) -> crate::Result<()> { let FlushedScanCtx { collection, @@ -179,10 +172,14 @@ impl CoreLoop { } } - // Build the row as Vec using the shared decoder. + // Build the row as Vec using the shared decoder, + // typing each cell by its declared column type. let row: Vec = decoded_cols .iter() - .map(|dc| decoded_col_to_value(dc, row_idx)) + .zip(&schema.columns) + .map(|(dc, col_def)| { + decoded_col_to_value(dc, row_idx, &col_def.column_type) + }) .collect(); if !bitemporal_row_visible( @@ -201,43 +198,16 @@ impl CoreLoop { continue; } - let mut obj = serde_json::Map::new(); - for (i, col_def) in schema.columns.iter().enumerate() { - let force_system_col = all_versions && col_def.name == TS_SYSTEM; - if !projection.is_empty() - && !force_system_col - && !projection.iter().any(|p| p == &col_def.name) - && !computed_cols.iter().any(|cc| cc.alias == col_def.name) - { - continue; - } - if i < row.len() { - obj.insert(col_def.name.clone(), value_to_json(&row[i])); - } - } - if !computed_cols.is_empty() { - let doc_val = - nodedb_types::Value::from(serde_json::Value::Object(obj.clone())); - for cc in computed_cols { - let existing = obj.get(&cc.alias); - if matches!(existing, Some(v) if !v.is_null()) { - continue; - } - // Computed-column projection is - // projection-shaped: a division/modulo-by-zero - // fails the whole scan. - let v = cc.expr.eval(&doc_val)?; - obj.insert(cc.alias.clone(), serde_json::Value::from(v)); - } - if !projection.is_empty() { - obj.retain(|k, _| { - projection.iter().any(|p| p == k) - || computed_cols.iter().any(|cc| &cc.alias == k) - || (all_versions && k == TS_SYSTEM) - }); - } - } - matched.push((row_surrogate, row, serde_json::Value::Object(obj))); + // Computed-column projection is projection-shaped: a + // division/modulo-by-zero fails the whole scan. + let obj = row_to_projected_value( + &row, + schema, + projection, + computed_cols, + all_versions, + )?; + matched.push((row_surrogate, row, obj)); if sort_keys.is_empty() && matched.len() >= limit { break; } diff --git a/nodedb/src/data/executor/handlers/columnar_write/read_prior.rs b/nodedb/src/data/executor/handlers/columnar_write/read_prior.rs index 761748692..e10d77043 100644 --- a/nodedb/src/data/executor/handlers/columnar_write/read_prior.rs +++ b/nodedb/src/data/executor/handlers/columnar_write/read_prior.rs @@ -62,11 +62,12 @@ impl CoreLoop { usize::MAX, )?; let mut row = Vec::with_capacity(row_capacity); - for col_idx in 0..column_count { + for (col_idx, col_def) in schema.columns.iter().enumerate() { let decoded = reader.read_column(col_idx).ok()?; row.push(crate::data::executor::scan_normalize::decoded_col_to_value( &decoded, loc.row_index as usize, + &col_def.column_type, )); } Some(row) diff --git a/nodedb/src/data/executor/handlers/transaction/overlay/columnar_merge.rs b/nodedb/src/data/executor/handlers/transaction/overlay/columnar_merge.rs index 365e91fb4..e2c38f4dc 100644 --- a/nodedb/src/data/executor/handlers/transaction/overlay/columnar_merge.rs +++ b/nodedb/src/data/executor/handlers/transaction/overlay/columnar_merge.rs @@ -29,17 +29,17 @@ use nodedb_types::value::Value; use crate::bridge::expr_eval::ComputedColumn; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::handlers::columnar_read::convert::row_to_projected_json; +use crate::data::executor::handlers::columnar_read::convert::row_to_projected_value; use crate::data::executor::handlers::columnar_read::filter::row_matches_filters; use crate::data::executor::handlers::transaction::overlay::Staged; use crate::types::{DatabaseId, TenantId, TxnId}; /// One matched columnar row: its cross-engine surrogate (when known), the /// decoded schema-ordered column values, and the already-projected response -/// JSON. Shared shape between the base scan (`execute_columnar_scan`) and -/// this overlay merge. -pub(in crate::data::executor) type ColumnarMatchedRow = - (Option, Vec, serde_json::Value); +/// object (`Value::Object`). Shared shape between the base scan +/// (`execute_columnar_scan`), the predicate DML row read, and this overlay +/// merge. +pub(in crate::data::executor) type ColumnarMatchedRow = (Option, Vec, Value); /// Inputs for [`CoreLoop::merge_overlay_into_columnar_scan`]. pub(in crate::data::executor) struct ColumnarOverlayMergeParams<'a> { @@ -109,7 +109,7 @@ impl CoreLoop { .collect(); // Base-minus-superseded: a tombstoned row is dropped; a staged put - // replaces the row's decoded values + JSON and is re-checked against + // replaces the row's decoded values + object and is re-checked against // the scan predicate (an update may have moved the row out of the // result). A row with no recorded surrogate has no overlay identity // to resolve and is left untouched, matching the base scan's own @@ -121,7 +121,7 @@ impl CoreLoop { // checked once the retain pass finishes, aborting the merge before // the overlay-addition pass below runs. let mut first_err: Option = None; - matched.retain_mut(|(surrogate, row, json)| { + matched.retain_mut(|(surrogate, row, obj)| { if first_err.is_some() { return true; } @@ -140,14 +140,14 @@ impl CoreLoop { return true; } } - match row_to_projected_json( + match row_to_projected_value( &new_row, schema, projection, computed_cols, all_versions, ) { - Ok(v) => *json = v, + Ok(v) => *obj = v, Err(e) => { first_err = Some(e); return true; @@ -183,9 +183,9 @@ impl CoreLoop { if !predicate(&new_row)? { continue; } - let json = - row_to_projected_json(&new_row, schema, projection, computed_cols, all_versions)?; - matched.push((Some(Surrogate::new(surrogate)), new_row, json)); + let obj = + row_to_projected_value(&new_row, schema, projection, computed_cols, all_versions)?; + matched.push((Some(Surrogate::new(surrogate)), new_row, obj)); seen.insert(surrogate); } Ok(()) diff --git a/nodedb/src/data/executor/handlers/transaction/overlay/mod.rs b/nodedb/src/data/executor/handlers/transaction/overlay/mod.rs index f8e5719ff..66bd40c12 100644 --- a/nodedb/src/data/executor/handlers/transaction/overlay/mod.rs +++ b/nodedb/src/data/executor/handlers/transaction/overlay/mod.rs @@ -10,7 +10,9 @@ mod staged; mod timeseries_merge; mod vector_merge; -pub(in crate::data::executor) use columnar_merge::{ColumnarOverlayMergeParams, decode_staged_row}; +pub(in crate::data::executor) use columnar_merge::{ + ColumnarMatchedRow, ColumnarOverlayMergeParams, decode_staged_row, +}; pub(in crate::data::executor) use fts_merge::FtsMergeParams; pub use graph_staged::{GraphCollKey, GraphTxnOverlay, NodeLabelDelta}; pub(in crate::data::executor) use merge::IndexOverlayMergeParams; diff --git a/nodedb/src/data/executor/handlers/transaction/overlay/spatial_merge.rs b/nodedb/src/data/executor/handlers/transaction/overlay/spatial_merge.rs index 34c32a810..8520fe9eb 100644 --- a/nodedb/src/data/executor/handlers/transaction/overlay/spatial_merge.rs +++ b/nodedb/src/data/executor/handlers/transaction/overlay/spatial_merge.rs @@ -37,7 +37,7 @@ use nodedb_types::value::Value; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::handlers::columnar_read::convert::row_to_projected_json; +use crate::data::executor::handlers::columnar_read::convert::row_to_projected_value; use crate::data::executor::handlers::spatial_refine::{ apply_predicate, extract_geometry, project_doc, }; @@ -64,20 +64,17 @@ pub(in crate::data::executor) struct SpatialOverlayMergeParams<'a> { /// staged row whose collection has no known columnar schema (defensively /// treated as "does not match" rather than surfacing a panic). Returns /// `Err` only when the row *does* decode but its computed-column projection -/// hits a division/modulo-by-zero — `row_to_projected_json` is called with +/// hits a division/modulo-by-zero — `row_to_projected_value` is called with /// no computed columns here (`&[]`), so this is currently unreachable, but /// the `Result` return keeps the signature honest about what -/// `row_to_projected_json` can do. +/// `row_to_projected_value` can do. fn decode_staged_spatial_row( body: &[u8], schema: Option<&ColumnarSchema>, ) -> crate::Result> { Ok(match nodedb_types::value_from_msgpack(body).ok() { Some(Value::Array(row)) => match schema { - Some(schema) => { - let json = row_to_projected_json(&row, schema, &[], &[], false)?; - Some(Value::from(json)) - } + Some(schema) => Some(row_to_projected_value(&row, schema, &[], &[], false)?), None => None, }, Some(obj @ Value::Object(_)) => Some(obj), diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_columnar_dml.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_columnar_dml.rs index 6c9e09b5d..0f3ff4e49 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_columnar_dml.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_columnar_dml.rs @@ -50,9 +50,11 @@ use nodedb_types::{RowIdentity, Surrogate, value_to_pk_string}; use crate::bridge::envelope::{ErrorCode, Response}; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::handlers::columnar_read::convert::row_to_projected_json; +use crate::data::executor::handlers::columnar_read::convert::row_to_projected_value; use crate::data::executor::handlers::columnar_read::filter::row_matches_filters; -use crate::data::executor::handlers::transaction::overlay::ColumnarOverlayMergeParams; +use crate::data::executor::handlers::transaction::overlay::{ + ColumnarMatchedRow, ColumnarOverlayMergeParams, +}; use crate::data::executor::response_codec; use crate::data::executor::task::ExecutionTask; use crate::types::{TenantId, TxnId}; @@ -297,7 +299,7 @@ impl CoreLoop { // shared `ColumnarMatchedRow` tuple the overlay merge consumes. A // missing engine means the only affected rows are overlay-only staged // inserts, which the merge appends below. - let mut matched: Vec<(Option, Vec, serde_json::Value)> = Vec::new(); + let mut matched: Vec = Vec::new(); if let Some(engine) = self.columnar_engines.get(&coll_key) { for (surrogate, row) in engine.scan_memtable_rows_with_surrogates() { if !filter_predicates.is_empty() { @@ -311,15 +313,15 @@ impl CoreLoop { } // No computed columns on this path (`&[]` below), so this // can never actually raise `DivisionByZero` today — handled - // uniformly with every other `row_to_projected_json` caller + // uniformly with every other `row_to_projected_value` caller // instead of assuming that invariant with an `unwrap`. - let json = match row_to_projected_json(&row, &schema, &[], &[], false) { + let obj = match row_to_projected_value(&row, &schema, &[], &[], false) { Ok(v) => v, Err(_e) => { return Err(self.response_error(task, ErrorCode::DivisionByZero)); } }; - matched.push((surrogate, row, json)); + matched.push((surrogate, row, obj)); } } diff --git a/nodedb/src/data/executor/row_shape.rs b/nodedb/src/data/executor/row_shape.rs index 16e89fa9d..e56a8a494 100644 --- a/nodedb/src/data/executor/row_shape.rs +++ b/nodedb/src/data/executor/row_shape.rs @@ -84,18 +84,32 @@ pub(in crate::data::executor) fn sparse_row_to_doc( /// Convert a single row from a `DecodedColumn` to a `nodedb_types::value::Value`. /// +/// `declared` is the column's declared type from the collection schema. It +/// decides how an eight-byte integer cell is typed, because the segment +/// reader infers a column's physical kind from its codec and decodes every +/// time column as `DecodedColumn::Int64`: a `Timestamp` column yields +/// `Value::NaiveDateTime`, a `Timestamptz` column `Value::DateTime`, both from +/// the epoch microseconds the segment stores. Every other declared type +/// yields the integer stored: a `SystemTimestamp` column is an engine-assigned +/// system-time count, and the bitemporal `_ts_system`, `_ts_valid_from`, +/// `_ts_valid_until` columns are declared `Int64` and hold epoch milliseconds +/// with `i64::MIN` / `i64::MAX` as the unbounded sentinels, which no instant +/// can carry. The live memtable applies the same rule, so a row reads +/// identically before and after a flush. +/// /// Returns `Value::Null` if the row index is out of range or the validity bit is false. pub(in crate::data::executor) fn decoded_col_to_value( col: &nodedb_columnar::reader::DecodedColumn, row_idx: usize, + declared: &nodedb_types::columnar::ColumnType, ) -> nodedb_types::value::Value { use nodedb_columnar::reader::DecodedColumn; use nodedb_types::value::Value; match col { - DecodedColumn::Int64 { values, valid } => { + DecodedColumn::Int64 { values, valid } | DecodedColumn::Timestamp { values, valid } => { if row_idx < valid.len() && valid[row_idx] { - Value::Integer(values[row_idx]) + declared.time_cell(values[row_idx]) } else { Value::Null } @@ -107,14 +121,6 @@ pub(in crate::data::executor) fn decoded_col_to_value( Value::Null } } - DecodedColumn::Timestamp { values, valid } => { - if row_idx < valid.len() && valid[row_idx] { - // Represent as integer microseconds (same as Value::Integer for timestamps). - Value::Integer(values[row_idx]) - } else { - Value::Null - } - } DecodedColumn::Bool { values, valid } => { if row_idx < valid.len() && valid[row_idx] { Value::Bool(values[row_idx]) @@ -160,13 +166,94 @@ pub(in crate::data::executor) fn decoded_col_to_value( Value::Null } } - _ => Value::Null, } } #[cfg(test)] mod tests { - use super::{kv_row_to_doc, msgpack_scan}; + use nodedb_columnar::reader::DecodedColumn; + use nodedb_types::columnar::ColumnType; + use nodedb_types::value::Value; + use nodedb_types::{InstantKind, NdbDateTime}; + + use super::{decoded_col_to_value, kv_row_to_doc, msgpack_scan}; + + const MICROS: i64 = 1_583_402_400_000_000; + + /// A time column as the segment reader decodes it: the reader infers the + /// physical kind from the codec, so a time column arrives as `Int64`. + fn time_column() -> DecodedColumn { + DecodedColumn::Int64 { + values: vec![MICROS, 0], + valid: vec![true, false], + } + } + + /// A `TIMESTAMP` column reads back as a naive instant, a `TIMESTAMPTZ` + /// column as a UTC instant, each carrying the stored microseconds, + /// whether the reader decoded the column as `Int64` or as `Timestamp`. + #[test] + fn a_declared_instant_column_reads_back_as_its_instant_variant() { + let col = time_column(); + assert_eq!( + decoded_col_to_value(&col, 0, &ColumnType::Timestamp), + Value::NaiveDateTime(NdbDateTime::from_micros(MICROS)) + ); + assert_eq!( + decoded_col_to_value(&col, 0, &ColumnType::Timestamptz), + Value::DateTime(NdbDateTime::from_micros(MICROS)) + ); + assert_eq!( + decoded_col_to_value(&col, 0, &ColumnType::Timestamp), + InstantKind::Naive.from_micros(MICROS) + ); + let typed = DecodedColumn::Timestamp { + values: vec![MICROS], + valid: vec![true], + }; + assert_eq!( + decoded_col_to_value(&typed, 0, &ColumnType::Timestamptz), + Value::DateTime(NdbDateTime::from_micros(MICROS)) + ); + } + + /// A system-time column and a duration column share time storage but are + /// not instants: they read back as the integer stored. + #[test] + fn a_non_instant_time_column_reads_back_as_the_integer_stored() { + let col = time_column(); + assert_eq!( + decoded_col_to_value(&col, 0, &ColumnType::SystemTimestamp), + Value::Integer(MICROS) + ); + assert_eq!( + decoded_col_to_value(&col, 0, &ColumnType::Duration), + Value::Integer(MICROS) + ); + } + + /// The declared type never changes how a non-time column reads, and an + /// invalid or out-of-range row is NULL for every declared type. + #[test] + fn a_null_time_cell_is_null_for_every_declared_type() { + let col = time_column(); + for ty in [ + ColumnType::Timestamp, + ColumnType::Timestamptz, + ColumnType::SystemTimestamp, + ] { + assert_eq!(decoded_col_to_value(&col, 1, &ty), Value::Null); + assert_eq!(decoded_col_to_value(&col, 2, &ty), Value::Null); + } + let ints = DecodedColumn::Int64 { + values: vec![7], + valid: vec![true], + }; + assert_eq!( + decoded_col_to_value(&ints, 0, &ColumnType::Int64), + Value::Integer(7) + ); + } /// A raw (non-msgpack) KV value must be wrapped as a msgpack STRING, not /// appended verbatim. diff --git a/nodedb/src/data/executor/scan_normalize.rs b/nodedb/src/data/executor/scan_normalize.rs index fb6f09277..1a30300fd 100644 --- a/nodedb/src/data/executor/scan_normalize.rs +++ b/nodedb/src/data/executor/scan_normalize.rs @@ -351,7 +351,11 @@ impl CoreLoop { let mut map = std::collections::HashMap::new(); let mut id = String::new(); for (col_idx, col_def) in schema.columns.iter().enumerate() { - let val = decoded_col_to_value(&decoded_cols[col_idx], row_idx); + let val = decoded_col_to_value( + &decoded_cols[col_idx], + row_idx, + &col_def.column_type, + ); if col_def.name == "id" && let nodedb_types::value::Value::String(s) = &val { diff --git a/nodedb/tests/wire/cases/sql_division_by_zero.rs b/nodedb/tests/wire/cases/sql_division_by_zero.rs index 52db70018..2ac7c0a2d 100644 --- a/nodedb/tests/wire/cases/sql_division_by_zero.rs +++ b/nodedb/tests/wire/cases/sql_division_by_zero.rs @@ -312,11 +312,11 @@ async fn columnar_where_clause_division_by_zero_errors_22012() { } /// Columnar engine, computed SELECT column division by zero: -/// `row_to_projected_json`'s `Err` arm in `execute_columnar_scan`'s +/// `row_to_projected_value`'s `Err` arm in `execute_columnar_scan`'s /// live-memtable phase (a real, non-empty `computed_cols`). /// /// `denom` must be listed explicitly alongside the computed `ratio` column -/// (not just referenced inside the expression): `row_to_projected_json` +/// (not just referenced inside the expression): `row_to_projected_value` /// only includes a stored column in the row object it hands to the computed /// expression evaluator when that column is itself in the projection list /// (or force-included) — an explicit, non-computed `SELECT` entry for every diff --git a/nodedb/tests/wire/cases/strict_typed_column_rendering.rs b/nodedb/tests/wire/cases/strict_typed_column_rendering.rs index 9886c65c0..4063e78bd 100644 --- a/nodedb/tests/wire/cases/strict_typed_column_rendering.rs +++ b/nodedb/tests/wire/cases/strict_typed_column_rendering.rs @@ -2,6 +2,8 @@ //! A `document_strict` `TIMESTAMP` column renders the stored instant the same //! way whichever route reads it, and the same way a timeseries time key does. +//! A `columnar` `TIMESTAMP` column renders the same instant from the live +//! memtable and from a flushed segment. use crate::harness::TestServer; @@ -145,3 +147,92 @@ async fn a_strict_timestamp_column_renders_the_same_as_a_timeseries_time_key() { key: strict={strict_reading:?} timeseries={timeseries_reading:?}" ); } + +/// A `columnar` collection carrying a `TIMESTAMP` column, read back with a +/// direct `SELECT` while the row is still in the live memtable, renders the +/// stored instant as ISO-8601. The columnar engine stores the cell as epoch +/// microseconds and reads it back as a typed instant, so the millisecond and +/// microsecond integer forms are both rendering defects here. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_columnar_timestamp_column_renders_the_stored_instant() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION columnar_ts_direct \ + (id TEXT PRIMARY KEY, created_at TIMESTAMP) \ + WITH (engine='columnar')", + ) + .await + .expect("create columnar_ts_direct"); + server + .exec(&format!( + "INSERT INTO columnar_ts_direct (id, created_at) VALUES ('r1', '{EARLY}')" + )) + .await + .expect("insert into columnar_ts_direct"); + + let rows = server + .query_text("SELECT created_at FROM columnar_ts_direct WHERE id = 'r1'") + .await + .expect("SELECT of a columnar TIMESTAMP column must succeed"); + assert_eq!(rows.len(), 1, "one stored row: {rows:?}"); + assert_eq!( + rows[0], EARLY_ISO, + "a columnar TIMESTAMP column must render {EARLY} as {EARLY_ISO}, got {rows:?}" + ); +} + +/// The columnar engine has two read paths for one column: the live memtable +/// and the flushed segments a full memtable drains into. With a flush +/// threshold of two, the first insert is read from the memtable, and after +/// two more inserts the first rows live only in a flushed segment while the +/// last stays in the memtable. Every row renders the one instant identically +/// on both paths. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_columnar_timestamp_column_renders_the_same_before_and_after_flush() { + let server = TestServer::start_with_columnar_flush_threshold(2).await; + server + .exec( + "CREATE COLLECTION columnar_ts_flush \ + (id TEXT PRIMARY KEY, created_at TIMESTAMP) \ + WITH (engine='columnar')", + ) + .await + .expect("create columnar_ts_flush"); + server + .exec(&format!( + "INSERT INTO columnar_ts_flush (id, created_at) VALUES ('r1', '{EARLY}')" + )) + .await + .expect("insert r1 into columnar_ts_flush"); + + let before_flush = server + .query_text("SELECT created_at FROM columnar_ts_flush WHERE id = 'r1'") + .await + .expect("SELECT from the live memtable must succeed"); + assert_eq!( + before_flush, + vec![EARLY_ISO.to_string()], + "the live-memtable read must render {EARLY} as {EARLY_ISO}" + ); + + for id in ["r2", "r3"] { + server + .exec(&format!( + "INSERT INTO columnar_ts_flush (id, created_at) VALUES ('{id}', '{EARLY}')" + )) + .await + .unwrap_or_else(|e| panic!("insert {id} into columnar_ts_flush: {e}")); + } + + let after_flush = server + .query_text("SELECT created_at FROM columnar_ts_flush ORDER BY id") + .await + .expect("SELECT across a flushed segment and the memtable must succeed"); + assert_eq!( + after_flush, + vec![EARLY_ISO.to_string(); 3], + "a flushed-segment cell and a live-memtable cell must render the one \ + instant identically: before={before_flush:?} after={after_flush:?}" + ); +} From 19dc49ae2a648f4693cba46d911dba064fb7a0d9 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 16 Sep 2026 18:59:12 +0800 Subject: [PATCH 08/21] feat(rls): enforce read policies on columnar and spatial scans Row-level-security FOR READ policies now govern SELECT results on columnar and spatial collections, not just columnar writes. The scan decodes the policy's ScanFilter payload once up front and rejects a statement outright when the payload is undecodable, rather than treating it as no policy. Matching rows are filtered by the query's WHERE predicates first, then the read policy, on flushed segments, the live memtable, and the in-transaction overlay alike, so excluded rows never consume a LIMIT slot or reach a join partner. The spatial scan handler splits from a single file into full_scan, prefilter, and rtree_scan modules to carry the new policy parameter alongside the existing full-scan and R-tree-prefiltered paths. Adds a wire test covering read-policy enforcement on columnar reads. --- .../executor/handlers/columnar_read/filter.rs | 33 +++ .../handlers/columnar_read/scan/execute.rs | 157 ++++++++--- .../handlers/columnar_read/scan/params.rs | 9 +- .../handlers/columnar_read/scan_flushed.rs | 17 +- .../executor/handlers/spatial/full_scan.rs | 155 +++++++++++ .../src/data/executor/handlers/spatial/mod.rs | 15 ++ .../executor/handlers/spatial/prefilter.rs | 44 +++ .../{spatial.rs => spatial/rtree_scan.rs} | 220 +++------------ .../transaction/overlay/columnar_merge.rs | 13 +- .../stage_write/stage_columnar_dml.rs | 1 + nodedb/src/data/executor/scan_normalize.rs | 10 +- nodedb/src/engine/spatial/mod.rs | 2 +- .../cases/columnar_read_row_level_security.rs | 250 ++++++++++++++++++ nodedb/tests/wire/cases/mod.rs | 1 + 14 files changed, 683 insertions(+), 244 deletions(-) create mode 100644 nodedb/src/data/executor/handlers/spatial/full_scan.rs create mode 100644 nodedb/src/data/executor/handlers/spatial/mod.rs create mode 100644 nodedb/src/data/executor/handlers/spatial/prefilter.rs rename nodedb/src/data/executor/handlers/{spatial.rs => spatial/rtree_scan.rs} (74%) create mode 100644 nodedb/tests/wire/cases/columnar_read_row_level_security.rs diff --git a/nodedb/src/data/executor/handlers/columnar_read/filter.rs b/nodedb/src/data/executor/handlers/columnar_read/filter.rs index 66851b0e3..d15492c32 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/filter.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/filter.rs @@ -5,6 +5,8 @@ use nodedb_query::EvalError; use nodedb_query::scan_filter::{FilterOp, ScanFilter}; +use crate::bridge::scan_filter::decode_scan_filters; + /// Check whether a memtable row satisfies all filter predicates. /// /// Returns `Ok(true)` if every filter passes (AND semantics). Uses the full @@ -49,3 +51,34 @@ pub(in crate::data::executor) fn value_matches_filters( } Ok(true) } + +/// Decode the planner's row-level-security slot into predicates. +/// +/// Empty bytes mean no policy governs the caller on this collection and every +/// row is admitted. Bytes that do not decode as a MessagePack +/// `Vec` are an error that fails the scan: a policy the Data +/// Plane cannot read never admits the rows it governs. +pub(in crate::data::executor) fn decode_rls_filters( + bytes: &[u8], +) -> crate::Result> { + decode_scan_filters(bytes, "RLS filter") +} + +/// Whether a memtable row passes the query's WHERE predicates and then the +/// caller's read policy. +/// +/// The WHERE predicates run first so a policy predicate is only evaluated on +/// rows the query itself selects. Either set being empty is a pass for that +/// set. Errors follow [`row_matches_filters`]. +pub(in crate::data::executor) fn row_matches_filters_and_policy( + row: &[nodedb_types::value::Value], + schema: &nodedb_types::columnar::ColumnarSchema, + filters: &[ScanFilter], + rls_filters: &[ScanFilter], +) -> Result { + if filters.is_empty() && rls_filters.is_empty() { + return Ok(true); + } + let doc = crate::data::executor::handlers::columnar_write::row_values_to_object(schema, row); + Ok(value_matches_filters(&doc, filters)? && value_matches_filters(&doc, rls_filters)?) +} diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs index 2bafe1dc1..78cc2de69 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs @@ -11,7 +11,9 @@ use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::handlers::columnar_read::bitemporal::bitemporal_row_visible; use crate::data::executor::handlers::columnar_read::convert::row_to_projected_value; -use crate::data::executor::handlers::columnar_read::filter::row_matches_filters; +use crate::data::executor::handlers::columnar_read::filter::{ + decode_rls_filters, row_matches_filters_and_policy, +}; use crate::data::executor::handlers::columnar_read::scan_flushed::FlushedScanCtx; use crate::data::executor::handlers::transaction::overlay::{ ColumnarMatchedRow, ColumnarOverlayMergeParams, @@ -47,7 +49,7 @@ impl CoreLoop { projection, limit, filters, - rls_filters: _, + rls_filters, sort_keys, system_time, valid_at_ms, @@ -81,6 +83,13 @@ impl CoreLoop { } else { Vec::new() }; + // The read policy is decoded before any row is read: a payload the + // Data Plane cannot decode fails the statement instead of admitting + // the rows it governs. + let rls_predicates: Vec = match decode_rls_filters(rls_filters) { + Ok(p) => p, + Err(e) => return self.response_error(task, e), + }; // A no-LIMIT SQL `SELECT * FROM ` arrives as // `limit == usize::MAX`. Capture that before the `limit == 0` rewrite // so the budget bound applies only to the unbounded path. Spatial @@ -153,13 +162,10 @@ impl CoreLoop { // triples. The raw `Vec` is kept for sort-key comparison — the // projected object is emitted only after ORDER BY + limit are // applied. When no sort is requested the limit is enforced inside the - // loop so the whole memtable is not materialised. + // loop so the whole memtable is not materialised. The limit counts + // rows that pass every predicate, so a row the WHERE clause or the + // read policy excludes never consumes a slot. let mut matched: Vec = Vec::new(); - let scan_budget = if sort_keys.is_empty() { - limit.saturating_mul(10).max(limit) - } else { - usize::MAX - }; // Resolve hidden bitemporal column positions once; `None` means // the collection is not bitemporal, so the per-row filter is a // no-op regardless of `system_as_of_ms` / `valid_at_ms` values. @@ -197,6 +203,7 @@ impl CoreLoop { limit, sort_keys, filter_predicates: &filter_predicates, + rls_predicates: &rls_predicates, prefilter, computed_cols: &computed_cols, all_versions, @@ -224,15 +231,11 @@ impl CoreLoop { } // ── Phase 2: live memtable ────────────────────────────────────────── - // Rows still in the active memtable (not yet flushed). - // Reduce the over-fetch budget by however many rows flushed segments - // already contributed so we do not materialise more than needed. - let memtable_budget = scan_budget.saturating_sub(matched.len()); - if !block_skipped && memtable_budget > 0 { - for (row_surrogate, row) in engine - .scan_memtable_rows_with_surrogates() - .take(memtable_budget) - { + // Rows still in the active memtable (not yet flushed). Skipped when + // the flushed pass already filled an unsorted limit; otherwise the + // loop stops at the limit or the deadline, like the flushed pass. + if !block_skipped && (!sort_keys.is_empty() || matched.len() < limit) { + for (row_surrogate, row) in engine.scan_memtable_rows_with_surrogates() { // Row-boundary prefilter: skip this row when its surrogate is // absent from the bitmap. Rows without a recorded surrogate // are always included when no prefilter is active; when a @@ -255,19 +258,24 @@ impl CoreLoop { ) { continue; } - if !filter_predicates.is_empty() { - match row_matches_filters(&row, schema, &filter_predicates) { - Ok(true) => {} - Ok(false) => continue, - // `row_matches_filters` returns `EvalError`, which - // has exactly one variant and no direct - // `Into` — mirrors - // `stage_columnar_dml.rs`'s identical call site, - // which hardcodes the same typed code rather than - // collapsing to `Internal`/`XX000`. - Err(_e) => { - return self.response_error(task, ErrorCode::DivisionByZero); - } + // The query's WHERE predicates, then the caller's read + // policy: a row the policy excludes is dropped here, before + // projection, sort, and limit, so a LIMIT counts admitted + // rows only. + match row_matches_filters_and_policy( + &row, + schema, + &filter_predicates, + &rls_predicates, + ) { + Ok(true) => {} + Ok(false) => continue, + // `EvalError` has exactly one variant and no direct + // `Into` — mirrors `stage_columnar_dml.rs`'s + // identical call site, which hardcodes the same typed + // code rather than collapsing to `Internal`/`XX000`. + Err(_e) => { + return self.response_error(task, ErrorCode::DivisionByZero); } } let obj = match row_to_projected_value( @@ -279,7 +287,7 @@ impl CoreLoop { ) { Ok(v) => v, // `row_to_projected_value` returns `crate::Result<_>` - // (unlike `row_matches_filters` above) — its only + // (unlike `row_matches_filters_and_policy` above) — its only // fallible step is a computed-column expression eval, // and `computed_cols` here is real, not `&[]` like the // DML staging path, so propagate the actual typed error @@ -328,6 +336,7 @@ impl CoreLoop { schema, projection, filter_predicates: &filter_predicates, + rls_predicates: &rls_predicates, computed_cols: &computed_cols, all_versions, }, @@ -562,13 +571,21 @@ mod tests { fn scan_params<'a>( collection: &'a str, prefilter: Option<&'a SurrogateBitmap>, + ) -> ColumnarScanParams<'a> { + scan_params_with_rls(collection, prefilter, &[]) + } + + fn scan_params_with_rls<'a>( + collection: &'a str, + prefilter: Option<&'a SurrogateBitmap>, + rls_filters: &'a [u8], ) -> ColumnarScanParams<'a> { ColumnarScanParams { collection, projection: &[], limit: 0, filters: &[], - rls_filters: &[], + rls_filters, sort_keys: &[], system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, @@ -578,6 +595,18 @@ mod tests { } } + /// A read policy `name = `, encoded the way the planner ships it. + fn policy_name_eq(name: &str) -> Vec { + let filter = crate::bridge::scan_filter::ScanFilter { + field: "name".into(), + op: "eq".into(), + value: Value::String(name.into()), + clauses: vec![], + expr: None, + }; + zerompk::to_msgpack_vec(&vec![filter]).expect("encode policy") + } + fn decode_rows(payload: &[u8]) -> Vec { match nodedb_types::value_from_msgpack(payload).expect("decode scan payload") { Value::Array(rows) => rows, @@ -719,6 +748,70 @@ mod tests { ); } + /// A read policy governs rows from the live memtable and from flushed + /// segments alike: only the rows it admits come back. + #[test] + fn a_read_policy_admits_only_matching_rows_from_both_phases() { + let (mut core, _dir) = make_core(); + let coll = "cf_rls_both"; + let key = insert_and_flush( + &mut core, + coll, + &[(1, "mine", Surrogate(801)), (2, "theirs", Surrogate(802))], + ); + let mut engine = core + .columnar_engines + .remove(&key) + .expect("engine registered"); + engine + .insert_with_surrogate(&row(3, "mine"), Surrogate(803)) + .expect("insert_with_surrogate"); + engine + .insert_with_surrogate(&row(4, "theirs"), Surrogate(804)) + .expect("insert_with_surrogate"); + core.columnar_engines.insert(key, engine); + + let policy = policy_name_eq("mine"); + let task = make_task(); + let resp = core.execute_columnar_scan(&task, scan_params_with_rls(coll, None, &policy)); + assert_eq!(resp.status, crate::bridge::envelope::Status::Ok); + assert_eq!( + ids(&decode_rows(resp.payload.as_bytes())), + vec![1, 3], + "one flushed and one live row are admitted; the others are excluded" + ); + } + + /// A read policy payload the Data Plane cannot decode fails the scan. + /// The alternative — treating it as "no policy" — would return every + /// row the policy exists to hide. + #[test] + fn a_malformed_read_policy_payload_fails_the_scan() { + let (mut core, _dir) = make_core(); + let coll = "cf_rls_malformed"; + insert_and_flush( + &mut core, + coll, + &[(1, "mine", Surrogate(901)), (2, "theirs", Surrogate(902))], + ); + + // `0xC1` is the one byte MessagePack reserves and never emits. + let malformed = [0xC1u8]; + let task = make_task(); + let resp = core.execute_columnar_scan(&task, scan_params_with_rls(coll, None, &malformed)); + + assert_eq!( + resp.status, + crate::bridge::envelope::Status::Error, + "an unreadable policy must not admit rows" + ); + assert!( + resp.payload.is_empty(), + "a failed scan carries no rows, got {} bytes", + resp.payload.len() + ); + } + /// Build a task whose deadline is already in the past. fn expired_task() -> ExecutionTask { let mut task = make_task(); diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan/params.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/params.rs index 824df3613..bceba5428 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/scan/params.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/params.rs @@ -11,10 +11,11 @@ pub(in crate::data::executor) struct ColumnarScanParams<'a> { pub projection: &'a [String], pub limit: usize, pub filters: &'a [u8], - /// RLS filter bytes — wiring is the responsibility of a separate - /// enforcement pass; the base scan handler itself does not consume - /// them (hence the `_` destructure). - #[allow(dead_code)] + /// The caller's read policy as a MessagePack `Vec`, injected + /// by the planner. Empty when no policy governs the caller. The scan + /// evaluates it per row after block pruning and `filters`, before + /// projection, sort, and limit, on the flushed-segment, live-memtable, + /// and in-transaction overlay rows alike. pub rls_filters: &'a [u8], pub sort_keys: &'a [nodedb_physical::physical_plan::SortKeySpec], /// Bitemporal system-time selection. `Current` is a current-state read; diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan_flushed.rs b/nodedb/src/data/executor/handlers/columnar_read/scan_flushed.rs index 38116a597..eb7df962a 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/scan_flushed.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/scan_flushed.rs @@ -10,7 +10,7 @@ use crate::data::executor::scan_normalize::decoded_col_to_value; use super::bitemporal::bitemporal_row_visible; use super::convert::row_to_projected_value; -use super::filter::row_matches_filters; +use super::filter::row_matches_filters_and_policy; /// Read-only context for the flushed-segment scan phase. All fields are /// borrowed from locals already computed in `execute_columnar_scan`. @@ -22,6 +22,8 @@ pub(in crate::data::executor) struct FlushedScanCtx<'a> { pub limit: usize, pub sort_keys: &'a [nodedb_physical::physical_plan::SortKeySpec], pub filter_predicates: &'a [ScanFilter], + /// The caller's decoded read policy; empty admits every row. + pub rls_predicates: &'a [ScanFilter], pub prefilter: Option<&'a nodedb_types::surrogate_bitmap::SurrogateBitmap>, pub computed_cols: &'a [ComputedColumn], pub all_versions: bool, @@ -54,6 +56,7 @@ impl CoreLoop { limit, sort_keys, filter_predicates, + rls_predicates, prefilter, computed_cols, all_versions, @@ -192,9 +195,15 @@ impl CoreLoop { ) { continue; } - if !filter_predicates.is_empty() - && !row_matches_filters(&row, schema, filter_predicates)? - { + // The query's WHERE predicates, then the caller's read + // policy, before projection so a limit counts admitted + // rows only. + if !row_matches_filters_and_policy( + &row, + schema, + filter_predicates, + rls_predicates, + )? { continue; } diff --git a/nodedb/src/data/executor/handlers/spatial/full_scan.rs b/nodedb/src/data/executor/handlers/spatial/full_scan.rs new file mode 100644 index 000000000..5da855a10 --- /dev/null +++ b/nodedb/src/data/executor/handlers/spatial/full_scan.rs @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: BUSL-1.1 + +use tracing::debug; + +use crate::bridge::envelope::{ErrorCode, Response}; +use crate::bridge::scan_filter::ScanFilter; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::doc_format; +use crate::data::executor::handlers::spatial_refine::{ + apply_predicate, extract_geometry, project_doc, +}; +use crate::data::executor::handlers::transaction::overlay::SpatialOverlayMergeParams; +use crate::data::executor::response_codec; +use crate::data::executor::task::ExecutionTask; +use nodedb_physical::physical_plan::SpatialPredicate; +use nodedb_types::SurrogateBitmap; + +use super::prefilter::prefilter_admits; + +/// Parameters for [`CoreLoop::spatial_full_scan`]. +pub(super) struct SpatialFullScanParams<'a> { + pub task: &'a ExecutionTask, + pub tid: u64, + pub collection: &'a str, + pub field: &'a str, + pub predicate: &'a SpatialPredicate, + pub query_geom: &'a nodedb_types::geometry::Geometry, + pub distance_meters: f64, + pub limit: usize, + pub projection: &'a [String], + pub attr_filters: &'a [ScanFilter], + pub rls_filters: &'a [ScanFilter], + pub prefilter: Option<&'a SurrogateBitmap>, +} + +impl CoreLoop { + /// Full scan when no R-tree exists for the field. + pub(super) fn spatial_full_scan(&self, params: SpatialFullScanParams<'_>) -> Response { + let SpatialFullScanParams { + task, + tid, + collection, + field, + predicate, + query_geom, + distance_meters, + limit, + projection, + attr_filters, + rls_filters, + prefilter, + } = params; + debug!(core = self.core_id, %collection, "spatial full scan (no R-tree index yet)"); + + let scan_limit = limit * 10; + let entries = match self.scan_collection( + task.request.database_id.as_u64(), + tid, + collection, + scan_limit, + ) { + Ok(e) => e, + Err(e) => { + return self.response_error( + task, + ErrorCode::Internal { + detail: e.to_string(), + }, + ); + } + }; + + let mut results = Vec::new(); + for (doc_id, doc_bytes) in &entries { + if results.len() >= limit { + break; + } + + // Prefilter: skip non-members before geometry evaluation. + if let Some(bitmap) = prefilter + && !prefilter_admits(bitmap, doc_id) + { + continue; + } + + // A row skipped here silently drops out of the spatial result set, + // which reads as "no row matched the geometry" rather than "a row + // could not be read". + let doc = match doc_format::decode_document_value(doc_bytes) { + Ok(d) => d, + Err(e) => return self.response_error(task, e), + }; + + let doc_geom = match extract_geometry(&doc, field) { + Some(g) => g, + None => continue, + }; + + if !apply_predicate(predicate, query_geom, &doc_geom, distance_meters) { + continue; + } + + match ScanFilter::all_match_value(attr_filters, &doc) { + Ok(true) => {} + Ok(false) => continue, + Err(_e) => { + return self.response_error(task, ErrorCode::DivisionByZero); + } + } + match ScanFilter::all_match_value(rls_filters, &doc) { + Ok(true) => {} + Ok(false) => continue, + Err(_e) => { + return self.response_error(task, ErrorCode::DivisionByZero); + } + } + + results.push(project_doc(&doc, doc_id, projection)); + } + + if let Some(txn_id) = task.request.txn_id { + let coll_key = ( + task.request.database_id, + crate::types::TenantId::new(tid), + collection.to_string(), + ); + if let Err(e) = self.merge_overlay_into_spatial_scan( + SpatialOverlayMergeParams { + txn_id, + coll_key: &coll_key, + field, + predicate, + query_geom, + distance_meters, + projection, + attr_filters, + row_level_filters: rls_filters, + }, + &mut results, + ) { + return self.response_error(task, e); + } + } + + match response_codec::encode_value_vec(&results) { + Ok(payload) => self.response_with_payload(task, payload), + Err(e) => self.response_error( + task, + ErrorCode::Internal { + detail: e.to_string(), + }, + ), + } + } +} diff --git a/nodedb/src/data/executor/handlers/spatial/mod.rs b/nodedb/src/data/executor/handlers/spatial/mod.rs new file mode 100644 index 000000000..ce5c3d708 --- /dev/null +++ b/nodedb/src/data/executor/handlers/spatial/mod.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Spatial query handler: R-tree index scan with predicate refinement. +//! +//! Documents with geometry fields are auto-indexed into per-field R-trees +//! on insert (see `handlers/point.rs`). Spatial queries use the R-tree for +//! fast bbox candidate selection, then refine with exact predicates. +//! +//! Internal document representation: `nodedb_types::Value` (no JSON intermediary). + +mod full_scan; +mod prefilter; +mod rtree_scan; + +pub(in crate::data::executor) use rtree_scan::SpatialScanParams; diff --git a/nodedb/src/data/executor/handlers/spatial/prefilter.rs b/nodedb/src/data/executor/handlers/spatial/prefilter.rs new file mode 100644 index 000000000..ebeb85f16 --- /dev/null +++ b/nodedb/src/data/executor/handlers/spatial/prefilter.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: BUSL-1.1 + +use nodedb_types::SurrogateBitmap; + +/// Whether `doc_id`'s surrogate is a member of `prefilter`. +/// +/// `doc_id` is the hex-encoded surrogate for a document-collection row, or +/// a columnar-family user `id` that never parses as a storage key — the +/// latter is never admitted, matching a sparse miss on a parsed key. +pub(super) fn prefilter_admits(prefilter: &SurrogateBitmap, doc_id: &str) -> bool { + match nodedb_types::StorageKey::parse(doc_id) { + Some(key) => prefilter.contains(key.surrogate()), + None => false, + } +} + +#[cfg(test)] +mod tests { + use super::prefilter_admits; + use nodedb_types::{Surrogate, SurrogateBitmap}; + + fn doc_id(surrogate: u32) -> String { + nodedb_types::StorageKey::for_surrogate(Surrogate::new(surrogate)).to_string() + } + + #[test] + fn prefilter_skips_non_member_doc_ids() { + // Direct unit on the production prefilter check (`prefilter_admits`), + // not a re-implementation of it. + let mut bitmap = SurrogateBitmap::new(); + bitmap.insert(Surrogate(2)); + + let candidate_doc_ids = [doc_id(1), doc_id(2), doc_id(3)]; + + let kept: Vec<_> = candidate_doc_ids + .iter() + .filter(|doc_id| prefilter_admits(&bitmap, doc_id)) + .cloned() + .collect(); + + assert_eq!(kept.len(), 1); + assert_eq!(kept[0], doc_id(2)); + } +} diff --git a/nodedb/src/data/executor/handlers/spatial.rs b/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs similarity index 74% rename from nodedb/src/data/executor/handlers/spatial.rs rename to nodedb/src/data/executor/handlers/spatial/rtree_scan.rs index cd90daece..1c2b09e53 100644 --- a/nodedb/src/data/executor/handlers/spatial.rs +++ b/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs @@ -1,36 +1,23 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Spatial query handler: R-tree index scan with predicate refinement. -//! -//! Documents with geometry fields are auto-indexed into per-field R-trees -//! on insert (see `handlers/point.rs`). Spatial queries use the R-tree for -//! fast bbox candidate selection, then refine with exact predicates. -//! -//! Internal document representation: `nodedb_types::Value` (no JSON intermediary). - use tracing::debug; -use super::super::response_codec; use crate::bridge::envelope::{ErrorCode, Response}; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::doc_format; +use crate::data::executor::handlers::columnar_read::filter::decode_rls_filters; +use crate::data::executor::handlers::spatial_refine::{ + apply_predicate, expand_bbox, extract_geometry, project_doc, +}; +use crate::data::executor::handlers::transaction::overlay::SpatialOverlayMergeParams; +use crate::data::executor::response_codec; use crate::data::executor::task::ExecutionTask; use nodedb_physical::physical_plan::SpatialPredicate; use nodedb_types::SurrogateBitmap; -use super::spatial_refine::{apply_predicate, expand_bbox, extract_geometry, project_doc}; - -/// Whether `doc_id`'s surrogate is a member of `prefilter`. -/// -/// `doc_id` is the hex-encoded surrogate for a document-collection row, or -/// a columnar-family user `id` that never parses as a storage key — the -/// latter is never admitted, matching a sparse miss on a parsed key. -fn prefilter_admits(prefilter: &SurrogateBitmap, doc_id: &str) -> bool { - match nodedb_types::StorageKey::parse(doc_id) { - Some(key) => prefilter.contains(key.surrogate()), - None => false, - } -} +use super::full_scan::SpatialFullScanParams; +use super::prefilter::prefilter_admits; /// Parameters for [`CoreLoop::execute_spatial_scan`]. pub(in crate::data::executor) struct SpatialScanParams<'a> { @@ -48,22 +35,6 @@ pub(in crate::data::executor) struct SpatialScanParams<'a> { pub prefilter: Option<&'a SurrogateBitmap>, } -/// Parameters for [`CoreLoop::spatial_full_scan`]. -struct SpatialFullScanParams<'a> { - task: &'a ExecutionTask, - tid: u64, - collection: &'a str, - field: &'a str, - predicate: &'a SpatialPredicate, - query_geom: &'a nodedb_types::geometry::Geometry, - distance_meters: f64, - limit: usize, - projection: &'a [String], - attr_filters: &'a [ScanFilter], - rls_filters: &'a [ScanFilter], - prefilter: Option<&'a SurrogateBitmap>, -} - impl CoreLoop { /// Execute a spatial scan using the R-tree index. /// @@ -106,16 +77,27 @@ impl CoreLoop { // The query geometry was parsed and validated on the Control Plane. let query_geom = query_geometry; - // 2. Deserialize attribute and RLS filters. + // 2. Deserialize attribute and RLS filters. A payload that does not + // decode fails the scan: an unreadable predicate never admits the + // rows it would have excluded. let attr_filters: Vec = if attribute_filters.is_empty() { Vec::new() } else { - zerompk::from_msgpack(attribute_filters).unwrap_or_default() + match zerompk::from_msgpack(attribute_filters) { + Ok(f) => f, + Err(e) => { + return self.response_error( + task, + ErrorCode::Internal { + detail: format!("malformed spatial attribute filters: {e}"), + }, + ); + } + } }; - let row_level_filters: Vec = if rls_filters.is_empty() { - Vec::new() - } else { - zerompk::from_msgpack(rls_filters).unwrap_or_default() + let row_level_filters: Vec = match decode_rls_filters(rls_filters) { + Ok(f) => f, + Err(e) => return self.response_error(task, e), }; // 3. Compute search bbox (expand by distance for ST_DWithin). @@ -260,7 +242,7 @@ impl CoreLoop { // A candidate skipped here silently drops out of the // spatial result set, which reads as "no row matched the // geometry" rather than "a row could not be read". - match super::super::doc_format::decode_document_value(&doc_mp) { + match doc_format::decode_document_value(&doc_mp) { Ok(d) => d, Err(e) => return self.response_error(task, e), } @@ -288,7 +270,7 @@ impl CoreLoop { let Some(doc_mp) = columnar_docs.as_ref().and_then(|m| m.get(&doc_id)) else { continue; }; - match super::super::doc_format::decode_document_value(doc_mp) { + match doc_format::decode_document_value(doc_mp) { Ok(d) => d, Err(e) => return self.response_error(task, e), } @@ -333,7 +315,7 @@ impl CoreLoop { if let Some(txn_id) = task.request.txn_id && let Err(e) = self.merge_overlay_into_spatial_scan( - super::transaction::overlay::SpatialOverlayMergeParams { + SpatialOverlayMergeParams { txn_id, coll_key: &coll_key, field, @@ -360,130 +342,11 @@ impl CoreLoop { ), } } - - /// Full scan when no R-tree exists for the field. - fn spatial_full_scan(&self, params: SpatialFullScanParams<'_>) -> Response { - let SpatialFullScanParams { - task, - tid, - collection, - field, - predicate, - query_geom, - distance_meters, - limit, - projection, - attr_filters, - rls_filters, - prefilter, - } = params; - debug!(core = self.core_id, %collection, "spatial full scan (no R-tree index yet)"); - - let scan_limit = limit * 10; - let entries = match self.scan_collection( - task.request.database_id.as_u64(), - tid, - collection, - scan_limit, - ) { - Ok(e) => e, - Err(e) => { - return self.response_error( - task, - ErrorCode::Internal { - detail: e.to_string(), - }, - ); - } - }; - - let mut results = Vec::new(); - for (doc_id, doc_bytes) in &entries { - if results.len() >= limit { - break; - } - - // Prefilter: skip non-members before geometry evaluation. - if let Some(bitmap) = prefilter - && !prefilter_admits(bitmap, doc_id) - { - continue; - } - - // A row skipped here silently drops out of the spatial result set, - // which reads as "no row matched the geometry" rather than "a row - // could not be read". - let doc = match super::super::doc_format::decode_document_value(doc_bytes) { - Ok(d) => d, - Err(e) => return self.response_error(task, e), - }; - - let doc_geom = match extract_geometry(&doc, field) { - Some(g) => g, - None => continue, - }; - - if !apply_predicate(predicate, query_geom, &doc_geom, distance_meters) { - continue; - } - - match ScanFilter::all_match_value(attr_filters, &doc) { - Ok(true) => {} - Ok(false) => continue, - Err(_e) => { - return self.response_error(task, ErrorCode::DivisionByZero); - } - } - match ScanFilter::all_match_value(rls_filters, &doc) { - Ok(true) => {} - Ok(false) => continue, - Err(_e) => { - return self.response_error(task, ErrorCode::DivisionByZero); - } - } - - results.push(project_doc(&doc, doc_id, projection)); - } - - if let Some(txn_id) = task.request.txn_id { - let coll_key = ( - task.request.database_id, - crate::types::TenantId::new(tid), - collection.to_string(), - ); - if let Err(e) = self.merge_overlay_into_spatial_scan( - super::transaction::overlay::SpatialOverlayMergeParams { - txn_id, - coll_key: &coll_key, - field, - predicate, - query_geom, - distance_meters, - projection, - attr_filters, - row_level_filters: rls_filters, - }, - &mut results, - ) { - return self.response_error(task, e); - } - } - - match response_codec::encode_value_vec(&results) { - Ok(payload) => self.response_with_payload(task, payload), - Err(e) => self.response_error( - task, - ErrorCode::Internal { - detail: e.to_string(), - }, - ), - } - } } #[cfg(test)] mod tests { - use super::{SpatialScanParams, prefilter_admits}; + use super::SpatialScanParams; use crate::bridge::envelope::{PhysicalPlan, Priority, Request, Status}; use crate::data::executor::task::ExecutionTask; use crate::engine::spatial::RTreeEntry; @@ -622,29 +485,6 @@ mod tests { }) } - fn doc_id(surrogate: u32) -> String { - nodedb_types::StorageKey::for_surrogate(Surrogate::new(surrogate)).to_string() - } - - #[test] - fn prefilter_skips_non_member_doc_ids() { - // Direct unit on the production prefilter check (`prefilter_admits`), - // not a re-implementation of it. - let mut bitmap = SurrogateBitmap::new(); - bitmap.insert(Surrogate(2)); - - let candidate_doc_ids = [doc_id(1), doc_id(2), doc_id(3)]; - - let kept: Vec<_> = candidate_doc_ids - .iter() - .filter(|doc_id| prefilter_admits(&bitmap, doc_id)) - .cloned() - .collect(); - - assert_eq!(kept.len(), 1); - assert_eq!(kept[0], doc_id(2)); - } - // Note: the R-tree-branch by-surrogate candidate hydration // (execute_spatial_scan → sparse.get → sparse_row_to_doc) is covered // end-to-end by `tests/engine_surface_spatial.rs:: diff --git a/nodedb/src/data/executor/handlers/transaction/overlay/columnar_merge.rs b/nodedb/src/data/executor/handlers/transaction/overlay/columnar_merge.rs index e2c38f4dc..69252f328 100644 --- a/nodedb/src/data/executor/handlers/transaction/overlay/columnar_merge.rs +++ b/nodedb/src/data/executor/handlers/transaction/overlay/columnar_merge.rs @@ -30,7 +30,7 @@ use crate::bridge::expr_eval::ComputedColumn; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::handlers::columnar_read::convert::row_to_projected_value; -use crate::data::executor::handlers::columnar_read::filter::row_matches_filters; +use crate::data::executor::handlers::columnar_read::filter::row_matches_filters_and_policy; use crate::data::executor::handlers::transaction::overlay::Staged; use crate::types::{DatabaseId, TenantId, TxnId}; @@ -48,6 +48,11 @@ pub(in crate::data::executor) struct ColumnarOverlayMergeParams<'a> { pub schema: &'a ColumnarSchema, pub projection: &'a [String], pub filter_predicates: &'a [ScanFilter], + /// The caller's decoded read policy. A staged row the policy excludes is + /// dropped from the result exactly like a base row; empty admits every + /// row. The predicate DML row read passes an empty slice because its + /// plan carries a write check, not a read policy. + pub rls_predicates: &'a [ScanFilter], pub computed_cols: &'a [ComputedColumn], pub all_versions: bool, } @@ -82,6 +87,7 @@ impl CoreLoop { schema, projection, filter_predicates, + rls_predicates, computed_cols, all_versions, } = params; @@ -94,10 +100,7 @@ impl CoreLoop { }; let predicate = |row: &[Value]| -> Result { - if filter_predicates.is_empty() { - return Ok(true); - } - row_matches_filters(row, schema, filter_predicates) + row_matches_filters_and_policy(row, schema, filter_predicates, rls_predicates) }; // Surrogates already represented in the base result. Additions diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_columnar_dml.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_columnar_dml.rs index 0f3ff4e49..f55d34856 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_columnar_dml.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_columnar_dml.rs @@ -337,6 +337,7 @@ impl CoreLoop { schema: &schema, projection: &[], filter_predicates: &filter_predicates, + rls_predicates: &[], computed_cols: &[], all_versions: false, }, diff --git a/nodedb/src/data/executor/scan_normalize.rs b/nodedb/src/data/executor/scan_normalize.rs index 1a30300fd..564516c58 100644 --- a/nodedb/src/data/executor/scan_normalize.rs +++ b/nodedb/src/data/executor/scan_normalize.rs @@ -55,10 +55,7 @@ impl CoreLoop { return Ok(docs); } - let filters: Vec = - zerompk::from_msgpack(filter_bytes).map_err(|e| crate::Error::PlanError { - detail: format!("{context} deserialization failed: {e}"), - })?; + let filters = crate::bridge::scan_filter::decode_scan_filters(filter_bytes, context)?; let mut kept = Vec::with_capacity(docs.len()); for (id, bytes) in docs { @@ -83,10 +80,7 @@ impl CoreLoop { if rls_filters.is_empty() { return Ok(true); } - let filters: Vec = - zerompk::from_msgpack(rls_filters).map_err(|e| crate::Error::PlanError { - detail: format!("RLS filter deserialization failed: {e}"), - })?; + let filters = crate::bridge::scan_filter::decode_scan_filters(rls_filters, "RLS filter")?; Ok(crate::bridge::scan_filter::ScanFilter::all_match_binary( &filters, row, )?) diff --git a/nodedb/src/engine/spatial/mod.rs b/nodedb/src/engine/spatial/mod.rs index 6dfdf3ef3..83959cb00 100644 --- a/nodedb/src/engine/spatial/mod.rs +++ b/nodedb/src/engine/spatial/mod.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 // Re-export shared spatial engine from nodedb-spatial crate. -// Origin's spatial handlers (data/executor/handlers/spatial.rs) and +// Origin's spatial handlers (data/executor/handlers/spatial/) and // checkpoint logic (data/executor/spatial_checkpoint/) use these directly. pub use nodedb_spatial::GeohashIndex; pub use nodedb_spatial::RTree; diff --git a/nodedb/tests/wire/cases/columnar_read_row_level_security.rs b/nodedb/tests/wire/cases/columnar_read_row_level_security.rs new file mode 100644 index 000000000..cddeeac96 --- /dev/null +++ b/nodedb/tests/wire/cases/columnar_read_row_level_security.rs @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Row-level security on columnar reads. +//! +//! A `FOR READ` policy on a columnar collection governs every row a `SELECT` +//! returns, from the live memtable and from flushed segments alike. The scan +//! applies the policy after block pruning and the query's own WHERE, before +//! projection, sort, and limit, so a row the policy excludes never reaches +//! the client, never consumes a `LIMIT` slot, and never joins a partner. +//! A spatial collection runs the same scan and is governed the same way. + +use crate::harness::TestServer; + +const PASSWORD: &str = "probe-secret-99"; + +/// Create a columnar `collection` holding one row owned by `user` and two +/// owned by someone else, plus `user` with the readwrite role. +async fn seed(server: &TestServer, collection: &str, user: &str) { + server + .exec(&format!( + "CREATE COLLECTION {collection} \ + (id TEXT PRIMARY KEY, owner TEXT, note TEXT) \ + WITH (engine='columnar')" + )) + .await + .unwrap_or_else(|e| panic!("create {collection}: {e}")); + server + .exec(&format!( + "INSERT INTO {collection} (id, owner, note) VALUES \ + ('r_mine', '{user}', 'mine'), \ + ('r_a', 'someone_else', 'theirs'), \ + ('r_b', 'someone_else', 'theirs too')" + )) + .await + .unwrap_or_else(|e| panic!("seed {collection}: {e}")); + server + .exec(&format!("CREATE USER {user} PASSWORD '{PASSWORD}'")) + .await + .unwrap_or_else(|e| panic!("create user {user}: {e}")); + server + .exec(&format!("GRANT ROLE readwrite TO {user}")) + .await + .unwrap_or_else(|e| panic!("grant readwrite to {user}: {e}")); + server + .exec(&format!( + "CREATE RLS POLICY {collection}_owner ON {collection} FOR READ \ + USING (owner = $auth.username)" + )) + .await + .unwrap_or_else(|e| panic!("create read policy on {collection}: {e}")); +} + +/// Run `sql` as `user` and return each row's cells joined by `|`. +async fn rows_as(server: &TestServer, user: &str, sql: &str) -> Vec { + let (client, handle) = server + .connect_as(user, PASSWORD) + .await + .unwrap_or_else(|e| panic!("connect as {user}: {e}")); + let messages = client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("{user} runs {sql}: {e}")); + let mut out = Vec::new(); + for message in messages { + if let tokio_postgres::SimpleQueryMessage::Row(row) = message { + let mut cells = Vec::new(); + for i in 0..row.len() { + cells.push(row.get(i).unwrap_or("").to_string()); + } + out.push(cells.join("|")); + } + } + drop(client); + handle.abort(); + out +} + +/// A `SELECT` over the live memtable returns only the rows the read policy +/// admits for the caller. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_columnar_select_returns_only_policy_admitted_rows() { + let server = TestServer::start().await; + seed(&server, "col_rls_read", "col_rls_reader").await; + + let rows = rows_as( + &server, + "col_rls_reader", + "SELECT id, note FROM col_rls_read ORDER BY id", + ) + .await; + assert_eq!( + rows, + vec!["r_mine|mine".to_string()], + "the read policy admits one row for this caller: {rows:?}" + ); +} + +/// A `SELECT` over flushed segments returns only the rows the read policy +/// admits for the caller. The policy applies after block pruning, so a +/// flushed row is governed the same as a live one. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_columnar_select_over_flushed_segments_returns_only_policy_admitted_rows() { + let server = TestServer::start_with_columnar_flush_threshold(2).await; + seed(&server, "col_rls_read_flushed", "col_rls_flushed_reader").await; + + let rows = rows_as( + &server, + "col_rls_flushed_reader", + "SELECT id, note FROM col_rls_read_flushed ORDER BY id", + ) + .await; + assert_eq!( + rows, + vec!["r_mine|mine".to_string()], + "the read policy admits one row for this caller: {rows:?}" + ); +} + +/// A `COUNT(*)` over a governed columnar collection counts only the rows the +/// read policy admits. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_columnar_count_counts_only_policy_admitted_rows() { + let server = TestServer::start().await; + seed(&server, "col_rls_count", "col_rls_counter").await; + + let rows = rows_as( + &server, + "col_rls_counter", + "SELECT COUNT(*) FROM col_rls_count", + ) + .await; + assert_eq!( + rows, + vec!["1".to_string()], + "the read policy admits one row for this caller: {rows:?}" + ); +} + +/// A `LIMIT` counts admitted rows only: with one admitted row and two +/// excluded rows seeded, `LIMIT 1` returns the admitted row, never an +/// excluded row that happened to be scanned first. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_columnar_select_with_a_limit_counts_only_admitted_rows() { + let server = TestServer::start().await; + seed(&server, "col_rls_limit", "col_rls_limiter").await; + + let rows = rows_as( + &server, + "col_rls_limiter", + "SELECT id FROM col_rls_limit ORDER BY id LIMIT 1", + ) + .await; + assert_eq!( + rows, + vec!["r_mine".to_string()], + "the limit applies to admitted rows only: {rows:?}" + ); +} + +/// A join reads the governed columnar side on the caller's behalf, so its +/// policy applies to that side before the join: excluded rows neither match +/// a partner nor reach the client. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_join_over_a_governed_columnar_collection_excludes_policy_filtered_rows() { + let server = TestServer::start().await; + seed(&server, "col_rls_join_c", "col_rls_joiner").await; + server + .exec( + "CREATE COLLECTION col_rls_join_d (id TEXT PRIMARY KEY, tag TEXT) \ + WITH (engine='document_strict')", + ) + .await + .expect("create document side"); + server + .exec( + "INSERT INTO col_rls_join_d (id, tag) VALUES \ + ('r_mine', 't_mine'), ('r_a', 't_a'), ('r_b', 't_b')", + ) + .await + .expect("seed document side"); + + let rows = rows_as( + &server, + "col_rls_joiner", + "SELECT c.id, d.tag FROM col_rls_join_c c \ + JOIN col_rls_join_d d ON c.id = d.id ORDER BY c.id", + ) + .await; + assert_eq!( + rows, + vec!["r_mine|t_mine".to_string()], + "the join surfaced columnar rows the read policy excludes: {rows:?}" + ); +} + +/// A plain `SELECT` over a spatial collection runs the same columnar scan, +/// so the read policy governs it the same way. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_spatial_select_returns_only_policy_admitted_rows() { + let server = TestServer::start().await; + let (collection, user) = ("sp_rls_read", "sp_rls_reader"); + server + .exec(&format!( + "CREATE COLLECTION {collection} \ + COLUMNS (id TEXT, owner TEXT, loc GEOMETRY) \ + WITH (engine='spatial')" + )) + .await + .unwrap_or_else(|e| panic!("create {collection}: {e}")); + for (id, owner, wkt) in [ + ("r_mine", user, "POINT(1 1)"), + ("r_a", "someone_else", "POINT(2 2)"), + ("r_b", "someone_else", "POINT(3 3)"), + ] { + server + .exec(&format!( + "INSERT INTO {collection} (id, owner, loc) \ + VALUES ('{id}', '{owner}', ST_GeomFromText('{wkt}'))" + )) + .await + .unwrap_or_else(|e| panic!("seed {collection} row {id}: {e}")); + } + server + .exec(&format!("CREATE USER {user} PASSWORD '{PASSWORD}'")) + .await + .unwrap_or_else(|e| panic!("create user {user}: {e}")); + server + .exec(&format!("GRANT ROLE readwrite TO {user}")) + .await + .unwrap_or_else(|e| panic!("grant readwrite to {user}: {e}")); + server + .exec(&format!( + "CREATE RLS POLICY {collection}_owner ON {collection} FOR READ \ + USING (owner = $auth.username)" + )) + .await + .unwrap_or_else(|e| panic!("create read policy on {collection}: {e}")); + + let rows = rows_as( + &server, + user, + &format!("SELECT id FROM {collection} ORDER BY id"), + ) + .await; + assert_eq!( + rows, + vec!["r_mine".to_string()], + "the read policy admits one row for this caller: {rows:?}" + ); +} diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index 958c12265..9b23ad6eb 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -37,6 +37,7 @@ mod clone_tombstone_hides_source_row; mod clone_write_isolation; mod clone_write_isolation_fuzz; mod clone_write_suppresses_source_row; +mod columnar_read_row_level_security; mod columnar_write_row_level_security; mod command_complete_tag_conformance; mod crdt_write_rls_database_scope; From 1f56a494222b4610b6053d615a0e3833e0d6e103 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 16 Sep 2026 19:41:37 +0800 Subject: [PATCH 09/21] feat(rls): enforce read policies on timeseries scans Row-level-security FOR READ policies now govern raw and grouped timeseries reads, matching the enforcement already in place for columnar and spatial. TimeseriesOp::Scan carries the caller's rls_filters payload through dispatch; the handler decodes it once up front and fails the statement outright on an undecodable payload rather than treating it as no policy. The raw scan evaluates the policy per row, after the WHERE predicates and before a row takes a LIMIT slot, on the live memtable, flushed partitions, and the in-transaction overlay alike. The aggregate path instead joins the policy onto the WHERE predicates and pushes the combined set into the grouped scan, so excluded rows never reach an accumulator; the COUNT(*) metadata fast path is skipped whenever a policy is present. eval_filters_to_bitmask and the aggregate_memtable/aggregate_partition callers now return Result instead of silently treating an unlowerable predicate as "no filter": a policy shape the grouped scan cannot express fails the aggregate rather than aggregating ungoverned rows. UnsupportedPredicate wires into the crate's central error type. Adds a wire test covering read-policy enforcement across the memtable and flushed-partition paths, with harness support for lowering the timeseries memtable budget so a test can force a flush. --- .../src/data/executor/dispatch/timeseries.rs | 210 ++++++++++++++- .../executor/handlers/timeseries/aggregate.rs | 92 ++++--- .../timeseries/raw_scan/partition_scan.rs | 66 +++-- .../handlers/timeseries/raw_scan/row_emit.rs | 32 +++ .../handlers/timeseries/raw_scan/scan.rs | 23 +- .../data/executor/handlers/timeseries/scan.rs | 49 +++- .../transaction/overlay/timeseries_merge.rs | 14 +- .../src/engine/timeseries/grouped_filter.rs | 93 +++++-- .../timeseries/grouped_scan/partition.rs | 71 +++-- nodedb/src/error_from.rs | 8 + nodedb/tests/wire/cases/mod.rs | 1 + .../timeseries_read_row_level_security.rs | 247 ++++++++++++++++++ nodedb/tests/wire/harness/config_toml.rs | 16 ++ nodedb/tests/wire/harness/lifecycle.rs | 14 + 14 files changed, 822 insertions(+), 114 deletions(-) create mode 100644 nodedb/tests/wire/cases/timeseries_read_row_level_security.rs diff --git a/nodedb/src/data/executor/dispatch/timeseries.rs b/nodedb/src/data/executor/dispatch/timeseries.rs index 40fb260b4..e20afac92 100644 --- a/nodedb/src/data/executor/dispatch/timeseries.rs +++ b/nodedb/src/data/executor/dispatch/timeseries.rs @@ -16,9 +16,13 @@ impl CoreLoop { op: &TimeseriesOp, ) -> Response { match op { + // `projection` is not destructured: the raw scan emits every + // stored column and the aggregate branch derives its own column + // set from GROUP BY, the aggregates, and the predicates. TimeseriesOp::Scan { collection, time_range, + projection: _, limit, filters, sort_keys, @@ -27,9 +31,9 @@ impl CoreLoop { aggregates, gap_fill, computed_columns, + rls_filters, system_time, valid_at_ms, - .. } => self.execute_timeseries_scan(TimeseriesScanParams { task, tid: task.request.tenant_id, @@ -37,6 +41,7 @@ impl CoreLoop { time_range: *time_range, limit: *limit, filters, + rls_filters, sort_keys, bucket_interval_ms: *bucket_interval_ms, group_by, @@ -130,17 +135,91 @@ mod tests { /// An autocommit ILP ingest exactly as the SQL and ILP planners build it: /// the plan carries NO LSN, the request envelope carries the minted one. fn autocommit_ingest_task(envelope_lsn: Option) -> ExecutionTask { - let plan = PhysicalPlan::Timeseries(TimeseriesOp::Ingest { - collection: nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, COLLECTION), - payload: format!("{COLLECTION},host=h0 value=1i\n").into_bytes(), - format: "ilp".to_string(), - wal_lsn: None, - surrogates: Vec::new(), - provenance: None, - rls_write_check: nodedb_types::RlsWriteCheck::NoPolicyApplies, - returning: None, - rls_filters: Vec::new(), - }); + ingest_task( + format!("{COLLECTION},host=h0 value=1i\n").into_bytes(), + envelope_lsn, + ) + } + + fn ingest_task(payload: Vec, envelope_lsn: Option) -> ExecutionTask { + task_for( + PhysicalPlan::Timeseries(TimeseriesOp::Ingest { + collection: nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, COLLECTION), + payload, + format: "ilp".to_string(), + wal_lsn: None, + surrogates: Vec::new(), + provenance: None, + rls_write_check: nodedb_types::RlsWriteCheck::NoPolicyApplies, + returning: None, + rls_filters: Vec::new(), + }), + envelope_lsn, + ) + } + + /// A read of every stored row, carrying `rls_filters` exactly as the + /// planner ships a read policy. `aggregates` empty is a raw scan. + fn scan_task(rls_filters: Vec, aggregates: Vec<(String, String)>) -> ExecutionTask { + task_for( + PhysicalPlan::Timeseries(TimeseriesOp::Scan { + collection: nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, COLLECTION), + time_range: nodedb_physical::physical_plan::UNBOUNDED_TIME_RANGE, + projection: Vec::new(), + limit: usize::MAX, + filters: Vec::new(), + sort_keys: Vec::new(), + bucket_interval_ms: 0, + group_by: Vec::new(), + aggregates, + gap_fill: String::new(), + computed_columns: Vec::new(), + rls_filters, + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + }), + None, + ) + } + + /// A read policy `owner = `, encoded the way the planner ships it. + fn policy_owner_eq(owner: &str) -> Vec { + let filter = crate::bridge::scan_filter::ScanFilter { + field: "owner".into(), + op: "eq".into(), + value: nodedb_types::Value::String(owner.into()), + clauses: vec![], + expr: None, + }; + zerompk::to_msgpack_vec(&vec![filter]).expect("encode policy") + } + + fn decode_rows(payload: &[u8]) -> Vec { + match nodedb_types::value_from_msgpack(payload).expect("decode scan payload") { + nodedb_types::Value::Array(rows) => rows, + other => panic!("scan payload is not an array: {other:?}"), + } + } + + /// Ingest one row owned by `mine` and two owned by `theirs`. + fn seed_owned_rows(h: &mut Harness) { + let task = ingest_task( + format!( + "{COLLECTION},owner=mine value=1i\n\ + {COLLECTION},owner=theirs value=2i\n\ + {COLLECTION},owner=theirs value=3i\n" + ) + .into_bytes(), + Some(1), + ); + let PhysicalPlan::Timeseries(op) = task.request.plan.clone() else { + panic!("timeseries plan"); + }; + let response = h.core.dispatch_timeseries(&task, &op); + assert_eq!(response.status, Status::Ok, "seed ingest must succeed"); + } + + fn task_for(plan: PhysicalPlan, envelope_lsn: Option) -> ExecutionTask { ExecutionTask::new(Request { request_id: RequestId::new(1), tenant_id: TenantId::new(TENANT), @@ -198,6 +277,113 @@ mod tests { ); } + /// A read policy governs a raw timeseries scan: only the rows it admits + /// come back, from the live memtable and from a flushed partition alike. + #[test] + fn a_read_policy_admits_only_matching_rows_from_memtable_and_partition() { + let mut h = make_core(); + seed_owned_rows(&mut h); + + let scan = scan_task(policy_owner_eq("mine"), Vec::new()); + let PhysicalPlan::Timeseries(op) = scan.request.plan.clone() else { + panic!("timeseries plan"); + }; + let live = h.core.dispatch_timeseries(&scan, &op); + assert_eq!(live.status, Status::Ok); + let rows = decode_rows(live.payload.as_bytes()); + assert_eq!(rows.len(), 1, "one live row is admitted: {rows:?}"); + + h.core + .flush_ts_collection(TenantId::new(TENANT), DatabaseId::DEFAULT, COLLECTION, 0) + .expect("flush"); + let flushed = h.core.dispatch_timeseries(&scan, &op); + assert_eq!(flushed.status, Status::Ok); + let rows = decode_rows(flushed.payload.as_bytes()); + assert_eq!(rows.len(), 1, "one flushed row is admitted: {rows:?}"); + } + + /// A governed `COUNT(*)` leaves the metadata fast path and counts only + /// the rows the policy admits. + #[test] + fn a_read_policy_governs_a_count_star_aggregate() { + let mut h = make_core(); + seed_owned_rows(&mut h); + + let scan = scan_task( + policy_owner_eq("mine"), + vec![("count".to_string(), "*".to_string())], + ); + let PhysicalPlan::Timeseries(op) = scan.request.plan.clone() else { + panic!("timeseries plan"); + }; + let response = h.core.dispatch_timeseries(&scan, &op); + assert_eq!(response.status, Status::Ok); + let rows = decode_rows(response.payload.as_bytes()); + let count_key = nodedb_query::agg_key::canonical_agg_key("count", "*"); + let counted = match rows.first() { + Some(nodedb_types::Value::Object(map)) => map.get(&count_key).cloned(), + other => panic!("expected one aggregate row, got {other:?}"), + }; + assert_eq!( + counted, + Some(nodedb_types::Value::Integer(1)), + "the policy admits one row for this caller: {rows:?}" + ); + } + + /// A read policy the grouped scan cannot lower fails the aggregate + /// instead of aggregating the rows the policy governs. + #[test] + fn an_unlowerable_read_policy_fails_an_aggregate() { + let mut h = make_core(); + seed_owned_rows(&mut h); + + let filter = crate::bridge::scan_filter::ScanFilter { + field: "owner".into(), + op: "like".into(), + value: nodedb_types::Value::String("mi%".into()), + clauses: vec![], + expr: None, + }; + let policy = zerompk::to_msgpack_vec(&vec![filter]).expect("encode policy"); + let scan = scan_task(policy, vec![("count".to_string(), "*".to_string())]); + let PhysicalPlan::Timeseries(op) = scan.request.plan.clone() else { + panic!("timeseries plan"); + }; + let response = h.core.dispatch_timeseries(&scan, &op); + assert_eq!( + response.status, + Status::Error, + "a policy the grouped scan cannot evaluate must not aggregate rows" + ); + assert!(response.payload.is_empty()); + } + + /// A read policy payload the Data Plane cannot decode fails the scan. + /// Treating it as "no policy" would return every row the policy hides. + #[test] + fn a_malformed_read_policy_payload_fails_the_scan() { + let mut h = make_core(); + seed_owned_rows(&mut h); + + // `0xC1` is the one byte MessagePack reserves and never emits. + let scan = scan_task(vec![0xC1], Vec::new()); + let PhysicalPlan::Timeseries(op) = scan.request.plan.clone() else { + panic!("timeseries plan"); + }; + let response = h.core.dispatch_timeseries(&scan, &op); + assert_eq!( + response.status, + Status::Error, + "an unreadable policy must not admit rows" + ); + assert!( + response.payload.is_empty(), + "a failed scan carries no rows, got {} bytes", + response.payload.len() + ); + } + /// Nothing minted an LSN, so nothing may be claimed as flushed: a stamp /// invented here would gate away records that are genuinely un-flushed. #[test] diff --git a/nodedb/src/data/executor/handlers/timeseries/aggregate.rs b/nodedb/src/data/executor/handlers/timeseries/aggregate.rs index b762d0c49..023bb735c 100644 --- a/nodedb/src/data/executor/handlers/timeseries/aggregate.rs +++ b/nodedb/src/data/executor/handlers/timeseries/aggregate.rs @@ -6,10 +6,16 @@ //! for low-cardinality symbol GROUP BY, parallel partition processing via //! std::thread::scope, sparse index block-level skip, and single metadata //! read per partition. +//! +//! The caller's read policy travels as part of `filter_predicates`: the +//! grouped scan lowers every predicate onto typed column vectors before any +//! row is aggregated, and a predicate it cannot lower fails the statement. +//! An unlowerable policy therefore never aggregates the rows it governs. use crate::bridge::envelope::Response; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; +use crate::engine::timeseries::grouped_filter::UnsupportedPredicate; use crate::engine::timeseries::grouped_scan::{ GroupedAggResult, PartitionAggParams, aggregate_memtable, aggregate_partition, }; @@ -23,6 +29,9 @@ pub(in crate::data::executor) struct TsAggregateParams<'a> { pub collection: &'a str, pub time_range: (i64, i64), pub limit: usize, + /// The query's WHERE predicates followed by the caller's read policy. + /// Every predicate is pushed into the grouped scan, so the policy + /// excludes rows before they reach an accumulator. pub filter_predicates: &'a [crate::bridge::scan_filter::ScanFilter], pub bucket_interval_ms: i64, pub group_by: &'a [String], @@ -62,15 +71,17 @@ impl CoreLoop { let mut merged = if let Some(mt) = self.columnar_memtables.get(&key) && !mt.is_empty() { - aggregate_memtable( + match aggregate_memtable( mt, group_by, aggregates, filter_predicates, time_range, bucket_interval_ms, - ) - .unwrap_or_else(|| GroupedAggResult::new(num_aggs)) + ) { + Ok(result) => result.unwrap_or_else(|| GroupedAggResult::new(num_aggs)), + Err(e) => return self.response_error(task, crate::Error::from(e)), + } } else { GroupedAggResult::new(num_aggs) }; @@ -98,7 +109,7 @@ impl CoreLoop { let io_priority = Some(task.request.priority); let io_metrics: &crate::data::io::IoMetrics = &self.io_metrics; for dir in &partition_dirs { - if let Some(part_result) = aggregate_partition(PartitionAggParams { + match aggregate_partition(PartitionAggParams { partition_dir: dir, group_by, aggregates, @@ -110,7 +121,9 @@ impl CoreLoop { io_priority, io_metrics: Some(io_metrics), }) { - merged.merge(&part_result); + Ok(Some(part_result)) => merged.merge(&part_result), + Ok(None) => {} + Err(e) => return self.response_error(task, crate::Error::from(e)), } } } else { @@ -127,41 +140,48 @@ impl CoreLoop { let thread_count = available.min(partition_dirs.len()).min(8); let chunk_size = partition_dirs.len().div_ceil(thread_count); - let partition_results: Vec = std::thread::scope(|s| { - let handles: Vec<_> = partition_dirs - .chunks(chunk_size) - .map(|chunk| { - let gb = &group_by_owned; - let ag = &agg_owned; - let fl = &filters_owned; - let nc = &needed_owned; - s.spawn(move || { - let mut local = GroupedAggResult::new(ag.len()); - for dir in chunk { - // Parallel threads: no io_uring (fadvise fallback). - if let Some(r) = aggregate_partition(PartitionAggParams { - partition_dir: dir, - group_by: gb, - aggregates: ag, - filters: fl, - time_range, - needed_columns: nc, - bucket_interval_ms, - uring_reader: None, - io_priority: None, - io_metrics: None, - }) { - local.merge(&r); + let partition_results: Result, UnsupportedPredicate> = + std::thread::scope(|s| { + let handles: Vec<_> = partition_dirs + .chunks(chunk_size) + .map(|chunk| { + let gb = &group_by_owned; + let ag = &agg_owned; + let fl = &filters_owned; + let nc = &needed_owned; + s.spawn(move || -> Result<_, UnsupportedPredicate> { + let mut local = GroupedAggResult::new(ag.len()); + for dir in chunk { + // Parallel threads: no io_uring (fadvise fallback). + if let Some(r) = + aggregate_partition(PartitionAggParams { + partition_dir: dir, + group_by: gb, + aggregates: ag, + filters: fl, + time_range, + needed_columns: nc, + bucket_interval_ms, + uring_reader: None, + io_priority: None, + io_metrics: None, + })? + { + local.merge(&r); + } } - } - local + Ok(local) + }) }) - }) - .collect(); + .collect(); - handles.into_iter().filter_map(|h| h.join().ok()).collect() - }); + handles.into_iter().filter_map(|h| h.join().ok()).collect() + }); + let partition_results = match partition_results { + Ok(results) => results, + Err(e) => return self.response_error(task, crate::Error::from(e)), + }; for r in &partition_results { merged.merge(r); } diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/partition_scan.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/partition_scan.rs index 375c80609..135c66dfb 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/partition_scan.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/partition_scan.rs @@ -4,25 +4,37 @@ use std::collections::HashMap; +use crate::bridge::scan_filter::ScanFilter; use crate::engine::timeseries::columnar_agg::timestamp_range_filter; use crate::engine::timeseries::columnar_memtable::{ColumnData, ColumnType}; use crate::engine::timeseries::columnar_segment::ColumnarSegmentReader; -use super::row_emit::{emit_partition_row, extract_timestamp}; +use super::row_emit::{emit_partition_row, extract_timestamp, row_admitted}; /// Scan disk partitions in parallel, returning rmpv rows sorted by timestamp. /// -/// `Err` when a stored time cell cannot be read as its column's instant. +/// `rls_predicates` is the caller's read policy: every row is checked +/// against it before it counts toward `limit`. `Err` when a stored time +/// cell cannot be read as its column's instant or a predicate expression +/// divides by zero. pub(super) fn scan_partitions_parallel( partition_dirs: &[std::path::PathBuf], time_range: (i64, i64), limit: usize, - filter_predicates: &[crate::bridge::scan_filter::ScanFilter], + filter_predicates: &[ScanFilter], has_filters: bool, + rls_predicates: &[ScanFilter], ) -> crate::Result> { if partition_dirs.len() <= 1 { return match partition_dirs.first() { - Some(dir) => scan_one_partition(dir, time_range, limit, filter_predicates, has_filters), + Some(dir) => scan_one_partition( + dir, + time_range, + limit, + filter_predicates, + has_filters, + rls_predicates, + ), None => Ok(Vec::new()), }; } @@ -41,11 +53,13 @@ pub(super) fn scan_partitions_parallel( limit, filter_predicates, has_filters, + rls_predicates, ); } let chunk_size = partition_dirs.len().div_ceil(thread_count); let filters_ref = filter_predicates; + let rls_ref = rls_predicates; let mut thread_results: Vec> = std::thread::scope(|s| { let handles: Vec<_> = partition_dirs @@ -58,6 +72,7 @@ pub(super) fn scan_partitions_parallel( limit, filters_ref, has_filters, + rls_ref, ) }) }) @@ -90,6 +105,7 @@ pub(super) fn scan_partitions_parallel( limit, filter_predicates, has_filters, + rls_predicates, ) } } @@ -98,8 +114,9 @@ pub(super) fn scan_partitions_sequential( partition_dirs: &[std::path::PathBuf], time_range: (i64, i64), limit: usize, - filter_predicates: &[crate::bridge::scan_filter::ScanFilter], + filter_predicates: &[ScanFilter], has_filters: bool, + rls_predicates: &[ScanFilter], ) -> crate::Result> { let mut results = Vec::new(); for dir in partition_dirs { @@ -107,7 +124,14 @@ pub(super) fn scan_partitions_sequential( break; } let remaining = limit - results.len(); - let rows = scan_one_partition(dir, time_range, remaining, filter_predicates, has_filters)?; + let rows = scan_one_partition( + dir, + time_range, + remaining, + filter_predicates, + has_filters, + rls_predicates, + )?; results.extend(rows); } results.truncate(limit); @@ -116,13 +140,18 @@ pub(super) fn scan_partitions_sequential( /// Scan a single disk partition, returning rmpv rows. /// -/// `Err` when a stored time cell cannot be read as its column's instant. +/// The WHERE predicates run on the typed columns when the evaluator can +/// lower them and per row otherwise. The read policy runs per row after +/// them, before the row counts toward `limit`. `Err` when a stored time +/// cell cannot be read as its column's instant or a predicate expression +/// divides by zero. pub(super) fn scan_one_partition( part_dir: &std::path::Path, time_range: (i64, i64), limit: usize, - filter_predicates: &[crate::bridge::scan_filter::ScanFilter], + filter_predicates: &[ScanFilter], has_filters: bool, + rls_predicates: &[ScanFilter], ) -> crate::Result> { let schema = match ColumnarSegmentReader::read_schema(part_dir, None) { Ok(s) => s, @@ -165,8 +194,11 @@ pub(super) fn scan_one_partition( sym_dicts: &sym_dicts, }; + // `row_filters` carries the WHERE predicates only when the typed-column + // evaluator could not lower them, so they still apply per row instead + // of being dropped. let row_count = timestamps.len(); - let filtered_indices = if has_filters { + let (filtered_indices, row_filters): (Vec, &[ScanFilter]) = if has_filters { if let Some(bitmask) = crate::data::executor::handlers::columnar_filter::eval_filters_bitmask( &part_src, @@ -174,21 +206,22 @@ pub(super) fn scan_one_partition( row_count, ) { - nodedb_query::simd_filter::bitmask_to_indices(&bitmask) + (nodedb_query::simd_filter::bitmask_to_indices(&bitmask), &[]) } else { match crate::data::executor::handlers::columnar_filter::eval_filters_sparse( &part_src, filter_predicates, &indices, ) { - Some(mask) => { - crate::data::executor::handlers::columnar_filter::apply_mask(&indices, &mask) - } - None => indices, + Some(mask) => ( + crate::data::executor::handlers::columnar_filter::apply_mask(&indices, &mask), + &[], + ), + None => (indices, filter_predicates), } } } else { - indices + (indices, &[]) }; let mut rows = Vec::with_capacity(filtered_indices.len().min(limit)); @@ -197,6 +230,9 @@ pub(super) fn scan_one_partition( break; } let row = emit_partition_row(&schema_vec, &col_data, &sym_dicts, idx as usize)?; + if !row_admitted(&row, row_filters, rls_predicates)? { + continue; + } rows.push(row); } diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs index 4e72d1a48..8fe5f2100 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs @@ -6,10 +6,42 @@ use std::collections::HashMap; use nodedb_types::columnar::schema::TS_SYSTEM; +use crate::bridge::scan_filter::ScanFilter; +use crate::data::executor::handlers::columnar_read::filter::value_matches_filters; use crate::data::executor::handlers::columnar_read::{emit_column_value, rmpv_time_cell}; use crate::engine::timeseries::columnar_memtable::{ColumnData, ColumnType}; use crate::util::rmpv_value::{rmpv_to_value, value_to_rmpv}; +/// Whether an emitted row passes `where_predicates` and then the caller's +/// read policy. +/// +/// Both sets empty admits the row without touching it. Otherwise the row is +/// converted once and both sets are evaluated by the predicate matcher the +/// columnar read path and the write gate use, so a policy means one thing +/// on every engine. `where_predicates` carries the WHERE clause only when +/// the typed-column evaluator could not lower it. `Err` when a predicate +/// expression divides by zero. +pub(super) fn row_admitted( + row: &rmpv::Value, + where_predicates: &[ScanFilter], + rls_predicates: &[ScanFilter], +) -> crate::Result { + if where_predicates.is_empty() && rls_predicates.is_empty() { + return Ok(true); + } + let doc = rmpv_to_value(row); + Ok(value_matches_filters(&doc, where_predicates)? + && value_matches_filters(&doc, rls_predicates)?) +} + +/// [`row_admitted`] for a row whose WHERE clause was already applied. +pub(super) fn row_admitted_by_policy( + row: &rmpv::Value, + rls_predicates: &[ScanFilter], +) -> crate::Result { + row_admitted(row, &[], rls_predicates) +} + /// Extract the `_ts_system` value from an rmpv-encoded row for audit-log /// ordering. Rows without the column sort first (treated as `i64::MIN`). pub(super) fn rmpv_system_time(row: &rmpv::Value) -> i64 { diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs index 48249acf0..824d7f204 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs @@ -3,12 +3,15 @@ //! Raw scan entry point: `RawScanParams` and `execute_ts_raw_scan`. use crate::bridge::envelope::{ErrorCode, Response}; +use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; use crate::engine::timeseries::columnar_agg::timestamp_range_filter; use super::partition_scan::scan_partitions_parallel; -use super::row_emit::{apply_computed_columns_rmpv, emit_memtable_row, rmpv_system_time}; +use super::row_emit::{ + apply_computed_columns_rmpv, emit_memtable_row, rmpv_system_time, row_admitted_by_policy, +}; /// Parameters for a timeseries raw scan (no aggregation). pub(in crate::data::executor) struct RawScanParams<'a> { @@ -17,8 +20,13 @@ pub(in crate::data::executor) struct RawScanParams<'a> { pub collection: &'a str, pub time_range: (i64, i64), pub limit: usize, - pub filter_predicates: &'a [crate::bridge::scan_filter::ScanFilter], + pub filter_predicates: &'a [ScanFilter], pub has_filters: bool, + /// The caller's decoded read policy; empty admits every row. Evaluated + /// per row after `filter_predicates` and before a row is gathered, so a + /// row the policy excludes never counts toward `limit`, on memtable, + /// partition, and in-transaction overlay rows alike. + pub rls_predicates: &'a [ScanFilter], pub computed_columns: &'a [u8], /// `AS OF SYSTEM TIME NULL`: emit every `_ts_system` version ordered /// ascending by system time (audit-log semantics). @@ -49,6 +57,7 @@ impl CoreLoop { limit, filter_predicates, has_filters, + rls_predicates, computed_columns: computed_columns_bytes, all_versions, txn_id, @@ -163,6 +172,14 @@ impl CoreLoop { } } } + // The caller's read policy, after the WHERE predicates and + // before the row is gathered, so the limit counts admitted + // rows only. + match row_admitted_by_policy(&row, rls_predicates) { + Ok(true) => {} + Ok(false) => continue, + Err(e) => return self.response_error(task, e), + } results.push(row); } } @@ -195,6 +212,7 @@ impl CoreLoop { remaining, filter_predicates, has_filters, + rls_predicates, ) { Ok(rows) => rows, Err(e) => return self.response_error(task, e), @@ -220,6 +238,7 @@ impl CoreLoop { time_range, filter_predicates, has_filters, + rls_predicates, limit: gather_limit, }, &mut results, diff --git a/nodedb/src/data/executor/handlers/timeseries/scan.rs b/nodedb/src/data/executor/handlers/timeseries/scan.rs index 942d89e17..5b2851975 100644 --- a/nodedb/src/data/executor/handlers/timeseries/scan.rs +++ b/nodedb/src/data/executor/handlers/timeseries/scan.rs @@ -3,7 +3,9 @@ //! Data Plane timeseries scan parameters and execution. use crate::bridge::envelope::{Payload, Response, Status}; +use crate::bridge::scan_filter::{ScanFilter, decode_scan_filters}; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::columnar_read::filter::decode_rls_filters; use crate::data::executor::task::ExecutionTask; use nodedb_query::agg_key::canonical_agg_key; use nodedb_types::columnar::schema::{TS_SYSTEM, TS_VALID_FROM, TS_VALID_UNTIL}; @@ -18,6 +20,14 @@ pub(in crate::data::executor) struct TimeseriesScanParams<'a> { pub time_range: (i64, i64), pub limit: usize, pub filters: &'a [u8], + /// The caller's read policy as a MessagePack `Vec`, injected + /// by the planner. Empty when no policy governs the caller. The raw scan + /// evaluates it per row after `filters`, before computed columns, sort, + /// and limit, on memtable, partition, and in-transaction overlay rows + /// alike. The aggregate branch pushes it into the grouped scan so it + /// excludes rows before aggregation, and fails when the grouped scan + /// cannot lower it. + pub rls_filters: &'a [u8], /// `ORDER BY` keys as `(column, ascending)`. Applied to the materialized /// result before `limit`, on both the raw and the aggregate branch. pub sort_keys: &'a [nodedb_physical::physical_plan::SortKeySpec], @@ -54,6 +64,7 @@ impl CoreLoop { time_range, limit, filters, + rls_filters, sort_keys, bucket_interval_ms, group_by, @@ -82,12 +93,20 @@ impl CoreLoop { ); } - let mut filter_predicates: Vec = - if filters.is_empty() { - Vec::new() - } else { - zerompk::from_msgpack(filters).unwrap_or_default() + // Both predicate sets are decoded before any row is read. A payload + // the Data Plane cannot decode fails the statement: for the policy + // that is the difference between hiding governed rows and returning + // them, and for the WHERE clause it is the difference between the + // asked-for rows and every row. + let mut filter_predicates: Vec = + match decode_scan_filters(filters, "timeseries filter") { + Ok(p) => p, + Err(e) => return self.response_error(task, e), }; + let rls_predicates: Vec = match decode_rls_filters(rls_filters) { + Ok(p) => p, + Err(e) => return self.response_error(task, e), + }; // Bitemporal cutoffs: translate to column-level predicates on // `_ts_system` / `_ts_valid_from` / `_ts_valid_until`. The // segment reader's block-skip infrastructure applies these @@ -130,11 +149,14 @@ impl CoreLoop { let is_aggregate = !aggregates.is_empty(); let has_time_range = time_range.0 > 0 || time_range.1 < i64::MAX; - // Fast path: COUNT(*) with no GROUP BY, no filters. + // Fast path: COUNT(*) with no GROUP BY, no filters, no read policy. + // The metadata count covers every stored row, so a governed read + // takes the grouped scan, where the policy excludes rows first. if is_aggregate && bucket_interval_ms == 0 && group_by.is_empty() && !has_filters + && rls_predicates.is_empty() && !has_time_range && aggregates.len() == 1 && aggregates[0].0 == "count" @@ -158,7 +180,7 @@ impl CoreLoop { needed.push(field.clone()); } } - for fp in &filter_predicates { + for fp in filter_predicates.iter().chain(&rls_predicates) { if !needed.contains(&fp.field) { needed.push(fp.field.clone()); } @@ -170,13 +192,23 @@ impl CoreLoop { // Mode dispatch. if is_aggregate || bucket_interval_ms > 0 { + // The policy joins the WHERE predicates so the grouped scan + // excludes governed rows before any accumulator sees them. The + // grouped scan fails on a predicate it cannot lower, so a policy + // shape it does not support fails the statement rather than + // aggregating ungoverned rows. + let governed_predicates: Vec = filter_predicates + .iter() + .chain(&rls_predicates) + .cloned() + .collect(); self.execute_ts_aggregate(aggregate::TsAggregateParams { task, tid, collection, time_range, limit, - filter_predicates: &filter_predicates, + filter_predicates: &governed_predicates, bucket_interval_ms, group_by, aggregates, @@ -202,6 +234,7 @@ impl CoreLoop { limit, filter_predicates: &filter_predicates, has_filters, + rls_predicates: &rls_predicates, computed_columns, all_versions, txn_id: overlay_txn, diff --git a/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs b/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs index 3450409fb..5e9a7c05b 100644 --- a/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs +++ b/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs @@ -24,6 +24,7 @@ use nodedb_types::value::Value; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::columnar_read::filter::value_matches_filters; use crate::data::executor::handlers::transaction::overlay::Staged; use crate::types::{DatabaseId, TenantId, TxnId}; use crate::util::rmpv_value::value_to_rmpv; @@ -37,6 +38,9 @@ pub(in crate::data::executor) struct TimeseriesOverlayMergeParams<'a> { pub time_range: (i64, i64), pub filter_predicates: &'a [ScanFilter], pub has_filters: bool, + /// The caller's decoded read policy. A staged row the policy excludes is + /// dropped exactly like a base row; empty admits every row. + pub rls_predicates: &'a [ScanFilter], /// Row ceiling for the whole scan (base + staged). Staged rows are only /// appended while the result is below this bound, so the merge never /// exceeds the SQL `LIMIT`. @@ -82,8 +86,8 @@ fn staged_row_to_rmpv(row: &Value) -> rmpv::Value { impl CoreLoop { /// Append this transaction's staged `TimeseriesOp::Ingest` rows to /// `results` (base raw-scan rows), each subject to the scan's time-range, - /// WHERE predicate, and row limit. No-op when the transaction has no - /// overlay entries for this collection. + /// WHERE predicate, read policy, and row limit. No-op when the + /// transaction has no overlay entries for this collection. pub(in crate::data::executor) fn merge_overlay_into_timeseries_scan( &self, params: TimeseriesOverlayMergeParams<'_>, @@ -95,6 +99,7 @@ impl CoreLoop { time_range, filter_predicates, has_filters, + rls_predicates, limit, } = params; @@ -133,6 +138,11 @@ impl CoreLoop { if has_filters && !ScanFilter::all_match_binary(filter_predicates, body)? { continue; } + // The caller's read policy, on the decoded row, after the WHERE + // predicate and before the row takes a limit slot. + if !value_matches_filters(&row, rls_predicates)? { + continue; + } results.push(staged_row_to_rmpv(&row)); } diff --git a/nodedb/src/engine/timeseries/grouped_filter.rs b/nodedb/src/engine/timeseries/grouped_filter.rs index 0b8503b7b..6c94a5af4 100644 --- a/nodedb/src/engine/timeseries/grouped_filter.rs +++ b/nodedb/src/engine/timeseries/grouped_filter.rs @@ -6,22 +6,50 @@ //! returning packed `Vec` bitmasks. Uses SIMD kernels from //! `nodedb_query::simd_filter` for numeric and symbol comparisons. +use nodedb_query::scan_filter::value_as_timestamp_ms; use nodedb_query::simd_filter; use super::columnar_memtable::{ColumnData, ColumnType}; use crate::bridge::scan_filter::ScanFilter; +/// A predicate the grouped scan cannot lower onto typed column vectors. +/// +/// The grouped scan evaluates predicates only as SIMD bitmasks, so a shape +/// it cannot lower fails the aggregate instead of aggregating rows the +/// predicate never excluded. The message names the predicate and why. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("predicate `{field} {op}` cannot be evaluated by the timeseries grouped scan: {reason}")] +pub struct UnsupportedPredicate { + pub field: String, + pub op: &'static str, + pub reason: &'static str, +} + +impl UnsupportedPredicate { + fn new(f: &ScanFilter, reason: &'static str) -> Self { + Self { + field: f.field.clone(), + op: f.op.as_str(), + reason, + } + } +} + /// Evaluate ScanFilter predicates on columnar data, returning a bitmask. /// -/// Bit *i* is set iff row *i* passes ALL filters. -/// Returns `None` for unsupported patterns (OR clauses, contains, etc.). +/// Bit *i* is set iff row *i* passes ALL filters. Supported shapes: a +/// single `(column, op, literal)` comparison with `eq`/`ne` on symbol +/// columns and `eq`/`ne`/`gt`/`gte`/`lt`/`lte` on numeric and time +/// columns. Every other shape (OR clauses, expressions, `in`, `like`, an +/// unknown column, a literal of the wrong type) is an +/// [`UnsupportedPredicate`] error. pub fn eval_filters_to_bitmask<'a>( filters: &[ScanFilter], schema: &[(String, ColumnType)], columns: &[Option<&'a ColumnData>], sym_lookup: &dyn Fn(usize) -> Option<&'a nodedb_types::timeseries::SymbolDictionary>, row_count: usize, -) -> Option> { +) -> Result, UnsupportedPredicate> { let rt = simd_filter::filter_runtime(); let mut mask = simd_filter::bitmask_all(row_count); @@ -30,16 +58,26 @@ pub fn eval_filters_to_bitmask<'a>( continue; } if !f.clauses.is_empty() { - return None; + return Err(UnsupportedPredicate::new(f, "OR clauses")); + } + if f.expr.is_some() { + return Err(UnsupportedPredicate::new(f, "expression predicate")); } - let col_pos = schema.iter().position(|(n, _)| n == &f.field)?; + let col_pos = schema + .iter() + .position(|(n, _)| n == &f.field) + .ok_or(UnsupportedPredicate::new(f, "column not in the schema"))?; let (_, col_type) = &schema[col_pos]; - let col_data = columns[col_pos]?; + let col_data = + columns[col_pos].ok_or(UnsupportedPredicate::new(f, "column data not loaded"))?; let filter_mask = match col_type { ColumnType::Float64 => { - let fv = f.value.as_f64()?; + let fv = f + .value + .as_f64() + .ok_or(UnsupportedPredicate::new(f, "literal is not a float"))?; let vals = col_data.as_f64(); let slice = &vals[..row_count.min(vals.len())]; match f.op.as_str() { @@ -57,15 +95,25 @@ pub fn eval_filters_to_bitmask<'a>( let b = (rt.lte_f64)(slice, fv + f64::EPSILON); simd_filter::bitmask_not(&simd_filter::bitmask_and(&a, &b), row_count) } - _ => return None, + _ => { + return Err(UnsupportedPredicate::new(f, "operator on a float column")); + } } } ColumnType::Int64 | ColumnType::Timestamp(_) => { - let fv = f.value.as_i64()?; - let vals = if *col_type == ColumnType::Int64 { - col_data.as_i64() + // A time column stores epoch milliseconds, so its literal is + // read as an instant: an integer, a datetime, or a datetime + // string all lower to the stored form. + let (fv, vals) = if *col_type == ColumnType::Int64 { + let fv = f + .value + .as_i64() + .ok_or(UnsupportedPredicate::new(f, "literal is not an integer"))?; + (fv, col_data.as_i64()) } else { - col_data.as_timestamps() + let fv = value_as_timestamp_ms(&f.value) + .ok_or(UnsupportedPredicate::new(f, "literal is not an instant"))?; + (fv, col_data.as_timestamps()) }; let slice = &vals[..row_count.min(vals.len())]; match f.op.as_str() { @@ -83,12 +131,21 @@ pub fn eval_filters_to_bitmask<'a>( let b = (rt.lte_i64)(slice, fv); simd_filter::bitmask_not(&simd_filter::bitmask_and(&a, &b), row_count) } - _ => return None, + _ => { + return Err(UnsupportedPredicate::new( + f, + "operator on an integer or time column", + )); + } } } ColumnType::Symbol => { - let filter_str = f.value.as_str()?; - let dict = sym_lookup(col_pos)?; + let filter_str = f + .value + .as_str() + .ok_or(UnsupportedPredicate::new(f, "literal is not a string"))?; + let dict = sym_lookup(col_pos) + .ok_or(UnsupportedPredicate::new(f, "symbol dictionary not loaded"))?; let sym_ids = col_data.as_symbols(); let slice = &sym_ids[..row_count.min(sym_ids.len())]; match f.op.as_str() { @@ -106,7 +163,9 @@ pub fn eval_filters_to_bitmask<'a>( simd_filter::bitmask_all(row_count) } } - _ => return None, + _ => { + return Err(UnsupportedPredicate::new(f, "operator on a symbol column")); + } } } }; @@ -114,7 +173,7 @@ pub fn eval_filters_to_bitmask<'a>( mask = simd_filter::bitmask_and(&mask, &filter_mask); } - Some(mask) + Ok(mask) } /// Apply sparse index block-level skip to a bitmask. diff --git a/nodedb/src/engine/timeseries/grouped_scan/partition.rs b/nodedb/src/engine/timeseries/grouped_scan/partition.rs index fdcd7e311..670afab00 100644 --- a/nodedb/src/engine/timeseries/grouped_scan/partition.rs +++ b/nodedb/src/engine/timeseries/grouped_scan/partition.rs @@ -12,7 +12,7 @@ use nodedb_query::simd_filter; use super::super::columnar_memtable::{ColumnData, ColumnType, ColumnarMemtable}; use super::super::columnar_segment::ColumnarSegmentReader; -use super::super::grouped_filter; +use super::super::grouped_filter::{self, UnsupportedPredicate}; use super::strategies::dispatch_grouping; use super::types::{GroupedAggResult, resolve_schema}; use crate::bridge::envelope::Priority; @@ -20,6 +20,11 @@ use crate::bridge::scan_filter::ScanFilter; use crate::data::io::IoMetrics; /// Aggregate from a columnar memtable with GROUP BY + optional time_bucket. +/// +/// `Ok(None)` when a GROUP BY or aggregate column is not in the memtable's +/// schema. `Err` when a predicate in `filters` cannot be lowered onto the +/// typed columns: the caller fails the statement rather than aggregating +/// rows the predicate never excluded. pub fn aggregate_memtable( mt: &ColumnarMemtable, group_by: &[String], @@ -27,15 +32,19 @@ pub fn aggregate_memtable( filters: &[ScanFilter], time_range: (i64, i64), bucket_interval_ms: i64, -) -> Option { +) -> Result, UnsupportedPredicate> { let schema = mt.schema(); let num_aggs = aggregates.len(); let row_count = mt.row_count() as usize; if row_count == 0 { - return Some(GroupedAggResult::new(num_aggs)); + return Ok(Some(GroupedAggResult::new(num_aggs))); } - let resolved = resolve_schema(&schema.columns, schema.timestamp_idx, group_by, aggregates)?; + let Some(resolved) = + resolve_schema(&schema.columns, schema.timestamp_idx, group_by, aggregates) + else { + return Ok(None); + }; let col_refs: Vec> = (0..schema.columns.len()) .map(|i| Some(mt.column(i))) @@ -63,7 +72,7 @@ pub fn aggregate_memtable( } if simd_filter::popcount(&mask) == 0 { - return Some(GroupedAggResult::new(num_aggs)); + return Ok(Some(GroupedAggResult::new(num_aggs))); } let timestamps = if bucket_interval_ms > 0 { @@ -72,7 +81,7 @@ pub fn aggregate_memtable( None }; - Some(dispatch_grouping( + Ok(Some(dispatch_grouping( super::strategies::GroupedScanInputs { resolved: &resolved, columns: &col_refs, @@ -84,7 +93,7 @@ pub fn aggregate_memtable( &sym_lookup, timestamps, bucket_interval_ms, - )) + ))) } /// Parameters for partition-level grouped aggregation. @@ -109,22 +118,36 @@ pub struct PartitionAggParams<'a> { /// /// When `uring_reader` is `Some`, column files are batch-read via io_uring /// (parallel kernel I/O). When `None`, falls back to fadvise + std::fs::read. -pub fn aggregate_partition(p: PartitionAggParams<'_>) -> Option { +/// +/// `Ok(None)` when the partition's schema or metadata cannot be read or does +/// not carry a GROUP BY / aggregate column. `Err` when a predicate in +/// `filters` cannot be lowered onto the partition's typed columns: the +/// caller fails the statement rather than aggregating rows the predicate +/// never excluded. +pub fn aggregate_partition( + p: PartitionAggParams<'_>, +) -> Result, UnsupportedPredicate> { let num_aggs = p.aggregates.len(); - let schema = ColumnarSegmentReader::read_schema(p.partition_dir, None).ok()?; - let meta = ColumnarSegmentReader::read_meta(p.partition_dir, None).ok()?; + let Ok(schema) = ColumnarSegmentReader::read_schema(p.partition_dir, None) else { + return Ok(None); + }; + let Ok(meta) = ColumnarSegmentReader::read_meta(p.partition_dir, None) else { + return Ok(None); + }; let row_count = meta.row_count as usize; if row_count == 0 { - return Some(GroupedAggResult::new(num_aggs)); + return Ok(Some(GroupedAggResult::new(num_aggs))); } - let resolved = resolve_schema( + let Some(resolved) = resolve_schema( &schema.columns, schema.timestamp_idx, p.group_by, p.aggregates, - )?; + ) else { + return Ok(None); + }; // Load sparse index for block-level skip. let sparse_idx = ColumnarSegmentReader::read_sparse_index(p.partition_dir, None) @@ -203,7 +226,9 @@ pub fn aggregate_partition(p: PartitionAggParams<'_>) -> Option) -> Option) -> Option> = col_data.iter().map(|c| c.as_ref()).collect(); let sym_lookup_tmp = |col_idx: usize| -> Option<&nodedb_types::timeseries::SymbolDictionary> { sym_dicts.get(&col_idx) }; - if let Some(filter_mask) = grouped_filter::eval_filters_to_bitmask( + let filter_mask = grouped_filter::eval_filters_to_bitmask( p.filters, &schema.columns, &col_refs_tmp, &sym_lookup_tmp, effective_row_count, - ) { - mask = simd_filter::bitmask_and(&mask, &filter_mask); - } + )?; + mask = simd_filter::bitmask_and(&mask, &filter_mask); } if simd_filter::popcount(&mask) == 0 { - return Some(GroupedAggResult::new(num_aggs)); + return Ok(Some(GroupedAggResult::new(num_aggs))); } let col_refs: Vec> = col_data.iter().map(|c| c.as_ref()).collect(); @@ -285,7 +312,7 @@ pub fn aggregate_partition(p: PartitionAggParams<'_>) -> Option { diff --git a/nodedb/src/error_from.rs b/nodedb/src/error_from.rs index 7a5fd470e..67eaf8316 100644 --- a/nodedb/src/error_from.rs +++ b/nodedb/src/error_from.rs @@ -45,6 +45,14 @@ impl From for Error { } } +impl From for Error { + fn from(e: crate::engine::timeseries::grouped_filter::UnsupportedPredicate) -> Self { + Self::FeatureNotSupported { + detail: e.to_string(), + } + } +} + impl From for Error { fn from(e: crate::engine::timeseries::query::QueryError) -> Self { Self::Storage { diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index 9b23ad6eb..25c9b0ae5 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -271,6 +271,7 @@ mod strict_schema_restart; mod strict_typed_column_rendering; mod timeseries_declared_time_key; mod timeseries_join_time_rendering; +mod timeseries_read_row_level_security; mod timeseries_write_row_level_security; mod transactional_ddl_atomicity; mod transactional_ddl_compensation; diff --git a/nodedb/tests/wire/cases/timeseries_read_row_level_security.rs b/nodedb/tests/wire/cases/timeseries_read_row_level_security.rs new file mode 100644 index 000000000..92c41dac6 --- /dev/null +++ b/nodedb/tests/wire/cases/timeseries_read_row_level_security.rs @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Row-level security on timeseries reads. +//! +//! A `FOR READ` policy on a timeseries collection governs every row a +//! `SELECT` returns, from the live memtable and from flushed partitions +//! alike. The raw scan applies the policy after time-range pruning and the +//! query's own WHERE, before computed columns, sort, and limit, so a row the +//! policy excludes never reaches the client, never consumes a `LIMIT` slot, +//! and never joins a partner. The aggregate path pushes the policy into the +//! grouped scan, so an excluded row never reaches an accumulator. + +use crate::harness::TestServer; + +const PASSWORD: &str = "ts-read-rls-secret-7"; + +/// Epoch-millisecond time keys, one per seeded row. +const TS_MINE: i64 = 1_700_000_000_000; +const TS_A: i64 = 1_700_000_001_000; +const TS_B: i64 = 1_700_000_002_000; + +/// Create a timeseries `collection` holding one row owned by `user` and two +/// owned by someone else, plus `user` with the readwrite role and a read +/// policy admitting only the caller's own rows. +async fn seed(server: &TestServer, collection: &str, user: &str) { + server + .exec(&format!( + "CREATE COLLECTION {collection} \ + (ts BIGINT TIME_KEY, owner TEXT, value FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .unwrap_or_else(|e| panic!("create {collection}: {e}")); + for (ts, owner, value) in [ + (TS_MINE, user, 1.0), + (TS_A, "someone_else", 2.0), + (TS_B, "someone_else", 3.0), + ] { + server + .exec(&format!( + "INSERT INTO {collection} (ts, owner, value) \ + VALUES ({ts}, '{owner}', {value})" + )) + .await + .unwrap_or_else(|e| panic!("seed {collection} row {ts}: {e}")); + } + server + .exec(&format!("CREATE USER {user} PASSWORD '{PASSWORD}'")) + .await + .unwrap_or_else(|e| panic!("create user {user}: {e}")); + server + .exec(&format!("GRANT ROLE readwrite TO {user}")) + .await + .unwrap_or_else(|e| panic!("grant readwrite to {user}: {e}")); + server + .exec(&format!( + "CREATE RLS POLICY {collection}_owner ON {collection} FOR READ \ + USING (owner = $auth.username)" + )) + .await + .unwrap_or_else(|e| panic!("create read policy on {collection}: {e}")); +} + +/// Run `sql` as `user` and return each row's cells joined by `|`. +async fn rows_as(server: &TestServer, user: &str, sql: &str) -> Vec { + let (client, handle) = server + .connect_as(user, PASSWORD) + .await + .unwrap_or_else(|e| panic!("connect as {user}: {e}")); + let messages = client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("{user} runs {sql}: {e}")); + let mut out = Vec::new(); + for message in messages { + if let tokio_postgres::SimpleQueryMessage::Row(row) = message { + let mut cells = Vec::new(); + for i in 0..row.len() { + cells.push(row.get(i).unwrap_or("").to_string()); + } + out.push(cells.join("|")); + } + } + drop(client); + handle.abort(); + out +} + +/// A `SELECT` over the live memtable returns only the rows the read policy +/// admits for the caller. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_timeseries_select_returns_only_policy_admitted_rows() { + let server = TestServer::start().await; + let user = "ts_rls_reader"; + seed(&server, "ts_rls_read", user).await; + + let rows = rows_as( + &server, + user, + "SELECT ts, owner FROM ts_rls_read ORDER BY ts", + ) + .await; + assert_eq!( + rows, + vec![format!("{TS_MINE}|{user}")], + "the read policy admits one row for this caller: {rows:?}" + ); +} + +/// A `SELECT` over flushed partitions returns only the rows the read policy +/// admits for the caller. The policy applies inside the partition reader, +/// so a flushed row is governed the same as a live one. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_timeseries_select_over_flushed_partitions_returns_only_policy_admitted_rows() { + let server = TestServer::start_with_timeseries_memtable_budget(1).await; + let user = "ts_rls_flushed_reader"; + seed(&server, "ts_rls_read_flushed", user).await; + + let rows = rows_as( + &server, + user, + "SELECT ts, owner FROM ts_rls_read_flushed ORDER BY ts", + ) + .await; + assert_eq!( + rows, + vec![format!("{TS_MINE}|{user}")], + "the read policy admits one row for this caller: {rows:?}" + ); +} + +/// A `LIMIT` counts admitted rows only: with one admitted row and two +/// excluded rows seeded, `LIMIT 1` returns the admitted row, never an +/// excluded row that happened to be scanned first. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_timeseries_select_with_a_limit_counts_only_admitted_rows() { + let server = TestServer::start().await; + let user = "ts_rls_limiter"; + seed(&server, "ts_rls_limit", user).await; + + let rows = rows_as( + &server, + user, + "SELECT ts, owner FROM ts_rls_limit ORDER BY ts LIMIT 1", + ) + .await; + assert_eq!( + rows, + vec![format!("{TS_MINE}|{user}")], + "the limit applies to admitted rows only: {rows:?}" + ); +} + +/// A `COUNT(*)` over a governed timeseries collection leaves the metadata +/// fast path and counts only the rows the read policy admits. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_timeseries_count_counts_only_policy_admitted_rows() { + let server = TestServer::start().await; + let user = "ts_rls_counter"; + seed(&server, "ts_rls_count", user).await; + + let rows = rows_as(&server, user, "SELECT COUNT(*) FROM ts_rls_count").await; + assert_eq!( + rows, + vec!["1".to_string()], + "the read policy admits one row for this caller: {rows:?}" + ); +} + +/// A `time_bucket` aggregate counts only admitted rows: the three seeded +/// rows share one hourly bucket, and the bucket counts one. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_timeseries_time_bucket_aggregate_counts_only_policy_admitted_rows() { + let server = TestServer::start().await; + let user = "ts_rls_bucketer"; + seed(&server, "ts_rls_bucket", user).await; + + let rows = rows_as( + &server, + user, + "SELECT time_bucket('1 hour', ts) AS b, COUNT(*) FROM ts_rls_bucket GROUP BY b", + ) + .await; + assert_eq!(rows.len(), 1, "one bucket holds every seeded row: {rows:?}"); + assert!( + rows[0].ends_with("|1"), + "the bucket counts the one admitted row: {rows:?}" + ); +} + +/// A `GROUP BY` aggregate groups only admitted rows: the excluded owner's +/// group never appears. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_timeseries_group_by_aggregate_groups_only_policy_admitted_rows() { + let server = TestServer::start().await; + let user = "ts_rls_grouper"; + seed(&server, "ts_rls_group", user).await; + + let rows = rows_as( + &server, + user, + "SELECT owner, AVG(value) FROM ts_rls_group GROUP BY owner", + ) + .await; + assert_eq!(rows.len(), 1, "one group is admitted: {rows:?}"); + assert!( + rows[0].starts_with(&format!("{user}|")), + "the admitted group is the caller's own: {rows:?}" + ); +} + +/// A join reads the governed timeseries side on the caller's behalf, so its +/// policy applies to that side before the join: excluded rows neither match +/// a partner nor reach the client. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_join_over_a_governed_timeseries_collection_excludes_policy_filtered_rows() { + let server = TestServer::start().await; + let user = "ts_rls_joiner"; + seed(&server, "ts_rls_join_t", user).await; + server + .exec( + "CREATE COLLECTION ts_rls_join_d (id TEXT PRIMARY KEY, tag TEXT) \ + WITH (engine='document_strict')", + ) + .await + .expect("create document side"); + server + .exec(&format!( + "INSERT INTO ts_rls_join_d (id, tag) VALUES \ + ('{user}', 't_mine'), ('someone_else', 't_theirs')" + )) + .await + .expect("seed document side"); + + let rows = rows_as( + &server, + user, + "SELECT t.ts, d.tag FROM ts_rls_join_t t \ + JOIN ts_rls_join_d d ON t.owner = d.id ORDER BY t.ts", + ) + .await; + assert_eq!( + rows, + vec![format!("{TS_MINE}|t_mine")], + "the join surfaced timeseries rows the read policy excludes: {rows:?}" + ); +} diff --git a/nodedb/tests/wire/harness/config_toml.rs b/nodedb/tests/wire/harness/config_toml.rs index 20582a762..9a14446c4 100644 --- a/nodedb/tests/wire/harness/config_toml.rs +++ b/nodedb/tests/wire/harness/config_toml.rs @@ -43,6 +43,9 @@ pub(super) struct TuningOverrides { /// Overrides `[tuning.query] stream_chunk_size` so a test can drive the /// chunked-streaming scan path without seeding 1000 rows. pub(super) stream_chunk_size: Option, + /// Overrides `[tuning.timeseries] memtable_budget_bytes` so a test can + /// observe timeseries partition flushes on a handful of rows. + pub(super) timeseries_memtable_budget_bytes: Option, } impl TuningOverrides { @@ -74,6 +77,14 @@ impl TuningOverrides { ..Self::default() } } + + /// Boot with a lowered timeseries memtable budget. + pub(super) fn timeseries_memtable_budget(bytes: usize) -> Self { + Self { + timeseries_memtable_budget_bytes: Some(bytes), + ..Self::default() + } + } } /// Write `nodedb.toml` into `dir` and return its path. @@ -111,6 +122,11 @@ pub(super) fn write_config(dir: &Path, auth_mode: AuthMode, tuning: TuningOverri if let Some(rows) = tuning.stream_chunk_size { toml.push_str(&format!("\n[tuning.query]\nstream_chunk_size = {rows}\n")); } + if let Some(bytes) = tuning.timeseries_memtable_budget_bytes { + toml.push_str(&format!( + "\n[tuning.timeseries]\nmemtable_budget_bytes = {bytes}\n" + )); + } toml.push_str(&format!( "\n[backup_encryption]\nkey_path = {}\n", toml_quote(&write_backup_kek(dir)) diff --git a/nodedb/tests/wire/harness/lifecycle.rs b/nodedb/tests/wire/harness/lifecycle.rs index 9c13179a7..683f1c54f 100644 --- a/nodedb/tests/wire/harness/lifecycle.rs +++ b/nodedb/tests/wire/harness/lifecycle.rs @@ -101,6 +101,20 @@ impl TestServer { Self::connect_and_build(spawned, dir, AuthMode::Trust).await } + /// Spawn a single-core NodeDB server with a lowered timeseries memtable + /// budget so every ingest flushes its rows to a partition, and a read + /// exercises the partition path on a handful of rows. + pub async fn start_with_timeseries_memtable_budget(bytes: usize) -> Self { + let dir = tempfile::tempdir().expect("tempdir"); + let spawned = process::spawn( + dir.path(), + AuthMode::Trust, + TuningOverrides::timeseries_memtable_budget(bytes), + 1, + ); + Self::connect_and_build(spawned, dir, AuthMode::Trust).await + } + /// Open a server backed by an existing data directory, reopened in place /// so a previous server's data is visible after boot. `dir` is not /// consumed — ownership stays with the caller. From 40a724fc25743c1c6ecb8ce228ad054259b3a412 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 16 Sep 2026 20:09:21 +0800 Subject: [PATCH 10/21] refactor(ddl): build ShapedRows via constructors, not struct literals Replace direct ShapedRows struct-literal construction across neutral DDL handlers with the from_json_rows/text_rows/with_notice helpers, removing repeated notice: None and column_types boilerplate at each call site. --- .../control/server/response_shape/types.rs | 32 +++++++++++++++++++ .../server/shared/ddl/neutral/alert/show.rs | 10 +++--- .../shared/ddl/neutral/apikey/create.rs | 12 +++---- .../shared/ddl/neutral/apikey/manage.rs | 5 ++- .../server/shared/ddl/neutral/auth_key.rs | 30 ++++++----------- .../server/shared/ddl/neutral/auth_user.rs | 8 +---- .../server/shared/ddl/neutral/blacklist.rs | 8 +---- .../shared/ddl/neutral/change_stream/show.rs | 8 +---- .../server/shared/ddl/neutral/chunk_text.rs | 16 ++-------- .../shared/ddl/neutral/cluster/health.rs | 5 ++- .../shared/ddl/neutral/cluster/migration.rs | 5 ++- .../server/shared/ddl/neutral/cluster/raft.rs | 13 ++------ .../shared/ddl/neutral/cluster/ranges.rs | 5 ++- .../ddl/neutral/cluster/rebalance_cmd.rs | 15 ++++----- .../ddl/neutral/cluster/routing_hint.rs | 5 ++- .../ddl/neutral/cluster/schema_version.rs | 10 ++---- .../shared/ddl/neutral/cluster/topology.rs | 21 +++--------- .../shared/ddl/neutral/collection/describe.rs | 14 ++------ .../ddl/neutral/collection/show_indexes.rs | 17 ++-------- .../ddl/neutral/collection/vector_metadata.rs | 17 +++------- .../shared/ddl/neutral/conflict_policy.rs | 20 +++++------- .../shared/ddl/neutral/constraint/show.rs | 8 +---- .../shared/ddl/neutral/consumer_group/show.rs | 16 ++-------- .../shared/ddl/neutral/continuous_agg/show.rs | 5 ++- .../server/shared/ddl/neutral/crdt_ops.rs | 28 ++++++---------- .../server/shared/ddl/neutral/custom_type.rs | 8 +---- .../shared/ddl/neutral/database/support.rs | 10 ++---- .../shared/ddl/neutral/estimate_count.rs | 11 +++---- .../server/shared/ddl/neutral/explain_ddl.rs | 24 ++++---------- .../shared/ddl/neutral/explain_tiers.rs | 10 ++---- .../shared/ddl/neutral/function/show.rs | 8 +---- .../shared/ddl/neutral/graph_ops/algo.rs | 20 ++++-------- .../ddl/neutral/graph_ops/rag_fusion.rs | 12 +++---- .../shared/ddl/neutral/graph_ops/response.rs | 17 ++-------- .../shared/ddl/neutral/graph_ops/stats.rs | 14 ++++---- .../shared/ddl/neutral/impersonation.rs | 8 +---- .../shared/ddl/neutral/inspect/grants.rs | 24 ++------------ .../shared/ddl/neutral/inspect/tenants.rs | 15 ++++----- .../shared/ddl/neutral/inspect/trigger_dlq.rs | 9 +++--- .../shared/ddl/neutral/inspect/users.rs | 17 ++++------ .../shared/ddl/neutral/inspect_audit.rs | 15 ++++----- .../shared/ddl/neutral/kv_atomic/dispatch.rs | 7 +--- .../ddl/neutral/kv_sorted_index/dispatch.rs | 18 ++++------- .../server/shared/ddl/neutral/last_value.rs | 18 +++++------ .../ddl/neutral/maintenance/storage_info.rs | 20 ++++-------- .../ddl/neutral/maintenance/vector_index.rs | 10 ++---- .../server/shared/ddl/neutral/match_ops.rs | 18 ++++------- .../ddl/neutral/materialized_view/show.rs | 8 +---- .../server/shared/ddl/neutral/metering_ddl.rs | 31 ++++-------------- .../shared/ddl/neutral/observability.rs | 13 +++----- .../control/server/shared/ddl/neutral/oidc.rs | 8 +---- .../server/shared/ddl/neutral/org_ddl.rs | 16 ++-------- .../shared/ddl/neutral/procedure/call.rs | 8 +---- .../shared/ddl/neutral/procedure/show.rs | 8 +---- .../ddl/neutral/query_functions/helpers.rs | 20 +++++------- .../server/shared/ddl/neutral/quota_ddl.rs | 8 +---- .../shared/ddl/neutral/redaction/drop_show.rs | 8 +---- .../ddl/neutral/retention_policy/show.rs | 5 ++- .../control/server/shared/ddl/neutral/rls.rs | 8 +---- .../shared/ddl/neutral/schedule/show.rs | 16 ++-------- .../shared/ddl/neutral/scope_ddl/grant.rs | 8 +---- .../shared/ddl/neutral/scope_ddl/show.rs | 16 ++-------- .../shared/ddl/neutral/scope_query_ddl.rs | 16 ++-------- .../server/shared/ddl/neutral/sequence.rs | 16 ++-------- .../shared/ddl/neutral/session_admin.rs | 17 +++------- .../server/shared/ddl/neutral/show_changes.rs | 16 ++-------- .../shared/ddl/neutral/stream_select.rs | 17 +++------- .../shared/ddl/neutral/synonym_group/show.rs | 8 +---- .../shared/ddl/neutral/tenant/support.rs | 10 ++---- .../shared/ddl/neutral/timeseries/show.rs | 9 +++--- .../server/shared/ddl/neutral/topic/show.rs | 8 +---- .../shared/ddl/neutral/topic_subscribe.rs | 9 ++---- .../shared/ddl/neutral/tree_ops/children.rs | 8 ++--- .../ddl/neutral/tree_ops/create_index.rs | 10 +++--- .../server/shared/ddl/neutral/tree_ops/sum.rs | 10 +++--- .../server/shared/ddl/neutral/trigger/show.rs | 8 +---- .../shared/ddl/neutral/typeguard/handlers.rs | 16 ++-------- .../shared/ddl/neutral/typeguard/validate.rs | 17 +++------- .../ddl/neutral/version_history/at_version.rs | 9 ++---- .../ddl/neutral/version_history/diff.rs | 7 ++-- .../neutral/version_history/show_versions.rs | 5 ++- .../shared/ddl/neutral/weighted_pick.rs | 15 +++------ 82 files changed, 300 insertions(+), 758 deletions(-) diff --git a/nodedb/src/control/server/response_shape/types.rs b/nodedb/src/control/server/response_shape/types.rs index b70d97615..d9d35c2e0 100644 --- a/nodedb/src/control/server/response_shape/types.rs +++ b/nodedb/src/control/server/response_shape/types.rs @@ -308,6 +308,38 @@ impl ShapedRows { vec![DdlColType::Text; n] } + /// A result set from decoded JSON rows, with one catalog type per column + /// and no notice. + pub fn from_json_rows( + columns: Vec, + column_types: Vec, + rows: Vec>, + ) -> Self { + Self { + columns, + column_types, + rows, + notice: None, + } + } + + /// A result set whose every column is `Text`: the shape a DDL or + /// inspection statement answers with. The type list is sized from + /// `columns`, so the two cannot disagree. + pub fn text_rows( + columns: Vec, + rows: Vec>, + ) -> Self { + let column_types = Self::text_types(columns.len()); + Self::from_json_rows(columns, column_types, rows) + } + + /// Attach a client-facing notice. + pub fn with_notice(mut self, notice: impl Into) -> Self { + self.notice = Some(notice.into()); + self + } + /// Fold another shaped result into this one so N tasks answer with ONE result /// set — some drivers reject multiple result sets. Columns are the union of /// every contributor's; rows read by key so a missing column encodes NULL. diff --git a/nodedb/src/control/server/shared/ddl/neutral/alert/show.rs b/nodedb/src/control/server/shared/ddl/neutral/alert/show.rs index 02a4c4b94..5720a65d7 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/alert/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/alert/show.rs @@ -107,12 +107,11 @@ pub fn show_alerts( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW ALERT STATUS ON — per-group active/cleared state. @@ -209,12 +208,11 @@ pub fn show_alert_status( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } fn format_duration_ms(ms: u64) -> String { diff --git a/nodedb/src/control/server/shared/ddl/neutral/apikey/create.rs b/nodedb/src/control/server/shared/ddl/neutral/apikey/create.rs index 28ced72dd..ba471fc00 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/apikey/create.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/apikey/create.rs @@ -13,7 +13,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::control::security::audit::AuditEvent; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use super::super::super::result::{DdlError, DdlResult}; @@ -146,10 +146,8 @@ pub fn create_api_key( let mut row = Map::new(); row.insert("api_key".to_string(), JsonValue::String(token)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["api_key".to_string()], - column_types: vec![DdlColType::Text], - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["api_key".to_string()], + vec![row], + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/apikey/manage.rs b/nodedb/src/control/server/shared/ddl/neutral/apikey/manage.rs index 035c04671..3069870d2 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/apikey/manage.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/apikey/manage.rs @@ -174,10 +174,9 @@ pub fn list_api_keys( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/auth_key.rs b/nodedb/src/control/server/shared/ddl/neutral/auth_key.rs index 7cafb8a7e..57db25aef 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/auth_key.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/auth_key.rs @@ -17,7 +17,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use super::super::result::{DdlError, DdlResult}; @@ -99,12 +99,10 @@ pub fn create_auth_key( let mut row = Map::new(); row.insert("auth_api_key".to_string(), JsonValue::String(token)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["auth_api_key".to_string()], - column_types: vec![DdlColType::Text], - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["auth_api_key".to_string()], + vec![row], + ))]) } /// ROTATE AUTH KEY '' [OVERLAP 24h] @@ -144,12 +142,10 @@ pub fn rotate_auth_key( let mut row = Map::new(); row.insert("new_auth_api_key".to_string(), JsonValue::String(new_token)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["new_auth_api_key".to_string()], - column_types: vec![DdlColType::Text], - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["new_auth_api_key".to_string()], + vec![row], + ))]) } /// LIST AUTH KEYS [FOR AUTH USER ''] @@ -179,7 +175,6 @@ pub fn list_auth_keys( "last_used_at".to_string(), "last_used_ip".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = keys .iter() @@ -219,10 +214,5 @@ pub fn list_auth_keys( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/auth_user.rs b/nodedb/src/control/server/shared/ddl/neutral/auth_user.rs index 34729cc3b..66e84db8f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/auth_user.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/auth_user.rs @@ -199,7 +199,6 @@ pub fn show_auth_users( "is_active".to_string(), "last_seen".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = users .iter() @@ -235,12 +234,7 @@ pub fn show_auth_users( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Public re-export of duration parser for use by other DDL modules. diff --git a/nodedb/src/control/server/shared/ddl/neutral/blacklist.rs b/nodedb/src/control/server/shared/ddl/neutral/blacklist.rs index 1e8203a16..5ce7301da 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/blacklist.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/blacklist.rs @@ -268,7 +268,6 @@ pub fn show_blacklist( "created_at".to_string(), "expires_at".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = entries .iter() @@ -297,12 +296,7 @@ pub fn show_blacklist( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Extract UNTIL timestamp from parts. Returns 0 (permanent) if not present. diff --git a/nodedb/src/control/server/shared/ddl/neutral/change_stream/show.rs b/nodedb/src/control/server/shared/ddl/neutral/change_stream/show.rs index 0b39f2319..1316326a7 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/change_stream/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/change_stream/show.rs @@ -61,11 +61,5 @@ pub fn show_change_streams( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/chunk_text.rs b/nodedb/src/control/server/shared/ddl/neutral/chunk_text.rs index 6872b14b4..5702298dc 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/chunk_text.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/chunk_text.rs @@ -10,7 +10,7 @@ use nodedb_sql::parser::preprocess::lex::find_ascii_case_insensitive; use serde_json::{Map, Value as JsonValue}; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use super::super::result::{DdlError, DdlResult}; @@ -135,13 +135,6 @@ pub fn execute_chunk_text(sql: &str) -> Result, DdlError> { "end".to_string(), "text".to_string(), ]; - let column_types = vec![ - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - ]; - let rows: Vec> = chunks .iter() .map(|c| { @@ -154,12 +147,7 @@ pub fn execute_chunk_text(sql: &str) -> Result, DdlError> { }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Build a [`DdlError`] from a SQLSTATE + message. diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/health.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/health.rs index 22f5375ec..4824b0371 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/health.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/health.rs @@ -103,10 +103,9 @@ pub fn show_peer_health( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/migration.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/migration.rs index f62d2e8db..8002aa2a3 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/migration.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/migration.rs @@ -76,10 +76,9 @@ pub fn show_migrations( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/raft.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/raft.rs index 014b8dbae..19c2977da 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/raft.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/raft.rs @@ -100,12 +100,11 @@ pub fn show_raft_groups( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW RAFT GROUP — detailed info for a specific Raft group. @@ -153,7 +152,6 @@ pub fn show_raft_group( }; let columns = vec!["property".to_string(), "value".to_string()]; - let column_types = vec![DdlColType::Text, DdlColType::Text]; let props = [ ("group_id", group.group_id.to_string()), @@ -203,12 +201,7 @@ pub fn show_raft_group( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// ALTER RAFT GROUP ADD|REMOVE NODE diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/ranges.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/ranges.rs index a3cc88651..c7de6a4b5 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/ranges.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/ranges.rs @@ -128,10 +128,9 @@ pub fn show_ranges( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/rebalance_cmd.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/rebalance_cmd.rs index cd2fe9fd5..593379f46 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/rebalance_cmd.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/rebalance_cmd.rs @@ -59,12 +59,10 @@ pub fn rebalance( "status".to_string(), JsonValue::String("cluster is balanced — no moves needed".to_string()), ); - return Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["status".to_string()], - column_types: vec![DdlColType::Text], - rows: vec![row], - notice: None, - })]); + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["status".to_string()], + vec![row], + ))]); } let columns = vec![ @@ -102,10 +100,9 @@ pub fn rebalance( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/routing_hint.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/routing_hint.rs index 1ac0dbe13..58291ac80 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/routing_hint.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/routing_hint.rs @@ -85,10 +85,9 @@ pub fn show_routing( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/schema_version.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/schema_version.rs index 99a94f561..47143629f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/schema_version.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/schema_version.rs @@ -11,7 +11,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use super::super::super::result::{DdlError, DdlResult}; @@ -31,7 +31,6 @@ pub fn show_schema_version( } let columns = vec!["property".to_string(), "value".to_string()]; - let column_types = vec![DdlColType::Text, DdlColType::Text]; let mut rows = Vec::new(); @@ -73,10 +72,5 @@ pub fn show_schema_version( ); rows.push(row); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/cluster/topology.rs b/nodedb/src/control/server/shared/ddl/neutral/cluster/topology.rs index 86093127d..e46c11976 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/cluster/topology.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/cluster/topology.rs @@ -92,12 +92,11 @@ pub fn show_nodes( } } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW NODE — detailed info for a specific node. @@ -124,7 +123,6 @@ pub fn show_node( .map_err(|_| ddl_err("42601", format!("invalid node_id: '{}'", parts[2])))?; let columns = vec!["property".to_string(), "value".to_string()]; - let column_types = vec![DdlColType::Text, DdlColType::Text]; let props = match &state.cluster_topology { Some(t) => { @@ -182,12 +180,7 @@ pub fn show_node( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// REMOVE NODE — mark a node as decommissioned. @@ -255,7 +248,6 @@ pub fn show_cluster( } let columns = vec!["property".to_string(), "value".to_string()]; - let column_types = vec![DdlColType::Text, DdlColType::Text]; let mut props = vec![("node_id", state.node_id.to_string())]; @@ -292,10 +284,5 @@ pub fn show_cluster( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/describe.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/describe.rs index 3c26f7d9e..62731497a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/describe.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/describe.rs @@ -66,8 +66,6 @@ pub fn describe_collection( "type".to_string(), "nullable".to_string(), ]; - let column_types = vec![DdlColType::Text, DdlColType::Text, DdlColType::Text]; - let mut rows = Vec::new(); // Synthesize the implicit 'id' field only when the collection does not @@ -147,12 +145,7 @@ pub fn describe_collection( } } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// SHOW COLLECTIONS @@ -242,10 +235,9 @@ pub fn show_collections( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/show_indexes.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/show_indexes.rs index a016807e9..199757094 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/show_indexes.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/show_indexes.rs @@ -16,7 +16,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::control::security::catalog::StoredIndexRecord; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::ddl::sql_parse::parse_ident_token; use crate::control::state::SharedState; use crate::types::DatabaseId; @@ -51,14 +51,6 @@ pub fn show_indexes( "fields".to_string(), "owner".to_string(), ]; - let column_types = vec![ - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - ]; - let mut records = state .credentials .catalog() @@ -105,10 +97,5 @@ pub fn show_indexes( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/vector_metadata.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/vector_metadata.rs index 9b23ad6c3..e77c3a0ba 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/vector_metadata.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/vector_metadata.rs @@ -218,12 +218,7 @@ pub fn handle_show_vector_models( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types: ShapedRows::text_types(6), - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Handle `SELECT VECTOR_METADATA('collection', 'column')` — return JSON. @@ -258,12 +253,10 @@ pub fn handle_vector_metadata_query( let mut row = Map::new(); row.insert("vector_metadata".to_string(), JsonValue::String(json)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["vector_metadata".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["vector_metadata".to_string()], + vec![row], + ))]) } /// Format a Unix timestamp (seconds) as an ISO-8601 UTC date string (YYYY-MM-DD). diff --git a/nodedb/src/control/server/shared/ddl/neutral/conflict_policy.rs b/nodedb/src/control/server/shared/ddl/neutral/conflict_policy.rs index 945b9132a..daaa8571f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/conflict_policy.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/conflict_policy.rs @@ -84,12 +84,10 @@ pub async fn alter_set_on_conflict( let mut row = Map::new(); row.insert("result".to_string(), JsonValue::String("OK".to_string())); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["result".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["result".to_string()], + vec![row], + ))]) } /// Handle `SHOW CONFLICT POLICY ON `. @@ -118,12 +116,10 @@ pub async fn show_conflict_policy( let mut row = Map::new(); row.insert("policy".to_string(), JsonValue::String(text)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["policy".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["policy".to_string()], + vec![row], + ))]) } fn resolve_policy_kind(kind: &ConflictPolicyKind) -> ConflictPolicy { diff --git a/nodedb/src/control/server/shared/ddl/neutral/constraint/show.rs b/nodedb/src/control/server/shared/ddl/neutral/constraint/show.rs index 5d418abe8..995fed066 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/constraint/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/constraint/show.rs @@ -109,13 +109,7 @@ pub fn show_constraints( rows.push(constraint_row(&cc.name, kind_str, "", &cc.check_sql)); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Build a single `SHOW CONSTRAINTS` row keyed by the four text columns. diff --git a/nodedb/src/control/server/shared/ddl/neutral/consumer_group/show.rs b/nodedb/src/control/server/shared/ddl/neutral/consumer_group/show.rs index ec2eb48c0..8b5f63cc5 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/consumer_group/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/consumer_group/show.rs @@ -72,13 +72,7 @@ pub fn show_consumer_groups( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Handle `SHOW PARTITIONS ON ` @@ -159,11 +153,5 @@ pub fn show_partitions( } } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/continuous_agg/show.rs b/nodedb/src/control/server/shared/ddl/neutral/continuous_agg/show.rs index 393dc5baf..2a4a098da 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/continuous_agg/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/continuous_agg/show.rs @@ -171,10 +171,9 @@ pub async fn show_continuous_aggregates( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs b/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs index 3627af386..832adb071 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/crdt_ops.rs @@ -17,7 +17,7 @@ use crate::control::crdt_post_image_policy::ExternalCrdtPostImagePolicy; use crate::control::planner::sql_plan_convert::convert::db_qualified; use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::{AuthenticatedIdentity, Permission}; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::authorization::{authorize_collection, authorize_task_set}; use crate::control::server::shared::ddl::sql_parse::hex_decode; use crate::control::state::SharedState; @@ -99,27 +99,22 @@ pub async fn crdt_state( .map_err(|e| DdlError::new("XX000", e.to_string()))?; let columns = vec!["crdt_state".to_string()]; - let column_types = vec![DdlColType::Text]; if result.is_empty() { - return Ok(vec![DdlResult::Rows(ShapedRows { + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: Vec::new(), - notice: None, - })]); + Vec::new(), + ))]); } let text = String::from_utf8_lossy(&result).into_owned(); let mut row = Map::new(); row.insert("crdt_state".to_string(), JsonValue::String(text)); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } /// `SELECT crdt_apply('collection', 'doc_id', 'delta_hex')` @@ -234,14 +229,11 @@ pub async fn crdt_apply( .map_err(|e| DdlError::new("XX000", e.to_string()))?; let columns = vec!["result".to_string()]; - let column_types = vec![DdlColType::Text]; let mut row = Map::new(); row.insert("result".to_string(), JsonValue::String("OK".to_string())); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/custom_type.rs b/nodedb/src/control/server/shared/ddl/neutral/custom_type.rs index 69eb4f5f9..f23ea6683 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/custom_type.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/custom_type.rs @@ -272,13 +272,7 @@ pub fn show_types( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } // ── Helpers ─────────────────────────────────────────────────────────────── diff --git a/nodedb/src/control/server/shared/ddl/neutral/database/support.rs b/nodedb/src/control/server/shared/ddl/neutral/database/support.rs index 6d8c52d7d..04269fc67 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/database/support.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/database/support.rs @@ -3,7 +3,7 @@ //! Shared error / result constructors for the protocol-neutral database DDL //! handlers. -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use super::super::super::result::{DdlError, DdlResult}; @@ -32,11 +32,5 @@ pub(super) fn text_rows( columns: Vec, rows: Vec>, ) -> Vec { - let column_types = vec![DdlColType::Text; columns.len()]; - vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })] + vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))] } diff --git a/nodedb/src/control/server/shared/ddl/neutral/estimate_count.rs b/nodedb/src/control/server/shared/ddl/neutral/estimate_count.rs index e37d4026c..b1d278557 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/estimate_count.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/estimate_count.rs @@ -10,7 +10,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::bridge::envelope::PhysicalPlan; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use crate::types::{DatabaseId, TraceId}; use nodedb_physical::physical_plan::DocumentOp; @@ -72,18 +72,15 @@ pub async fn estimate_count( &resp.payload, ); let columns = vec!["estimate_count".to_string()]; - let column_types = vec![DdlColType::Text]; let mut row = Map::new(); row.insert( "estimate_count".to_string(), JsonValue::String(payload_text), ); - return Ok(vec![DdlResult::Rows(ShapedRows { + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]); + vec![row], + ))]); } Err(e) => { return Err(DdlError::new("XX000", e.to_string())); diff --git a/nodedb/src/control/server/shared/ddl/neutral/explain_ddl.rs b/nodedb/src/control/server/shared/ddl/neutral/explain_ddl.rs index 1bc1afaca..e3b0ce4bd 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/explain_ddl.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/explain_ddl.rs @@ -110,12 +110,7 @@ pub fn explain_permission( ); rows.push(row); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types: ShapedRows::text_types(3), - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// EXPLAIN SCOPE FOR AUTH USER '' @@ -169,12 +164,7 @@ pub fn explain_scope( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types: ShapedRows::text_types(3), - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// `SELECT nodedb_assert_visible('', '', '')` @@ -226,10 +216,8 @@ pub fn assert_visible( JsonValue::String(visible.to_string()), ); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["visible".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["visible".to_string()], + vec![row], + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/explain_tiers.rs b/nodedb/src/control/server/shared/ddl/neutral/explain_tiers.rs index 224e47953..cc9e2d414 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/explain_tiers.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/explain_tiers.rs @@ -8,7 +8,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::ddl::sql_parse::parse_ident_token; use crate::control::state::SharedState; use crate::types::DatabaseId; @@ -65,7 +65,6 @@ pub fn explain_tiers( crate::control::planner::auto_tier::explain_tier_selection(&policy, time_range); let columns = vec!["plan".to_string()]; - let column_types = vec![DdlColType::Text]; let mut rows = Vec::new(); for line in explanation.lines() { let mut row = Map::new(); @@ -73,12 +72,7 @@ pub fn explain_tiers( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Build a [`DdlError`] from a SQLSTATE + message. diff --git a/nodedb/src/control/server/shared/ddl/neutral/function/show.rs b/nodedb/src/control/server/shared/ddl/neutral/function/show.rs index 87b6ad54f..edb512dff 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/function/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/function/show.rs @@ -86,11 +86,5 @@ pub fn show_functions( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/algo.rs b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/algo.rs index bc7b4ab77..5826ce89f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/algo.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/algo.rs @@ -7,7 +7,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::bridge::envelope::PhysicalPlan; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::broadcast; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use crate::data::executor::response_codec; use crate::engine::graph::algo::GraphAlgorithm; @@ -235,15 +235,11 @@ fn algo_payload_to_rows( .iter() .map(|&(name, _)| name.to_string()) .collect(); - let column_types = vec![DdlColType::Text; columns.len()]; - if payload.is_empty() { - return Ok(vec![DdlResult::Rows(ShapedRows { + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: Vec::new(), - notice: None, - })]); + Vec::new(), + ))]); } let json_text = response_codec::decode_payload_to_json(payload); @@ -268,12 +264,10 @@ fn algo_payload_to_rows( shaped_rows.push(out); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: shaped_rows, - notice: None, - })]) + shaped_rows, + ))]) } #[cfg(test)] diff --git a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs index ec36332d6..2d5f1b1c9 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs @@ -16,7 +16,7 @@ use nodedb_sql::ddl_ast::GraphDirection; use crate::bridge::envelope::PhysicalPlan; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::ddl::user_dispatch; use crate::control::state::SharedState; use crate::data::executor::response_codec; @@ -140,10 +140,8 @@ pub async fn rag_fusion( let mut row = Map::new(); row.insert("result".to_string(), JsonValue::String(json_text)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["result".to_string()], - column_types: vec![DdlColType::Text], - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["result".to_string()], + vec![row], + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/response.rs b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/response.rs index d2c4e28b0..1056926ed 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/response.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/response.rs @@ -4,7 +4,7 @@ use serde_json::{Map, Value as JsonValue}; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::data::executor::response_codec; use super::super::super::result::DdlResult; @@ -16,25 +16,14 @@ use super::super::super::result::DdlResult; /// handler's empty `QueryResponse`. pub(super) fn payload_to_rows(payload: &crate::bridge::envelope::Payload) -> Vec { let columns = vec!["result".to_string()]; - let column_types = vec![DdlColType::Text]; if payload.is_empty() { - return vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows: Vec::new(), - notice: None, - })]; + return vec![DdlResult::Rows(ShapedRows::text_rows(columns, Vec::new()))]; } let json_text = response_codec::decode_payload_to_json(payload); let mut row = Map::new(); row.insert("result".to_string(), JsonValue::String(json_text)); - vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows: vec![row], - notice: None, - })] + vec![DdlResult::Rows(ShapedRows::text_rows(columns, vec![row]))] } diff --git a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/stats.rs b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/stats.rs index 24ce8f205..0f0e07a63 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/stats.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/stats.rs @@ -305,12 +305,11 @@ fn encode_compact_response(rows: Vec) -> Vec { data_rows.push(row); } - vec![DdlResult::Rows(ShapedRows { + vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, - rows: data_rows, - notice: None, - })] + data_rows, + ))] } fn encode_verbose_response(rows: Vec) -> Vec { @@ -338,10 +337,9 @@ fn encode_verbose_response(rows: Vec) -> Vec { } } - vec![DdlResult::Rows(ShapedRows { + vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, - rows: data_rows, - notice: None, - })] + data_rows, + ))] } diff --git a/nodedb/src/control/server/shared/ddl/neutral/impersonation.rs b/nodedb/src/control/server/shared/ddl/neutral/impersonation.rs index a0e860604..03a57094a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/impersonation.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/impersonation.rs @@ -236,11 +236,5 @@ pub fn show_delegations( }) .collect(); - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/inspect/grants.rs b/nodedb/src/control/server/shared/ddl/neutral/inspect/grants.rs index facd9a38a..621438822 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/inspect/grants.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/inspect/grants.rs @@ -11,7 +11,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use crate::types::DatabaseId; @@ -46,7 +46,6 @@ pub fn show_grants( }; let columns = vec!["username".to_string(), "role".to_string()]; - let column_types = vec![DdlColType::Text, DdlColType::Text]; let user = state.credentials.get_user(&target_user); let mut rows = Vec::new(); @@ -63,12 +62,7 @@ pub fn show_grants( } } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// `SHOW PERMISSIONS [ON ] [FOR ]` @@ -105,13 +99,6 @@ pub fn show_permissions( "target".to_string(), "type".to_string(), ]; - let column_types = vec![ - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - ]; - let mut rows = Vec::new(); if let Some(collection) = on_collection { @@ -224,10 +211,5 @@ pub fn show_permissions( } } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/inspect/tenants.rs b/nodedb/src/control/server/shared/ddl/neutral/inspect/tenants.rs index ae34b8769..8dad4a30e 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/inspect/tenants.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/inspect/tenants.rs @@ -33,12 +33,11 @@ pub fn show_tenants( } let (columns, column_types, rows) = tenant_rows(state, |_, _| true); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW TENANT — single-tenant introspection by identifier. @@ -71,12 +70,11 @@ pub fn show_tenant_by_identifier( return Err(ddl_err("42704", format!("tenant '{ident}' not found"))); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW TENANTS WITH NAME — filtered list form. Returns a row @@ -102,12 +100,11 @@ pub fn show_tenants_filtered_by_name( return Err(ddl_err("42704", format!("tenant '{name}' not found"))); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// Build the `(columns, column_types, rows)` triple shared by `SHOW TENANTS` diff --git a/nodedb/src/control/server/shared/ddl/neutral/inspect/trigger_dlq.rs b/nodedb/src/control/server/shared/ddl/neutral/inspect/trigger_dlq.rs index 4b66795b4..0a964091b 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/inspect/trigger_dlq.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/inspect/trigger_dlq.rs @@ -79,12 +79,11 @@ pub fn show_trigger_dlq( } } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: COLUMNS.iter().map(|c| (*c).to_owned()).collect(), - column_types: column_types(), + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( + COLUMNS.iter().map(|c| (*c).to_owned()).collect(), + column_types(), rows, - notice: None, - })]) + ))]) } /// REQUEUE TRIGGER DLQ — hand one dead-lettered action back to the diff --git a/nodedb/src/control/server/shared/ddl/neutral/inspect/users.rs b/nodedb/src/control/server/shared/ddl/neutral/inspect/users.rs index 8c6d4d71f..bfe7e0f56 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/inspect/users.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/inspect/users.rs @@ -68,12 +68,11 @@ pub fn show_users( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW ROLES — list all custom roles. Built-in role enum is fixed @@ -122,12 +121,11 @@ pub fn show_roles( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW SESSION — display current session identity. @@ -178,10 +176,9 @@ pub fn show_session(identity: &AuthenticatedIdentity) -> Result, JsonValue::String(if identity.is_superuser { "t" } else { "f" }.to_string()), ); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/inspect_audit.rs b/nodedb/src/control/server/shared/ddl/neutral/inspect_audit.rs index cb8ec5004..6c8aba052 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/inspect_audit.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/inspect_audit.rs @@ -111,12 +111,11 @@ pub fn show_audit_log( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// SHOW AUDIT WHERE event_type = '' @@ -206,12 +205,11 @@ pub fn show_audit_where( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// `SHOW AUDIT IN DATABASE [LIMIT ]` @@ -329,12 +327,11 @@ pub fn show_audit_in_database( } } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// Audit entries are read with a regular `SELECT` query against diff --git a/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs index b13a3f112..2316c3aa1 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs @@ -178,12 +178,7 @@ fn data_plane_error(code: Option) -> DdlErro pub(crate) fn single_text_col(col: &str, text: String) -> DdlResult { let mut row = Map::new(); row.insert(col.to_string(), JsonValue::String(text)); - DdlResult::Rows(ShapedRows { - columns: vec![col.to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - }) + DdlResult::Rows(ShapedRows::text_rows(vec![col.to_string()], vec![row])) } /// Parse function arguments from `SELECT FUNC_NAME(arg1, arg2, ...)`. diff --git a/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs index afd482069..66a91a2d2 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs @@ -205,12 +205,10 @@ pub(super) async fn dispatch_and_respond_json( let payload_text = crate::data::executor::response_codec::decode_payload_to_json(&resp.payload); let mut row = Map::new(); row.insert(col_name.to_string(), JsonValue::String(payload_text)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec![col_name.to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec![col_name.to_string()], + vec![row], + ))]) } /// Dispatch plan and return multi-row response (for TOPK, RANGE). @@ -240,10 +238,8 @@ pub(super) async fn dispatch_and_respond_rows( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["rank".to_string(), "key".to_string()], - column_types: ShapedRows::text_types(2), + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["rank".to_string(), "key".to_string()], rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/last_value.rs b/nodedb/src/control/server/shared/ddl/neutral/last_value.rs index dc26cef8c..9d12a5c17 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/last_value.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/last_value.rs @@ -80,16 +80,15 @@ pub async fn query_last_values( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec![ + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( + vec![ "series_id".to_string(), "timestamp_ms".to_string(), "value".to_string(), ], - column_types: vec![DdlColType::Int8, DdlColType::Int8, DdlColType::Text], + vec![DdlColType::Int8, DdlColType::Int8, DdlColType::Text], rows, - notice: None, - })]) + ))]) } /// `SELECT LAST_VALUE('', )` — returns single series value. @@ -143,12 +142,11 @@ pub async fn query_last_value( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["timestamp_ms".to_string(), "value".to_string()], - column_types: vec![DdlColType::Int8, DdlColType::Text], + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( + vec!["timestamp_ms".to_string(), "value".to_string()], + vec![DdlColType::Int8, DdlColType::Text], rows, - notice: None, - })]) + ))]) } fn ddl_err(sqlstate: &str, message: impl Into) -> DdlError { diff --git a/nodedb/src/control/server/shared/ddl/neutral/maintenance/storage_info.rs b/nodedb/src/control/server/shared/ddl/neutral/maintenance/storage_info.rs index 73124314a..1eb210070 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/maintenance/storage_info.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/maintenance/storage_info.rs @@ -13,7 +13,7 @@ use nodedb_types::DatabaseId; use serde_json::{Map, Value as JsonValue}; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::ddl::sql_parse::parse_ident_token; use crate::control::state::SharedState; @@ -65,7 +65,6 @@ pub fn handle_show_storage( "row_count".to_string(), "last_analyzed".to_string(), ]; - let column_types = vec![DdlColType::Text; 4]; let row_count = stats.first().map(|s| s.row_count).unwrap_or(0); let last_analyzed = stats @@ -97,12 +96,10 @@ pub fn handle_show_storage( JsonValue::String(last_analyzed), ); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } /// Handle `SHOW COMPACTION STATUS`. @@ -115,7 +112,6 @@ pub fn handle_show_compaction_status( "pending_jobs".to_string(), "compaction_debt".to_string(), ]; - let column_types = vec![DdlColType::Text; 3]; // Compaction runs automatically in the Data Plane. We report the current // state as "idle" — detailed stats require Data Plane query support. @@ -130,12 +126,10 @@ pub fn handle_show_compaction_status( JsonValue::String("0".to_string()), ); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } fn now_ms() -> u64 { diff --git a/nodedb/src/control/server/shared/ddl/neutral/maintenance/vector_index.rs b/nodedb/src/control/server/shared/ddl/neutral/maintenance/vector_index.rs index ea7306fc6..faf24988b 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/maintenance/vector_index.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/maintenance/vector_index.rs @@ -20,7 +20,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::bridge::envelope::PhysicalPlan; use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use crate::types::DatabaseId; use crate::types::TraceId; @@ -72,7 +72,6 @@ pub async fn handle_show_vector_index( .map_err(|e| ddl_err("XX000", format!("decode vector stats: {e}")))?; let columns = vec!["property".to_string(), "value".to_string()]; - let column_types = vec![DdlColType::Text; 2]; let pairs: Vec<(&str, String)> = vec![ ("dimensions", stats.dimensions.to_string()), @@ -115,12 +114,7 @@ pub async fn handle_show_vector_index( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Handle `ALTER VECTOR INDEX ON collection.column SEAL`. diff --git a/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs b/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs index 35366d352..ab45188ef 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/match_ops.rs @@ -232,15 +232,12 @@ fn match_payload_to_rows( column_names: &[String], ) -> Result, DdlError> { let columns = column_names.to_vec(); - let column_types = ShapedRows::text_types(column_names.len()); if payload.is_empty() { - return Ok(vec![DdlResult::Rows(ShapedRows { + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: Vec::new(), - notice: None, - })]); + Vec::new(), + ))]); } let json_text = response_codec::decode_payload_to_json(payload); @@ -257,12 +254,9 @@ fn match_payload_to_rows( out_rows.push(map); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows: out_rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + columns, out_rows, + ))]) } // Tenant-prefix stripping lives in the Data Plane, in diff --git a/nodedb/src/control/server/shared/ddl/neutral/materialized_view/show.rs b/nodedb/src/control/server/shared/ddl/neutral/materialized_view/show.rs index b119ee07d..b3f1db060 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/materialized_view/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/materialized_view/show.rs @@ -94,11 +94,5 @@ pub fn show_materialized_views( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/metering_ddl.rs b/nodedb/src/control/server/shared/ddl/neutral/metering_ddl.rs index db5f12990..fcd4e7d0c 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/metering_ddl.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/metering_ddl.rs @@ -115,12 +115,7 @@ pub fn show_usage( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types: ShapedRows::text_types(6), - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// SHOW QUOTA FOR AUTH USER '' / SHOW QUOTA FOR ORG '' @@ -183,12 +178,7 @@ pub fn show_quota( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types: ShapedRows::text_types(7), - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// SHOW USAGE FOR TENANT @@ -237,12 +227,7 @@ pub fn show_usage_for_tenant( } } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types: ShapedRows::text_types(2), - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// EXPORT USAGE FOR TENANT [PERIOD ''] FORMAT 'json' @@ -293,12 +278,10 @@ pub fn export_usage( let mut row = Map::new(); row.insert("usage_json".to_string(), JsonValue::String(json)); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["usage_json".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["usage_json".to_string()], + vec![row], + ))]) } /// Parse FOR AUTH USER '' or FOR ORG '' from parts. diff --git a/nodedb/src/control/server/shared/ddl/neutral/observability.rs b/nodedb/src/control/server/shared/ddl/neutral/observability.rs index 8d66a2ed8..94cfa2a4f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/observability.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/observability.rs @@ -38,12 +38,10 @@ fn key_value_result(rows_in: Vec<(String, String)>) -> Result, Dd row.insert("value".to_string(), JsonValue::String(v)); rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["name".to_string(), "value".to_string()], - column_types: ShapedRows::text_types(2), + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["name".to_string(), "value".to_string()], rows, - notice: None, - })]) + ))]) } /// Build the canonical `(name, value)` rows for `SHOW STATS` and @@ -250,10 +248,9 @@ pub fn show_memory( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/oidc.rs b/nodedb/src/control/server/shared/ddl/neutral/oidc.rs index d2b63d366..44820fae6 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/oidc.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/oidc.rs @@ -354,11 +354,5 @@ pub fn show_oidc_providers( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/org_ddl.rs b/nodedb/src/control/server/shared/ddl/neutral/org_ddl.rs index 9f11bc426..2c4fb528a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/org_ddl.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/org_ddl.rs @@ -176,7 +176,6 @@ pub fn show_orgs( "tenant_id".to_string(), "status".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = orgs .iter() @@ -193,12 +192,7 @@ pub fn show_orgs( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// SHOW MEMBERS OF ORG '' @@ -223,7 +217,6 @@ pub fn show_members( "role".to_string(), "joined_at".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = members .iter() @@ -243,10 +236,5 @@ pub fn show_members( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/procedure/call.rs b/nodedb/src/control/server/shared/ddl/neutral/procedure/call.rs index 05713c84a..71bf5f7fe 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/procedure/call.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/procedure/call.rs @@ -130,13 +130,7 @@ fn build_out_response( row.insert(param.name.clone(), serde_json::Value::String(text)); } - let column_types = ShapedRows::text_types(columns.len()); - vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows: vec![row], - notice: None, - })] + vec![DdlResult::Rows(ShapedRows::text_rows(columns, vec![row]))] } /// Parse `CALL (arg1, arg2, ...)`. diff --git a/nodedb/src/control/server/shared/ddl/neutral/procedure/show.rs b/nodedb/src/control/server/shared/ddl/neutral/procedure/show.rs index 0f5de4b8c..169fed8ff 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/procedure/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/procedure/show.rs @@ -70,11 +70,5 @@ pub fn show_procedures( } } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/query_functions/helpers.rs b/nodedb/src/control/server/shared/ddl/neutral/query_functions/helpers.rs index 8cbfcb7e5..ec7e41f84 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/query_functions/helpers.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/query_functions/helpers.rs @@ -73,12 +73,10 @@ pub fn json_to_decimal(v: &serde_json::Value) -> Option { pub fn single_result(value: &str) -> Vec { let mut row = Map::new(); row.insert("result".to_string(), JsonValue::String(value.to_string())); - vec![DdlResult::Rows(ShapedRows { - columns: vec!["result".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })] + vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["result".to_string()], + vec![row], + ))] } /// Unwrap the `DocumentOp::Scan` raw-passthrough envelope (`{"id": .., @@ -132,10 +130,8 @@ pub fn unwrap_scan_doc_with_id(doc: JsonValue) -> (String, Map Vec { - vec![DdlResult::Rows(ShapedRows { - columns: vec!["result".to_string()], - column_types: ShapedRows::text_types(1), - rows: Vec::new(), - notice: None, - })] + vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["result".to_string()], + Vec::new(), + ))] } diff --git a/nodedb/src/control/server/shared/ddl/neutral/quota_ddl.rs b/nodedb/src/control/server/shared/ddl/neutral/quota_ddl.rs index 02aec9bc7..566c09f7b 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/quota_ddl.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/quota_ddl.rs @@ -221,7 +221,6 @@ pub fn show_quotas( "enforcement".to_string(), "warning_threshold".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = state .quota_manager @@ -250,10 +249,5 @@ pub fn show_quotas( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/redaction/drop_show.rs b/nodedb/src/control/server/shared/ddl/neutral/redaction/drop_show.rs index fddabe4af..c5323cc87 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/redaction/drop_show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/redaction/drop_show.rs @@ -130,13 +130,7 @@ pub fn show_redaction_policies( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Render a redaction mode for `SHOW`. The mask literal is deliberately diff --git a/nodedb/src/control/server/shared/ddl/neutral/retention_policy/show.rs b/nodedb/src/control/server/shared/ddl/neutral/retention_policy/show.rs index b61a2b7fe..b67dfd866 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/retention_policy/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/retention_policy/show.rs @@ -118,12 +118,11 @@ pub fn show_retention_policy( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// Format tiers into a compact human-readable string. diff --git a/nodedb/src/control/server/shared/ddl/neutral/rls.rs b/nodedb/src/control/server/shared/ddl/neutral/rls.rs index 900010334..df57309cf 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/rls.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/rls.rs @@ -342,13 +342,7 @@ pub fn show_rls_policies( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } #[cfg(test)] diff --git a/nodedb/src/control/server/shared/ddl/neutral/schedule/show.rs b/nodedb/src/control/server/shared/ddl/neutral/schedule/show.rs index 500a6f431..4bd6dfc72 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/schedule/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/schedule/show.rs @@ -95,13 +95,7 @@ pub fn show_schedules( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Handle `SHOW SCHEDULE HISTORY name` @@ -164,13 +158,7 @@ pub fn show_schedule_history( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Format epoch seconds as ISO 8601 UTC string. diff --git a/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/grant.rs b/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/grant.rs index 39c6b3b6d..a746f6795 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/grant.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/grant.rs @@ -213,7 +213,6 @@ pub fn show_scope_grants( "conditions".to_string(), "granted_by".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = grants .iter() @@ -254,12 +253,7 @@ pub fn show_scope_grants( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } // ── Parse helpers for time-bound GRANT SCOPE syntax ──────────────── diff --git a/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/show.rs b/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/show.rs index 0711cb9a8..38d6ec78a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/scope_ddl/show.rs @@ -22,7 +22,6 @@ pub fn show_scopes( let name = parts[2].trim_matches('\''); let resolved = state.scope_defs.resolve(name); let columns = vec!["permission".to_string(), "collection".to_string()]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = resolved .iter() .map(|(perm, coll)| { @@ -32,12 +31,7 @@ pub fn show_scopes( row }) .collect(); - return Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]); + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]); } // SHOW SCOPES — list all scope definitions. @@ -48,7 +42,6 @@ pub fn show_scopes( "includes".to_string(), "created_by".to_string(), ]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = scopes .iter() @@ -76,10 +69,5 @@ pub fn show_scopes( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/scope_query_ddl.rs b/nodedb/src/control/server/shared/ddl/neutral/scope_query_ddl.rs index c9a29031b..178c66df8 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/scope_query_ddl.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/scope_query_ddl.rs @@ -129,7 +129,6 @@ pub fn show_my_scopes( let effective = state.scope_grants.effective_scopes(&user_id, &org_ids); let columns = vec!["scope".to_string(), "source".to_string()]; - let column_types = ShapedRows::text_types(columns.len()); let mut rows = Vec::new(); for scope_name in &effective { @@ -148,12 +147,7 @@ pub fn show_my_scopes( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// SHOW SCOPES FOR USER '' / SHOW SCOPES FOR ORG '' @@ -184,7 +178,6 @@ pub fn show_scopes_for( }; let columns = vec!["scope".to_string()]; - let column_types = ShapedRows::text_types(columns.len()); let rows: Vec<_> = scopes .iter() .map(|s| { @@ -194,10 +187,5 @@ pub fn show_scopes_for( }) .collect(); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/sequence.rs b/nodedb/src/control/server/shared/ddl/neutral/sequence.rs index b8f7995f0..6622fec5d 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/sequence.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/sequence.rs @@ -342,13 +342,7 @@ pub fn show_sequences( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Handle `DESCRIBE SEQUENCE `, resolved in the current database. @@ -408,11 +402,5 @@ pub fn describe_sequence( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/session_admin.rs b/nodedb/src/control/server/shared/ddl/neutral/session_admin.rs index c69b00dd8..ccc413664 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/session_admin.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/session_admin.rs @@ -145,13 +145,7 @@ pub fn show_sessions( }) .collect(); - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// KILL SESSION '' @@ -286,13 +280,10 @@ pub fn verify_audit_chain( "entries".to_string(), JsonValue::String(audit.len().to_string()), ); - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } Err(broken_seq) => Err(err( "XX001", diff --git a/nodedb/src/control/server/shared/ddl/neutral/show_changes.rs b/nodedb/src/control/server/shared/ddl/neutral/show_changes.rs index d57ab15f2..c5d1f3518 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/show_changes.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/show_changes.rs @@ -15,7 +15,7 @@ use serde_json::{Map, Value as JsonValue}; use crate::control::change_stream::ReplayStart; use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::{AuthenticatedIdentity, Permission}; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::server::shared::authorization::authorize_collection; use crate::control::state::SharedState; use crate::types::DatabaseId; @@ -95,13 +95,6 @@ pub fn show_changes( "timestamp_ms".to_string(), "lsn".to_string(), ]; - let column_types = vec![ - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - DdlColType::Text, - ]; let mut rows = Vec::with_capacity(changes.len()); for change in &changes { @@ -129,12 +122,7 @@ pub fn show_changes( rows.push(row); } - return Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]); + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]); } Err(DdlError::new( diff --git a/nodedb/src/control/server/shared/ddl/neutral/stream_select.rs b/nodedb/src/control/server/shared/ddl/neutral/stream_select.rs index 43a3f5bc6..1db3f4a29 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/stream_select.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/stream_select.rs @@ -146,13 +146,10 @@ pub async fn select_from_stream( Err(ConsumeError::BufferEmpty(_)) => { // Return empty result set. let columns = result_columns(); - let column_types = ShapedRows::text_types(columns.len()); - return Ok(vec![DdlResult::Rows(ShapedRows { + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: Vec::new(), - notice: None, - })]); + Vec::new(), + ))]); } Err(e) => { return Err(err("42704", e.to_string())); @@ -221,13 +218,7 @@ pub async fn select_from_stream( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Column schema for stream SELECT results. diff --git a/nodedb/src/control/server/shared/ddl/neutral/synonym_group/show.rs b/nodedb/src/control/server/shared/ddl/neutral/synonym_group/show.rs index c0334f06b..94ca9c7ae 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/synonym_group/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/synonym_group/show.rs @@ -36,11 +36,5 @@ pub fn show_synonym_groups( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/tenant/support.rs b/nodedb/src/control/server/shared/ddl/neutral/tenant/support.rs index 08e09c967..5d73fecf0 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/tenant/support.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/tenant/support.rs @@ -7,7 +7,7 @@ //! and `tenant_exists` are byte-identical except for the error type //! (`DdlError` instead of `PgWireError`). -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use crate::types::TenantId; @@ -31,13 +31,7 @@ pub(super) fn text_rows( columns: Vec, rows: Vec>, ) -> Vec { - let column_types = vec![DdlColType::Text; columns.len()]; - vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })] + vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))] } /// Resolve a tenant reference token to a [`TenantId`], accepting either a diff --git a/nodedb/src/control/server/shared/ddl/neutral/timeseries/show.rs b/nodedb/src/control/server/shared/ddl/neutral/timeseries/show.rs index 1680f047f..3035a4a50 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/timeseries/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/timeseries/show.rs @@ -91,8 +91,8 @@ pub fn show_partitions( } } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec![ + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( + vec![ "partition".to_string(), "min_ts".to_string(), "max_ts".to_string(), @@ -100,7 +100,7 @@ pub fn show_partitions( "size".to_string(), "state".to_string(), ], - column_types: vec![ + vec![ DdlColType::Text, DdlColType::Int8, DdlColType::Int8, @@ -109,6 +109,5 @@ pub fn show_partitions( DdlColType::Text, ], rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/topic/show.rs b/nodedb/src/control/server/shared/ddl/neutral/topic/show.rs index 5f398fe2d..b11ba143f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/topic/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/topic/show.rs @@ -57,11 +57,5 @@ pub fn show_topics( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/topic_subscribe.rs b/nodedb/src/control/server/shared/ddl/neutral/topic_subscribe.rs index f0c159ddc..140e1a424 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/topic_subscribe.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/topic_subscribe.rs @@ -127,13 +127,10 @@ pub fn subscribe_to( JsonValue::String(backlog.len().to_string()), ); - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } #[cfg(test)] diff --git a/nodedb/src/control/server/shared/ddl/neutral/tree_ops/children.rs b/nodedb/src/control/server/shared/ddl/neutral/tree_ops/children.rs index 60fe6b0af..043920dea 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/tree_ops/children.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/tree_ops/children.rs @@ -96,10 +96,8 @@ pub async fn tree_children( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["child_id".to_string()], - column_types: ShapedRows::text_types(1), + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["child_id".to_string()], rows, - notice: None, - })]) + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/tree_ops/create_index.rs b/nodedb/src/control/server/shared/ddl/neutral/tree_ops/create_index.rs index dce6deaf5..6f20b8b27 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/tree_ops/create_index.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/tree_ops/create_index.rs @@ -282,12 +282,10 @@ pub async fn create_graph_index( JsonValue::String(total_edges.to_string()), ); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["edges_created".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["edges_created".to_string()], + vec![row], + ))]) } /// Surface a build-time failure. diff --git a/nodedb/src/control/server/shared/ddl/neutral/tree_ops/sum.rs b/nodedb/src/control/server/shared/ddl/neutral/tree_ops/sum.rs index 982fda124..ed725078a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/tree_ops/sum.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/tree_ops/sum.rs @@ -179,10 +179,8 @@ pub async fn tree_sum( let mut row = Map::new(); row.insert("tree_sum".to_string(), JsonValue::String(total.to_string())); - Ok(vec![DdlResult::Rows(ShapedRows { - columns: vec!["tree_sum".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![row], - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + vec!["tree_sum".to_string()], + vec![row], + ))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/trigger/show.rs b/nodedb/src/control/server/shared/ddl/neutral/trigger/show.rs index f5302fbf6..c2d40371a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/trigger/show.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/trigger/show.rs @@ -90,11 +90,5 @@ pub fn show_triggers( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/typeguard/handlers.rs b/nodedb/src/control/server/shared/ddl/neutral/typeguard/handlers.rs index 19ab0c828..52e3fe66b 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/typeguard/handlers.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/typeguard/handlers.rs @@ -303,13 +303,7 @@ pub fn show_typeguard( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } /// Handle `SHOW TYPEGUARDS` — list all collections with active type guards. @@ -343,11 +337,5 @@ pub fn show_typeguards( rows.push(row); } - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/typeguard/validate.rs b/nodedb/src/control/server/shared/ddl/neutral/typeguard/validate.rs index 141675359..fe4c86bc2 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/typeguard/validate.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/typeguard/validate.rs @@ -52,13 +52,10 @@ pub async fn validate_typeguard( if coll.type_guards.is_empty() { // No type guards — return empty result. - let column_types = ShapedRows::text_types(columns.len()); - return Ok(vec![DdlResult::Rows(ShapedRows { + return Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: Vec::new(), - notice: None, - })]); + Vec::new(), + ))]); } let guards = coll.type_guards.clone(); @@ -163,11 +160,5 @@ pub async fn validate_typeguard( }) .collect(); - let column_types = ShapedRows::text_types(columns.len()); - Ok(vec![DdlResult::Rows(ShapedRows { - columns, - column_types, - rows, - notice: None, - })]) + Ok(vec![DdlResult::Rows(ShapedRows::text_rows(columns, rows))]) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/at_version.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/at_version.rs index 084ad94d6..bb215c239 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/at_version.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/at_version.rs @@ -71,16 +71,13 @@ pub async fn select_at_version( let text = String::from_utf8_lossy(&payload).into_owned(); let columns = vec!["document".to_string()]; - let column_types = ShapedRows::text_types(columns.len()); let mut row = Map::new(); row.insert("document".to_string(), JsonValue::String(text)); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( columns, - column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } /// Resolve a checkpoint name to its version vector JSON. diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/diff.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/diff.rs index 23669abe6..7359d3042 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/diff.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/diff.rs @@ -113,12 +113,11 @@ pub async fn select_diff( ); row.insert("delta_hex".to_string(), JsonValue::String(hex)); - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, - rows: vec![row], - notice: None, - })]) + vec![row], + ))]) } /// Parse function arguments from `SELECT DIFF('a', 'b', 'c', 'd')`. diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/show_versions.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/show_versions.rs index 74ea7446b..ea6b97dcb 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/show_versions.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/show_versions.rs @@ -97,12 +97,11 @@ pub fn show_versions( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { + Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( columns, column_types, rows, - notice: None, - })]) + ))]) } /// Parse: SHOW VERSIONS OF collection WHERE id = 'doc-id' [LIMIT N] diff --git a/nodedb/src/control/server/shared/ddl/neutral/weighted_pick.rs b/nodedb/src/control/server/shared/ddl/neutral/weighted_pick.rs index 847cb1519..53edba9b0 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/weighted_pick.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/weighted_pick.rs @@ -220,12 +220,10 @@ pub async fn weighted_pick( rows.push(row); } - Ok(vec![DdlResult::Rows(ShapedRows { - columns: pick_columns(), - column_types: ShapedRows::text_types(3), + Ok(vec![DdlResult::Rows(ShapedRows::text_rows( + pick_columns(), rows, - notice: None, - })]) + ))]) } // ── Helpers ──────────────────────────────────────────────────────────── @@ -325,12 +323,7 @@ fn pick_columns() -> Vec { /// Empty (no-rows) result set with the WEIGHTED_PICK schema. fn empty_pick_rows() -> DdlResult { - DdlResult::Rows(ShapedRows { - columns: pick_columns(), - column_types: ShapedRows::text_types(3), - rows: Vec::new(), - notice: None, - }) + DdlResult::Rows(ShapedRows::text_rows(pick_columns(), Vec::new())) } fn unquote(s: &str) -> String { From a9f44454b5054ac3c10a1ff92cf4b652ab949e85 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 16 Sep 2026 22:00:33 +0800 Subject: [PATCH 11/21] refactor(response-shape): carry shaped rows as typed Value, not JSON Route response shaping through nodedb_types::Value end to end instead of serde_json::Value, so a cell keeps its typed form (instant, bytes, decimal, etc.) until a protocol renders it at its own edge. Adds decode_payload_value as the typed counterpart of decode_payload_to_json, and a single value_to_wire_json/row_to_wire_json conversion in util::wire_json + response_shape::cell that every protocol (pgwire, HTTP, native) and the redaction path now render through. Splits response_shape::compose and response_shape::types into directories (compose/kernel.rs, compose/materialized.rs, compose/array_slice.rs; types/plan_kind.rs, types/shaped.rs) to keep each concern in its own file as the row representation changes. --- .../src/control/security/redaction/apply.rs | 202 ++++- .../server/http/routes/query_stream.rs | 21 +- .../server/http/routes/result_shape.rs | 18 +- .../server/native/dispatch/conversion.rs | 15 +- .../server/native/session/session_stream.rs | 27 +- .../src/control/server/pgwire/ddl_encode.rs | 6 +- .../server/pgwire/handler/shape_encode.rs | 110 ++- .../server/pgwire/handler/stream_response.rs | 18 +- .../src/control/server/response_shape/cell.rs | 20 + .../control/server/response_shape/compose.rs | 732 ------------------ .../response_shape/compose/array_slice.rs | 49 ++ .../server/response_shape/compose/kernel.rs | 492 ++++++++++++ .../response_shape/compose/materialized.rs | 256 ++++++ .../server/response_shape/compose/mod.rs | 11 + .../src/control/server/response_shape/kv.rs | 63 +- .../src/control/server/response_shape/mod.rs | 2 + .../control/server/response_shape/project.rs | 74 +- .../server/response_shape/redaction/shapes.rs | 4 +- .../server/response_shape/returning.rs | 135 ++-- .../server/response_shape/types/mod.rs | 9 + .../{types.rs => types/plan_kind.rs} | 188 ----- .../server/response_shape/types/shaped.rs | 267 +++++++ .../ddl/neutral/query_functions/helpers.rs | 24 +- .../server/shared/ddl/neutral/show_changes.rs | 4 +- .../shared/ddl/neutral/topic_subscribe.rs | 5 +- .../data/executor/response_codec/encode.rs | 58 +- .../src/data/executor/response_codec/mod.rs | 5 +- nodedb/src/util.rs | 1 + nodedb/src/util/wire_json.rs | 95 +++ .../tests/inproc/cases/oidc_provider_ddl.rs | 2 +- 30 files changed, 1768 insertions(+), 1145 deletions(-) create mode 100644 nodedb/src/control/server/response_shape/cell.rs delete mode 100644 nodedb/src/control/server/response_shape/compose.rs create mode 100644 nodedb/src/control/server/response_shape/compose/array_slice.rs create mode 100644 nodedb/src/control/server/response_shape/compose/kernel.rs create mode 100644 nodedb/src/control/server/response_shape/compose/materialized.rs create mode 100644 nodedb/src/control/server/response_shape/compose/mod.rs create mode 100644 nodedb/src/control/server/response_shape/types/mod.rs rename nodedb/src/control/server/response_shape/{types.rs => types/plan_kind.rs} (74%) create mode 100644 nodedb/src/control/server/response_shape/types/shaped.rs create mode 100644 nodedb/src/util/wire_json.rs diff --git a/nodedb/src/control/security/redaction/apply.rs b/nodedb/src/control/security/redaction/apply.rs index 685a732b3..4f488f54d 100644 --- a/nodedb/src/control/security/redaction/apply.rs +++ b/nodedb/src/control/security/redaction/apply.rs @@ -7,13 +7,21 @@ //! per-row [`RedactionStore::apply_flat_row`] go through it, so the mask / hash //! / null semantics exist exactly once. //! -//! `apply_flat_row` is the SELECT-path entry point: it rewrites one already -//! flattened result row, whose columns may come from several source -//! collections at once (a join), rather than one document belonging to a -//! single collection. +//! `apply_flat_row` (JSON cells) and `apply_flat_row_typed` (typed +//! `nodedb_types::Value` cells) are the SELECT-path entry points: each +//! rewrites one already flattened result row, whose columns may come from +//! several source collections at once (a join), rather than one document +//! belonging to a single collection. Both share one rule-resolution body +//! through [`FlatRow`], and a typed cell hashes through the same JSON text +//! its wire rendering produces, so the two cannot drift. +use std::collections::BTreeMap; + +use nodedb_types::Value; use serde_json::{Map, Value as JsonValue}; +use crate::util::wire_json::value_to_wire_json; + use super::store::RedactionStore; use super::types::{RedactionMode, RedactionRule, policy_key}; @@ -45,13 +53,66 @@ fn hash_value(value: &JsonValue) -> String { format!("hash:{digest:x}") } -/// Rewrite `row[key]` per `mode`, if the row actually carries that key. -fn redact_key(row: &mut Map, key: &str, mode: &RedactionMode) { - if !row.contains_key(key) { - return; +/// The value `mode` produces for a typed field whose present value is +/// `current`. A mask or hash renders through the JSON path: the hash input +/// is the cell's wire JSON, exactly what the JSON row would have held. +fn redacted_typed_value(mode: &RedactionMode, current: Option<&Value>) -> Value { + match mode { + RedactionMode::Mask(mask) => Value::String(mask.clone()), + RedactionMode::Hash => { + let wire = current.map(value_to_wire_json); + Value::String(hash_value(wire.as_ref().unwrap_or(&JsonValue::Null))) + } + RedactionMode::Null => Value::Null, + } +} + +/// One flattened result row, whatever its cell type. +/// +/// The rule-resolution body in `RedactionStore::apply_flat_row_impl` is +/// written once against this trait; the JSON and typed row maps each +/// implement it. +trait FlatRow { + fn is_empty(&self) -> bool; + fn keys(&self) -> impl Iterator; + /// Rewrite `self[key]` per `mode`, if the row actually carries that key. + fn redact_key(&mut self, key: &str, mode: &RedactionMode); +} + +impl FlatRow for Map { + fn is_empty(&self) -> bool { + Map::is_empty(self) + } + + fn keys(&self) -> impl Iterator { + Map::keys(self).map(String::as_str) + } + + fn redact_key(&mut self, key: &str, mode: &RedactionMode) { + if !self.contains_key(key) { + return; + } + let value = redacted_value(mode, self.get(key)); + self.insert(key.to_string(), value); + } +} + +impl FlatRow for BTreeMap { + fn is_empty(&self) -> bool { + BTreeMap::is_empty(self) + } + + fn keys(&self) -> impl Iterator { + BTreeMap::keys(self).map(String::as_str) + } + + fn redact_key(&mut self, key: &str, mode: &RedactionMode) { + if !self.contains_key(key) { + return; + } + let value = redacted_typed_value(mode, self.get(key)); + self.insert(key.to_string(), value); } - let value = redacted_value(mode, row.get(key)); - row.insert(key.to_string(), value); } /// How many of the plan's sources a row-map key can be attributed to. @@ -73,7 +134,7 @@ fn attribution_count(key: &str, sources: &[(&str, Vec<&RedactionRule>)]) -> usiz } impl RedactionStore { - /// Redact one already-flattened SELECT result row in place. + /// Redact one already-flattened SELECT result row of JSON cells in place. /// /// `collections` lists the plan's source collections as /// `(qualifier, collection)`, where `qualifier` is the prefix that appears @@ -100,6 +161,30 @@ impl RedactionStore { roles: &[String], collections: &[(String, String)], row: &mut Map, + ) { + self.apply_flat_row_impl(tenant_id, roles, collections, row); + } + + /// Redact one already-flattened SELECT result row of typed cells in + /// place. Same matching rules as [`RedactionStore::apply_flat_row`]; a + /// `Null` mode writes `Value::Null`, a mask or hash writes the same text + /// the JSON path writes. + pub fn apply_flat_row_typed( + &self, + tenant_id: u64, + roles: &[String], + collections: &[(String, String)], + row: &mut BTreeMap, + ) { + self.apply_flat_row_impl(tenant_id, roles, collections, row); + } + + fn apply_flat_row_impl( + &self, + tenant_id: u64, + roles: &[String], + collections: &[(String, String)], + row: &mut R, ) { if roles.is_empty() || collections.is_empty() || row.is_empty() { return; @@ -122,14 +207,14 @@ impl RedactionStore { if let [(_, rules)] = sources.as_slice() { for rule in rules { - redact_key(row, &rule.field, &rule.mode); + row.redact_key(&rule.field, &rule.mode); } return; } for (qualifier, rules) in &sources { for rule in rules { - redact_key(row, &format!("{qualifier}.{}", rule.field), &rule.mode); + row.redact_key(&format!("{qualifier}.{}", rule.field), &rule.mode); } } @@ -137,7 +222,7 @@ impl RedactionStore { let unattributed: Vec = row .keys() .filter(|key| attribution_count(key, &sources) != 1) - .cloned() + .map(str::to_owned) .collect(); for key in unattributed { let bare = key.rfind('.').map_or(key.as_str(), |dot| &key[dot + 1..]); @@ -147,7 +232,7 @@ impl RedactionStore { .find(|rule| rule.field == bare || rule.field == key) .map(|rule| rule.mode.clone()); if let Some(mode) = mode { - redact_key(row, &key, &mode); + row.redact_key(&key, &mode); } } } @@ -273,6 +358,93 @@ mod tests { assert_eq!(r["email"], JsonValue::Null); } + fn typed_row(pairs: &[(&str, Value)]) -> BTreeMap { + pairs + .iter() + .map(|(k, v)| ((*k).to_string(), v.clone())) + .collect() + } + + /// The typed entry point resolves rules exactly as the JSON one: same + /// mask on the ruled side of a join, clear value on the other, and a + /// `Null` mode keeps the key present. + #[test] + fn typed_row_matches_the_json_rules() { + let store = store_with( + "workspaces", + "support", + vec![ + mask("id", "***"), + RedactionRule { + field: "note".into(), + mode: RedactionMode::Null, + }, + ], + ); + let mut r = typed_row(&[ + ("w.id", Value::String("w1".into())), + ("b.id", Value::String("b1".into())), + ("w.note", Value::Integer(7)), + ]); + store.apply_flat_row_typed( + 1, + &["support".into()], + &[ + ("w".into(), "workspaces".into()), + ("b".into(), "boards".into()), + ], + &mut r, + ); + assert_eq!(r["w.id"], Value::String("***".into())); + assert_eq!(r["b.id"], Value::String("b1".into())); + assert_eq!(r["w.note"], Value::Null); + } + + /// A hashed typed cell yields the digest the JSON path yields for the + /// same cell: a string hashes its bytes, a number hashes its JSON text, + /// and bytes hash their base64 wire text. + #[test] + fn typed_hash_matches_the_json_hash() { + let store = store_with( + "users", + "support", + vec![ + RedactionRule { + field: "email".into(), + mode: RedactionMode::Hash, + }, + RedactionRule { + field: "n".into(), + mode: RedactionMode::Hash, + }, + RedactionRule { + field: "blob".into(), + mode: RedactionMode::Hash, + }, + ], + ); + let roles = ["support".to_string()]; + let sources = [(String::new(), "users".to_string())]; + + let mut typed = typed_row(&[ + ("email", Value::String("a@b.c".into())), + ("n", Value::Integer(42)), + ("blob", Value::Bytes(vec![0, 255, 7])), + ]); + store.apply_flat_row_typed(1, &roles, &sources, &mut typed); + + let mut json = row(json!({"email": "a@b.c", "n": 42, "blob": "AP8H"})); + store.apply_flat_row(1, &roles, &sources, &mut json); + + for key in ["email", "n", "blob"] { + assert_eq!( + typed[key], + Value::String(json[key].as_str().expect("hashed text").to_string()), + "{key}" + ); + } + } + #[test] fn no_policy_leaves_the_row_untouched() { let store = RedactionStore::new(); diff --git a/nodedb/src/control/server/http/routes/query_stream.rs b/nodedb/src/control/server/http/routes/query_stream.rs index df1e8b265..c05149c97 100644 --- a/nodedb/src/control/server/http/routes/query_stream.rs +++ b/nodedb/src/control/server/http/routes/query_stream.rs @@ -26,12 +26,13 @@ use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::exchange::gather::gather_all_cores_stream_authorized; use crate::control::server::exchange::streamable::streamable_gather_child; +use crate::control::server::response_shape::cell::row_to_wire_json; use crate::control::server::response_shape::compose::shape_decoded_rows; use crate::control::server::response_shape::redaction::QueryRedaction; use crate::control::server::response_shape::schema::OutputSchema; use crate::control::server::result_stream::ResultStream; use crate::control::server::shared::metering::DetachedMeterGuard; -use crate::data::executor::response_codec::decode_payload_to_json; +use crate::data::executor::response_codec::decode_payload_value; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use super::super::auth::AppState; @@ -171,8 +172,7 @@ pub(super) fn ndjson_body_stream( } }; - let json_str = decode_payload_to_json(&batch.payload); - let value = match sonic_rs::from_str::(&json_str) { + let value = match decode_payload_value(&batch.payload) { Ok(v) => v, Err(e) => { // A malformed batch payload is surfaced as an in-band error @@ -187,12 +187,13 @@ pub(super) fn ndjson_body_stream( } }; // Row maps are keyed by `ShapedRows::cell_keys`, so each NDJSON - // line serializes as-is; two output columns sharing a name emit - // `{"id": …, "id_1": …}` rather than collapsing to one cell. + // line serializes key for key, each typed cell converted at this + // edge; two output columns sharing a name emit `{"id": …, + // "id_1": …}` rather than collapsing to one cell. // Only re-borrows the once-resolved inputs, so the very first // batch is redacted under the same policy as the last. let shaped = match shape_decoded_rows( - &value, + value, projection.as_ref(), redaction.as_ref().map(|r| r.ctx(&state.redaction)), ) { @@ -205,11 +206,11 @@ pub(super) fn ndjson_body_stream( return; } }; - for row in shaped.rows { + for row in &shaped.rows { if emitted >= limit { break; } - let line = format!("{}\n", serde_json::Value::Object(row)); + let line = format!("{}\n", serde_json::Value::Object(row_to_wire_json(row))); emitted += 1; // Incremented for the row this line actually carries, right // before it is handed to the body sink below — a client that @@ -232,8 +233,8 @@ mod tests { use crate::control::state::SharedState; use crate::types::Lsn; - /// A JSON-text array of `n` `{"id": i}` objects. `decode_payload_to_json` - /// returns a JSON-leading payload as-is, so this exercises the same + /// A JSON-text array of `n` `{"id": i}` objects. `decode_payload_value` + /// parses a JSON-leading payload as JSON, so this exercises the same /// array-of-objects → per-line decode path a Data-Plane scan chunk drives. fn json_object_batch(start: usize, n: usize) -> Vec { let items: Vec = (start..start + n) diff --git a/nodedb/src/control/server/http/routes/result_shape.rs b/nodedb/src/control/server/http/routes/result_shape.rs index b953c1052..32ee1f6ae 100644 --- a/nodedb/src/control/server/http/routes/result_shape.rs +++ b/nodedb/src/control/server/http/routes/result_shape.rs @@ -10,6 +10,7 @@ //! (`Execution`, `DmlResult`) come back as [`HttpShaped::Passthrough`]; the //! caller keeps its existing raw decode/base64 fallback for those. +use crate::control::server::response_shape::cell::row_to_wire_json; use crate::control::server::response_shape::compose::{ShapeOutcome, shape_response_materialized}; use crate::control::server::response_shape::request::MaterializedShapeRequest; use nodedb_types::NodeDbError; @@ -31,15 +32,16 @@ pub(super) fn shape_http_payload( ) -> Result { match shape_response_materialized(request)? { // Each row map is already keyed by `ShapedRows::cell_keys`, so it - // serializes to JSON as-is. When two output columns share a name the - // later one carries a `_` suffix (`SELECT w.id, b.id` → - // `{"id": …, "id_1": …}`) — a JSON object cannot repeat a key, and - // dropping the duplicate would silently lose a projected column. + // serializes to JSON key for key, each typed cell converted at this + // edge. When two output columns share a name the later one carries a + // `_` suffix (`SELECT w.id, b.id` → `{"id": …, "id_1": …}`) — a + // JSON object cannot repeat a key, and dropping the duplicate would + // silently lose a projected column. ShapeOutcome::Rows(shaped) => Ok(HttpShaped::Rows( shaped .rows - .into_iter() - .map(serde_json::Value::Object) + .iter() + .map(|row| serde_json::Value::Object(row_to_wire_json(row))) .collect(), )), ShapeOutcome::Passthrough => Ok(HttpShaped::Passthrough), @@ -102,8 +104,8 @@ pub(super) fn ddl_results_to_json( // Keyed by `ShapedRows::cell_keys`, same JSON contract as // `shape_http_payload` above (duplicate names take a `_` key). DdlResult::Rows(shaped) => { - for row in shaped.rows { - rows.push(serde_json::Value::Object(row)); + for row in &shaped.rows { + rows.push(serde_json::Value::Object(row_to_wire_json(row))); } } DdlResult::Empty => { diff --git a/nodedb/src/control/server/native/dispatch/conversion.rs b/nodedb/src/control/server/native/dispatch/conversion.rs index bf6ed5c04..3c4decf63 100644 --- a/nodedb/src/control/server/native/dispatch/conversion.rs +++ b/nodedb/src/control/server/native/dispatch/conversion.rs @@ -3,7 +3,6 @@ //! Shared conversion helpers for native protocol dispatch. use nodedb_types::Value; -use nodedb_types::conversion::json_to_value_ref; use nodedb_types::protocol::NativeResponse; use crate::bridge::envelope::Response; @@ -282,8 +281,8 @@ pub(crate) fn calvin_native_response( /// Convert protocol-neutral `ShapedRows` (produced by /// `response_shape::compose::shape_response_materialized`) into native wire -/// columns/rows: each JSON cell becomes a typed `Value` via `json_to_value_ref`; -/// a column absent from a given row's map becomes `Value::Null`. +/// columns/rows: each typed cell is carried as-is; a column absent from a +/// given row's map becomes `Value::Null`. /// /// Structure is preserved all the way down, including nested objects and /// arrays. The native protocol is MessagePack and `Value` has `Object` and @@ -293,8 +292,8 @@ pub(crate) fn calvin_native_response( /// its JSON, so deserializing the row into the struct it was written from /// fails with a type error, while a field that genuinely holds a JSON string /// is indistinguishable from one that was flattened. Text rendering belongs -/// to pgwire, whose wire format is textual and which has its own -/// `json_value_to_text` for exactly that. +/// to pgwire, whose wire format is textual and which converts each cell to +/// JSON at its own edge for exactly that. pub(crate) fn to_native_columns_rows(shaped: &ShapedRows) -> (Vec, Vec>) { // Cells live in the row maps under per-column keys (display names may // repeat across columns, e.g. `SELECT w.id, b.id`), so read through the @@ -306,11 +305,7 @@ pub(crate) fn to_native_columns_rows(shaped: &ShapedRows) -> (Vec, Vec, redaction: Option>, ) -> crate::Result<(Vec, Vec>)> { - match sonic_rs::from_str::(json_text) { + match decode_payload_value(payload) { Ok(decoded) => { - let shaped = shape_decoded_rows(&decoded, projection, redaction)?; + let shaped = shape_decoded_rows(decoded, projection, redaction)?; Ok(to_native_columns_rows(&shaped)) } Err(_) => Ok(( vec!["result".into()], - vec![vec![Value::String(json_text.to_string())]], + vec![vec![Value::String( + String::from_utf8_lossy(payload).into_owned(), + )]], )), } } @@ -98,12 +100,11 @@ pub(super) async fn emit_sql_stream( last_lsn = batch.watermark_lsn.as_u64(); - let json_text = decode_payload_to_json(&batch.payload); // The redaction inputs were resolved once when the stream opened; this // only re-borrows them, so every batch — including the first — is // shaped under the same policy. let (cols, mut batch_rows) = decode_batch_to_columns_rows( - &json_text, + &batch.payload, projection.as_ref(), redaction.as_ref().map(|r| r.ctx(&state.redaction)), )?; @@ -187,8 +188,8 @@ mod tests { SharedState::new(dispatcher, wal).expect("test shared state") } - /// A JSON-text array of `n` `{"id": i}` objects — `decode_payload_to_json` - /// returns JSON-leading bytes as-is, exercising the array → rows decode. + /// A JSON-text array of `n` `{"id": i}` objects — `decode_payload_value` + /// parses JSON-leading bytes as JSON, exercising the array → rows decode. fn json_batch(start: usize, n: usize) -> Vec { let items: Vec = (start..start + n) .map(|i| serde_json::json!({ "id": i })) diff --git a/nodedb/src/control/server/pgwire/ddl_encode.rs b/nodedb/src/control/server/pgwire/ddl_encode.rs index 0509b5ab9..c61ff1630 100644 --- a/nodedb/src/control/server/pgwire/ddl_encode.rs +++ b/nodedb/src/control/server/pgwire/ddl_encode.rs @@ -18,6 +18,7 @@ use pgwire::api::results::{DataRowEncoder, FieldInfo, QueryResponse, Response, T use pgwire::error::{ErrorInfo, PgWireError, PgWireResult}; use serde_json::Value as JsonValue; +use crate::control::server::response_shape::cell::value_to_wire_json; use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; use crate::control::server::shared::ddl::result::{DdlError, DdlResult}; @@ -110,7 +111,8 @@ fn rows_to_response(shaped: ShapedRows) -> PgWireResult { let mut encoder = DataRowEncoder::new(schema.clone()); for (idx, name) in columns.iter().enumerate() { let ct = column_types.get(idx).copied().unwrap_or(DdlColType::Text); - match row.get(name) { + // Each typed cell converts to JSON at this edge before it renders. + match row.get(name).map(value_to_wire_json) { // Captured text (the transitional wrapper path, and text-typed // migrated cells): re-emit verbatim so the DataRow bytes match. Some(JsonValue::String(s)) => encoder.encode_field(&s)?, @@ -123,7 +125,7 @@ fn rows_to_response(shaped: ShapedRows) -> PgWireResult { // produced — string pre-rendering (`f64::to_string`) diverges // (e.g. `0.0` → "0" vs "0.0"). Integer/other numerics render to // the same decimal text either way. - Some(value @ JsonValue::Number(n)) => match ct { + Some(ref value @ JsonValue::Number(ref n)) => match ct { DdlColType::Float8 => match n.as_f64() { Some(f) => encoder.encode_field(&f)?, None => encoder.encode_field(&None::)?, diff --git a/nodedb/src/control/server/pgwire/handler/shape_encode.rs b/nodedb/src/control/server/pgwire/handler/shape_encode.rs index 15788da3c..a86554567 100644 --- a/nodedb/src/control/server/pgwire/handler/shape_encode.rs +++ b/nodedb/src/control/server/pgwire/handler/shape_encode.rs @@ -14,6 +14,9 @@ //! render as ISO-8601 text, and everything else (`Text`, integers, `Bool`) //! falls back to `json_value_to_text` — notably `Bool` as `t`/`f`, not //! `true`/`false`. +//! +//! Each typed cell converts to JSON at this edge through +//! [`value_to_wire_json`] before it is rendered. use std::sync::Arc; @@ -24,8 +27,9 @@ use pgwire::messages::data::DataRow; use nodedb_types::NdbDateTime; use nodedb_types::columnar::IntWidth; +use crate::control::server::response_shape::cell::value_to_wire_json; use crate::control::server::response_shape::project::json_value_to_text; -use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::server::response_shape::types::{DdlColType, ShapedRow, ShapedRows}; use super::super::ddl_encode::col_type_to_field_with_format; use super::super::numeric_narrow::{checked_narrow, checked_narrow_f32}; @@ -39,25 +43,27 @@ use super::super::numeric_narrow::{checked_narrow, checked_narrow_f32}; /// display names except where those repeat (`SELECT w.id, b.id`), in which /// case later duplicates carry a `_n` suffix so both cells survive the map. /// -/// Missing keys and explicit JSON `null` both encode as SQL NULL. Every other -/// cell renders per its column type via [`encode_typed_cell`]; a -/// missing/short `column_types` entry defaults to `Text`. +/// Each cell converts to JSON at this edge. Missing keys and a cell whose +/// JSON is `null` (an explicit `Value::Null`, or a float with no JSON form) +/// both encode as SQL NULL. Every other cell renders per its column type via +/// [`encode_typed_cell`]; a missing/short `column_types` entry defaults to +/// `Text`. pub(in crate::control::server::pgwire) fn encode_shaped_row( schema: &Arc>, cell_keys: &[String], column_types: &[DdlColType], formats: &[FieldFormat], - row: &serde_json::Map, + row: &ShapedRow, ) -> PgWireResult { let mut encoder = DataRowEncoder::new(schema.clone()); for (idx, name) in cell_keys.iter().enumerate() { let ct = column_types.get(idx).copied().unwrap_or(DdlColType::Text); let format = formats.get(idx).copied().unwrap_or(FieldFormat::Text); - match row.get(name) { + match row.get(name).map(value_to_wire_json) { None | Some(serde_json::Value::Null) => { encoder.encode_field(&None::<&str>)?; } - Some(v) => encode_typed_cell(&mut encoder, ct, format, v)?, + Some(v) => encode_typed_cell(&mut encoder, ct, format, &v)?, } } Ok(encoder.take_row()) @@ -218,11 +224,11 @@ pub(in crate::control::server::pgwire) fn shaped_query_response( #[cfg(test)] mod tests { use futures::StreamExt; + use nodedb_types::Value; use pgwire::api::results::{QueryResponse, Response}; - use serde_json::json; use super::shaped_query_response; - use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; + use crate::control::server::response_shape::types::{DdlColType, ShapedRow, ShapedRows}; /// Drain a `QueryResponse` stream into a `Vec` of `DataRow`s. async fn drain(mut qr: QueryResponse) -> Vec { @@ -273,10 +279,7 @@ mod tests { None } - fn make_shaped( - columns: &[&str], - rows: Vec>, - ) -> ShapedRows { + fn make_shaped(columns: &[&str], rows: Vec) -> ShapedRows { let columns: Vec = columns.iter().map(|s| s.to_string()).collect(); let column_types = ShapedRows::text_types(columns.len()); ShapedRows { @@ -287,16 +290,20 @@ mod tests { } } - fn obj(pairs: &[(&str, serde_json::Value)]) -> serde_json::Map { + fn obj(pairs: &[(&str, Value)]) -> ShapedRow { pairs .iter() .map(|(k, v)| (k.to_string(), v.clone())) .collect() } + fn text(s: &str) -> Value { + Value::String(s.to_string()) + } + #[tokio::test] async fn string_cell_renders_verbatim() { - let shaped = make_shaped(&["a"], vec![obj(&[("a", json!("hello"))])]); + let shaped = make_shaped(&["a"], vec![obj(&[("a", text("hello"))])]); let (response, notice) = shaped_query_response(shaped, &[]); assert!(notice.is_none()); let Response::Query(qr) = response else { @@ -310,7 +317,10 @@ mod tests { async fn bool_cells_render_as_t_f_not_true_false() { let shaped = make_shaped( &["a"], - vec![obj(&[("a", json!(true))]), obj(&[("a", json!(false))])], + vec![ + obj(&[("a", Value::Bool(true))]), + obj(&[("a", Value::Bool(false))]), + ], ); let (response, _notice) = shaped_query_response(shaped, &[]); let Response::Query(qr) = response else { @@ -325,7 +335,10 @@ mod tests { async fn number_cells_render_via_to_string() { let shaped = make_shaped( &["a"], - vec![obj(&[("a", json!(42))]), obj(&[("a", json!(0.0))])], + vec![ + obj(&[("a", Value::Integer(42))]), + obj(&[("a", Value::Float(0.0))]), + ], ); let (response, _notice) = shaped_query_response(shaped, &[]); let Response::Query(qr) = response else { @@ -338,13 +351,13 @@ mod tests { #[tokio::test] async fn null_and_missing_column_both_encode_as_sql_null() { - let shaped = make_shaped(&["a", "b"], vec![obj(&[("a", serde_json::Value::Null)])]); + let shaped = make_shaped(&["a", "b"], vec![obj(&[("a", Value::Null)])]); let (response, _notice) = shaped_query_response(shaped, &[]); let Response::Query(qr) = response else { panic!("expected Query response"); }; let rows = drain(qr).await; - // "a" was explicit JSON null. + // "a" was an explicit NULL cell. assert_eq!(field_text(&rows[0], 0), None); // "b" was entirely absent from the row object. assert_eq!(field_text(&rows[0], 1), None); @@ -354,7 +367,7 @@ mod tests { async fn column_order_is_preserved() { let shaped = make_shaped( &["b", "a"], - vec![obj(&[("a", json!("first")), ("b", json!("second"))])], + vec![obj(&[("a", text("first")), ("b", text("second"))])], ); let (response, _notice) = shaped_query_response(shaped, &[]); let Response::Query(qr) = response else { @@ -383,13 +396,13 @@ mod tests { DdlColType::Timestamp, ]; let row = obj(&[ - ("i", json!(42)), + ("i", Value::Integer(42)), // Integral float renders Postgres-style "0" (shortest form) via the // native float encoder, not serde's "0.0". - ("f", json!(0.0)), - ("b", json!(true)), + ("f", Value::Float(0.0)), + ("b", Value::Bool(true)), // Epoch microseconds → ISO-8601 text (0 == Unix epoch). - ("ts", json!(0)), + ("ts", Value::Integer(0)), ]); let shaped = ShapedRows { columns, @@ -419,9 +432,40 @@ mod tests { ); } + /// An instant cell reaches the wire as its ISO-8601 text through the + /// edge conversion, under a TEXT column and under a TIMESTAMP column + /// alike, and a byte cell as the transcoder's unpadded base64. + #[tokio::test] + async fn instant_and_byte_cells_render_through_the_edge_conversion() { + let at = nodedb_types::NdbDateTime::from_micros(1_583_402_400_000_000); + let shaped = shaped_typed( + &["t", "ts", "blob"], + vec![DdlColType::Text, DdlColType::Timestamp, DdlColType::Text], + obj(&[ + ("t", Value::NaiveDateTime(at)), + ("ts", Value::NaiveDateTime(at)), + ("blob", Value::Bytes(vec![0, 255, 7])), + ]), + ); + let (response, _notice) = shaped_query_response(shaped, &[]); + let Response::Query(qr) = response else { + panic!("expected Query response"); + }; + let rows = drain(qr).await; + assert_eq!( + field_text(&rows[0], 0).as_deref(), + Some("2020-03-05T10:00:00.000000Z") + ); + assert_eq!( + field_text(&rows[0], 1).as_deref(), + Some("2020-03-05T10:00:00.000000Z") + ); + assert_eq!(field_text(&rows[0], 2).as_deref(), Some("AP8H")); + } + #[tokio::test] async fn notice_is_preserved_not_dropped() { - let mut shaped = make_shaped(&["a"], vec![obj(&[("a", json!("x"))])]); + let mut shaped = make_shaped(&["a"], vec![obj(&[("a", text("x"))])]); shaped.notice = Some("heads up".to_owned()); let (_response, notice) = shaped_query_response(shaped, &[]); assert_eq!(notice.as_deref(), Some("heads up")); @@ -461,11 +505,7 @@ mod tests { None } - fn shaped_typed( - columns: &[&str], - column_types: Vec, - row: serde_json::Map, - ) -> ShapedRows { + fn shaped_typed(columns: &[&str], column_types: Vec, row: ShapedRow) -> ShapedRows { ShapedRows { columns: columns.iter().map(|s| s.to_string()).collect(), column_types, @@ -490,10 +530,10 @@ mod tests { DdlColType::Text, ], obj(&[ - ("i", json!(42)), - ("f", json!(1.5)), - ("b", json!(true)), - ("t", json!("hello")), + ("i", Value::Integer(42)), + ("f", Value::Float(1.5)), + ("b", Value::Bool(true)), + ("t", text("hello")), ]), ); let formats = vec![FieldFormat::Binary; 4]; @@ -528,7 +568,7 @@ mod tests { let shaped = shaped_typed( &["i", "j"], vec![DdlColType::Int8, DdlColType::Int8], - obj(&[("i", json!(7)), ("j", json!(9))]), + obj(&[("i", Value::Integer(7)), ("j", Value::Integer(9))]), ); let formats = vec![FieldFormat::Binary, FieldFormat::Text]; let (response, _notice) = shaped_query_response(shaped, &formats); diff --git a/nodedb/src/control/server/pgwire/handler/stream_response.rs b/nodedb/src/control/server/pgwire/handler/stream_response.rs index 84103a59a..847242dfb 100644 --- a/nodedb/src/control/server/pgwire/handler/stream_response.rs +++ b/nodedb/src/control/server/pgwire/handler/stream_response.rs @@ -19,7 +19,7 @@ use crate::control::server::response_shape::types::DdlColType; use crate::control::server::result_stream::ResultStream; use crate::control::server::shared::metering::DetachedMeterGuard; use crate::control::state::SharedState; -use crate::data::executor::response_codec::decode_payload_to_json; +use crate::data::executor::response_codec::{decode_payload_to_json, decode_payload_value}; use super::super::ddl_encode::col_type_to_field_with_format; use super::super::types::{error_to_sqlstate, text_field}; @@ -214,8 +214,7 @@ pub(crate) fn streaming_shaped_response( break; } - let text = decode_payload_to_json(&batch.payload); - let value = sonic_rs::from_str::(&text).map_err(|e| { + let value = decode_payload_value(&batch.payload).map_err(|e| { PgWireError::UserError(Box::new(ErrorInfo::new( "ERROR".to_owned(), "XX000".to_owned(), @@ -225,7 +224,7 @@ pub(crate) fn streaming_shaped_response( // Resolved once before the first batch was pulled; this only // re-borrows it, so no batch can slip out ahead of the policy. let shaped = shape_decoded_rows( - &value, + value, Some(&schema_out), redaction.as_ref().map(|r| r.ctx(&state.redaction)), ) @@ -279,7 +278,7 @@ pub(crate) async fn streaming_star_response( ) -> Response { use futures::StreamExt; - let mut values: Vec = Vec::new(); + let mut values: Vec = Vec::new(); let mut batches = stream; while let Some(batch) = batches.next().await { let batch = match batch { @@ -298,9 +297,8 @@ pub(crate) async fn streaming_star_response( break; } - let text = decode_payload_to_json(&batch.payload); - match sonic_rs::from_str::(&text) { - Ok(serde_json::Value::Array(items)) => { + match decode_payload_value(&batch.payload) { + Ok(nodedb_types::Value::Array(items)) => { for item in items { if values.len() >= limit { break; @@ -312,7 +310,7 @@ pub(crate) async fn streaming_star_response( return single_pgwire_error(PgWireError::UserError(Box::new(ErrorInfo::new( "ERROR".to_owned(), "XX000".to_owned(), - "streamed batch payload was not a JSON array".to_owned(), + "streamed batch payload was not a row array".to_owned(), )))); } Err(e) => { @@ -340,7 +338,7 @@ pub(crate) async fn streaming_star_response( } let shaped = match shape_decoded_rows( - &serde_json::Value::Array(values), + nodedb_types::Value::Array(values), None, redaction.as_ref().map(|r| r.ctx(&state.redaction)), ) { diff --git a/nodedb/src/control/server/response_shape/cell.rs b/nodedb/src/control/server/response_shape/cell.rs new file mode 100644 index 000000000..4d497ff9f --- /dev/null +++ b/nodedb/src/control/server/response_shape/cell.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Row-level wrapper around the shared wire-JSON cell conversion. +//! +//! The scalar conversion itself, [`value_to_wire_json`], lives in +//! [`crate::util::wire_json`] — a neutral home so `control::security` +//! (which `response_shape` depends on for redaction) never has to depend +//! back on `control::server`. This module re-exports it and adds the +//! row-level helper, which depends on [`ShapedRow`]. + +use super::types::ShapedRow; + +pub use crate::util::wire_json::value_to_wire_json; + +/// Render one shaped row as a JSON object, cell by cell. +pub fn row_to_wire_json(row: &ShapedRow) -> serde_json::Map { + row.iter() + .map(|(k, v)| (k.clone(), value_to_wire_json(v))) + .collect() +} diff --git a/nodedb/src/control/server/response_shape/compose.rs b/nodedb/src/control/server/response_shape/compose.rs deleted file mode 100644 index 5e1286d97..000000000 --- a/nodedb/src/control/server/response_shape/compose.rs +++ /dev/null @@ -1,732 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Composed, protocol-neutral materialized response shaping. -//! -//! `shape_response_materialized` and `shape_decoded_rows` are the canonical -//! SELECT-read shaping used by every protocol entrypoint. `shape_response_materialized` -//! performs the full per-payload shaping order (`apply_kv_wrap` -> -//! `translate_search_response` -> decode -> scan-envelope unwrap -> optional -//! SELECT-list projection) as a single call, producing an already-shaped, -//! already-projected [`ShapeOutcome`]. Every SELECT-read producer — pgwire's -//! non-streaming dispatch, native's dispatch loop — calls this directly and -//! hands the resulting `ShapedRows` to its own protocol encoder; each -//! protocol then encodes those rows in its own wire format (pgwire's -//! RowDescription/DataRow, native's MessagePack, http's JSON). -//! -//! Producers with no `PhysicalPlan` in scope (ClusterArray, set-op merges, -//! gateway forwarding, clone merges) call [`shape_payload_no_plan`], which -//! skips the plan-dependent `apply_kv_wrap` / `translate_search_response` transforms -//! those callers never ran. The pure kernel [`shape_decoded_rows`] is shared -//! with per-batch lazy streaming callers, which have an already-decoded batch -//! and only need the envelope-unwrap + projection logic. - -use std::collections::HashSet; - -use serde_json::{Map, Value as JsonValue}; - -use crate::control::server::response_translate::dispatch::translate_search_response; -use crate::data::executor::response_codec::{ArraySliceResponse, decode_payload_to_json}; -use nodedb_types::NodeDbError; -use nodedb_types::columnar::schema::is_reserved_bitemporal_column; - -use super::kv::apply_kv_wrap; -use super::project::push_flat_rows; -use super::redaction::RedactionCtx; -use super::request::MaterializedShapeRequest; -use super::returning::shape_returning_rows; -use super::schema::OutputSchema; -use super::types::{DdlColType, PlanKind, ShapedRows}; - -/// NOTICE text for an `AS OF SYSTEM TIME` cutoff older than the oldest -/// retained tile version. This is the canonical definition, surfaced to -/// every protocol via [`ShapedRows::notice`]. -const TRUNCATED_BEFORE_HORIZON_NOTICE: &str = "AS OF SYSTEM TIME cutoff is older than the oldest retained tile version; \ - results may be incomplete"; - -/// Outcome of materialized response shaping. -/// -/// Row-producing plan kinds (`SingleDocument`, `MultiRow`, `ReturningRows`, -/// `ArraySlice`) yield `Rows`. Tag/execution kinds (`Execution`, -/// `DmlResult`) yield `Passthrough` — a `ShapedRows` cannot represent a bare -/// `CommandComplete` tag or affected-row count, so callers keep their -/// existing tag / `rows_affected` handling for those. -pub enum ShapeOutcome { - Rows(ShapedRows), - Passthrough, -} - -/// Shape a single Data-Plane payload into protocol-neutral rows, applying -/// the canonical shaping order: KV point-get wrap, vector surrogate->PK -/// translation, payload decode, scan-envelope unwrap, and (when -/// `projection` names columns) SELECT-list column selection. -pub fn shape_response_materialized( - request: MaterializedShapeRequest<'_>, -) -> Result { - let MaterializedShapeRequest { - payload, - plan, - plan_kind, - projection, - state, - database_id, - tenant_id, - redaction, - } = request; - - match plan_kind { - PlanKind::Execution | PlanKind::DmlResult(_) => return Ok(ShapeOutcome::Passthrough), - PlanKind::ArraySlice - | PlanKind::ReturningRows - | PlanKind::SingleDocument - | PlanKind::MultiRow => {} - } - - // Seam-1 order, exactly as pgwire's `dispatch_task_loop` applies it - // (apply_kv_wrap -> translate_search_response) before any decode/shape step. - let wrapped = apply_kv_wrap(plan, payload); - let translated = translate_search_response(&wrapped, plan, state, database_id, tenant_id); - - let shaped = match plan_kind { - PlanKind::ArraySlice => shape_array_slice(&translated, redaction)?, - // `RETURNING` rows are held to the columns already announced to the - // client, when any were — see `super::returning`. - PlanKind::ReturningRows => shape_returning_rows(&translated, projection, redaction)?, - PlanKind::SingleDocument | PlanKind::MultiRow => { - shape_generic_rows(&translated, projection, redaction)? - } - // Handled by the early return above; kept exhaustive (no catch-all, - // no panic) so a future PlanKind desync degrades to passthrough - // rather than crashing the connection. - PlanKind::Execution | PlanKind::DmlResult(_) => return Ok(ShapeOutcome::Passthrough), - }; - Ok(ShapeOutcome::Rows(shaped)) -} - -/// Shape a Data-Plane payload with no `PhysicalPlan` in scope. -/// -/// Producers that never had a plan to KV-wrap or vector-translate -/// (ClusterArray, set-op merges, gateway forwarding, clone merges) call this -/// instead of [`shape_response_materialized`]: it applies only the decode + -/// scan-envelope unwrap + optional SELECT-list projection steps, skipping the -/// plan-dependent `apply_kv_wrap` / `translate_search_response` transforms those -/// callers never ran. -pub fn shape_payload_no_plan( - payload: &[u8], - plan_kind: PlanKind, - projection: Option<&OutputSchema>, - redaction: Option>, -) -> Result { - Ok(match plan_kind { - PlanKind::Execution | PlanKind::DmlResult(_) => ShapeOutcome::Passthrough, - PlanKind::ArraySlice => ShapeOutcome::Rows(shape_array_slice(payload, redaction)?), - PlanKind::ReturningRows => { - ShapeOutcome::Rows(shape_returning_rows(payload, projection, redaction)?) - } - PlanKind::SingleDocument | PlanKind::MultiRow => { - ShapeOutcome::Rows(shape_generic_rows(payload, projection, redaction)?) - } - }) -} - -/// Shape an `ArrayOp::Slice` response: decode the `ArraySliceResponse` -/// envelope (falling back to a plain payload decode for legacy shapes), -/// unwrap the row envelope, and surface `truncated_before_horizon` as a -/// notice. -/// -/// Array slices never carry a SELECT-list projection today (matching the -/// pre-extraction behavior), so `shape_decoded_rows` is always called with -/// a `None` projection here — but redaction still applies to the cells. -fn shape_array_slice( - payload: &[u8], - redaction: Option>, -) -> crate::Result { - if payload.is_empty() { - return Ok(empty_shaped()); - } - let (rows_json, truncated) = - if let Ok(resp) = zerompk::from_msgpack::(payload) { - ( - decode_payload_to_json(&resp.rows_msgpack), - resp.truncated_before_horizon, - ) - } else { - (decode_payload_to_json(payload), false) - }; - let notice = truncated.then(|| TRUNCATED_BEFORE_HORIZON_NOTICE.to_string()); - - let mut shaped = match sonic_rs::from_str::(&rows_json) { - Ok(value) => shape_decoded_rows(&value, None, redaction)?, - Err(_) => empty_shaped(), - }; - shaped.notice = notice; - Ok(shaped) -} - -/// Shape a `SingleDocument` / `MultiRow` response: decode to JSON, then -/// hand the parsed value to the pure [`shape_decoded_rows`] core. -/// -/// Non-JSON scalar payloads (undecodable envelope) fall back to a single -/// "result" column, matching pgwire's single-row fallback. -fn shape_generic_rows( - payload: &[u8], - projection: Option<&OutputSchema>, - redaction: Option>, -) -> crate::Result { - if payload.is_empty() { - return Ok(empty_shaped()); - } - let text = decode_payload_to_json(payload); - match sonic_rs::from_str::(&text) { - Ok(value) => shape_decoded_rows(&value, projection, redaction), - Err(_) => Ok(single_result_row(text)), - } -} - -/// Pure shaping core: given an already-decoded Data-Plane JSON payload, -/// unwrap the `{id, data}` scan envelope via `push_flat_rows`, then either -/// select the named SELECT-list columns when a projection is given, or -/// derive the id-first column union across all rows when no named -/// projection applies. -/// -/// Callers needing the composed materialized-shaping order (KV wrap, vector -/// translation, payload decode) should use [`shape_response_materialized`]; -/// this function does none of that — it is the shared core both the -/// materialized path and a per-batch lazy streaming caller (native's -/// `emit_sql_stream`) call directly, since a streamed scan batch has no plan -/// to KV-wrap or vector-translate but still needs the same envelope-unwrap + -/// projection logic applied per batch. -pub fn shape_decoded_rows( - decoded: &JsonValue, - projection: Option<&OutputSchema>, - redaction: Option>, -) -> crate::Result { - let mut rows = Vec::new(); - push_flat_rows(decoded.clone(), &mut rows)?; - - // Column-level redaction runs on the flat row maps, AFTER the scan - // envelope is unwrapped and BEFORE any projection or column derivation. - // - // After projection, `SELECT email AS contact` would have renamed the - // field out from under its rule; after column derivation, a - // `RedactionMode::Null` column would be missing from a `SELECT *` result - // instead of present and null. Both orderings deliver data the policy - // says to withhold, so the hook belongs exactly here. - redact_rows(redaction.as_ref(), &mut rows); - - match projection { - Some(s) if !s.is_star && !s.columns.is_empty() => { - let lookup_keys: Vec = s.columns.iter().map(|c| c.lookup_key.clone()).collect(); - let display_names: Vec = - s.columns.iter().map(|c| c.display_name.clone()).collect(); - // Row cells are stored under per-column unique keys, not display - // names: duplicate display names (`SELECT w.id, b.id` → `id`, - // `id`) would collide in the row map and collapse both wire - // columns to the last value. Encoders re-derive the same keys via - // `cell_keys` when reading cells. - let keys = super::project::cell_keys(&display_names); - let projected_rows = rows - .iter() - .map(|row| project_row(row, &lookup_keys, &display_names, &keys)) - .collect(); - // Carry each projected column's real catalog type, aligned in - // order with `display_names`. Only the pgwire encoder consumes - // these — mapping them to typed RowDescription OIDs and rendering - // each cell in that type's PostgreSQL text form; native/http - // ignore column types entirely. - let column_types: Vec = s.columns.iter().map(|c| c.ty).collect(); - Ok(ShapedRows { - columns: display_names, - column_types, - rows: projected_rows, - notice: None, - }) - } - _ => { - // Star / derived columns come from JSON rows with no catalog type, - // so they stay TEXT — typing them would regress `SELECT *` on - // schemaless collections. - let columns = derive_columns(&rows); - let column_types = ShapedRows::text_types(columns.len()); - Ok(ShapedRows { - columns, - column_types, - rows, - notice: None, - }) - } - } -} - -/// Apply the statement's column-level redaction policy to every flat row. -/// -/// A `None` context means the producer has no requester identity in scope and -/// therefore no roles to evaluate a policy against. -pub(super) fn redact_rows( - redaction: Option<&RedactionCtx<'_>>, - rows: &mut [Map], -) { - let Some(ctx) = redaction else { - return; - }; - for row in rows.iter_mut() { - ctx.store - .apply_flat_row(ctx.tenant_id, ctx.roles, ctx.collections, row); - } -} - -/// Select and rename one flat row's fields per the projection lists, trying -/// each candidate key in order: the full lookup key, then the bare -/// (post-dot) column name, then the SELECT alias. -/// -/// Cells are inserted under `cell_keys` (unique per column, see -/// [`super::project::cell_keys`]) rather than the display names, which may -/// repeat across columns and would otherwise collapse in the output map. -pub(super) fn project_row( - row: &Map, - lookup_keys: &[String], - display_names: &[String], - cell_keys: &[String], -) -> Map { - let mut out = Map::new(); - for (i, lookup_key) in lookup_keys.iter().enumerate() { - let bare = lookup_key - .rfind('.') - .map(|dot_pos| &lookup_key[dot_pos + 1..]) - .unwrap_or(lookup_key.as_str()); - let display_name = display_names - .get(i) - .map(String::as_str) - .unwrap_or(lookup_key.as_str()); - let value = row - .get(lookup_key.as_str()) - .or_else(|| { - if bare != lookup_key { - row.get(bare) - } else { - None - } - }) - .or_else(|| { - if display_name != lookup_key.as_str() && display_name != bare { - row.get(display_name) - } else { - None - } - }) - .cloned() - .unwrap_or(JsonValue::Null); - let cell_key = cell_keys.get(i).map(String::as_str).unwrap_or(display_name); - out.insert(cell_key.to_string(), value); - } - out -} - -/// Derive the id-first column union across all rows: `id` first (if -/// present), then each row's remaining keys in first-seen order. -/// -/// The order is user-visible wire column order and is pinned by callers and -/// tests, so it stays exactly first-seen. Membership lives in a set beside the -/// vec rather than being answered by rescanning the vec: the vec alone makes -/// the union quadratic in the number of distinct columns, on a path that runs -/// once per result set. -fn derive_columns(rows: &[Map]) -> Vec { - let mut cols: Vec = Vec::new(); - let mut seen: HashSet<&str> = HashSet::new(); - if let Some(first) = rows.first() { - if first.contains_key("id") { - cols.push("id".to_string()); - seen.insert("id"); - } - for key in first.keys() { - if key != "id" && !is_reserved_bitemporal_column(key) { - cols.push(key.clone()); - seen.insert(key.as_str()); - } - } - } - for row in rows.iter().skip(1) { - for key in row.keys() { - if !is_reserved_bitemporal_column(key) && seen.insert(key.as_str()) { - cols.push(key.clone()); - } - } - } - cols -} - -fn empty_shaped() -> ShapedRows { - ShapedRows { - columns: Vec::new(), - column_types: Vec::new(), - rows: Vec::new(), - notice: None, - } -} - -pub(super) fn single_result_row(text: String) -> ShapedRows { - let mut map = Map::new(); - map.insert("result".to_string(), JsonValue::String(text)); - ShapedRows { - columns: vec!["result".to_string()], - column_types: ShapedRows::text_types(1), - rows: vec![map], - notice: None, - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use crate::wal::WalManager; - use nodedb_types::CrdtPreviewResult; - - use super::*; - use crate::bridge::dispatch::Dispatcher; - use crate::bridge::envelope::PhysicalPlan; - use crate::control::server::response_shape::types::describe_plan; - use crate::control::state::SharedState; - use nodedb_types::{DatabaseId, TenantId}; - - fn preview_plan() -> PhysicalPlan { - PhysicalPlan::Crdt(nodedb_physical::physical_plan::CrdtOp::PreviewApply { - collection: nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, "tasks"), - document_id: "task-1".to_string(), - delta: vec![0x92, 0x01], - }) - } - - fn preview_payload() -> (CrdtPreviewResult, Vec) { - let result = CrdtPreviewResult { - post_image_msgpack: vec![0xc0], - imported_ops: 17, - trimmed_ops: 0, - frontier_digest: [0x5a; 32], - }; - let payload = zerompk::to_msgpack_vec(&result).expect("preview result serializes"); - (result, payload) - } - - /// Build the minimum real shared state needed by the materialized entry - /// point. Execution plans return before consulting it, which is exactly - /// the property this test protects. - fn shared_state() -> Arc { - let directory = tempfile::tempdir().expect("temporary WAL directory"); - let wal = Arc::new( - WalManager::open_for_testing(&directory.path().join("response-shape.wal")) - .expect("test WAL"), - ); - let (dispatcher, _) = Dispatcher::new(1, 1); - SharedState::new(dispatcher, wal).expect("test shared state") - } - - #[tokio::test] - async fn crdt_preview_is_byte_preserving_through_both_shaping_entry_points() { - let plan = preview_plan(); - let kind = describe_plan(&plan); - assert!(matches!(kind, PlanKind::Execution)); - let (expected, payload) = preview_payload(); - let original_payload = payload.clone(); - - let state = shared_state(); - let materialized = shape_response_materialized(MaterializedShapeRequest { - payload: &payload, - plan: &plan, - plan_kind: kind, - projection: None, - state: &state, - database_id: DatabaseId::new(1), - tenant_id: TenantId::new(1), - redaction: None, - }) - .expect("execution plan passthrough"); - assert!(matches!(materialized, ShapeOutcome::Passthrough)); - assert_eq!( - payload, original_payload, - "materialized path must not rewrite bytes" - ); - assert_eq!( - zerompk::from_msgpack::(&payload) - .expect("materialized passthrough remains decodable"), - expected - ); - - let no_plan = shape_payload_no_plan(&payload, kind, None, None); - assert!(matches!(no_plan, Ok(ShapeOutcome::Passthrough))); - assert_eq!( - payload, original_payload, - "no-plan path must not rewrite bytes" - ); - assert_eq!( - zerompk::from_msgpack::(&payload) - .expect("no-plan passthrough remains decodable"), - expected - ); - } - - /// Two projected columns sharing the display name `id` (`SELECT w.id, - /// b.id`) must keep both values in the shaped row instead of collapsing - /// to the last table's value. - #[test] - fn project_row_keeps_both_columns_with_duplicate_display_names() { - let mut row = Map::new(); - row.insert("w.id".to_string(), JsonValue::String("w1".to_string())); - row.insert("b.id".to_string(), JsonValue::String("b1".to_string())); - - let lookup_keys = vec!["w.id".to_string(), "b.id".to_string()]; - let display_names = vec!["id".to_string(), "id".to_string()]; - let keys = super::super::project::cell_keys(&display_names); - - let out = project_row(&row, &lookup_keys, &display_names, &keys); - assert_eq!(out.len(), 2, "both cells must survive the projection"); - assert_eq!(out.get("id"), Some(&JsonValue::String("w1".to_string()))); - assert_eq!(out.get("id_1"), Some(&JsonValue::String("b1".to_string()))); - } - - // ── Column-level redaction ────────────────────────────────────────── - - use crate::control::security::redaction::{ - RedactionMode, RedactionPolicy, RedactionRule, RedactionStore, - }; - use crate::control::server::response_shape::schema::OutputColumn; - - fn policy(collection: &str, role: &str, field: &str, mode: RedactionMode) -> RedactionPolicy { - RedactionPolicy { - name: format!("{collection}_{role}_{field}"), - tenant_id: 1, - collection: collection.into(), - display_collection: collection.into(), - for_role: role.into(), - rules: vec![RedactionRule { - field: field.into(), - mode, - }], - } - } - - fn store_with(policies: Vec) -> RedactionStore { - let store = RedactionStore::new(); - for p in policies { - store.create_policy(p); - } - store - } - - fn ctx<'a>( - store: &'a RedactionStore, - roles: &'a [String], - collections: &'a [(String, String)], - ) -> RedactionCtx<'a> { - RedactionCtx { - store, - tenant_id: 1, - roles, - collections, - } - } - - fn named_projection(pairs: &[(&str, &str)]) -> OutputSchema { - OutputSchema { - columns: pairs - .iter() - .map(|(lookup, display)| OutputColumn { - display_name: (*display).to_string(), - lookup_key: (*lookup).to_string(), - ty: DdlColType::Text, - }) - .collect(), - is_star: false, - } - } - - fn one_row(fields: JsonValue) -> JsonValue { - JsonValue::Array(vec![fields]) - } - - /// A `Mask` rule redacts for the role that holds the policy. - #[test] - fn mask_rule_redacts_for_the_policy_role() { - let store = store_with(vec![policy( - "users", - "support", - "email", - RedactionMode::Mask("***".into()), - )]); - let roles = vec!["support".to_string()]; - let sources = vec![(String::new(), "users".to_string())]; - let decoded = one_row(serde_json::json!({"email": "a@b.c", "name": "Alice"})); - - let shaped = shape_decoded_rows(&decoded, None, Some(ctx(&store, &roles, &sources))) - .expect("shape rows"); - assert_eq!(shaped.rows[0]["email"], JsonValue::String("***".into())); - assert_eq!(shaped.rows[0]["name"], JsonValue::String("Alice".into())); - } - - /// A role with no policy sees the value in the clear, and the rows are - /// otherwise identical to the unredacted shaping. - #[test] - fn role_without_a_policy_passes_rows_through_unchanged() { - let store = store_with(vec![policy( - "users", - "support", - "email", - RedactionMode::Mask("***".into()), - )]); - let roles = vec!["analyst".to_string()]; - let sources = vec![(String::new(), "users".to_string())]; - let decoded = one_row(serde_json::json!({"email": "a@b.c", "name": "Alice"})); - - let baseline = shape_decoded_rows(&decoded, None, None).expect("shape rows"); - let shaped = shape_decoded_rows(&decoded, None, Some(ctx(&store, &roles, &sources))) - .expect("shape rows"); - assert_eq!(shaped.rows, baseline.rows); - assert_eq!(shaped.columns, baseline.columns); - } - - /// `SELECT email AS contact` must still be redacted: the rule names the - /// stored field, and redaction runs before the projection renames it. - #[test] - fn select_alias_does_not_escape_the_rule() { - let store = store_with(vec![policy( - "users", - "support", - "email", - RedactionMode::Mask("***".into()), - )]); - let roles = vec!["support".to_string()]; - let sources = vec![(String::new(), "users".to_string())]; - let decoded = one_row(serde_json::json!({"email": "a@b.c"})); - let projection = named_projection(&[("email", "contact")]); - - let shaped = shape_decoded_rows( - &decoded, - Some(&projection), - Some(ctx(&store, &roles, &sources)), - ) - .expect("shape rows"); - assert_eq!(shaped.columns, vec!["contact".to_string()]); - assert_eq!(shaped.rows[0]["contact"], JsonValue::String("***".into())); - } - - /// Two joined collections both carry `id`, but only the left side has a - /// rule. Matching the bare name would redact the right side too. - #[test] - fn join_redacts_only_the_side_the_rule_belongs_to() { - let store = store_with(vec![policy( - "workspaces", - "support", - "id", - RedactionMode::Mask("***".into()), - )]); - let roles = vec!["support".to_string()]; - let sources = vec![ - ("w".to_string(), "workspaces".to_string()), - ("b".to_string(), "boards".to_string()), - ]; - let decoded = one_row(serde_json::json!({"w.id": "w1", "b.id": "b1"})); - let projection = named_projection(&[("w.id", "id"), ("b.id", "id")]); - - let shaped = shape_decoded_rows( - &decoded, - Some(&projection), - Some(ctx(&store, &roles, &sources)), - ) - .expect("shape rows"); - // `cell_keys` suffixes the duplicate display name. - assert_eq!(shaped.rows[0]["id"], JsonValue::String("***".into())); - assert_eq!(shaped.rows[0]["id_1"], JsonValue::String("b1".into())); - } - - /// `RedactionMode::Null` must leave the column in a `SELECT *` result, - /// valued null — removing the key would drop it from the derived schema. - #[test] - fn star_keeps_a_null_redacted_column_in_the_schema() { - let store = store_with(vec![policy( - "users", - "support", - "email", - RedactionMode::Null, - )]); - let roles = vec!["support".to_string()]; - let sources = vec![(String::new(), "users".to_string())]; - let decoded = one_row(serde_json::json!({"id": "u1", "email": "a@b.c"})); - - let shaped = shape_decoded_rows(&decoded, None, Some(ctx(&store, &roles, &sources))) - .expect("shape rows"); - assert!( - shaped.columns.contains(&"email".to_string()), - "redacted column must stay in the derived SELECT * schema: {:?}", - shaped.columns - ); - assert_eq!(shaped.rows[0]["email"], JsonValue::Null); - } - - /// Duplicate-free projections still store cells under the display name - /// (`cell_keys` is the identity), so existing readers are unaffected. - #[test] - fn project_row_uses_display_names_when_unique() { - let mut row = Map::new(); - row.insert("w.id".to_string(), JsonValue::String("w1".to_string())); - row.insert("b.title".to_string(), JsonValue::String("t".to_string())); - - let lookup_keys = vec!["w.id".to_string(), "b.title".to_string()]; - let display_names = vec!["id".to_string(), "title".to_string()]; - let keys = super::super::project::cell_keys(&display_names); - - let out = project_row(&row, &lookup_keys, &display_names, &keys); - assert_eq!(out.get("id"), Some(&JsonValue::String("w1".to_string()))); - assert_eq!(out.get("title"), Some(&JsonValue::String("t".to_string()))); - } - - fn row_of(keys: &[&str]) -> Map { - let mut row = Map::new(); - for k in keys { - row.insert((*k).to_string(), JsonValue::String((*k).to_string())); - } - row - } - - /// Column order is user-visible: `id` first when the first row has it, - /// then every other column in the order it is first seen, scanning rows in - /// order. Overlapping and disjoint rows must not reorder or duplicate. - /// - /// Every `row_of` list here is already in ascending key order, so the - /// within-row iteration order is the same whichever map backs - /// `serde_json::Map`; what this pins is the cross-row order. - #[test] - fn derive_columns_pins_id_first_then_first_seen_order() { - let rows = vec![ - row_of(&["a", "b", "id"]), - row_of(&["a", "z"]), - row_of(&["b", "y"]), - row_of(&["q"]), - ]; - - assert_eq!( - derive_columns(&rows), - vec!["id", "a", "b", "z", "y", "q"], - "id leads; later rows append only their newly-seen columns" - ); - } - - #[test] - fn derive_columns_without_id_keeps_the_first_rows_columns_leading() { - let rows = vec![row_of(&["a", "b"]), row_of(&["c", "id"])]; - - assert_eq!( - derive_columns(&rows), - vec!["a", "b", "c", "id"], - "a late `id` appends where it is first seen; it is not hoisted" - ); - } - - #[test] - fn derive_columns_skips_reserved_bitemporal_columns() { - let rows = vec![ - row_of(&["__system_from_ms", "id"]), - row_of(&["__valid_from_ms", "name"]), - ]; - - assert_eq!(derive_columns(&rows), vec!["id", "name"]); - } -} diff --git a/nodedb/src/control/server/response_shape/compose/array_slice.rs b/nodedb/src/control/server/response_shape/compose/array_slice.rs new file mode 100644 index 000000000..e9d921ea6 --- /dev/null +++ b/nodedb/src/control/server/response_shape/compose/array_slice.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Shaping for an `ArrayOp::Slice` response. + +use crate::data::executor::response_codec::{ArraySliceResponse, decode_payload_value}; + +use super::super::redaction::RedactionCtx; +use super::super::types::ShapedRows; +use super::kernel::{empty_shaped, shape_decoded_rows}; + +/// NOTICE text for an `AS OF SYSTEM TIME` cutoff older than the oldest +/// retained tile version. This is the canonical definition, surfaced to +/// every protocol via [`ShapedRows::notice`]. +const TRUNCATED_BEFORE_HORIZON_NOTICE: &str = "AS OF SYSTEM TIME cutoff is older than the oldest retained tile version; \ + results may be incomplete"; + +/// Shape an `ArrayOp::Slice` response: decode the `ArraySliceResponse` +/// envelope (falling back to a plain payload decode for legacy shapes), +/// unwrap the row envelope, and surface `truncated_before_horizon` as a +/// notice. +/// +/// Array slices never carry a SELECT-list projection, so `shape_decoded_rows` +/// is always called with a `None` projection here — but redaction still +/// applies to the cells. A payload that decodes to no value shapes as an +/// empty result set. +pub(super) fn shape_array_slice( + payload: &[u8], + redaction: Option>, +) -> crate::Result { + if payload.is_empty() { + return Ok(empty_shaped()); + } + let (rows, truncated) = if let Ok(resp) = zerompk::from_msgpack::(payload) { + ( + decode_payload_value(&resp.rows_msgpack), + resp.truncated_before_horizon, + ) + } else { + (decode_payload_value(payload), false) + }; + let notice = truncated.then(|| TRUNCATED_BEFORE_HORIZON_NOTICE.to_string()); + + let mut shaped = match rows { + Ok(value) => shape_decoded_rows(value, None, redaction)?, + Err(_) => empty_shaped(), + }; + shaped.notice = notice; + Ok(shaped) +} diff --git a/nodedb/src/control/server/response_shape/compose/kernel.rs b/nodedb/src/control/server/response_shape/compose/kernel.rs new file mode 100644 index 000000000..d0c0942e3 --- /dev/null +++ b/nodedb/src/control/server/response_shape/compose/kernel.rs @@ -0,0 +1,492 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Pure shaping kernel over an already-decoded Data-Plane value. +//! +//! [`shape_decoded_rows`] unwraps the `{id, data}` scan envelope, redacts, +//! and either projects the SELECT list or derives the column union. It is +//! the shared core both the materialized path and every per-batch lazy +//! streaming caller (pgwire, native, http) call directly. + +use std::collections::HashSet; + +use nodedb_types::Value; +use nodedb_types::columnar::schema::is_reserved_bitemporal_column; + +use super::super::project::push_flat_rows; +use super::super::redaction::RedactionCtx; +use super::super::schema::OutputSchema; +use super::super::types::{DdlColType, ShapedRow, ShapedRows}; + +/// Pure shaping core: given an already-decoded Data-Plane value, unwrap the +/// `{id, data}` scan envelope via `push_flat_rows`, then either select the +/// named SELECT-list columns when a projection is given, or derive the +/// id-first column union across all rows when no named projection applies. +/// +/// Callers needing the composed materialized-shaping order (KV wrap, vector +/// translation, payload decode) use `shape_response_materialized`; this +/// function does none of that. A streamed scan batch has no plan to KV-wrap +/// or vector-translate but still needs the same envelope-unwrap + projection +/// logic applied per batch, so streaming callers call this directly. +pub fn shape_decoded_rows( + decoded: Value, + projection: Option<&OutputSchema>, + redaction: Option>, +) -> crate::Result { + let mut rows = Vec::new(); + push_flat_rows(decoded, &mut rows)?; + + // Column-level redaction runs on the flat row maps, AFTER the scan + // envelope is unwrapped and BEFORE any projection or column derivation. + // + // After projection, `SELECT email AS contact` would have renamed the + // field out from under its rule; after column derivation, a + // `RedactionMode::Null` column would be missing from a `SELECT *` result + // instead of present and null. Both orderings deliver data the policy + // says to withhold, so the hook belongs exactly here. + redact_rows(redaction.as_ref(), &mut rows); + + match projection { + Some(s) if !s.is_star && !s.columns.is_empty() => { + let lookup_keys: Vec = s.columns.iter().map(|c| c.lookup_key.clone()).collect(); + let display_names: Vec = + s.columns.iter().map(|c| c.display_name.clone()).collect(); + // Row cells are stored under per-column unique keys, not display + // names: duplicate display names (`SELECT w.id, b.id` → `id`, + // `id`) would collide in the row map and collapse both wire + // columns to the last value. Encoders re-derive the same keys via + // `cell_keys` when reading cells. + let keys = super::super::project::cell_keys(&display_names); + let projected_rows = rows + .iter() + .map(|row| project_row(row, &lookup_keys, &display_names, &keys)) + .collect(); + // Carry each projected column's real catalog type, aligned in + // order with `display_names`. Only the pgwire encoder consumes + // these — mapping them to typed RowDescription OIDs and rendering + // each cell in that type's PostgreSQL text form; native/http + // ignore column types entirely. + let column_types: Vec = s.columns.iter().map(|c| c.ty).collect(); + Ok(ShapedRows::from_rows( + display_names, + column_types, + projected_rows, + )) + } + _ => { + // Star / derived columns come from rows with no catalog type, so + // they stay TEXT — typing them would regress `SELECT *` on + // schemaless collections. + let columns = derive_columns(&rows); + let column_types = ShapedRows::text_types(columns.len()); + Ok(ShapedRows::from_rows(columns, column_types, rows)) + } + } +} + +/// Apply the statement's column-level redaction policy to every flat row. +/// +/// A `None` context means the producer has no requester identity in scope and +/// therefore no roles to evaluate a policy against. +pub(in crate::control::server::response_shape) fn redact_rows( + redaction: Option<&RedactionCtx<'_>>, + rows: &mut [ShapedRow], +) { + let Some(ctx) = redaction else { + return; + }; + for row in rows.iter_mut() { + ctx.store + .apply_flat_row_typed(ctx.tenant_id, ctx.roles, ctx.collections, row); + } +} + +/// Select and rename one flat row's fields per the projection lists, trying +/// each candidate key in order: the full lookup key, then the bare +/// (post-dot) column name, then the SELECT alias. +/// +/// Cells are inserted under `cell_keys` (unique per column, see +/// [`super::super::project::cell_keys`]) rather than the display names, which +/// may repeat across columns and would otherwise collapse in the output map. +pub(in crate::control::server::response_shape) fn project_row( + row: &ShapedRow, + lookup_keys: &[String], + display_names: &[String], + cell_keys: &[String], +) -> ShapedRow { + let mut out = ShapedRow::new(); + for (i, lookup_key) in lookup_keys.iter().enumerate() { + let bare = lookup_key + .rfind('.') + .map(|dot_pos| &lookup_key[dot_pos + 1..]) + .unwrap_or(lookup_key.as_str()); + let display_name = display_names + .get(i) + .map(String::as_str) + .unwrap_or(lookup_key.as_str()); + let value = row + .get(lookup_key.as_str()) + .or_else(|| { + if bare != lookup_key { + row.get(bare) + } else { + None + } + }) + .or_else(|| { + if display_name != lookup_key.as_str() && display_name != bare { + row.get(display_name) + } else { + None + } + }) + .cloned() + .unwrap_or(Value::Null); + let cell_key = cell_keys.get(i).map(String::as_str).unwrap_or(display_name); + out.insert(cell_key.to_string(), value); + } + out +} + +/// Derive the id-first column union across all rows: `id` first (if +/// present), then each row's remaining keys in first-seen order. +/// +/// The order is user-visible wire column order and is pinned by callers and +/// tests, so it stays exactly first-seen. Membership lives in a set beside the +/// vec rather than being answered by rescanning the vec: the vec alone makes +/// the union quadratic in the number of distinct columns, on a path that runs +/// once per result set. +fn derive_columns(rows: &[ShapedRow]) -> Vec { + let mut cols: Vec = Vec::new(); + let mut seen: HashSet<&str> = HashSet::new(); + if let Some(first) = rows.first() { + if first.contains_key("id") { + cols.push("id".to_string()); + seen.insert("id"); + } + for key in first.keys() { + if key != "id" && !is_reserved_bitemporal_column(key) { + cols.push(key.clone()); + seen.insert(key.as_str()); + } + } + } + for row in rows.iter().skip(1) { + for key in row.keys() { + if !is_reserved_bitemporal_column(key) && seen.insert(key.as_str()) { + cols.push(key.clone()); + } + } + } + cols +} + +/// A result set with no columns and no rows. +pub(in crate::control::server::response_shape) fn empty_shaped() -> ShapedRows { + ShapedRows::from_rows(Vec::new(), Vec::new(), Vec::new()) +} + +/// A single `result` text column holding one row of `text`. +pub(in crate::control::server::response_shape) fn single_result_row(text: String) -> ShapedRows { + let mut row = ShapedRow::new(); + row.insert("result".to_string(), Value::String(text)); + ShapedRows::from_rows( + vec!["result".to_string()], + ShapedRows::text_types(1), + vec![row], + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::security::redaction::{ + RedactionMode, RedactionPolicy, RedactionRule, RedactionStore, + }; + use crate::control::server::response_shape::schema::OutputColumn; + use nodedb_types::NdbDateTime; + + fn text(s: &str) -> Value { + Value::String(s.to_string()) + } + + fn row_of_pairs(pairs: &[(&str, Value)]) -> ShapedRow { + pairs + .iter() + .map(|(k, v)| ((*k).to_string(), v.clone())) + .collect() + } + + /// Two projected columns sharing the display name `id` (`SELECT w.id, + /// b.id`) must keep both values in the shaped row instead of collapsing + /// to the last table's value. + #[test] + fn project_row_keeps_both_columns_with_duplicate_display_names() { + let row = row_of_pairs(&[("w.id", text("w1")), ("b.id", text("b1"))]); + + let lookup_keys = vec!["w.id".to_string(), "b.id".to_string()]; + let display_names = vec!["id".to_string(), "id".to_string()]; + let keys = crate::control::server::response_shape::project::cell_keys(&display_names); + + let out = project_row(&row, &lookup_keys, &display_names, &keys); + assert_eq!(out.len(), 2, "both cells must survive the projection"); + assert_eq!(out.get("id"), Some(&text("w1"))); + assert_eq!(out.get("id_1"), Some(&text("b1"))); + } + + // ── Column-level redaction ────────────────────────────────────────── + + fn policy(collection: &str, role: &str, field: &str, mode: RedactionMode) -> RedactionPolicy { + RedactionPolicy { + name: format!("{collection}_{role}_{field}"), + tenant_id: 1, + collection: collection.into(), + display_collection: collection.into(), + for_role: role.into(), + rules: vec![RedactionRule { + field: field.into(), + mode, + }], + } + } + + fn store_with(policies: Vec) -> RedactionStore { + let store = RedactionStore::new(); + for p in policies { + store.create_policy(p); + } + store + } + + fn ctx<'a>( + store: &'a RedactionStore, + roles: &'a [String], + collections: &'a [(String, String)], + ) -> RedactionCtx<'a> { + RedactionCtx { + store, + tenant_id: 1, + roles, + collections, + } + } + + fn named_projection(pairs: &[(&str, &str)]) -> OutputSchema { + OutputSchema { + columns: pairs + .iter() + .map(|(lookup, display)| OutputColumn { + display_name: (*display).to_string(), + lookup_key: (*lookup).to_string(), + ty: DdlColType::Text, + }) + .collect(), + is_star: false, + } + } + + /// One decoded row: a one-element array holding the object `pairs`. + fn one_row(pairs: &[(&str, Value)]) -> Value { + let object = pairs + .iter() + .map(|(k, v)| ((*k).to_string(), v.clone())) + .collect(); + Value::Array(vec![Value::Object(object)]) + } + + /// A `Mask` rule redacts for the role that holds the policy. + #[test] + fn mask_rule_redacts_for_the_policy_role() { + let store = store_with(vec![policy( + "users", + "support", + "email", + RedactionMode::Mask("***".into()), + )]); + let roles = vec!["support".to_string()]; + let sources = vec![(String::new(), "users".to_string())]; + let decoded = one_row(&[("email", text("a@b.c")), ("name", text("Alice"))]); + + let shaped = shape_decoded_rows(decoded, None, Some(ctx(&store, &roles, &sources))) + .expect("shape rows"); + assert_eq!(shaped.rows[0]["email"], text("***")); + assert_eq!(shaped.rows[0]["name"], text("Alice")); + } + + /// A role with no policy sees the value in the clear, and the rows are + /// otherwise identical to the unredacted shaping. + #[test] + fn role_without_a_policy_passes_rows_through_unchanged() { + let store = store_with(vec![policy( + "users", + "support", + "email", + RedactionMode::Mask("***".into()), + )]); + let roles = vec!["analyst".to_string()]; + let sources = vec![(String::new(), "users".to_string())]; + let decoded = one_row(&[("email", text("a@b.c")), ("name", text("Alice"))]); + + let baseline = shape_decoded_rows(decoded.clone(), None, None).expect("shape rows"); + let shaped = shape_decoded_rows(decoded, None, Some(ctx(&store, &roles, &sources))) + .expect("shape rows"); + assert_eq!(shaped.rows, baseline.rows); + assert_eq!(shaped.columns, baseline.columns); + } + + /// `SELECT email AS contact` must still be redacted: the rule names the + /// stored field, and redaction runs before the projection renames it. + #[test] + fn select_alias_does_not_escape_the_rule() { + let store = store_with(vec![policy( + "users", + "support", + "email", + RedactionMode::Mask("***".into()), + )]); + let roles = vec!["support".to_string()]; + let sources = vec![(String::new(), "users".to_string())]; + let decoded = one_row(&[("email", text("a@b.c"))]); + let projection = named_projection(&[("email", "contact")]); + + let shaped = shape_decoded_rows( + decoded, + Some(&projection), + Some(ctx(&store, &roles, &sources)), + ) + .expect("shape rows"); + assert_eq!(shaped.columns, vec!["contact".to_string()]); + assert_eq!(shaped.rows[0]["contact"], text("***")); + } + + /// Two joined collections both carry `id`, but only the left side has a + /// rule. Matching the bare name would redact the right side too. + #[test] + fn join_redacts_only_the_side_the_rule_belongs_to() { + let store = store_with(vec![policy( + "workspaces", + "support", + "id", + RedactionMode::Mask("***".into()), + )]); + let roles = vec!["support".to_string()]; + let sources = vec![ + ("w".to_string(), "workspaces".to_string()), + ("b".to_string(), "boards".to_string()), + ]; + let decoded = one_row(&[("w.id", text("w1")), ("b.id", text("b1"))]); + let projection = named_projection(&[("w.id", "id"), ("b.id", "id")]); + + let shaped = shape_decoded_rows( + decoded, + Some(&projection), + Some(ctx(&store, &roles, &sources)), + ) + .expect("shape rows"); + // `cell_keys` suffixes the duplicate display name. + assert_eq!(shaped.rows[0]["id"], text("***")); + assert_eq!(shaped.rows[0]["id_1"], text("b1")); + } + + /// `RedactionMode::Null` must leave the column in a `SELECT *` result, + /// valued null — removing the key would drop it from the derived schema. + #[test] + fn star_keeps_a_null_redacted_column_in_the_schema() { + let store = store_with(vec![policy( + "users", + "support", + "email", + RedactionMode::Null, + )]); + let roles = vec!["support".to_string()]; + let sources = vec![(String::new(), "users".to_string())]; + let decoded = one_row(&[("id", text("u1")), ("email", text("a@b.c"))]); + + let shaped = shape_decoded_rows(decoded, None, Some(ctx(&store, &roles, &sources))) + .expect("shape rows"); + assert!( + shaped.columns.contains(&"email".to_string()), + "redacted column must stay in the derived SELECT * schema: {:?}", + shaped.columns + ); + assert_eq!(shaped.rows[0]["email"], Value::Null); + } + + /// Duplicate-free projections still store cells under the display name + /// (`cell_keys` is the identity), so existing readers are unaffected. + #[test] + fn project_row_uses_display_names_when_unique() { + let row = row_of_pairs(&[("w.id", text("w1")), ("b.title", text("t"))]); + + let lookup_keys = vec!["w.id".to_string(), "b.title".to_string()]; + let display_names = vec!["id".to_string(), "title".to_string()]; + let keys = crate::control::server::response_shape::project::cell_keys(&display_names); + + let out = project_row(&row, &lookup_keys, &display_names, &keys); + assert_eq!(out.get("id"), Some(&text("w1"))); + assert_eq!(out.get("title"), Some(&text("t"))); + } + + /// A typed instant cell passes through projection as itself; its + /// ISO-8601 text is the protocol edge's rendering, not the kernel's. + #[test] + fn an_instant_cell_survives_projection_as_a_typed_value() { + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + let decoded = one_row(&[("at", Value::NaiveDateTime(at)), ("id", text("r1"))]); + let projection = named_projection(&[("at", "at")]); + + let shaped = shape_decoded_rows(decoded, Some(&projection), None).expect("shape rows"); + assert_eq!(shaped.rows[0]["at"], Value::NaiveDateTime(at)); + assert_eq!( + crate::control::server::response_shape::cell::value_to_wire_json(&shaped.rows[0]["at"]), + serde_json::Value::String("2020-03-05T10:00:00.000000Z".into()) + ); + } + + fn row_of(keys: &[&str]) -> ShapedRow { + keys.iter().map(|k| ((*k).to_string(), text(k))).collect() + } + + /// Column order is user-visible: `id` first when the first row has it, + /// then every other column in the order it is first seen, scanning rows in + /// order. Overlapping and disjoint rows must not reorder or duplicate. + /// + /// Every `row_of` list here is already in ascending key order, so the + /// within-row iteration order is the sorted order a `ShapedRow` iterates + /// in; what this pins is the cross-row order. + #[test] + fn derive_columns_pins_id_first_then_first_seen_order() { + let rows = vec![ + row_of(&["a", "b", "id"]), + row_of(&["a", "z"]), + row_of(&["b", "y"]), + row_of(&["q"]), + ]; + + assert_eq!( + derive_columns(&rows), + vec!["id", "a", "b", "z", "y", "q"], + "id leads; later rows append only their newly-seen columns" + ); + } + + #[test] + fn derive_columns_without_id_keeps_the_first_rows_columns_leading() { + let rows = vec![row_of(&["a", "b"]), row_of(&["c", "id"])]; + + assert_eq!( + derive_columns(&rows), + vec!["a", "b", "c", "id"], + "a late `id` appends where it is first seen; it is not hoisted" + ); + } + + #[test] + fn derive_columns_skips_reserved_bitemporal_columns() { + let rows = vec![ + row_of(&["__system_from_ms", "id"]), + row_of(&["__valid_from_ms", "name"]), + ]; + + assert_eq!(derive_columns(&rows), vec!["id", "name"]); + } +} diff --git a/nodedb/src/control/server/response_shape/compose/materialized.rs b/nodedb/src/control/server/response_shape/compose/materialized.rs new file mode 100644 index 000000000..6c5b30142 --- /dev/null +++ b/nodedb/src/control/server/response_shape/compose/materialized.rs @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Composed, protocol-neutral materialized response shaping. +//! +//! `shape_response_materialized` is the canonical SELECT-read shaping used by +//! every protocol entrypoint. It performs the full per-payload shaping order +//! (`apply_kv_wrap` -> `translate_search_response` -> decode -> scan-envelope +//! unwrap -> optional SELECT-list projection) as a single call, producing an +//! already-shaped, already-projected [`ShapeOutcome`]. Every SELECT-read +//! producer — pgwire's non-streaming dispatch, native's dispatch loop — calls +//! this directly and hands the resulting `ShapedRows` to its own protocol +//! encoder; each protocol then encodes those rows in its own wire format +//! (pgwire's RowDescription/DataRow, native's MessagePack, http's JSON). +//! +//! Producers with no `PhysicalPlan` in scope (ClusterArray, set-op merges, +//! gateway forwarding, clone merges) call [`shape_payload_no_plan`], which +//! skips the plan-dependent `apply_kv_wrap` / `translate_search_response` +//! transforms those callers never ran. The pure kernel `shape_decoded_rows` +//! is shared with per-batch lazy streaming callers, which have an +//! already-decoded batch and only need the envelope-unwrap + projection logic. + +use crate::control::server::response_translate::dispatch::translate_search_response; +use crate::data::executor::response_codec::decode_payload_value; +use nodedb_types::NodeDbError; + +use super::super::kv::apply_kv_wrap; +use super::super::redaction::RedactionCtx; +use super::super::request::MaterializedShapeRequest; +use super::super::returning::shape_returning_rows; +use super::super::schema::OutputSchema; +use super::super::types::{PlanKind, ShapedRows}; +use super::array_slice::shape_array_slice; +use super::kernel::{empty_shaped, shape_decoded_rows, single_result_row}; + +/// Outcome of materialized response shaping. +/// +/// Row-producing plan kinds (`SingleDocument`, `MultiRow`, `ReturningRows`, +/// `ArraySlice`) yield `Rows`. Tag/execution kinds (`Execution`, +/// `DmlResult`) yield `Passthrough` — a `ShapedRows` cannot represent a bare +/// `CommandComplete` tag or affected-row count, so callers keep their +/// existing tag / `rows_affected` handling for those. +pub enum ShapeOutcome { + Rows(ShapedRows), + Passthrough, +} + +/// Shape a single Data-Plane payload into protocol-neutral rows, applying +/// the canonical shaping order: KV point-get wrap, vector surrogate->PK +/// translation, payload decode, scan-envelope unwrap, and (when +/// `projection` names columns) SELECT-list column selection. +pub fn shape_response_materialized( + request: MaterializedShapeRequest<'_>, +) -> Result { + let MaterializedShapeRequest { + payload, + plan, + plan_kind, + projection, + state, + database_id, + tenant_id, + redaction, + } = request; + + match plan_kind { + PlanKind::Execution | PlanKind::DmlResult(_) => return Ok(ShapeOutcome::Passthrough), + PlanKind::ArraySlice + | PlanKind::ReturningRows + | PlanKind::SingleDocument + | PlanKind::MultiRow => {} + } + + // Seam-1 order, exactly as pgwire's `dispatch_task_loop` applies it + // (apply_kv_wrap -> translate_search_response) before any decode/shape step. + let wrapped = apply_kv_wrap(plan, payload); + let translated = translate_search_response(&wrapped, plan, state, database_id, tenant_id); + + let shaped = match plan_kind { + PlanKind::ArraySlice => shape_array_slice(&translated, redaction)?, + // `RETURNING` rows are held to the columns already announced to the + // client, when any were — see `super::returning`. + PlanKind::ReturningRows => shape_returning_rows(&translated, projection, redaction)?, + PlanKind::SingleDocument | PlanKind::MultiRow => { + shape_generic_rows(&translated, projection, redaction)? + } + // Handled by the early return above; kept exhaustive (no catch-all, + // no panic) so a future PlanKind desync degrades to passthrough + // rather than crashing the connection. + PlanKind::Execution | PlanKind::DmlResult(_) => return Ok(ShapeOutcome::Passthrough), + }; + Ok(ShapeOutcome::Rows(shaped)) +} + +/// Shape a Data-Plane payload with no `PhysicalPlan` in scope. +/// +/// Producers that never had a plan to KV-wrap or vector-translate +/// (ClusterArray, set-op merges, gateway forwarding, clone merges) call this +/// instead of [`shape_response_materialized`]: it applies only the decode + +/// scan-envelope unwrap + optional SELECT-list projection steps, skipping the +/// plan-dependent `apply_kv_wrap` / `translate_search_response` transforms those +/// callers never ran. +pub fn shape_payload_no_plan( + payload: &[u8], + plan_kind: PlanKind, + projection: Option<&OutputSchema>, + redaction: Option>, +) -> Result { + Ok(match plan_kind { + PlanKind::Execution | PlanKind::DmlResult(_) => ShapeOutcome::Passthrough, + PlanKind::ArraySlice => ShapeOutcome::Rows(shape_array_slice(payload, redaction)?), + PlanKind::ReturningRows => { + ShapeOutcome::Rows(shape_returning_rows(payload, projection, redaction)?) + } + PlanKind::SingleDocument | PlanKind::MultiRow => { + ShapeOutcome::Rows(shape_generic_rows(payload, projection, redaction)?) + } + }) +} + +/// Shape a `SingleDocument` / `MultiRow` response: decode the payload to a +/// typed value, then hand it to the pure [`shape_decoded_rows`] core. +/// +/// A payload that decodes as neither msgpack nor JSON falls back to a single +/// "result" column holding its lossy UTF-8 text, matching pgwire's +/// single-row fallback. +fn shape_generic_rows( + payload: &[u8], + projection: Option<&OutputSchema>, + redaction: Option>, +) -> crate::Result { + if payload.is_empty() { + return Ok(empty_shaped()); + } + match decode_payload_value(payload) { + Ok(value) => shape_decoded_rows(value, projection, redaction), + Err(_) => Ok(single_result_row( + String::from_utf8_lossy(payload).into_owned(), + )), + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use crate::wal::WalManager; + use nodedb_types::CrdtPreviewResult; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::bridge::envelope::PhysicalPlan; + use crate::control::server::response_shape::types::describe_plan; + use crate::control::state::SharedState; + use nodedb_types::{DatabaseId, TenantId}; + + fn preview_plan() -> PhysicalPlan { + PhysicalPlan::Crdt(nodedb_physical::physical_plan::CrdtOp::PreviewApply { + collection: nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, "tasks"), + document_id: "task-1".to_string(), + delta: vec![0x92, 0x01], + }) + } + + fn preview_payload() -> (CrdtPreviewResult, Vec) { + let result = CrdtPreviewResult { + post_image_msgpack: vec![0xc0], + imported_ops: 17, + trimmed_ops: 0, + frontier_digest: [0x5a; 32], + }; + let payload = zerompk::to_msgpack_vec(&result).expect("preview result serializes"); + (result, payload) + } + + /// Build the minimum real shared state needed by the materialized entry + /// point. Execution plans return before consulting it, which is exactly + /// the property this test protects. + fn shared_state() -> Arc { + let directory = tempfile::tempdir().expect("temporary WAL directory"); + let wal = Arc::new( + WalManager::open_for_testing(&directory.path().join("response-shape.wal")) + .expect("test WAL"), + ); + let (dispatcher, _) = Dispatcher::new(1, 1); + SharedState::new(dispatcher, wal).expect("test shared state") + } + + #[tokio::test] + async fn crdt_preview_is_byte_preserving_through_both_shaping_entry_points() { + let plan = preview_plan(); + let kind = describe_plan(&plan); + assert!(matches!(kind, PlanKind::Execution)); + let (expected, payload) = preview_payload(); + let original_payload = payload.clone(); + + let state = shared_state(); + let materialized = shape_response_materialized(MaterializedShapeRequest { + payload: &payload, + plan: &plan, + plan_kind: kind, + projection: None, + state: &state, + database_id: DatabaseId::new(1), + tenant_id: TenantId::new(1), + redaction: None, + }) + .expect("execution plan passthrough"); + assert!(matches!(materialized, ShapeOutcome::Passthrough)); + assert_eq!( + payload, original_payload, + "materialized path must not rewrite bytes" + ); + assert_eq!( + zerompk::from_msgpack::(&payload) + .expect("materialized passthrough remains decodable"), + expected + ); + + let no_plan = shape_payload_no_plan(&payload, kind, None, None); + assert!(matches!(no_plan, Ok(ShapeOutcome::Passthrough))); + assert_eq!( + payload, original_payload, + "no-plan path must not rewrite bytes" + ); + assert_eq!( + zerompk::from_msgpack::(&payload) + .expect("no-plan passthrough remains decodable"), + expected + ); + } + + /// A msgpack `bin` cell reaches the shaped row as `Value::Bytes`; its + /// text form is decided at the protocol edge. + #[test] + fn a_byte_cell_shapes_as_bytes() { + let mut row = std::collections::HashMap::new(); + row.insert( + "blob".to_string(), + nodedb_types::Value::Bytes(vec![0, 255, 7]), + ); + let payload = nodedb_types::value_to_msgpack(&nodedb_types::Value::Array(vec![ + nodedb_types::Value::Object(row), + ])) + .expect("encode"); + + let ShapeOutcome::Rows(shaped) = + shape_payload_no_plan(&payload, PlanKind::MultiRow, None, None).expect("shape") + else { + panic!("multi-row plan must yield rows"); + }; + assert_eq!( + shaped.rows[0]["blob"], + nodedb_types::Value::Bytes(vec![0, 255, 7]) + ); + } +} diff --git a/nodedb/src/control/server/response_shape/compose/mod.rs b/nodedb/src/control/server/response_shape/compose/mod.rs new file mode 100644 index 000000000..bf76a3773 --- /dev/null +++ b/nodedb/src/control/server/response_shape/compose/mod.rs @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Composed, protocol-neutral response shaping: the materialized entry +//! points, the pure decoded-value kernel, and the array-slice envelope. + +pub mod array_slice; +pub mod kernel; +pub mod materialized; + +pub use kernel::shape_decoded_rows; +pub use materialized::{ShapeOutcome, shape_payload_no_plan, shape_response_materialized}; diff --git a/nodedb/src/control/server/response_shape/kv.rs b/nodedb/src/control/server/response_shape/kv.rs index 15efd5c29..b30e75dc5 100644 --- a/nodedb/src/control/server/response_shape/kv.rs +++ b/nodedb/src/control/server/response_shape/kv.rs @@ -4,12 +4,13 @@ //! into the stored value(s) before the protocol layer turns them into //! SQL rows. -use serde_json::{Map, Value as JsonValue}; +use std::collections::HashMap; use crate::bridge::envelope::PhysicalPlan; -use crate::data::executor::response_codec::decode_payload_to_json; +use crate::data::executor::response_codec::decode_payload_value; use nodedb_physical::physical_plan::KvOp; use nodedb_query::msgpack_scan; +use nodedb_types::Value; /// When `plan` is a KV point-get or batch-get, turn the engine's stored /// bytes into row-shaped msgpack. @@ -47,32 +48,70 @@ pub fn apply_kv_wrap(plan: &PhysicalPlan, payload: &[u8]) -> Vec { /// Zip `KvOp::BatchGet`'s `keys` with the Data Plane's positional /// `[value_or_null, ...]` array and wrap each pair into a `{key, value}` /// row, msgpack-encoded so the rest of the shaping pipeline -/// (`decode_payload_to_json` -> `push_flat_rows`) treats it exactly like +/// (`decode_payload_value` -> `push_flat_rows`) treats it exactly like /// any other row-array payload. /// /// Falls back to the raw payload (rather than panicking) if the Data /// Plane payload is not the expected JSON/msgpack array — a malformed -/// upstream payload degrades to the pre-fix (empty-looking) shape instead -/// of taking down the connection. +/// upstream payload degrades to an empty-looking shape instead of taking +/// down the connection. fn wrap_batch_get(keys: &[Vec], payload: &[u8]) -> Vec { - let decoded = decode_payload_to_json(payload); - let Ok(JsonValue::Array(values)) = sonic_rs::from_str::(&decoded) else { + let Ok(Value::Array(values)) = decode_payload_value(payload) else { return payload.to_vec(); }; - let rows: Vec = keys + let rows: Vec = keys .iter() .zip(values) .map(|(key, value)| { - let mut row = Map::new(); + let mut row = HashMap::with_capacity(2); row.insert( "key".to_string(), - JsonValue::String(String::from_utf8_lossy(key).into_owned()), + Value::String(String::from_utf8_lossy(key).into_owned()), ); row.insert("value".to_string(), value); - JsonValue::Object(row) + Value::Object(row) }) .collect(); - nodedb_types::json_to_msgpack(&JsonValue::Array(rows)).unwrap_or_else(|_| payload.to_vec()) + nodedb_types::value_to_msgpack(&Value::Array(rows)).unwrap_or_else(|_| payload.to_vec()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Each positional result is paired with its key into a `{key, value}` + /// row; a missing key stays `null` and the value keeps its type. + #[test] + fn wrap_batch_get_pairs_keys_with_positional_values() { + let keys = vec![b"k1".to_vec(), b"k2".to_vec()]; + let payload = nodedb_types::value_to_msgpack(&Value::Array(vec![ + Value::String("dmFs".into()), + Value::Null, + ])) + .expect("encode"); + + let wrapped = wrap_batch_get(&keys, &payload); + let Value::Array(rows) = decode_payload_value(&wrapped).expect("decode") else { + panic!("wrapped payload must be an array"); + }; + let Value::Object(first) = &rows[0] else { + panic!("row must be an object"); + }; + assert_eq!(first["key"], Value::String("k1".into())); + assert_eq!(first["value"], Value::String("dmFs".into())); + let Value::Object(second) = &rows[1] else { + panic!("row must be an object"); + }; + assert_eq!(second["key"], Value::String("k2".into())); + assert_eq!(second["value"], Value::Null); + } + + /// A payload that is not an array passes through unchanged. + #[test] + fn wrap_batch_get_passes_a_non_array_payload_through() { + let payload = nodedb_types::value_to_msgpack(&Value::Integer(1)).expect("encode"); + assert_eq!(wrap_batch_get(&[b"k".to_vec()], &payload), payload); + } } diff --git a/nodedb/src/control/server/response_shape/mod.rs b/nodedb/src/control/server/response_shape/mod.rs index e578e3ea7..01f1b6b08 100644 --- a/nodedb/src/control/server/response_shape/mod.rs +++ b/nodedb/src/control/server/response_shape/mod.rs @@ -2,6 +2,7 @@ //! Shared, protocol-neutral response shaping helpers. +pub mod cell; pub mod compose; pub mod kv; pub mod project; @@ -11,4 +12,5 @@ pub mod returning; pub mod schema; pub mod types; +pub use cell::{row_to_wire_json, value_to_wire_json}; pub use redaction::{redact_decoded_value, redact_envelope_row, redact_stored_value_bytes}; diff --git a/nodedb/src/control/server/response_shape/project.rs b/nodedb/src/control/server/response_shape/project.rs index 646868027..4ee1e2dc1 100644 --- a/nodedb/src/control/server/response_shape/project.rs +++ b/nodedb/src/control/server/response_shape/project.rs @@ -1,11 +1,19 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Pure JSON/plan projection and flattening helpers for SELECT responses. +//! Pure projection and flattening helpers for SELECT responses. //! -//! These operate purely on parsed SQL and `serde_json::Value` — no pgwire -//! wire types — so they are shared across any protocol-specific response -//! shaper. Protocol-specific encode glue that turns these into wire rows -//! (e.g. pgwire's `DataRow`) lives in each protocol's own handler code. +//! These operate on decoded `nodedb_types::Value` rows — no pgwire wire +//! types — so they are shared across any protocol-specific response shaper. +//! Protocol-specific encode glue that turns these into wire rows (e.g. +//! pgwire's `DataRow`) lives in each protocol's own handler code. +//! `json_value_to_text` is the one JSON helper here: pgwire converts a cell +//! to JSON at its edge and renders that JSON as PostgreSQL text. + +use std::collections::HashMap; + +use nodedb_types::Value; + +use super::types::ShapedRow; /// Convert a JSON scalar value to its PostgreSQL text-format string. /// @@ -23,45 +31,65 @@ pub fn json_value_to_text(v: &serde_json::Value) -> String { } } -/// Flatten a parsed JSON value into row objects. +/// Flatten a decoded Data-Plane value into typed row objects. /// /// The envelope `id` is a rendered [`StorageKey`](crate::engine::document::store::StorageKey) /// by construction. /// A value that fails `StorageKey::parse` is surfaced as `Err`, never accommodated. -pub fn push_flat_rows( - value: serde_json::Value, - out: &mut Vec>, -) -> crate::Result<()> { +pub fn push_flat_rows(value: Value, out: &mut Vec) -> crate::Result<()> { match value { - serde_json::Value::Array(items) => { + Value::Array(items) => { for item in items { push_flat_rows(item, out)?; } } - serde_json::Value::Object(mut map) => { + Value::Object(mut map) => { if is_scan_wrapper(&map) - && let Some(serde_json::Value::Object(mut inner)) = map.remove("data") + && let Some(Value::Object(inner)) = map.remove("data") { + let mut inner: ShapedRow = inner.into_iter().collect(); // The envelope carries the row's storage key, which is // internal. A body with no `id` field carries identity // nowhere else, so the key renders to an identity at this // boundary. `or_insert` leaves a declared primary key as the // authority. - if let Some(serde_json::Value::String(key)) = map.remove("id") { + if let Some(Value::String(key)) = map.remove("id") { let identity = crate::engine::document::store::StorageKey::parse(&key) .ok_or_else(|| crate::Error::Internal { detail: format!("scan envelope id is not a storage key: '{key}'"), })? .to_identity(); inner - .entry("id") - .or_insert(serde_json::Value::String(identity.into_string())); + .entry("id".to_string()) + .or_insert(Value::String(identity.into_string())); } out.push(inner); return Ok(()); } - out.push(map); + out.push(map.into_iter().collect()); } + // A scalar is not a row: the shaper only ever answers with objects. + Value::Null + | Value::Bool(_) + | Value::Integer(_) + | Value::Float(_) + | Value::String(_) + | Value::Bytes(_) + | Value::Uuid(_) + | Value::Ulid(_) + | Value::DateTime(_) + | Value::NaiveDateTime(_) + | Value::Duration(_) + | Value::Decimal(_) + | Value::Geometry(_) + | Value::Set(_) + | Value::Regex(_) + | Value::Range { .. } + | Value::Record { .. } + | Value::ArrayCell(_) + | Value::Vector(_) => {} + // `Value` is `#[non_exhaustive]`: a variant this crate cannot name is + // not a row either. _ => {} } Ok(()) @@ -70,7 +98,17 @@ pub fn push_flat_rows( /// The Data Plane's raw document-scan codec emits objects with exactly /// the keys `id` (string) and `data` (object). This is the wire shape /// we unwrap before column projection. -pub fn is_scan_wrapper(map: &serde_json::Map) -> bool { +pub fn is_scan_wrapper(map: &HashMap) -> bool { + map.len() == 2 + && matches!(map.get("id"), Some(Value::String(_))) + && matches!(map.get("data"), Some(Value::Object(_))) +} + +/// [`is_scan_wrapper`] for a row still held as JSON: the same two-key +/// `{id: string, data: object}` shape, read off a `serde_json::Map`. The +/// JSON-only redaction paths (`redact_envelope_row`) check the envelope +/// without lifting the row to a typed value. +pub fn is_scan_wrapper_json(map: &serde_json::Map) -> bool { map.len() == 2 && matches!(map.get("id"), Some(serde_json::Value::String(_))) && matches!(map.get("data"), Some(serde_json::Value::Object(_))) diff --git a/nodedb/src/control/server/response_shape/redaction/shapes.rs b/nodedb/src/control/server/response_shape/redaction/shapes.rs index 29a5b69dd..fc89c7a54 100644 --- a/nodedb/src/control/server/response_shape/redaction/shapes.rs +++ b/nodedb/src/control/server/response_shape/redaction/shapes.rs @@ -22,7 +22,7 @@ //! a decoded payload of unknown shape. use crate::control::security::redaction::RedactionStore; -use crate::control::server::response_shape::project::is_scan_wrapper; +use crate::control::server::response_shape::project::is_scan_wrapper_json; use super::query::QueryRedaction; @@ -49,7 +49,7 @@ pub fn redact_envelope_row( let Some(map) = item.as_object_mut() else { return; }; - let target = if is_scan_wrapper(map) { + let target = if is_scan_wrapper_json(map) { map.get_mut("data") .and_then(serde_json::Value::as_object_mut) } else { diff --git a/nodedb/src/control/server/response_shape/returning.rs b/nodedb/src/control/server/response_shape/returning.rs index eb3395391..72124f3db 100644 --- a/nodedb/src/control/server/response_shape/returning.rs +++ b/nodedb/src/control/server/response_shape/returning.rs @@ -34,17 +34,15 @@ //! a schemaless row carries fields no catalog column declares — which is the //! same answer `SELECT *` gives for the same row. -use serde_json::{Map, Value as JsonValue}; - -use nodedb_types::{NativeCell, NodeDbError}; +use nodedb_types::{NativeCell, NodeDbError, Value}; use crate::data::executor::response_codec::{RowsPayload, decode_payload_to_json}; -use super::compose::{project_row, redact_rows, single_result_row}; +use super::compose::kernel::{project_row, redact_rows, single_result_row}; use super::project::cell_keys; use super::redaction::RedactionCtx; use super::schema::OutputSchema; -use super::types::{DdlColType, ShapedRows}; +use super::types::{DdlColType, ShapedRow, ShapedRows}; /// Shape a DML-with-`RETURNING` response. /// @@ -100,12 +98,7 @@ pub fn shape_returning_rows( let Some(schema) = announced else { let column_types = ShapedRows::text_types(columns.len()); - return Ok(ShapedRows { - columns, - column_types, - rows, - notice: None, - }); + return Ok(ShapedRows::from_rows(columns, column_types, rows)); }; Ok(project_onto_announced(schema, &rows)) } @@ -117,19 +110,16 @@ fn announced_columns(projection: Option<&OutputSchema>) -> Option<&OutputSchema> projection.filter(|schema| !schema.is_star && !schema.columns.is_empty()) } -/// Re-key each payload row from positional typed cells to a name-keyed JSON -/// map. This is the one step where a cell leaves its `Value` form: an -/// instant renders as ISO-8601 text, `Value::Null` (SQL NULL) as JSON `null`, -/// numbers and booleans as themselves. -fn rows_keyed_by_column( - columns: &[String], - rows: Vec>, -) -> Vec> { +/// Re-key each payload row from positional typed cells to a name-keyed row. +/// A cell keeps its `Value` form: an instant stays an instant, `Value::Null` +/// is SQL NULL, numbers and booleans are themselves. Each protocol renders +/// the cell at its own edge. +fn rows_keyed_by_column(columns: &[String], rows: Vec>) -> Vec { rows.into_iter() .map(|row_vals| { - let mut map = Map::new(); + let mut map = ShapedRow::new(); for (col, cell) in columns.iter().zip(row_vals) { - map.insert(col.clone(), JsonValue::from(cell.0)); + map.insert(col.clone(), cell.0); } map }) @@ -141,7 +131,7 @@ fn rows_keyed_by_column( /// Uses the same `project_row` + `cell_keys` pair the SELECT path uses, so a /// `RETURNING` result and a `SELECT` result of the same columns are laid out /// identically for the encoders. -fn project_onto_announced(schema: &OutputSchema, rows: &[Map]) -> ShapedRows { +fn project_onto_announced(schema: &OutputSchema, rows: &[ShapedRow]) -> ShapedRows { let lookup_keys: Vec = schema .columns .iter() @@ -168,12 +158,7 @@ fn project_onto_announced(schema: &OutputSchema, rows: &[Map] }) .collect(); - ShapedRows { - columns: display_names, - column_types, - rows: projected, - notice: None, - } + ShapedRows::from_rows(display_names, column_types, projected) } /// Re-read a text `RETURNING` cell as the column's announced type. @@ -188,8 +173,8 @@ fn project_onto_announced(schema: &OutputSchema, rows: &[Map] /// /// A cell that does not parse as its announced type is left as text, which /// both encoders render verbatim. -fn retype_cell(ct: DdlColType, cell: &mut JsonValue) { - let JsonValue::String(text) = cell else { +fn retype_cell(ct: DdlColType, cell: &mut Value) { + let Value::String(text) = cell else { return; }; let retyped = match ct { @@ -198,15 +183,17 @@ fn retype_cell(ct: DdlColType, cell: &mut JsonValue) { | DdlColType::Int2 // Epoch microseconds; the encoder formats the number as ISO-8601. | DdlColType::Timestamp - | DdlColType::Timestamptz => text.parse::().ok().map(JsonValue::from), + | DdlColType::Timestamptz => text.parse::().ok().map(Value::Integer), + // A float that parses but is not finite has no JSON form, so it stays + // text. DdlColType::Float8 | DdlColType::Float4 => text .parse::() .ok() - .and_then(serde_json::Number::from_f64) - .map(JsonValue::Number), + .filter(|f| f.is_finite()) + .map(Value::Float), DdlColType::Bool => match text.as_str() { - "t" | "true" | "TRUE" | "T" => Some(JsonValue::Bool(true)), - "f" | "false" | "FALSE" | "F" => Some(JsonValue::Bool(false)), + "t" | "true" | "TRUE" | "T" => Some(Value::Bool(true)), + "f" | "false" | "FALSE" | "F" => Some(Value::Bool(false)), _ => None, }, DdlColType::Text @@ -225,34 +212,37 @@ fn retype_cell(ct: DdlColType, cell: &mut JsonValue) { /// The announced columns with no rows — a write that matched nothing still /// answers with the result set the client was promised, just an empty one. fn empty_announced(schema: &OutputSchema) -> ShapedRows { - ShapedRows { - columns: schema + ShapedRows::from_rows( + schema .columns .iter() .map(|c| c.display_name.clone()) .collect(), - column_types: schema.columns.iter().map(|c| c.ty).collect(), - rows: Vec::new(), - notice: None, - } + schema.columns.iter().map(|c| c.ty).collect(), + Vec::new(), + ) } /// Single "result" column with zero rows, for an empty payload with nothing /// announced. fn single_result_column_empty() -> ShapedRows { - ShapedRows { - columns: vec!["result".to_string()], - column_types: ShapedRows::text_types(1), - rows: Vec::new(), - notice: None, - } + ShapedRows::from_rows( + vec!["result".to_string()], + ShapedRows::text_types(1), + Vec::new(), + ) } #[cfg(test)] mod tests { use super::*; + use crate::control::server::response_shape::cell::value_to_wire_json; use crate::control::server::response_shape::schema::OutputColumn; - use nodedb_types::{NdbDateTime, Value}; + use nodedb_types::NdbDateTime; + + fn text(s: &str) -> Value { + Value::String(s.to_string()) + } fn typed_payload(columns: &[&str], rows: Vec>) -> Vec { let rp = RowsPayload { @@ -325,8 +315,8 @@ mod tests { assert_eq!(shaped.column_types.len(), shaped.columns.len()); let keys = shaped.cell_keys(); assert_eq!(shaped.rows[0].len(), keys.len(), "one cell per column"); - assert_eq!(shaped.rows[0]["id"], JsonValue::String("a".into())); - assert_eq!(shaped.rows[0]["name"], JsonValue::String("x".into())); + assert_eq!(shaped.rows[0]["id"], text("a")); + assert_eq!(shaped.rows[0]["name"], text("x")); } /// An announced column the row does not carry is SQL NULL, in its @@ -341,9 +331,9 @@ mod tests { ]); let shaped = shape_returning_rows(&bytes, Some(&schema), None).expect("shape"); assert_eq!(shaped.columns, ["id", "name", "score"]); - assert_eq!(shaped.rows[0]["id"], JsonValue::String("a".into())); - assert_eq!(shaped.rows[0]["name"], JsonValue::Null); - assert_eq!(shaped.rows[0]["score"], JsonValue::from(7i64)); + assert_eq!(shaped.rows[0]["id"], text("a")); + assert_eq!(shaped.rows[0]["name"], Value::Null); + assert_eq!(shaped.rows[0]["score"], Value::Integer(7)); } /// Cells are re-read as the announced type so a binary-format request is @@ -361,16 +351,17 @@ mod tests { ("s", DdlColType::Text), ]); let shaped = shape_returning_rows(&bytes, Some(&schema), None).expect("shape"); - assert_eq!(shaped.rows[0]["i"], JsonValue::from(42i64)); - assert_eq!(shaped.rows[0]["f"], JsonValue::from(1.5f64)); - assert_eq!(shaped.rows[0]["b"], JsonValue::Bool(true)); + assert_eq!(shaped.rows[0]["i"], Value::Integer(42)); + assert_eq!(shaped.rows[0]["f"], Value::Float(1.5)); + assert_eq!(shaped.rows[0]["b"], Value::Bool(true)); // A TEXT column keeps its text even when it looks like a number. - assert_eq!(shaped.rows[0]["s"], JsonValue::String("42".into())); + assert_eq!(shaped.rows[0]["s"], text("42")); } /// A typed cell passes through as itself: an integer stays a number under - /// an INT column, an instant renders as the ISO-8601 text a `SELECT` of - /// the same column renders, and SQL NULL is JSON `null`. + /// an INT column, an instant stays an instant (and renders at the edge as + /// the ISO-8601 text a `SELECT` of the same column renders), and SQL NULL + /// stays NULL. #[test] fn typed_cells_reach_the_row_as_themselves() { let at = NdbDateTime::from_micros(1_583_402_400_000_000); @@ -390,25 +381,28 @@ mod tests { ("gone", DdlColType::Text), ]); let shaped = shape_returning_rows(&bytes, Some(&schema), None).expect("shape"); - assert_eq!(shaped.rows[0]["n"], JsonValue::from(7i64)); + assert_eq!(shaped.rows[0]["n"], Value::Integer(7)); + assert_eq!(shaped.rows[0]["at"], Value::NaiveDateTime(at)); assert_eq!( - shaped.rows[0]["at"], - JsonValue::String("2020-03-05T10:00:00.000000Z".into()) + value_to_wire_json(&shaped.rows[0]["at"]), + serde_json::Value::String("2020-03-05T10:00:00.000000Z".into()) ); - assert_eq!(shaped.rows[0]["f"], JsonValue::from(1.5f64)); - assert_eq!(shaped.rows[0]["gone"], JsonValue::Null); + assert_eq!(shaped.rows[0]["f"], Value::Float(1.5)); + assert_eq!(shaped.rows[0]["gone"], Value::Null); } - /// The same instant renders as ISO-8601 when nothing was announced, so the - /// simple-query protocol shows what the extended one does. + /// The same instant stays typed when nothing was announced, and renders + /// as ISO-8601 at the edge, so the simple-query protocol shows what the + /// extended one does. #[test] fn an_instant_cell_renders_iso8601_without_a_projection() { let at = NdbDateTime::from_micros(1_583_402_400_000_000); let bytes = typed_payload(&["at"], vec![vec![Value::DateTime(at)]]); let shaped = shape_returning_rows(&bytes, None, None).expect("shape"); + assert_eq!(shaped.rows[0]["at"], Value::DateTime(at)); assert_eq!( - shaped.rows[0]["at"], - JsonValue::String("2020-03-05T10:00:00.000000Z".into()) + value_to_wire_json(&shaped.rows[0]["at"]), + serde_json::Value::String("2020-03-05T10:00:00.000000Z".into()) ); } @@ -419,10 +413,7 @@ mod tests { let bytes = payload(&["i"], &[&[Some("not a number")]]); let schema = announced(&[("i", DdlColType::Int8)]); let shaped = shape_returning_rows(&bytes, Some(&schema), None).expect("shape"); - assert_eq!( - shaped.rows[0]["i"], - JsonValue::String("not a number".into()) - ); + assert_eq!(shaped.rows[0]["i"], text("not a number")); } /// A write that matched nothing still answers with the announced result diff --git a/nodedb/src/control/server/response_shape/types/mod.rs b/nodedb/src/control/server/response_shape/types/mod.rs new file mode 100644 index 000000000..9b0a0fb2d --- /dev/null +++ b/nodedb/src/control/server/response_shape/types/mod.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Protocol-neutral plan classification and shaped row-set types. + +pub mod plan_kind; +pub mod shaped; + +pub use plan_kind::{PlanKind, describe_plan}; +pub use shaped::{DdlColType, ShapedRow, ShapedRows}; diff --git a/nodedb/src/control/server/response_shape/types.rs b/nodedb/src/control/server/response_shape/types/plan_kind.rs similarity index 74% rename from nodedb/src/control/server/response_shape/types.rs rename to nodedb/src/control/server/response_shape/types/plan_kind.rs index d9d35c2e0..0f68cd00d 100644 --- a/nodedb/src/control/server/response_shape/types.rs +++ b/nodedb/src/control/server/response_shape/types/plan_kind.rs @@ -265,199 +265,11 @@ pub fn describe_plan(plan: &PhysicalPlan) -> PlanKind { // Bring the variant into scope for brevity in match arms above. use PlanKind::DmlResult; -/// Protocol-neutral SQL column type, mapped to each entrypoint's own wire -/// type. One variant per pgwire field-builder, so the mapping is lossless. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum DdlColType { - #[default] - Text, - Int8, - Int4, - Int2, - Float8, - Float4, - Bool, - Bytea, - Json, - Jsonb, - Timestamp, - Timestamptz, - Varchar, - Float4Array, - Float8Array, -} - -/// Protocol-neutral shaped row set: columns + row objects + an optional -/// client-facing notice. -#[derive(Debug, Clone)] -pub struct ShapedRows { - pub columns: Vec, - /// Per-column SQL type, parallel to `columns`. Only pgwire consumes this - /// (RowDescription OIDs); `Text` when the source type is unknown. - pub column_types: Vec, - /// One map per row, keyed by [`ShapedRows::cell_keys`] not `columns` — - /// SQL output names may repeat and a map can't hold two cells per key. - pub rows: Vec>, - pub notice: Option, -} - -impl ShapedRows { - /// Build a `column_types` vec of `n` `Text` entries, for non-DDL sites - /// whose consumers ignore column types. - pub fn text_types(n: usize) -> Vec { - vec![DdlColType::Text; n] - } - - /// A result set from decoded JSON rows, with one catalog type per column - /// and no notice. - pub fn from_json_rows( - columns: Vec, - column_types: Vec, - rows: Vec>, - ) -> Self { - Self { - columns, - column_types, - rows, - notice: None, - } - } - - /// A result set whose every column is `Text`: the shape a DDL or - /// inspection statement answers with. The type list is sized from - /// `columns`, so the two cannot disagree. - pub fn text_rows( - columns: Vec, - rows: Vec>, - ) -> Self { - let column_types = Self::text_types(columns.len()); - Self::from_json_rows(columns, column_types, rows) - } - - /// Attach a client-facing notice. - pub fn with_notice(mut self, notice: impl Into) -> Self { - self.notice = Some(notice.into()); - self - } - - /// Fold another shaped result into this one so N tasks answer with ONE result - /// set — some drivers reject multiple result sets. Columns are the union of - /// every contributor's; rows read by key so a missing column encodes NULL. - pub fn append(&mut self, other: ShapedRows) { - if self.notice.is_none() { - self.notice = other.notice; - } - if self.columns.is_empty() { - self.columns = other.columns; - self.column_types = other.column_types; - self.rows.extend(other.rows); - return; - } - for (index, name) in other.columns.iter().enumerate() { - if self.columns.iter().any(|existing| existing == name) { - continue; - } - self.columns.push(name.clone()); - self.column_types - .push(other.column_types.get(index).copied().unwrap_or_default()); - } - self.rows.extend(other.rows); - } - - /// Per-column keys for reading cells out of [`ShapedRows::rows`]. Identical - /// to `columns` unless names collide, then later duplicates take a `_` suffix - /// — visible in HTTP JSON, but pgwire/native stay positional. - pub fn cell_keys(&self) -> Vec { - super::project::cell_keys(&self.columns) - } -} - #[cfg(test)] mod tests { use super::*; use nodedb_types::{DatabaseId, QualifiedCollection}; - fn shaped(columns: &[&str], rows: &[&[(&str, &str)]]) -> ShapedRows { - ShapedRows { - columns: columns.iter().map(|c| (*c).to_string()).collect(), - column_types: ShapedRows::text_types(columns.len()), - rows: rows - .iter() - .map(|row| { - row.iter() - .map(|(k, v)| { - ( - (*k).to_string(), - serde_json::Value::String((*v).to_string()), - ) - }) - .collect() - }) - .collect(), - notice: None, - } - } - - /// A column only a later contributor carries must survive the fold — a key - /// absent from `columns` is never read by `cell_keys()`. - #[test] - fn append_unions_a_column_only_a_later_row_carries() { - let mut merged = shaped(&["id", "name"], &[&[("id", "r1"), ("name", "a")]]); - merged.append(shaped( - &["id", "name", "extra"], - &[&[("id", "r2"), ("name", "b"), ("extra", "x")]], - )); - - assert_eq!(merged.columns, vec!["id", "name", "extra"]); - assert_eq!( - merged.column_types.len(), - merged.columns.len(), - "column types must stay parallel to columns" - ); - - let keys = merged.cell_keys(); - assert_eq!(keys, vec!["id", "name", "extra"]); - assert_eq!( - merged.rows[1].get(keys[2].as_str()), - Some(&serde_json::Value::String("x".to_string())), - "the later row's extra value must be readable through the merged keys" - ); - assert!( - merged.rows[0].get(keys[2].as_str()).is_none(), - "the row that lacks the column encodes as NULL, not a shifted cell" - ); - } - - /// The first contributor's columns keep their positions and newly-seen - /// columns are appended in first-seen order, so a positional client never - /// sees a column move between rows. - #[test] - fn append_keeps_the_first_contributors_column_order_and_appends_the_rest() { - let mut merged = shaped(&["b", "a"], &[&[("b", "1"), ("a", "2")]]); - merged.append(shaped(&["a", "z"], &[&[("a", "3"), ("z", "4")]])); - merged.append(shaped(&["y", "b"], &[&[("y", "5"), ("b", "6")]])); - - assert_eq!( - merged.columns, - vec!["b", "a", "z", "y"], - "first contributor's positions are fixed; later columns append in \ - first-seen order" - ); - assert_eq!(merged.rows.len(), 3); - } - - /// A contributor with no columns at all — a task whose rows were entirely - /// removed by a read policy, which shapes as `RETURNING *` with an empty - /// column list — must not fix an empty shape for the statement. - #[test] - fn append_adopts_the_shape_of_the_first_contributor_that_has_columns() { - let mut merged = shaped(&[], &[]); - merged.append(shaped(&["id"], &[&[("id", "r1")]])); - - assert_eq!(merged.columns, vec!["id"]); - assert_eq!(merged.rows.len(), 1); - } - #[test] fn crdt_preview_is_an_opaque_execution_plan() { let plan = PhysicalPlan::Crdt(CrdtOp::PreviewApply { diff --git a/nodedb/src/control/server/response_shape/types/shaped.rs b/nodedb/src/control/server/response_shape/types/shaped.rs new file mode 100644 index 000000000..6774c8a0d --- /dev/null +++ b/nodedb/src/control/server/response_shape/types/shaped.rs @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Protocol-neutral shaped row set: typed cells keyed by column. + +use std::collections::BTreeMap; + +use nodedb_types::Value; + +/// Protocol-neutral SQL column type, mapped to each entrypoint's own wire +/// type. One variant per pgwire field-builder, so the mapping is lossless. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum DdlColType { + #[default] + Text, + Int8, + Int4, + Int2, + Float8, + Float4, + Bool, + Bytea, + Json, + Jsonb, + Timestamp, + Timestamptz, + Varchar, + Float4Array, + Float8Array, +} + +/// One shaped row: typed cells keyed by [`ShapedRows::cell_keys`]. +/// +/// A `BTreeMap` iterates in sorted key order, which is the order column +/// derivation reads a row's keys in. +pub type ShapedRow = BTreeMap; + +/// Protocol-neutral shaped row set: columns + row objects + an optional +/// client-facing notice. +#[derive(Debug, Clone)] +pub struct ShapedRows { + pub columns: Vec, + /// Per-column SQL type, parallel to `columns`. Only pgwire consumes this + /// (RowDescription OIDs); `Text` when the source type is unknown. + pub column_types: Vec, + /// One map per row, keyed by [`ShapedRows::cell_keys`] not `columns` — + /// SQL output names may repeat and a map can't hold two cells per key. + pub rows: Vec, + pub notice: Option, +} + +impl ShapedRows { + /// Build a `column_types` vec of `n` `Text` entries, for non-DDL sites + /// whose consumers ignore column types. + pub fn text_types(n: usize) -> Vec { + vec![DdlColType::Text; n] + } + + /// A result set from typed rows, with one catalog type per column and no + /// notice. + pub fn from_rows( + columns: Vec, + column_types: Vec, + rows: Vec, + ) -> Self { + Self { + columns, + column_types, + rows, + notice: None, + } + } + + /// A result set from decoded JSON rows, with one catalog type per column + /// and no notice. Each JSON cell becomes the matching [`Value`]: a JSON + /// string is a `Value::String`, never parsed further. + pub fn from_json_rows( + columns: Vec, + column_types: Vec, + rows: Vec>, + ) -> Self { + let rows = rows.into_iter().map(json_row_to_shaped).collect(); + Self::from_rows(columns, column_types, rows) + } + + /// A result set whose every column is `Text`: the shape a DDL or + /// inspection statement answers with. The type list is sized from + /// `columns`, so the two cannot disagree. + pub fn text_rows( + columns: Vec, + rows: Vec>, + ) -> Self { + let column_types = Self::text_types(columns.len()); + Self::from_json_rows(columns, column_types, rows) + } + + /// Attach a client-facing notice. + pub fn with_notice(mut self, notice: impl Into) -> Self { + self.notice = Some(notice.into()); + self + } + + /// Fold another shaped result into this one so N tasks answer with ONE result + /// set — some drivers reject multiple result sets. Columns are the union of + /// every contributor's; rows read by key so a missing column encodes NULL. + pub fn append(&mut self, other: ShapedRows) { + if self.notice.is_none() { + self.notice = other.notice; + } + if self.columns.is_empty() { + self.columns = other.columns; + self.column_types = other.column_types; + self.rows.extend(other.rows); + return; + } + for (index, name) in other.columns.iter().enumerate() { + if self.columns.iter().any(|existing| existing == name) { + continue; + } + self.columns.push(name.clone()); + self.column_types + .push(other.column_types.get(index).copied().unwrap_or_default()); + } + self.rows.extend(other.rows); + } + + /// Per-column keys for reading cells out of [`ShapedRows::rows`]. Identical + /// to `columns` unless names collide, then later duplicates take a `_` suffix + /// — visible in HTTP JSON, but pgwire/native stay positional. + pub fn cell_keys(&self) -> Vec { + crate::control::server::response_shape::project::cell_keys(&self.columns) + } +} + +/// Convert one JSON row map to a typed row, cell by cell. +fn json_row_to_shaped(row: serde_json::Map) -> ShapedRow { + row.into_iter().map(|(k, v)| (k, Value::from(v))).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn shaped(columns: &[&str], rows: &[&[(&str, &str)]]) -> ShapedRows { + ShapedRows { + columns: columns.iter().map(|c| (*c).to_string()).collect(), + column_types: ShapedRows::text_types(columns.len()), + rows: rows + .iter() + .map(|row| { + row.iter() + .map(|(k, v)| ((*k).to_string(), Value::String((*v).to_string()))) + .collect() + }) + .collect(), + notice: None, + } + } + + /// A column only a later contributor carries must survive the fold — a key + /// absent from `columns` is never read by `cell_keys()`. + #[test] + fn append_unions_a_column_only_a_later_row_carries() { + let mut merged = shaped(&["id", "name"], &[&[("id", "r1"), ("name", "a")]]); + merged.append(shaped( + &["id", "name", "extra"], + &[&[("id", "r2"), ("name", "b"), ("extra", "x")]], + )); + + assert_eq!(merged.columns, vec!["id", "name", "extra"]); + assert_eq!( + merged.column_types.len(), + merged.columns.len(), + "column types must stay parallel to columns" + ); + + let keys = merged.cell_keys(); + assert_eq!(keys, vec!["id", "name", "extra"]); + assert_eq!( + merged.rows[1].get(keys[2].as_str()), + Some(&Value::String("x".to_string())), + "the later row's extra value must be readable through the merged keys" + ); + assert!( + !merged.rows[0].contains_key(keys[2].as_str()), + "the row that lacks the column encodes as NULL, not a shifted cell" + ); + } + + /// The first contributor's columns keep their positions and newly-seen + /// columns are appended in first-seen order, so a positional client never + /// sees a column move between rows. + #[test] + fn append_keeps_the_first_contributors_column_order_and_appends_the_rest() { + let mut merged = shaped(&["b", "a"], &[&[("b", "1"), ("a", "2")]]); + merged.append(shaped(&["a", "z"], &[&[("a", "3"), ("z", "4")]])); + merged.append(shaped(&["y", "b"], &[&[("y", "5"), ("b", "6")]])); + + assert_eq!( + merged.columns, + vec!["b", "a", "z", "y"], + "first contributor's positions are fixed; later columns append in \ + first-seen order" + ); + assert_eq!(merged.rows.len(), 3); + } + + /// A contributor with no columns at all — a task whose rows were entirely + /// removed by a read policy, which shapes as `RETURNING *` with an empty + /// column list — must not fix an empty shape for the statement. + #[test] + fn append_adopts_the_shape_of_the_first_contributor_that_has_columns() { + let mut merged = shaped(&[], &[]); + merged.append(shaped(&["id"], &[&[("id", "r1")]])); + + assert_eq!(merged.columns, vec!["id"]); + assert_eq!(merged.rows.len(), 1); + } + + /// Every JSON cell kind lands as the matching typed cell; a JSON string + /// stays a string and is never parsed into an instant or a number. + #[test] + fn from_json_rows_maps_each_json_kind_to_the_matching_value() { + let row = match serde_json::json!({ + "n": 7, + "f": 1.5, + "s": "2020-03-05T10:00:00.000000Z", + "z": null, + "b": true, + "a": [1, "x"], + "o": {"k": false}, + }) { + serde_json::Value::Object(map) => map, + other => panic!("fixture must be an object, got {other}"), + }; + let columns: Vec = ["n", "f", "s", "z", "b", "a", "o"] + .iter() + .map(|c| (*c).to_string()) + .collect(); + let shaped = ShapedRows::from_json_rows( + columns.clone(), + ShapedRows::text_types(columns.len()), + vec![row], + ); + + let cells = &shaped.rows[0]; + assert_eq!(cells["n"], Value::Integer(7)); + assert_eq!(cells["f"], Value::Float(1.5)); + assert_eq!( + cells["s"], + Value::String("2020-03-05T10:00:00.000000Z".to_string()) + ); + assert_eq!(cells["z"], Value::Null); + assert_eq!(cells["b"], Value::Bool(true)); + assert_eq!( + cells["a"], + Value::Array(vec![Value::Integer(1), Value::String("x".to_string())]) + ); + assert_eq!( + cells["o"], + Value::Object(std::collections::HashMap::from([( + "k".to_string(), + Value::Bool(false) + )])) + ); + assert!(shaped.notice.is_none()); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/query_functions/helpers.rs b/nodedb/src/control/server/shared/ddl/neutral/query_functions/helpers.rs index ec7e41f84..28cc868e6 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/query_functions/helpers.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/query_functions/helpers.rs @@ -9,9 +9,11 @@ //! of a pgwire `QueryResponse`. SQLSTATE codes and messages are unchanged. use nodedb_sql::parser::preprocess::lex::find_ascii_case_insensitive; +use nodedb_types::Value; use serde_json::{Map, Value as JsonValue}; -use crate::control::server::response_shape::project::{is_scan_wrapper, push_flat_rows}; +use crate::control::server::response_shape::cell::row_to_wire_json; +use crate::control::server::response_shape::project::{is_scan_wrapper_json, push_flat_rows}; use crate::control::server::response_shape::types::ShapedRows; use super::super::super::result::{DdlError, DdlResult}; @@ -85,14 +87,16 @@ pub fn single_result(value: &str) -> Vec { /// /// Reuses `response_shape::project::push_flat_rows` — the same unwrap the /// pgwire/HTTP row shaper applies — so there is exactly one definition of -/// "unwrap a scan envelope" in the tree. Rows that are not `{id, data}` +/// "unwrap a scan envelope" in the tree. Each JSON document is lifted to a +/// typed value for the unwrap and its rows rendered back to JSON for the +/// callers, which read fields as JSON. Rows that are not `{id, data}` /// wrapped (already-flat producers) pass through unchanged. pub fn unwrap_scan_docs(docs: Vec) -> Result>, DdlError> { - let mut out = Vec::with_capacity(docs.len()); + let mut rows = Vec::with_capacity(docs.len()); for doc in docs { - push_flat_rows(doc, &mut out).map_err(|e| err("XX000", &e.to_string()))?; + push_flat_rows(Value::from(doc), &mut rows).map_err(|e| err("XX000", &e.to_string()))?; } - Ok(out) + Ok(rows.iter().map(row_to_wire_json).collect()) } /// Unwrap a `DocumentOp::Scan` envelope while also returning the row's wire @@ -102,15 +106,15 @@ pub fn unwrap_scan_docs(docs: Vec) -> Result (String, Map) { let JsonValue::Object(mut map) = doc else { return (String::new(), Map::new()); }; - if is_scan_wrapper(&map) { + if is_scan_wrapper_json(&map) { let id = map .get("id") .and_then(|v| v.as_str()) diff --git a/nodedb/src/control/server/shared/ddl/neutral/show_changes.rs b/nodedb/src/control/server/shared/ddl/neutral/show_changes.rs index c5d1f3518..08a8a6e99 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/show_changes.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/show_changes.rs @@ -235,7 +235,7 @@ mod tests { .iter() .map(|row| { row.get("document_id") - .and_then(JsonValue::as_str) + .and_then(nodedb_types::Value::as_str) .expect("document id column") }) .collect::>(), @@ -283,7 +283,7 @@ mod tests { .iter() .map(|row| { row.get("document_id") - .and_then(JsonValue::as_str) + .and_then(nodedb_types::Value::as_str) .expect("document id column") }) .collect(); diff --git a/nodedb/src/control/server/shared/ddl/neutral/topic_subscribe.rs b/nodedb/src/control/server/shared/ddl/neutral/topic_subscribe.rs index 140e1a424..dc058679a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/topic_subscribe.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/topic_subscribe.rs @@ -218,7 +218,10 @@ mod tests { let DdlResult::Rows(rows) = &result[0] else { panic!("expected rows"); }; - assert_eq!(rows.rows[0]["backlog"], JsonValue::String("1".into())); + assert_eq!( + rows.rows[0]["backlog"], + nodedb_types::Value::String("1".into()) + ); assert_eq!(state.ep_topic_registry.receiver_count(), 0); } } diff --git a/nodedb/src/data/executor/response_codec/encode.rs b/nodedb/src/data/executor/response_codec/encode.rs index 0f384112c..25cff1599 100644 --- a/nodedb/src/data/executor/response_codec/encode.rs +++ b/nodedb/src/data/executor/response_codec/encode.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: BUSL-1.1 //! Generic encoders for Data Plane response payloads, plus the -//! `decode_payload` / `decode_payload_to_json` counterparts used at the Control -//! Plane boundary. +//! `decode_payload` / `decode_payload_to_json` / `decode_payload_value` +//! counterparts used at the Control Plane boundary. //! //! # Every encoder here emits MessagePack //! @@ -152,6 +152,32 @@ pub fn decode_payload_to_json(payload: &[u8]) -> String { .unwrap_or_else(|_| String::from_utf8_lossy(payload).into_owned()) } +/// Decode a MessagePack or JSON payload to a typed [`nodedb_types::Value`]. +/// +/// The counterpart of [`decode_payload_to_json`] for the shaping kernel, +/// with the same format sniff: JSON text parses through `serde_json::Value` +/// (so a JSON string is a `Value::String`, never parsed further), anything +/// else reads as msgpack (a `bin` is `Value::Bytes`, an instant ext is a +/// `Value::DateTime` / `Value::NaiveDateTime`). An empty payload is +/// `Value::Null`. A non-empty payload that decodes as neither is an error. +pub fn decode_payload_value(payload: &[u8]) -> crate::Result { + let Some(&first) = payload.first() else { + return Ok(nodedb_types::Value::Null); + }; + + if looks_like_json(first) { + return sonic_rs::from_slice::(payload) + .map(nodedb_types::Value::from) + .map_err(|e| crate::Error::Codec { + detail: format!("response payload is not JSON text: {e}"), + }); + } + + nodedb_types::value_from_msgpack(payload).map_err(|e| crate::Error::Codec { + detail: format!("response payload is not msgpack: {e}"), + }) +} + /// True when `first` can open JSON text: an array, object, string, number or /// one of the `true` / `false` / `null` literals. No msgpack marker for a map /// or array shares these bytes. @@ -213,6 +239,34 @@ mod tests { ); } + /// `decode_payload_value` sniffs the same way `decode_payload_to_json` + /// does: JSON text and msgpack both land as the typed value, a msgpack + /// `bin` as bytes, and an empty payload as NULL. + #[test] + fn decode_payload_value_reads_json_text_and_msgpack_alike() { + use nodedb_types::Value; + + let from_json = decode_payload_value(br#"[{"id":1,"s":"x"}]"#).unwrap(); + let mut row = std::collections::HashMap::new(); + row.insert("id".to_string(), Value::Integer(1)); + row.insert("s".to_string(), Value::String("x".into())); + assert_eq!(from_json, Value::Array(vec![Value::Object(row.clone())])); + + let msgpack = + nodedb_types::value_to_msgpack(&Value::Array(vec![Value::Object(row)])).unwrap(); + assert_eq!(decode_payload_value(&msgpack).unwrap(), from_json); + + let bin = nodedb_types::value_to_msgpack(&Value::Bytes(vec![0, 255])).unwrap(); + assert_eq!( + decode_payload_value(&bin).unwrap(), + Value::Bytes(vec![0, 255]) + ); + + assert_eq!(decode_payload_value(&[]).unwrap(), Value::Null); + assert!(decode_payload_value(&[0xC1]).is_err()); + assert!(decode_payload_value(b"true-ish").is_err()); + } + /// `TOPK` / `RANGE` rows — `encode_json_vec_as_msgpack`. #[test] fn decode_payload_reads_back_json_vec_rows() { diff --git a/nodedb/src/data/executor/response_codec/mod.rs b/nodedb/src/data/executor/response_codec/mod.rs index 1cd862b07..82a38892a 100644 --- a/nodedb/src/data/executor/response_codec/mod.rs +++ b/nodedb/src/data/executor/response_codec/mod.rs @@ -13,7 +13,8 @@ //! Split by concern: //! //! - `encode` — generic encoders + the `decode_payload` / -//! `decode_payload_to_json` counterparts used at the Control Plane boundary. +//! `decode_payload_to_json` / `decode_payload_value` counterparts used at +//! the Control Plane boundary. //! - `decode` — payload→docs decoders for inline sub-plans (e.g. multi-way //! joins consuming an inner-join Response). //! - `raw` — raw-msgpack passthrough encoders (`encode_raw_document_rows`, @@ -30,7 +31,7 @@ mod raw; pub use arrow::encode_as_arrow_ipc; pub(in crate::data::executor) use decode::decode_response_to_docs; -pub use encode::{decode_payload, decode_payload_to_json}; +pub use encode::{decode_payload, decode_payload_to_json, decode_payload_value}; pub(in crate::data::executor) use encode::{ encode, encode_count, encode_json_as_msgpack, encode_json_vec_as_msgpack, encode_serde, encode_value_vec, diff --git a/nodedb/src/util.rs b/nodedb/src/util.rs index 652c63432..909036999 100644 --- a/nodedb/src/util.rs +++ b/nodedb/src/util.rs @@ -5,6 +5,7 @@ pub mod bounded_json; pub mod bounded_msgpack; pub mod rmpv_value; +pub mod wire_json; /// FNV-1a 64-bit hash of a byte slice. /// diff --git a/nodedb/src/util/wire_json.rs b/nodedb/src/util/wire_json.rs new file mode 100644 index 000000000..fef164e06 --- /dev/null +++ b/nodedb/src/util/wire_json.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Edge conversion of a typed cell to the JSON a text protocol emits. +//! +//! pgwire and HTTP render JSON text, and `control::security` redacts a +//! typed cell before rendering it into a redaction preview; each converts +//! through [`value_to_wire_json`], and nowhere else: the one place a byte +//! cell picks its text form is here. This lives outside +//! `control::server::response_shape` so `control::security` — which +//! `response_shape` itself depends on for redaction — never depends back +//! on `control::server`. +//! +//! A `Value::Bytes` cell renders as unpadded standard base64, the text the +//! msgpack → JSON transcoder (`nodedb_types::msgpack_to_json_string`) emits +//! for a msgpack `bin` — not the hex `serde_json::Value::from(Value)` +//! produces. Every other variant renders exactly as that `From` does. + +use base64::Engine; +use nodedb_types::Value; + +/// Render one typed cell as the JSON value a text protocol emits. +pub fn value_to_wire_json(cell: &Value) -> serde_json::Value { + match cell { + Value::Bytes(bytes) => serde_json::Value::String( + base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes), + ), + Value::Array(items) | Value::Set(items) => { + serde_json::Value::Array(items.iter().map(value_to_wire_json).collect()) + } + Value::Object(map) => serde_json::Value::Object( + map.iter() + .map(|(k, v)| (k.clone(), value_to_wire_json(v))) + .collect(), + ), + Value::Null + | Value::Bool(_) + | Value::Integer(_) + | Value::Float(_) + | Value::String(_) + | Value::Uuid(_) + | Value::Ulid(_) + | Value::DateTime(_) + | Value::NaiveDateTime(_) + | Value::Duration(_) + | Value::Decimal(_) + | Value::Geometry(_) + | Value::Regex(_) + | Value::Range { .. } + | Value::Record { .. } + | Value::ArrayCell(_) + | Value::Vector(_) => serde_json::Value::from(cell.clone()), + // `Value` is `#[non_exhaustive]`: a variant this crate cannot name + // renders through the shared `From`, like the scalar arm above. + _ => serde_json::Value::from(cell.clone()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_types::NdbDateTime; + + /// A byte cell renders as the same unpadded base64 the msgpack → JSON + /// transcoder emits for a `bin`, nested or not. + #[test] + fn bytes_render_as_unpadded_base64_like_the_transcoder() { + let bytes = vec![0u8, 255, 7, 1]; + let transcoded = nodedb_types::msgpack_to_json_string( + &nodedb_types::value_to_msgpack(&Value::Bytes(bytes.clone())).expect("encode"), + ) + .expect("transcode"); + let expected: serde_json::Value = serde_json::from_str(&transcoded).expect("json"); + + assert_eq!(value_to_wire_json(&Value::Bytes(bytes.clone())), expected); + assert_eq!( + value_to_wire_json(&Value::Array(vec![Value::Bytes(bytes)])), + serde_json::Value::Array(vec![expected]) + ); + } + + /// An instant renders as ISO-8601 text, a number as itself, NULL as null. + #[test] + fn scalars_render_through_the_shared_from() { + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + assert_eq!( + value_to_wire_json(&Value::NaiveDateTime(at)), + serde_json::Value::String("2020-03-05T10:00:00.000000Z".into()) + ); + assert_eq!( + value_to_wire_json(&Value::Integer(7)), + serde_json::Value::from(7i64) + ); + assert_eq!(value_to_wire_json(&Value::Null), serde_json::Value::Null); + } +} diff --git a/nodedb/tests/inproc/cases/oidc_provider_ddl.rs b/nodedb/tests/inproc/cases/oidc_provider_ddl.rs index 12be7b18a..afbd0db81 100644 --- a/nodedb/tests/inproc/cases/oidc_provider_ddl.rs +++ b/nodedb/tests/inproc/cases/oidc_provider_ddl.rs @@ -224,7 +224,7 @@ async fn show_oidc_providers_exposes_tenant_binding() { assert_eq!( rows.rows[0] .get("tenant_id") - .and_then(serde_json::Value::as_str), + .and_then(nodedb_types::Value::as_str), Some("42") ); } From b80f2a25eea0af09328b1a06e490045b2d6adc17 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 17 Sep 2026 00:02:58 +0800 Subject: [PATCH 12/21] feat(sql): coerce timestamp literals to a typed instant in the planner Timestamp and Timestamptz columns now go through coerce_to_instant alongside the existing Int64/Float64 coercion: a typed instant is retagged to the declared kind, ISO-8601 text is parsed, and a numeric literal is read as epoch milliseconds, the unit every engine's own ingest path already expects. A literal with no representable instant is refused at the statement, naming the column and the literal, and this is the exact result of taking the coercion path once instead of guessing at write or read time. --- .../src/planner/declared_type_coerce.rs | 283 +++++++++++++++++- 1 file changed, 268 insertions(+), 15 deletions(-) diff --git a/nodedb-sql/src/planner/declared_type_coerce.rs b/nodedb-sql/src/planner/declared_type_coerce.rs index cffdad297..0170ac218 100644 --- a/nodedb-sql/src/planner/declared_type_coerce.rs +++ b/nodedb-sql/src/planner/declared_type_coerce.rs @@ -31,21 +31,35 @@ //! //! # Scope //! -//! Only [`SqlDataType::Int64`] and [`SqlDataType::Float64`] columns are -//! coerced, and both are coerced symmetrically — this is not a float special -//! case. They are the two declared types whose stored representation is -//! decided by the declaration rather than by the value: nodedb keeps every -//! integer as an `i64` and every float as an `f64`, and the declared width -//! that drives the wire OID (see `ColumnInfo::int_width` / -//! `ColumnInfo::float_width`) is only honest if the stored cell is a number of -//! that family to begin with. Every other declared type either has one -//! unambiguous literal form already or (like `DECIMAL`) is deliberately -//! carried as text for exactness, and passes through untouched. +//! [`SqlDataType::Int64`] and [`SqlDataType::Float64`] columns are coerced +//! symmetrically — this is not a float special case. They are two declared +//! types whose stored representation is decided by the declaration rather +//! than by the value: nodedb keeps every integer as an `i64` and every float +//! as an `f64`, and the declared width that drives the wire OID (see +//! `ColumnInfo::int_width` / `ColumnInfo::float_width`) is only honest if the +//! stored cell is a number of that family to begin with. //! -//! The conversions mirror the strict document encoder's `coerce_value`, so the -//! two engines accept and reject exactly the same literals for a given -//! declared type. +//! [`SqlDataType::Timestamp`] and [`SqlDataType::Timestamptz`] columns are +//! coerced to a typed instant, once, here, for every engine. An integer time +//! literal carries no unit of its own, and the engines that re-type on write +//! disagree on one (the strict encoder reads an integer as microseconds, the +//! columnar family as milliseconds), while the engines that store the +//! planner's value verbatim would persist the bare integer and leave the read +//! side to guess. Resolving the literal here fixes the unit in one place: +//! SQL reads an integer time literal as epoch milliseconds, the unit every +//! engine already reads it as through its own ingest paths. A text literal is +//! parsed here for the same reason, so an unparseable spelling is refused at +//! the statement instead of being stored as text that no read can render. +//! +//! Every other declared type either has one unambiguous literal form already +//! or (like `DECIMAL`) is deliberately carried as text for exactness, and +//! passes through untouched. +//! +//! The numeric conversions mirror the strict document encoder's +//! `coerce_value`, so the two engines accept and reject exactly the same +//! literals for a given declared type. +use nodedb_types::datetime::NdbDateTime; use rust_decimal::prelude::ToPrimitive; use crate::error::{Result, SqlError}; @@ -146,11 +160,12 @@ fn coerce_value(column: &str, value: SqlValue, declared: &SqlDataType) -> Result match declared { SqlDataType::Int64 => coerce_to_int(column, value), SqlDataType::Float64 => coerce_to_float(column, value), + SqlDataType::Timestamp | SqlDataType::Timestamptz => { + coerce_to_instant(column, value, declared) + } SqlDataType::String | SqlDataType::Bool | SqlDataType::Bytes - | SqlDataType::Timestamp - | SqlDataType::Timestamptz | SqlDataType::Decimal | SqlDataType::Uuid | SqlDataType::Vector(_) @@ -205,6 +220,98 @@ fn coerce_to_float(column: &str, value: SqlValue) -> Result { } } +/// Timestamp column: every accepted literal becomes the one typed instant +/// the column stores, tagged as the declared kind. +/// +/// A typed literal keeps its instant and takes the declared tag, so a +/// `TIMESTAMP` column holds `SqlValue::Timestamp` and a `TIMESTAMPTZ` column +/// holds `SqlValue::Timestamptz` whichever spelling wrote it. Text is parsed +/// as ISO-8601 (`Z`, a `±HH:MM` offset, and fractional seconds are all +/// accepted). A numeric literal is epoch milliseconds — the unit every engine +/// reads an integer time literal as — and a fractional one contributes its +/// integer part, mirroring the columnar engine. `NULL` is left alone: +/// nullability is enforced elsewhere. Any other literal kind is refused +/// naming the column and the kind, because no instant can be read from it. +fn coerce_to_instant(column: &str, value: SqlValue, declared: &SqlDataType) -> Result { + let declared_name = instant_declared_name(declared); + let tag = |at: NdbDateTime| match declared { + SqlDataType::Timestamptz => SqlValue::Timestamptz(at), + SqlDataType::Int64 + | SqlDataType::Float64 + | SqlDataType::String + | SqlDataType::Bool + | SqlDataType::Bytes + | SqlDataType::Timestamp + | SqlDataType::Decimal + | SqlDataType::Uuid + | SqlDataType::Vector(_) + | SqlDataType::Geometry + | SqlDataType::Unknown => SqlValue::Timestamp(at), + }; + match value { + SqlValue::Null => Ok(SqlValue::Null), + SqlValue::Timestamp(at) | SqlValue::Timestamptz(at) => Ok(tag(at)), + SqlValue::String(s) => NdbDateTime::parse(&s) + .map(tag) + .ok_or_else(|| not_representable(column, &s, declared_name)), + SqlValue::Int(millis) => NdbDateTime::from_millis(millis) + .map(tag) + .map_err(|_| not_representable(column, &millis.to_string(), declared_name)), + SqlValue::Float(f) => whole_f64(f.trunc()) + .and_then(|millis| NdbDateTime::from_millis(millis).ok()) + .map(tag) + .ok_or_else(|| not_representable(column, &f.to_string(), declared_name)), + SqlValue::Decimal(d) => d + .trunc() + .to_i64() + .and_then(|millis| NdbDateTime::from_millis(millis).ok()) + .map(tag) + .ok_or_else(|| not_representable(column, &d.to_string(), declared_name)), + SqlValue::Bool(_) | SqlValue::Bytes(_) | SqlValue::Array(_) => { + Err(SqlError::TypeMismatch { + detail: format!( + "column '{column}': cannot store {} as {declared_name}", + literal_kind(&value) + ), + }) + } + } +} + +/// The declared type name an instant coercion error reports. +fn instant_declared_name(declared: &SqlDataType) -> &'static str { + match declared { + SqlDataType::Timestamptz => "TIMESTAMPTZ", + SqlDataType::Int64 + | SqlDataType::Float64 + | SqlDataType::String + | SqlDataType::Bool + | SqlDataType::Bytes + | SqlDataType::Timestamp + | SqlDataType::Decimal + | SqlDataType::Uuid + | SqlDataType::Vector(_) + | SqlDataType::Geometry + | SqlDataType::Unknown => "TIMESTAMP", + } +} + +/// The article-prefixed kind name an error names a refused literal by. +fn literal_kind(value: &SqlValue) -> &'static str { + match value { + SqlValue::Null => "null", + SqlValue::Bool(_) => "a boolean", + SqlValue::Int(_) => "an integer", + SqlValue::Float(_) => "a float", + SqlValue::Decimal(_) => "a decimal", + SqlValue::String(_) => "text", + SqlValue::Bytes(_) => "bytes", + SqlValue::Array(_) => "an array", + SqlValue::Timestamp(_) => "a timestamp", + SqlValue::Timestamptz(_) => "a timestamptz", + } +} + /// `Some(n)` when `f` is a whole number inside `i64`'s range. fn whole_f64(f: f64) -> Option { (f.fract() == 0.0 && f >= i64::MIN as f64 && f <= i64::MAX as f64).then_some(f as i64) @@ -361,6 +468,152 @@ mod tests { ); } + /// `2020-03-05T10:00:00Z` as microseconds since the Unix epoch. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + fn early() -> NdbDateTime { + NdbDateTime::from_micros(EARLY_MICROS) + } + + /// A typed literal keeps its instant and takes the declared tag, in both + /// directions. + #[test] + fn typed_instant_literals_are_retagged_to_the_declared_kind() { + let columns = [ + column("at", SqlDataType::Timestamp), + column("at_tz", SqlDataType::Timestamptz), + ]; + assert_eq!( + coerced(&columns, "at", SqlValue::Timestamptz(early())) + .expect("a typed instant fits a TIMESTAMP column"), + SqlValue::Timestamp(early()) + ); + assert_eq!( + coerced(&columns, "at_tz", SqlValue::Timestamp(early())) + .expect("a typed instant fits a TIMESTAMPTZ column"), + SqlValue::Timestamptz(early()) + ); + } + + /// Text is parsed as ISO-8601: a `Z` suffix, an offset, and fractional + /// seconds all resolve to the one instant, and the offset shifts it. + #[test] + fn text_literals_are_parsed_to_the_instant_they_spell() { + let columns = [column("at", SqlDataType::Timestamp)]; + for spelling in [ + "2020-03-05 10:00:00", + "2020-03-05T10:00:00Z", + "2020-03-05T10:00:00.000000Z", + "2020-03-05T15:30:00+05:30", + ] { + assert_eq!( + coerced(&columns, "at", SqlValue::String(spelling.into())) + .unwrap_or_else(|e| panic!("{spelling} parses: {e}")), + SqlValue::Timestamp(early()), + "{spelling}" + ); + } + } + + /// Text that spells no instant is refused, and the error names the + /// column and the literal. + #[test] + fn non_datetime_text_is_refused_naming_the_column_and_literal() { + let columns = [column("created_at", SqlDataType::Timestamp)]; + let err = coerced( + &columns, + "created_at", + SqlValue::String("not a date".into()), + ) + .expect_err("non-datetime text must be refused"); + let detail = err.to_string(); + assert!( + detail.contains("created_at") && detail.contains("not a date"), + "error must name the column and the literal: {detail}" + ); + } + + /// An integer literal is epoch milliseconds; a fractional literal + /// contributes its integer part. + #[test] + fn numeric_literals_are_epoch_milliseconds() { + let columns = [ + column("at", SqlDataType::Timestamp), + column("at_tz", SqlDataType::Timestamptz), + ]; + assert_eq!( + coerced(&columns, "at", SqlValue::Int(1_583_402_400_000)) + .expect("an integer is epoch milliseconds"), + SqlValue::Timestamp(early()) + ); + assert_eq!( + coerced(&columns, "at_tz", SqlValue::Int(1_583_402_400_000)) + .expect("an integer is epoch milliseconds"), + SqlValue::Timestamptz(early()) + ); + assert_eq!( + coerced(&columns, "at", SqlValue::Float(1_583_402_400_000.7)) + .expect("a float contributes its integer part"), + SqlValue::Timestamp(early()) + ); + assert_eq!( + coerced(&columns, "at", decimal("1583402400000.9")) + .expect("a decimal contributes its integer part"), + SqlValue::Timestamp(early()) + ); + } + + /// A numeric literal whose milliseconds overflow the instant's range, or + /// that is not a finite number, is refused rather than wrapped. + #[test] + fn out_of_range_numeric_literals_are_refused() { + let columns = [column("at", SqlDataType::Timestamp)]; + assert!(coerced(&columns, "at", SqlValue::Int(i64::MAX)).is_err()); + assert!(coerced(&columns, "at", SqlValue::Float(f64::NAN)).is_err()); + assert!(coerced(&columns, "at", SqlValue::Float(f64::INFINITY)).is_err()); + assert!(coerced(&columns, "at", decimal("99999999999999999999999")).is_err()); + } + + /// NULL is untouched, and every literal kind that carries no instant is + /// refused with an error naming the column and the kind. + #[test] + fn null_passes_and_non_instant_kinds_are_refused() { + let columns = [column("created_at", SqlDataType::Timestamptz)]; + assert_eq!( + coerced(&columns, "created_at", SqlValue::Null).expect("null is not an error"), + SqlValue::Null + ); + for (literal, kind) in [ + (SqlValue::Bool(true), "a boolean"), + (SqlValue::Bytes(vec![1, 2]), "bytes"), + (SqlValue::Array(vec![SqlValue::Int(1)]), "an array"), + ] { + let err = coerced(&columns, "created_at", literal) + .expect_err("a literal with no instant must be refused"); + let detail = err.to_string(); + assert!( + detail.contains("created_at") && detail.contains(kind), + "error must name the column and the kind: {detail}" + ); + } + } + + /// `SET at = ` runs the same instant coercion as `VALUES`. + #[test] + fn assignments_coerce_instants() { + let columns = [column("at", SqlDataType::Timestamp)]; + let mut assignments = vec![( + "at".to_string(), + SqlExpr::Literal(SqlValue::Int(1_583_402_400_000)), + )]; + coerce_assignments_to_declared_types(&columns, &mut assignments, None) + .expect("an integer assignment is epoch milliseconds"); + match &assignments[0].1 { + SqlExpr::Literal(value) => assert_eq!(*value, SqlValue::Timestamp(early())), + other => panic!("expected a literal assignment, got {other:?}"), + } + } + /// The primary key keeps its literal exactly as written even when its /// declared type would otherwise re-type it: the engines derive a row's /// identity from the literal's own rendering on both the write and the From 44b1de0e12a71e30c13567b2cdea8ad88707a3a5 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 17 Sep 2026 00:03:24 +0800 Subject: [PATCH 13/21] feat(pgwire): encode timestamp columns in binary result format Splits handler/shape_encode.rs into a directory (cell.rs / response.rs / mod.rs) and adds a binary encode arm for Timestamp/Timestamptz, so a client requesting binary results gets PostgreSQL binary timestamp (microseconds since 2000-01-01) instead of a text-format downgrade. result_format::binary_supported and resolve_result_formats now honour that arm, and ddl_encode / response_shape::project route every cell through the one shared encoder instead of a local JSON-to-text match. response_shape::cell gains cell_text and instant_of, the typed readings a timestamp column renders a cell through, and response_shape::returning retypes announced-timestamp text into a typed instant (NaiveDateTime for TIMESTAMP, DateTime for TIMESTAMPTZ) instead of parsing it as an epoch integer. nodedb_types::NdbDateTime::parse now accepts a trailing UTC offset (+05:30, -0800, +02) in addition to Z, refuses partial or malformed spellings that the previous parser silently truncated, and both timestamp binary decoding and offset parsing are covered by new wire and unit tests. --- nodedb-types/src/datetime/timestamp.rs | 177 +++++-- .../src/control/server/pgwire/ddl_encode.rs | 52 +- .../src/control/server/pgwire/handler/mod.rs | 2 +- .../server/pgwire/handler/prepared/execute.rs | 4 +- .../pgwire/handler/prepared/result_format.rs | 38 +- .../pgwire/handler/shape_encode/cell.rs | 470 ++++++++++++++++++ .../server/pgwire/handler/shape_encode/mod.rs | 10 + .../response.rs} | 408 ++++++--------- .../control/server/pgwire/numeric_narrow.rs | 85 +--- .../src/control/server/response_shape/cell.rs | 247 ++++++++- .../control/server/response_shape/project.rs | 21 +- .../server/response_shape/returning.rs | 51 +- nodedb/src/util/wire_json.rs | 3 +- .../tests/wire/cases/pgwire_extended_query.rs | 32 +- .../cases/strict_typed_column_rendering.rs | 207 +++++++- 15 files changed, 1326 insertions(+), 481 deletions(-) create mode 100644 nodedb/src/control/server/pgwire/handler/shape_encode/cell.rs create mode 100644 nodedb/src/control/server/pgwire/handler/shape_encode/mod.rs rename nodedb/src/control/server/pgwire/handler/{shape_encode.rs => shape_encode/response.rs} (55%) diff --git a/nodedb-types/src/datetime/timestamp.rs b/nodedb-types/src/datetime/timestamp.rs index d050924f7..c88daade6 100644 --- a/nodedb-types/src/datetime/timestamp.rs +++ b/nodedb-types/src/datetime/timestamp.rs @@ -133,53 +133,65 @@ impl NdbDateTime { ) } - /// Parse from ISO 8601 string (basic subset). + /// Parse from ISO 8601 text. /// - /// Supports: `"2024-03-15T10:30:00Z"`, `"2024-03-15T10:30:00.123456Z"`, - /// `"2024-03-15"` (midnight UTC). + /// Accepts `"2024-03-15T10:30:00Z"`, `"2024-03-15 10:30:00"`, + /// `"2024-03-15T10:30:00.123456Z"`, `"2024-03-15"` (midnight UTC), and + /// a trailing UTC offset in place of `Z` — `+05:30`, `-0800`, `+02` — + /// which shifts the result to UTC. Seconds are optional. Every component + /// must parse whole: trailing characters after the seconds, the fraction + /// or the offset are refused rather than dropped, so an unrecognised + /// spelling is `None`, never a silently different instant. pub fn parse(s: &str) -> Option { - let s = s.trim().trim_end_matches('Z').trim_end_matches('z'); + let (body, offset_secs) = split_utc_offset(s.trim())?; - if s.len() == 10 { + if body.len() == 10 { // Date only: "2024-03-15" → midnight UTC. - let parts: Vec<&str> = s.split('-').collect(); - if parts.len() != 3 { - return None; - } - let year: i32 = parts[0].parse().ok()?; - let month: u32 = parts[1].parse().ok()?; - let day: u32 = parts[2].parse().ok()?; - return Self::from_civil(year, month, day, 0, 0, 0, 0); + let (year, month, day) = parse_civil_date(body)?; + return Self::from_civil(year, month, day, 0, 0, 0, 0)?.shift_secs(-offset_secs); } // Full: "2024-03-15T10:30:00" or "2024-03-15T10:30:00.123456" - let (date_part, time_part) = s.split_once('T').or_else(|| s.split_once(' '))?; - let date_parts: Vec<&str> = date_part.split('-').collect(); - if date_parts.len() != 3 { - return None; - } - let year: i32 = date_parts[0].parse().ok()?; - let month: u32 = date_parts[1].parse().ok()?; - let day: u32 = date_parts[2].parse().ok()?; - - let (time_main, frac) = if let Some((t, f)) = time_part.split_once('.') { - (t, f) - } else { - (time_part, "0") + let (date_part, time_part) = body.split_once('T').or_else(|| body.split_once(' '))?; + let (year, month, day) = parse_civil_date(date_part)?; + + let (time_main, frac) = match time_part.split_once('.') { + Some((t, f)) => (t, Some(f)), + None => (time_part, None), }; let time_parts: Vec<&str> = time_main.split(':').collect(); - if time_parts.len() < 2 { + if time_parts.len() < 2 || time_parts.len() > 3 { return None; } let hour: u32 = time_parts[0].parse().ok()?; let minute: u32 = time_parts[1].parse().ok()?; - let second: u32 = time_parts.get(2).and_then(|s| s.parse().ok()).unwrap_or(0); + let second: u32 = match time_parts.get(2) { + Some(text) => text.parse().ok()?, + None => 0, + }; - // Parse fractional seconds (up to microseconds). - let frac_padded = format!("{frac:0<6}"); - let micros: u32 = frac_padded[..6].parse().unwrap_or(0); + // Fractional seconds: one to nine digits, read to microseconds. + let micros: u32 = match frac { + Some(digits) => { + if digits.is_empty() + || digits.len() > 9 + || !digits.bytes().all(|b| b.is_ascii_digit()) + { + return None; + } + let padded = format!("{digits:0<6}"); + padded[..6].parse().ok()? + } + None => 0, + }; - Self::from_civil(year, month, day, hour, minute, second, micros) + Self::from_civil(year, month, day, hour, minute, second, micros)?.shift_secs(-offset_secs) + } + + /// Shift by whole seconds, or `None` on overflow. + fn shift_secs(self, secs: i64) -> Option { + let micros = self.micros.checked_add(secs.checked_mul(1_000_000)?)?; + Some(Self { micros }) } /// Build from civil date components. @@ -262,6 +274,60 @@ impl NdbDateTime { } } +/// Split a trailing `Z` or `±HH[:MM]` / `±HHMM` UTC offset off `s`, returning +/// the remaining text and the offset in seconds east of UTC (`0` when there +/// is none). A `-` inside the date part is never read as an offset: only a +/// sign after the last `T` / ` ` time separator counts. +fn split_utc_offset(s: &str) -> Option<(&str, i64)> { + if let Some(body) = s.strip_suffix('Z').or_else(|| s.strip_suffix('z')) { + return Some((body, 0)); + } + // The offset can only follow the time part, or a bare `YYYY-MM-DD`. + let time_start = match s.rfind(['T', ' ']) { + Some(i) => i + 1, + None => { + let at = s.len().min(10); + if !s.is_char_boundary(at) { + return None; + } + at + } + }; + let Some(sign_at) = s[time_start..].rfind(['+', '-']).map(|i| time_start + i) else { + return Some((s, 0)); + }; + let sign: i64 = if s.as_bytes()[sign_at] == b'-' { -1 } else { 1 }; + let rest = &s[sign_at + 1..]; + let (hh, mm) = match (rest.len(), rest.as_bytes().get(2)) { + (2, None) => (rest, "0"), + (4, Some(b)) if b.is_ascii_digit() => (&rest[..2], &rest[2..]), + (5, Some(&b':')) => (&rest[..2], &rest[3..]), + _ => return None, + }; + if !hh.bytes().all(|b| b.is_ascii_digit()) || !mm.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let hours: i64 = hh.parse().ok()?; + let minutes: i64 = mm.parse().ok()?; + if hours > 23 || minutes > 59 { + return None; + } + Some((&s[..sign_at], sign * (hours * 3600 + minutes * 60))) +} + +/// Parse `YYYY-MM-DD` into its three components. +fn parse_civil_date(s: &str) -> Option<(i32, u32, u32)> { + let parts: Vec<&str> = s.split('-').collect(); + if parts.len() != 3 { + return None; + } + Some(( + parts[0].parse().ok()?, + parts[1].parse().ok()?, + parts[2].parse().ok()?, + )) +} + impl std::fmt::Display for NdbDateTime { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(&self.to_iso8601()) @@ -323,6 +389,53 @@ mod tests { assert_eq!(c.microsecond, 123456); } + /// A trailing offset shifts the instant to UTC; a space separator and + /// missing seconds are accepted. + #[test] + fn datetime_parse_applies_utc_offset() { + let utc = NdbDateTime::parse("2024-06-15T06:30:00Z").unwrap(); + assert_eq!( + NdbDateTime::parse("2024-06-15 12:00:00+05:30").unwrap(), + utc + ); + assert_eq!(NdbDateTime::parse("2024-06-15T12:00:00+0530").unwrap(), utc); + assert_eq!( + NdbDateTime::parse("2024-06-15T04:30:00-02:00").unwrap(), + utc + ); + assert_eq!(NdbDateTime::parse("2024-06-15T04:30-02").unwrap(), utc); + assert_eq!( + NdbDateTime::parse("2024-06-15T06:30:00.250+00:00").unwrap(), + NdbDateTime::from_micros(utc.micros + 250_000) + ); + assert_eq!( + NdbDateTime::parse("2024-06-16+05:30").unwrap(), + NdbDateTime::parse("2024-06-15T18:30:00Z").unwrap() + ); + } + + /// Text that is not a whole timestamp is refused, never read as a + /// different instant. + #[test] + fn datetime_parse_refuses_partial_spellings() { + for text in [ + "yesterday", + "1583402400000000", + "2024-06-15T12:00:00+05:30:00", + "2024-06-15T12:00:xx", + "2024-06-15T12:00:00.abc", + "2024-06-15T12:00:00.", + "2024-06-15T12:00:00.1234567890", + "2024-06-15T12", + "2024-06-15T12:00:00+25:00", + ] { + assert!( + NdbDateTime::parse(text).is_none(), + "{text:?} must not parse" + ); + } + } + #[test] fn datetime_date_only() { let dt = NdbDateTime::parse("2024-03-15").unwrap(); diff --git a/nodedb/src/control/server/pgwire/ddl_encode.rs b/nodedb/src/control/server/pgwire/ddl_encode.rs index c61ff1630..ea11f4194 100644 --- a/nodedb/src/control/server/pgwire/ddl_encode.rs +++ b/nodedb/src/control/server/pgwire/ddl_encode.rs @@ -4,26 +4,21 @@ //! `Response` values. //! //! This is the pgwire entrypoint's consumer of the shared, protocol-neutral -//! DDL dispatch result — the mirror of the native and http encoders. It -//! reproduces the exact wire shape (RowDescription type OIDs, DataRow text -//! bytes, CommandComplete tag) the pgwire DDL router produced directly, -//! because the neutral result captured each column's original OID (as a -//! [`DdlColType`]) and each cell's already-text-rendered value. Values are -//! re-emitted as captured text — never re-typed or re-parsed — so the field -//! bytes are byte-identical. +//! DDL dispatch result — the mirror of the native and http encoders. Each +//! column's `RowDescription` OID comes from the [`DdlColType`] the neutral +//! result captured, and each typed cell renders through the one pgwire cell +//! encoder (`handler::shape_encode::encode_cell`) in that type's text form. use std::sync::Arc; -use pgwire::api::results::{DataRowEncoder, FieldInfo, QueryResponse, Response, Tag}; +use pgwire::api::results::{DataRowEncoder, FieldFormat, FieldInfo, QueryResponse, Response, Tag}; use pgwire::error::{ErrorInfo, PgWireError, PgWireResult}; -use serde_json::Value as JsonValue; -use crate::control::server::response_shape::cell::value_to_wire_json; use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; use crate::control::server::shared::ddl::result::{DdlError, DdlResult}; use super::command_tag::dml_tag; -use super::numeric_narrow::checked_narrow_f32; +use super::handler::shape_encode::encode_cell; use super::types::{ bool_field, bytea_field, float4_array_field, float4_field, float8_array_field, float8_field, int2_field, int4_field, int8_field, json_field, jsonb_field, text_field, timestamp_field, @@ -111,37 +106,10 @@ fn rows_to_response(shaped: ShapedRows) -> PgWireResult { let mut encoder = DataRowEncoder::new(schema.clone()); for (idx, name) in columns.iter().enumerate() { let ct = column_types.get(idx).copied().unwrap_or(DdlColType::Text); - // Each typed cell converts to JSON at this edge before it renders. - match row.get(name).map(value_to_wire_json) { - // Captured text (the transitional wrapper path, and text-typed - // migrated cells): re-emit verbatim so the DataRow bytes match. - Some(JsonValue::String(s)) => encoder.encode_field(&s)?, - // Explicit NULL (or absent key) → -1 length field. - Some(JsonValue::Null) | None => encoder.encode_field(&None::<&str>)?, - // A migrated handler may carry a numeric cell typed rather than - // pre-rendered. Float columns MUST be encoded through pgwire's - // native float path (ryu + extra_float_digits) so the text - // bytes match what the original handler's `encode_field(&f64)` - // produced — string pre-rendering (`f64::to_string`) diverges - // (e.g. `0.0` → "0" vs "0.0"). Integer/other numerics render to - // the same decimal text either way. - Some(ref value @ JsonValue::Number(ref n)) => match ct { - DdlColType::Float8 => match n.as_f64() { - Some(f) => encoder.encode_field(&f)?, - None => encoder.encode_field(&None::)?, - }, - // Narrowing to `real` goes through the shared guard, so a - // finite value beyond f32's range surfaces as SQLSTATE - // 22003 here exactly as it does on the SELECT path, - // instead of reaching the client as `Infinity`. - DdlColType::Float4 => match checked_narrow_f32(value)? { - Some(f) => encoder.encode_field(&f)?, - None => encoder.encode_field(&None::)?, - }, - _ => encoder.encode_field(&n.to_string())?, - }, - // Defensive: any other scalar rendered to its text form. - Some(other) => encoder.encode_field(&other.to_string())?, + match row.get(name) { + // Absent key → -1 length field. + None => encoder.encode_field(&None::<&str>)?, + Some(v) => encode_cell(&mut encoder, name, ct, FieldFormat::Text, v)?, } } encoded_rows.push(Ok(encoder.take_row())); diff --git a/nodedb/src/control/server/pgwire/handler/mod.rs b/nodedb/src/control/server/pgwire/handler/mod.rs index 98631290a..6f4c275fc 100644 --- a/nodedb/src/control/server/pgwire/handler/mod.rs +++ b/nodedb/src/control/server/pgwire/handler/mod.rs @@ -19,7 +19,7 @@ mod routing; mod session_cmds; mod session_explain; mod session_show; -mod shape_encode; +pub(in crate::control::server::pgwire) mod shape_encode; mod sql_exec; mod sql_prepared; mod sql_split; diff --git a/nodedb/src/control/server/pgwire/handler/prepared/execute.rs b/nodedb/src/control/server/pgwire/handler/prepared/execute.rs index 6838c9221..61327b308 100644 --- a/nodedb/src/control/server/pgwire/handler/prepared/execute.rs +++ b/nodedb/src/control/server/pgwire/handler/prepared/execute.rs @@ -125,8 +125,8 @@ impl NodeDbPgHandler { // holds those rows to exactly the announced columns, so the DataRow // field count equals the RowDescription column count by construction. // Resolve the client's requested per-column result formats (from the - // Bind message), downgrading any column whose binary encoding is - // feature-blocked back to text. Parallel to `stmt.result_fields`. + // Bind message), downgrading any column the cell encoder has no + // binary arm for back to text. Parallel to `stmt.result_fields`. let result_formats = resolve_result_formats(&stmt.result_fields, &portal.result_column_format); diff --git a/nodedb/src/control/server/pgwire/handler/prepared/result_format.rs b/nodedb/src/control/server/pgwire/handler/prepared/result_format.rs index b608f75b4..15ffc6fda 100644 --- a/nodedb/src/control/server/pgwire/handler/prepared/result_format.rs +++ b/nodedb/src/control/server/pgwire/handler/prepared/result_format.rs @@ -5,13 +5,12 @@ //! //! A Bind message carries the client's requested result-column format codes //! (via `portal.result_column_format`). NodeDB honors a binary request only -//! for the scalar types whose binary wire encoding is available under the -//! current pgwire feature set: the integers, floats, `bool`, `bytea`, and the -//! string types. Columns whose binary encoding is feature-gated -//! (`Timestamp`/`Timestamptz`/`Json`/`Jsonb`) or that map to no dedicated -//! scalar wire type stay in text format even when binary was requested — this -//! is protocol-legal (the RowDescription advertises text, the client decodes -//! text). +//! for the scalar types the cell encoder (`shape_encode::encode_cell`) has a +//! binary arm for: the integers, floats, `bool`, the string types, and the +//! timestamps. Every other column (`Bytea`/`Json`/`Jsonb`, the arrays, and +//! anything mapping to no dedicated scalar wire type) stays in text format +//! even when binary was requested — this is protocol-legal (the +//! RowDescription advertises text, the client decodes text). use pgwire::api::Type; use pgwire::api::portal::Format; @@ -59,10 +58,9 @@ pub(super) fn pg_type_to_ddl_col_type(t: &Type) -> DdlColType { } } -/// Whether a column of this neutral type can be encoded in binary result -/// format under the current pgwire feature set. Timestamp/Numeric/Json/Jsonb -/// and the array types are excluded — their binary encoders are feature-gated -/// or client-library-specific — and stay text even when binary is requested. +/// Whether a column of this neutral type has a binary arm in the cell +/// encoder. Bytea/Json/Jsonb and the array types have none and stay text +/// even when binary is requested. pub(super) fn binary_supported(ct: DdlColType) -> bool { matches!( ct, @@ -74,6 +72,8 @@ pub(super) fn binary_supported(ct: DdlColType) -> bool { | DdlColType::Bool | DdlColType::Text | DdlColType::Varchar + | DdlColType::Timestamp + | DdlColType::Timestamptz ) } @@ -154,14 +154,15 @@ mod tests { } #[test] - fn binary_supported_excludes_feature_blocked() { + fn binary_supported_excludes_types_without_a_binary_arm() { assert!(binary_supported(DdlColType::Int8)); assert!(binary_supported(DdlColType::Bool)); assert!(binary_supported(DdlColType::Text)); - // bytea binary is not supported in v1 (ambiguous JSON representation): - // it downgrades to text-format like timestamp/numeric/json. + assert!(binary_supported(DdlColType::Timestamp)); + assert!(binary_supported(DdlColType::Timestamptz)); + // bytea has no binary arm (a byte cell renders as base64 text): it + // downgrades to text-format like json and the arrays. assert!(!binary_supported(DdlColType::Bytea)); - assert!(!binary_supported(DdlColType::Timestamp)); assert!(!binary_supported(DdlColType::Json)); assert!(!binary_supported(DdlColType::Float8Array)); } @@ -170,12 +171,15 @@ mod tests { fn unified_binary_downgrades_blocked_types() { let fields = vec![ FieldInfo::new("a".into(), None, None, Type::INT8, FieldFormat::Text), - FieldInfo::new("b".into(), None, None, Type::TIMESTAMP, FieldFormat::Text), + FieldInfo::new("b".into(), None, None, Type::JSON, FieldFormat::Text), + FieldInfo::new("c".into(), None, None, Type::TIMESTAMP, FieldFormat::Text), ]; let formats = resolve_result_formats(&fields, &Format::UnifiedBinary); assert_eq!(formats[0], FieldFormat::Binary); - // TIMESTAMP is feature-blocked -> stays text even under UnifiedBinary. + // JSON has no binary arm -> stays text even under UnifiedBinary. assert_eq!(formats[1], FieldFormat::Text); + // TIMESTAMP has one -> honoured. + assert_eq!(formats[2], FieldFormat::Binary); } #[test] diff --git a/nodedb/src/control/server/pgwire/handler/shape_encode/cell.rs b/nodedb/src/control/server/pgwire/handler/shape_encode/cell.rs new file mode 100644 index 000000000..19bb7429d --- /dev/null +++ b/nodedb/src/control/server/pgwire/handler/shape_encode/cell.rs @@ -0,0 +1,470 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Encode one typed cell into a pgwire `DataRow` per its column type. +//! +//! Every pgwire row encoder renders through [`encode_cell`], so a column +//! type has one rendering. Each arm reads the typed +//! [`nodedb_types::Value`] directly: a timestamp column renders only from an +//! instant — a typed `DateTime`/`NaiveDateTime` or ISO-8601 text — and an +//! integer under it is an error, never a guessed epoch unit. A binary +//! numeric or bool column likewise takes only its own scalar shape, because +//! the RowDescription already told the client how many bytes to read. + +use std::error::Error; + +use bytes::{BufMut, BytesMut}; +use pgwire::api::Type; +use pgwire::api::results::{DataRowEncoder, FieldFormat}; +use pgwire::error::{PgWireError, PgWireResult}; +use pgwire::types::ToSqlText; +use pgwire::types::format::FormatOptions; +use postgres_types::{IsNull, ToSql, accepts, to_sql_checked}; + +use nodedb_types::columnar::IntWidth; +use nodedb_types::error::NodeDbError; +use nodedb_types::{NdbDateTime, Value}; + +use crate::control::server::pgwire::numeric_narrow::{checked_narrow, checked_narrow_f32}; +use crate::control::server::pgwire::types::error_map::{numeric_code_to_sqlstate, sqlstate_error}; +use crate::control::server::response_shape::cell::{cell_text, instant_of, shape_mismatch}; +use crate::control::server::response_shape::types::DdlColType; + +/// Microseconds from the Unix epoch to the PostgreSQL epoch +/// (2000-01-01 00:00:00 UTC), which binary `timestamp`/`timestamptz` count +/// from. +const PG_EPOCH_OFFSET_MICROS: i64 = 946_684_800_000_000; + +/// Encode one cell of column `column` into `encoder` per its column type +/// `ct` and wire `format`. +/// +/// `Value::Null` is SQL NULL for every type and format. Under the text +/// format, `Float8`/`Float4` numbers go through pgwire's native float +/// encoder (ryu + `extra_float_digits`) so their bytes match PostgreSQL, +/// `Timestamp`/`Timestamptz` cells render the instant [`instant_of`] reads +/// as ISO-8601, and every other type renders [`cell_text`]. Under the binary +/// format each scalar type takes only its own shape; a mismatch is an error, +/// because the client reads the advertised type's bytes and a text value +/// under it would be misread. +pub(in crate::control::server::pgwire) fn encode_cell( + encoder: &mut DataRowEncoder, + column: &str, + ct: DdlColType, + format: FieldFormat, + v: &Value, +) -> PgWireResult<()> { + if matches!(v, Value::Null) { + return encoder.encode_field(&None::<&str>); + } + + // A timestamp column renders the one instant its cell denotes; the + // encoder picks text (ISO-8601) or binary (PostgreSQL-epoch micros) from + // the schema's format. + if matches!(ct, DdlColType::Timestamp | DdlColType::Timestamptz) { + let instant = instant_of(v, column).map_err(to_pg_error)?; + return encoder.encode_field(&PgTimestamp(instant)); + } + + // Binary result format: the column's `FieldInfo` is Binary, so + // `encode_field` emits the value's binary wire form. Only the + // binary-supported types reach a Binary format (the resolver downgrades + // the rest to Text upstream); any other `ct` under Binary falls through + // to the text arms below. + if format == FieldFormat::Binary { + match ct { + DdlColType::Int8 => return encoder.encode_field(&integer_of(column, v)?), + // Narrowing casts are fallible, so they are `try_from`, not `as`. + // A stored value wider than the column's declared width cannot be + // transmitted under a narrowed OID: the client reads exactly 2 or 4 + // bytes and would silently decode a wrapped number. Writes are + // range-checked (`nodedb_sql::planner::dml`), so this is + // unreachable for data written through SQL — but rows predating the + // declared width, or arriving via a non-SQL ingest path, can still + // be out of range, and those must surface as an error rather than + // corrupt a value in flight. + DdlColType::Int4 => { + // `as i32` is lossless here: `checked_narrow` has already + // proved the value is inside `IntWidth::I32`. + let n = checked_narrow(integer_of(column, v)?, IntWidth::I32)?; + return encoder.encode_field(&(n as i32)); + } + DdlColType::Int2 => { + let n = checked_narrow(integer_of(column, v)?, IntWidth::I16)?; + return encoder.encode_field(&(n as i16)); + } + DdlColType::Float8 => return encoder.encode_field(&float_of(column, v)?), + // Unlike the integer arms above this is not a range *constraint* + // check: narrowing an f64 rounds rather than wraps, so `1.1` + // arriving as `1.10000002` is correct PostgreSQL `real` behaviour + // and never an error. Only overflow-to-infinity is refused. + DdlColType::Float4 => { + let f = checked_narrow_f32(float_of(column, v)?)?; + return encoder.encode_field(&f); + } + DdlColType::Bool => match v { + Value::Bool(b) => return encoder.encode_field(b), + other => return Err(to_pg_error(shape_mismatch(column, "a bool", other))), + }, + // TEXT/VARCHAR binary wire bytes are identical to text bytes, so + // the cell renders its text form and is emitted as binary. + DdlColType::Text | DdlColType::Varchar => return encoder.encode_field(&cell_text(v)), + DdlColType::Bytea + | DdlColType::Json + | DdlColType::Jsonb + | DdlColType::Float4Array + | DdlColType::Float8Array + | DdlColType::Timestamp + | DdlColType::Timestamptz => {} + } + } + + match ct { + DdlColType::Float8 => match v { + Value::Float(f) => encoder.encode_field(f), + Value::Integer(i) => encoder.encode_field(&(*i as f64)), + other => encoder.encode_field(&cell_text(other)), + }, + // Same overflow guard as the binary arm: the text rendering of a + // `real` column must not silently read `Infinity` for a finite stored + // value either. + DdlColType::Float4 => match v { + Value::Float(f) => encoder.encode_field(&checked_narrow_f32(*f)?), + Value::Integer(i) => encoder.encode_field(&checked_narrow_f32(*i as f64)?), + other => encoder.encode_field(&cell_text(other)), + }, + DdlColType::Text + | DdlColType::Varchar + | DdlColType::Int8 + | DdlColType::Int4 + | DdlColType::Int2 + | DdlColType::Bool + | DdlColType::Bytea + | DdlColType::Json + | DdlColType::Jsonb + | DdlColType::Float4Array + | DdlColType::Float8Array + | DdlColType::Timestamp + | DdlColType::Timestamptz => encoder.encode_field(&cell_text(v)), + } +} + +/// The integer a binary integer column transmits, or the shape error. +fn integer_of(column: &str, v: &Value) -> PgWireResult { + match v { + Value::Integer(i) => Ok(*i), + other => Err(to_pg_error(shape_mismatch(column, "an integer", other))), + } +} + +/// The float a binary float column transmits, or the shape error. An +/// integer cell widens losslessly up to 2^53, as it does under the text +/// format. +fn float_of(column: &str, v: &Value) -> PgWireResult { + match v { + Value::Float(f) => Ok(*f), + Value::Integer(i) => Ok(*i as f64), + other => Err(to_pg_error(shape_mismatch(column, "a float", other))), + } +} + +/// Map a cell error to the pgwire error the client reads, with the SQLSTATE +/// its numeric code maps to. +fn to_pg_error(e: NodeDbError) -> PgWireError { + sqlstate_error(numeric_code_to_sqlstate(e.code()), e.message()) +} + +/// An instant as pgwire encodes it under a `timestamp`/`timestamptz` +/// column: ISO-8601 text under the text format, an `i64` of microseconds +/// since the PostgreSQL epoch under the binary format. +#[derive(Debug)] +struct PgTimestamp(NdbDateTime); + +impl ToSql for PgTimestamp { + fn to_sql( + &self, + _ty: &Type, + out: &mut BytesMut, + ) -> Result> { + let micros = self + .0 + .micros + .checked_sub(PG_EPOCH_OFFSET_MICROS) + .ok_or_else(|| { + Box::::from(format!( + "timestamp {} is out of range for the PostgreSQL binary encoding", + self.0.to_iso8601() + )) + })?; + out.put_i64(micros); + Ok(IsNull::No) + } + + accepts!(TIMESTAMP, TIMESTAMPTZ); + + to_sql_checked!(); +} + +impl ToSqlText for PgTimestamp { + fn to_sql_text( + &self, + _ty: &Type, + out: &mut BytesMut, + _format_options: &FormatOptions, + ) -> Result> { + out.put_slice(self.0.to_iso8601().as_bytes()); + Ok(IsNull::No) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use pgwire::api::results::FieldInfo; + + use super::*; + use crate::control::server::pgwire::ddl_encode::col_type_to_field_with_format; + + /// The one instant these tests use: 2020-03-05T10:00:00Z. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + /// Encode one cell under a one-column schema and return its raw field + /// bytes, or `None` for SQL NULL. + fn encode_one(ct: DdlColType, format: FieldFormat, v: &Value) -> PgWireResult>> { + let schema: Arc> = + Arc::new(vec![col_type_to_field_with_format("c", ct, format)]); + let mut encoder = DataRowEncoder::new(schema); + encode_cell(&mut encoder, "c", ct, format, v)?; + let row = encoder.take_row(); + let data = &row.data; + let len = i32::from_be_bytes([data[0], data[1], data[2], data[3]]); + if len < 0 { + return Ok(None); + } + Ok(Some(data[4..4 + len as usize].to_vec())) + } + + fn encode_text(ct: DdlColType, v: &Value) -> PgWireResult> { + Ok(encode_one(ct, FieldFormat::Text, v)? + .map(|b| String::from_utf8(b).expect("text cell is UTF-8"))) + } + + /// The SQLSTATE and message an encode error carries. + fn error_of(err: PgWireError) -> (String, String) { + let PgWireError::UserError(info) = err else { + panic!("expected a UserError, got {err:?}"); + }; + (info.code.clone(), info.message.clone()) + } + + #[test] + fn null_is_sql_null_under_every_type_and_format() { + for ct in [ + DdlColType::Text, + DdlColType::Int8, + DdlColType::Float8, + DdlColType::Bool, + DdlColType::Timestamp, + ] { + for format in [FieldFormat::Text, FieldFormat::Binary] { + assert_eq!( + encode_one(ct, format, &Value::Null).expect("null encodes"), + None, + "{ct:?}/{format:?} must encode NULL" + ); + } + } + } + + /// An integer under a timestamp column is an error naming the column, + /// under both formats. + #[test] + fn integer_under_timestamp_is_an_error_naming_the_column() { + for ct in [DdlColType::Timestamp, DdlColType::Timestamptz] { + for format in [FieldFormat::Text, FieldFormat::Binary] { + let err = encode_one(ct, format, &Value::Integer(EARLY_MICROS)) + .expect_err("an integer carries no unit"); + let (_, message) = error_of(err); + assert!( + message.contains("column \"c\" holds an integer where a timestamp is required"), + "{ct:?}/{format:?}: {message}" + ); + } + } + } + + #[test] + fn typed_instant_under_timestamp_renders_iso8601() { + let at = NdbDateTime::from_micros(EARLY_MICROS); + assert_eq!( + encode_text(DdlColType::Timestamp, &Value::NaiveDateTime(at)) + .expect("encodes") + .as_deref(), + Some("2020-03-05T10:00:00.000000Z") + ); + assert_eq!( + encode_text(DdlColType::Timestamptz, &Value::DateTime(at)) + .expect("encodes") + .as_deref(), + Some("2020-03-05T10:00:00.000000Z") + ); + } + + /// An ISO string re-parses, so its rendering is canonical, not verbatim. + #[test] + fn iso_text_under_timestamp_renders_canonical_iso8601() { + assert_eq!( + encode_text( + DdlColType::Timestamp, + &Value::String("2020-03-05 10:00:00".into()) + ) + .expect("encodes") + .as_deref(), + Some("2020-03-05T10:00:00.000000Z") + ); + } + + /// Binary `timestamp` is a big-endian `i64` of microseconds since + /// 2000-01-01, under both timestamp types. + #[test] + fn binary_timestamp_is_pg_epoch_micros_big_endian() { + let at = NdbDateTime::from_micros(EARLY_MICROS); + let expected = (EARLY_MICROS - PG_EPOCH_OFFSET_MICROS) + .to_be_bytes() + .to_vec(); + assert_eq!( + encode_one( + DdlColType::Timestamp, + FieldFormat::Binary, + &Value::NaiveDateTime(at) + ) + .expect("encodes"), + Some(expected.clone()) + ); + assert_eq!( + encode_one( + DdlColType::Timestamptz, + FieldFormat::Binary, + &Value::DateTime(at) + ) + .expect("encodes"), + Some(expected) + ); + } + + /// Binary `timestamp` reads back as the same instant through + /// postgres-types' own `SystemTime` decoder. + #[test] + fn binary_timestamp_round_trips_through_postgres_types() { + use postgres_types::FromSql; + + let at = NdbDateTime::from_micros(EARLY_MICROS); + let bytes = encode_one( + DdlColType::Timestamp, + FieldFormat::Binary, + &Value::NaiveDateTime(at), + ) + .expect("encodes") + .expect("not null"); + let decoded = std::time::SystemTime::from_sql(&Type::TIMESTAMP, &bytes).expect("decodes"); + assert_eq!( + decoded, + std::time::UNIX_EPOCH + std::time::Duration::from_micros(EARLY_MICROS as u64) + ); + } + + /// The text arms render the pinned PostgreSQL text forms. + #[test] + fn text_arms_render_postgres_text() { + assert_eq!( + encode_text(DdlColType::Bool, &Value::Bool(true)) + .expect("encodes") + .as_deref(), + Some("t") + ); + assert_eq!( + encode_text(DdlColType::Int8, &Value::Integer(42)) + .expect("encodes") + .as_deref(), + Some("42") + ); + // Float columns go through pgwire's float encoder: shortest form. + assert_eq!( + encode_text(DdlColType::Float8, &Value::Float(0.0)) + .expect("encodes") + .as_deref(), + Some("0") + ); + // A text column keeps the JSON text of a float. + assert_eq!( + encode_text(DdlColType::Text, &Value::Float(0.0)) + .expect("encodes") + .as_deref(), + Some("0.0") + ); + assert_eq!( + encode_text(DdlColType::Text, &Value::Bytes(vec![0, 255, 7])) + .expect("encodes") + .as_deref(), + Some("AP8H") + ); + } + + /// A finite `f64` beyond `f32` range is refused under `real`, in text + /// and binary alike. + #[test] + fn float4_overflow_is_refused_in_both_formats() { + for format in [FieldFormat::Text, FieldFormat::Binary] { + let err = encode_one(DdlColType::Float4, format, &Value::Float(1e39)) + .expect_err("overflow must be refused"); + assert_eq!(error_of(err).0, "22003"); + } + } + + /// A binary scalar column takes only its own shape. + #[test] + fn binary_scalar_shape_mismatch_is_an_error() { + let text = Value::String("42".into()); + for (ct, expected) in [ + (DdlColType::Int8, "an integer"), + (DdlColType::Int4, "an integer"), + (DdlColType::Float8, "a float"), + (DdlColType::Bool, "a bool"), + ] { + let err = encode_one(ct, FieldFormat::Binary, &text) + .expect_err("text under a binary scalar column must be refused"); + let (_, message) = error_of(err); + assert!( + message.contains(&format!("holds text where {expected} is required")), + "{ct:?}: {message}" + ); + } + } + + /// Text under a text-format integer column renders verbatim: a + /// schemaless row can hold `"42"` under a declared `INT`. + #[test] + fn text_under_text_format_integer_renders_verbatim() { + assert_eq!( + encode_text(DdlColType::Int8, &Value::String("42".into())) + .expect("encodes") + .as_deref(), + Some("42") + ); + } + + /// Binary integer narrowing is range-checked. + #[test] + fn binary_narrowing_rejects_out_of_range() { + let err = encode_one( + DdlColType::Int2, + FieldFormat::Binary, + &Value::Integer(i16::MAX as i64 + 1), + ) + .expect_err("out of range must be refused"); + assert_eq!(error_of(err).0, "22003"); + assert_eq!( + encode_one(DdlColType::Int2, FieldFormat::Binary, &Value::Integer(7)).expect("encodes"), + Some(7i16.to_be_bytes().to_vec()) + ); + } +} diff --git a/nodedb/src/control/server/pgwire/handler/shape_encode/mod.rs b/nodedb/src/control/server/pgwire/handler/shape_encode/mod.rs new file mode 100644 index 000000000..2c53367a2 --- /dev/null +++ b/nodedb/src/control/server/pgwire/handler/shape_encode/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Encode a protocol-neutral [`ShapedRows`](crate::control::server::response_shape::types::ShapedRows) +//! into pgwire `DataRow`s and a `Response::Query`. + +pub mod cell; +pub mod response; + +pub(in crate::control::server::pgwire) use cell::encode_cell; +pub(in crate::control::server::pgwire) use response::{encode_shaped_row, shaped_query_response}; diff --git a/nodedb/src/control/server/pgwire/handler/shape_encode.rs b/nodedb/src/control/server/pgwire/handler/shape_encode/response.rs similarity index 55% rename from nodedb/src/control/server/pgwire/handler/shape_encode.rs rename to nodedb/src/control/server/pgwire/handler/shape_encode/response.rs index a86554567..0e1249e05 100644 --- a/nodedb/src/control/server/pgwire/handler/shape_encode.rs +++ b/nodedb/src/control/server/pgwire/handler/shape_encode/response.rs @@ -7,16 +7,8 @@ //! This is the pgwire entrypoint's encoder for the canonical neutral shaping //! core: the SELECT-read path builds a `ShapedRows` once and every protocol //! entrypoint (pgwire, native, http) renders it in its own wire format. Here, -//! each cell renders in its column's PostgreSQL text form, driven by the -//! per-column `DdlColType` the shaper threaded through `ShapedRows`: -//! `Float8`/`Float4` go through pgwire's native float encoder (so `0.0` stays -//! `"0.0"`, not `"0"`), `Timestamp`/`Timestamptz` epoch-microsecond cells -//! render as ISO-8601 text, and everything else (`Text`, integers, `Bool`) -//! falls back to `json_value_to_text` — notably `Bool` as `t`/`f`, not -//! `true`/`false`. -//! -//! Each typed cell converts to JSON at this edge through -//! [`value_to_wire_json`] before it is rendered. +//! each typed cell renders per the `DdlColType` the shaper threaded through +//! `ShapedRows`, through the one cell encoder [`encode_cell`]. use std::sync::Arc; @@ -24,30 +16,23 @@ use pgwire::api::results::{DataRowEncoder, FieldFormat, FieldInfo, QueryResponse use pgwire::error::PgWireResult; use pgwire::messages::data::DataRow; -use nodedb_types::NdbDateTime; -use nodedb_types::columnar::IntWidth; - -use crate::control::server::response_shape::cell::value_to_wire_json; -use crate::control::server::response_shape::project::json_value_to_text; +use crate::control::server::pgwire::ddl_encode::col_type_to_field_with_format; use crate::control::server::response_shape::types::{DdlColType, ShapedRow, ShapedRows}; -use super::super::ddl_encode::col_type_to_field_with_format; -use super::super::numeric_narrow::{checked_narrow, checked_narrow_f32}; +use super::cell::encode_cell; /// Encode one flat row object into a pgwire `DataRow`, using `cell_keys` (in /// order) to look up cells in `row` and `column_types` (parallel to -/// `cell_keys`) to pick each cell's text rendering. +/// `cell_keys`) to pick each cell's rendering. /// /// `cell_keys` are the per-column unique row-map keys derived from the /// display names via `response_shape::project::cell_keys` — identical to the /// display names except where those repeat (`SELECT w.id, b.id`), in which /// case later duplicates carry a `_n` suffix so both cells survive the map. /// -/// Each cell converts to JSON at this edge. Missing keys and a cell whose -/// JSON is `null` (an explicit `Value::Null`, or a float with no JSON form) -/// both encode as SQL NULL. Every other cell renders per its column type via -/// [`encode_typed_cell`]; a missing/short `column_types` entry defaults to -/// `Text`. +/// A missing key encodes as SQL NULL. Every present cell renders per its +/// column type via [`encode_cell`]; a missing/short `column_types` entry +/// defaults to `Text`. pub(in crate::control::server::pgwire) fn encode_shaped_row( schema: &Arc>, cell_keys: &[String], @@ -59,123 +44,14 @@ pub(in crate::control::server::pgwire) fn encode_shaped_row( for (idx, name) in cell_keys.iter().enumerate() { let ct = column_types.get(idx).copied().unwrap_or(DdlColType::Text); let format = formats.get(idx).copied().unwrap_or(FieldFormat::Text); - match row.get(name).map(value_to_wire_json) { - None | Some(serde_json::Value::Null) => { - encoder.encode_field(&None::<&str>)?; - } - Some(v) => encode_typed_cell(&mut encoder, ct, format, &v)?, + match row.get(name) { + None => encoder.encode_field(&None::<&str>)?, + Some(v) => encode_cell(&mut encoder, name, ct, format, v)?, } } Ok(encoder.take_row()) } -/// Encode one non-NULL JSON cell into `encoder` per its column type `ct`. -/// -/// `Float8`/`Float4` numeric cells go through pgwire's native float encoder -/// (ryu + `extra_float_digits`) so their text bytes match PostgreSQL exactly; -/// `Timestamp`/`Timestamptz` epoch-microsecond numbers render as ISO-8601 -/// text. Any cell whose JSON shape doesn't match the typed arm (e.g. an -/// already-formatted timestamp string) falls back to `json_value_to_text`, as -/// does every other type — `Text`, integers, and `Bool` (`t`/`f`). -fn encode_typed_cell( - encoder: &mut DataRowEncoder, - ct: DdlColType, - format: FieldFormat, - v: &serde_json::Value, -) -> PgWireResult<()> { - use serde_json::Value; - - // Binary result format: the column's `FieldInfo` is Binary, so - // `encode_field` emits the value's binary wire form. Extract the correctly - // typed scalar from the JSON cell. A type/shape mismatch cannot fall back - // to the text arms here — the RowDescription already advertises this - // column's binary type, so a text value under it would be misread by the - // client; encode SQL NULL for the (well-typed data should never hit this) - // mismatch instead. Only the feature-supported scalar types reach a Binary - // format (the resolver downgrades the rest to Text upstream); any other - // `ct` under Binary falls through to the text arms below. - if format == FieldFormat::Binary { - match ct { - DdlColType::Int8 => return encoder.encode_field(&v.as_i64()), - // Narrowing casts are fallible, so they are `try_from`, not `as`. - // A stored value wider than the column's declared width cannot be - // transmitted under a narrowed OID: the client reads exactly 2 or 4 - // bytes and would silently decode a wrapped number. Writes are - // range-checked (`nodedb_sql::planner::dml`), so this is - // unreachable for data written through SQL — but rows predating the - // declared width, or arriving via a non-SQL ingest path, can still - // be out of range, and those must surface as an error rather than - // corrupt a value in flight. - DdlColType::Int4 => { - // `as i32` is lossless here: `checked_narrow` has already - // proved the value is inside `IntWidth::I32`. - return match checked_narrow(v, IntWidth::I32)? { - Some(n) => encoder.encode_field(&(n as i32)), - None => encoder.encode_field(&None::), - }; - } - DdlColType::Int2 => { - return match checked_narrow(v, IntWidth::I16)? { - Some(n) => encoder.encode_field(&(n as i16)), - None => encoder.encode_field(&None::), - }; - } - DdlColType::Float8 => return encoder.encode_field(&v.as_f64()), - // Unlike the integer arms above this is not a range *constraint* - // check: narrowing an f64 rounds rather than wraps, so `1.1` - // arriving as `1.10000002` is correct PostgreSQL `real` behaviour - // and never an error. Only overflow-to-infinity is refused. - DdlColType::Float4 => { - return match checked_narrow_f32(v)? { - Some(f) => encoder.encode_field(&f), - None => encoder.encode_field(&None::), - }; - } - DdlColType::Bool => return encoder.encode_field(&v.as_bool()), - DdlColType::Text | DdlColType::Varchar => { - // TEXT/VARCHAR binary wire bytes are identical to text bytes, - // so render any JSON scalar (numbers, bools, strings) to its - // text form exactly as the text arm does, then emit as binary. - return encoder.encode_field(&json_value_to_text(v)); - } - // Feature-blocked / non-scalar types are downgraded to Text by the - // format resolver and never reach here as Binary; if one somehow - // does, fall through to the text arms below. - _ => {} - } - } - - match ct { - DdlColType::Float8 => match v { - Value::Number(n) => match n.as_f64() { - Some(f) => encoder.encode_field(&f), - None => encoder.encode_field(&None::), - }, - _ => encoder.encode_field(&json_value_to_text(v)), - }, - // Same overflow guard as the binary arm: the text rendering of a - // `real` column must not silently read `Infinity` for a finite stored - // value either. - DdlColType::Float4 => match v { - Value::Number(_) => match checked_narrow_f32(v)? { - Some(f) => encoder.encode_field(&f), - None => encoder.encode_field(&None::), - }, - _ => encoder.encode_field(&json_value_to_text(v)), - }, - DdlColType::Timestamp | DdlColType::Timestamptz => match v { - Value::Number(n) => match n.as_i64() { - Some(micros) => { - encoder.encode_field(&NdbDateTime::from_micros(micros).to_iso8601()) - } - None => encoder.encode_field(&json_value_to_text(v)), - }, - _ => encoder.encode_field(&json_value_to_text(v)), - }, - _ => encoder.encode_field(&json_value_to_text(v)), - } -} - /// Build a `Response::Query` from a protocol-neutral [`ShapedRows`], plus its /// carried client-facing notice. /// @@ -225,25 +101,38 @@ pub(in crate::control::server::pgwire) fn shaped_query_response( mod tests { use futures::StreamExt; use nodedb_types::Value; - use pgwire::api::results::{QueryResponse, Response}; + use pgwire::api::results::{FieldFormat, QueryResponse, Response}; + use pgwire::error::PgWireError; use super::shaped_query_response; use crate::control::server::response_shape::types::{DdlColType, ShapedRow, ShapedRows}; - /// Drain a `QueryResponse` stream into a `Vec` of `DataRow`s. - async fn drain(mut qr: QueryResponse) -> Vec { + type DataRow = pgwire::messages::data::DataRow; + + /// Drain a `QueryResponse` stream into a `Vec` of per-row results. + async fn drain_results(mut qr: QueryResponse) -> Vec> { let mut rows = Vec::new(); while let Some(r) = qr.data_rows.next().await { - rows.push(r.unwrap()); + rows.push(r); } rows } - /// Read the text value of field `idx` from a `DataRow`'s raw wire buffer. + /// Drain a `QueryResponse` stream into a `Vec` of `DataRow`s. + async fn drain(qr: QueryResponse) -> Vec { + drain_results(qr) + .await + .into_iter() + .map(|r| r.unwrap()) + .collect() + } + + /// Read the raw bytes of field `idx` from a `DataRow` (or `None` for SQL + /// NULL), without assuming UTF-8 — used to inspect binary-format cells. /// /// Wire format: 4-byte big-endian length + bytes per field; a negative /// length denotes SQL NULL. - fn field_text(row: &pgwire::messages::data::DataRow, idx: usize) -> Option { + fn field_bytes(row: &DataRow, idx: usize) -> Option> { let data = &row.data; let mut offset = 0usize; for field_i in 0..=idx { @@ -268,17 +157,18 @@ mod tests { return None; } if field_i == idx { - return Some( - std::str::from_utf8(&data[offset..offset + len]) - .unwrap() - .to_owned(), - ); + return Some(data[offset..offset + len].to_vec()); } offset += len; } None } + /// Read the text value of field `idx` from a `DataRow`'s raw wire buffer. + fn field_text(row: &DataRow, idx: usize) -> Option { + field_bytes(row, idx).map(|b| String::from_utf8(b).unwrap()) + } + fn make_shaped(columns: &[&str], rows: Vec) -> ShapedRows { let columns: Vec = columns.iter().map(|s| s.to_string()).collect(); let column_types = ShapedRows::text_types(columns.len()); @@ -290,6 +180,15 @@ mod tests { } } + fn shaped_typed(columns: &[&str], column_types: Vec, row: ShapedRow) -> ShapedRows { + ShapedRows { + columns: columns.iter().map(|s| s.to_string()).collect(), + column_types, + rows: vec![row], + notice: None, + } + } + fn obj(pairs: &[(&str, Value)]) -> ShapedRow { pairs .iter() @@ -301,15 +200,19 @@ mod tests { Value::String(s.to_string()) } + fn query_of(response: Response) -> QueryResponse { + let Response::Query(qr) = response else { + panic!("expected Query response"); + }; + qr + } + #[tokio::test] async fn string_cell_renders_verbatim() { let shaped = make_shaped(&["a"], vec![obj(&[("a", text("hello"))])]); let (response, notice) = shaped_query_response(shaped, &[]); assert!(notice.is_none()); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - let rows = drain(qr).await; + let rows = drain(query_of(response)).await; assert_eq!(field_text(&rows[0], 0).as_deref(), Some("hello")); } @@ -323,10 +226,7 @@ mod tests { ], ); let (response, _notice) = shaped_query_response(shaped, &[]); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - let rows = drain(qr).await; + let rows = drain(query_of(response)).await; assert_eq!(field_text(&rows[0], 0).as_deref(), Some("t")); assert_eq!(field_text(&rows[1], 0).as_deref(), Some("f")); } @@ -341,10 +241,7 @@ mod tests { ], ); let (response, _notice) = shaped_query_response(shaped, &[]); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - let rows = drain(qr).await; + let rows = drain(query_of(response)).await; assert_eq!(field_text(&rows[0], 0).as_deref(), Some("42")); assert_eq!(field_text(&rows[1], 0).as_deref(), Some("0.0")); } @@ -353,10 +250,7 @@ mod tests { async fn null_and_missing_column_both_encode_as_sql_null() { let shaped = make_shaped(&["a", "b"], vec![obj(&[("a", Value::Null)])]); let (response, _notice) = shaped_query_response(shaped, &[]); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - let rows = drain(qr).await; + let rows = drain(query_of(response)).await; // "a" was an explicit NULL cell. assert_eq!(field_text(&rows[0], 0), None); // "b" was entirely absent from the row object. @@ -370,10 +264,7 @@ mod tests { vec![obj(&[("a", text("first")), ("b", text("second"))])], ); let (response, _notice) = shaped_query_response(shaped, &[]); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - let rows = drain(qr).await; + let rows = drain(query_of(response)).await; assert_eq!(field_text(&rows[0], 0).as_deref(), Some("second")); assert_eq!(field_text(&rows[0], 1).as_deref(), Some("first")); } @@ -385,36 +276,28 @@ mod tests { async fn typed_columns_report_correct_oid_and_text() { use pgwire::api::Type; - let columns: Vec = ["i", "f", "b", "ts"] - .iter() - .map(|s| s.to_string()) - .collect(); - let column_types = vec![ - DdlColType::Int8, - DdlColType::Float8, - DdlColType::Bool, - DdlColType::Timestamp, - ]; - let row = obj(&[ - ("i", Value::Integer(42)), - // Integral float renders Postgres-style "0" (shortest form) via the - // native float encoder, not serde's "0.0". - ("f", Value::Float(0.0)), - ("b", Value::Bool(true)), - // Epoch microseconds → ISO-8601 text (0 == Unix epoch). - ("ts", Value::Integer(0)), - ]); - let shaped = ShapedRows { - columns, - column_types, - rows: vec![row], - notice: None, - }; + let at = nodedb_types::NdbDateTime::from_micros(0); + let shaped = shaped_typed( + &["i", "f", "b", "ts"], + vec![ + DdlColType::Int8, + DdlColType::Float8, + DdlColType::Bool, + DdlColType::Timestamp, + ], + obj(&[ + ("i", Value::Integer(42)), + // Integral float renders Postgres-style "0" (shortest form) via + // the native float encoder, not serde's "0.0". + ("f", Value::Float(0.0)), + ("b", Value::Bool(true)), + // The Unix epoch as a typed instant → ISO-8601 text. + ("ts", Value::NaiveDateTime(at)), + ]), + ); let (response, _notice) = shaped_query_response(shaped, &[]); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; + let qr = query_of(response); // RowDescription OIDs are the typed ones, not TEXT. let schema = qr.row_schema.clone(); assert_eq!(schema[0].datatype(), &Type::INT8); @@ -432,9 +315,8 @@ mod tests { ); } - /// An instant cell reaches the wire as its ISO-8601 text through the - /// edge conversion, under a TEXT column and under a TIMESTAMP column - /// alike, and a byte cell as the transcoder's unpadded base64. + /// An instant cell renders as its ISO-8601 text under a TEXT column and + /// under a TIMESTAMP column alike, and a byte cell as unpadded base64. #[tokio::test] async fn instant_and_byte_cells_render_through_the_edge_conversion() { let at = nodedb_types::NdbDateTime::from_micros(1_583_402_400_000_000); @@ -448,10 +330,7 @@ mod tests { ]), ); let (response, _notice) = shaped_query_response(shaped, &[]); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - let rows = drain(qr).await; + let rows = drain(query_of(response)).await; assert_eq!( field_text(&rows[0], 0).as_deref(), Some("2020-03-05T10:00:00.000000Z") @@ -463,55 +342,83 @@ mod tests { assert_eq!(field_text(&rows[0], 2).as_deref(), Some("AP8H")); } + /// An ISO string under a TIMESTAMP column re-parses, so the row renders + /// the canonical ISO-8601 form, not the stored spelling. #[tokio::test] - async fn notice_is_preserved_not_dropped() { - let mut shaped = make_shaped(&["a"], vec![obj(&[("a", text("x"))])]); - shaped.notice = Some("heads up".to_owned()); - let (_response, notice) = shaped_query_response(shaped, &[]); - assert_eq!(notice.as_deref(), Some("heads up")); + async fn iso_text_under_timestamp_renders_canonical_iso8601() { + let shaped = shaped_typed( + &["ts"], + vec![DdlColType::Timestamp], + obj(&[("ts", text("2020-03-05 10:00:00"))]), + ); + let (response, _notice) = shaped_query_response(shaped, &[]); + let rows = drain(query_of(response)).await; + assert_eq!( + field_text(&rows[0], 0).as_deref(), + Some("2020-03-05T10:00:00.000000Z") + ); } - /// Read the raw bytes of field `idx` from a `DataRow` (or `None` for SQL - /// NULL), without assuming UTF-8 — used to inspect binary-format cells. - fn field_bytes(row: &pgwire::messages::data::DataRow, idx: usize) -> Option> { - let data = &row.data; - let mut offset = 0usize; - for field_i in 0..=idx { - if offset + 4 > data.len() { - return None; - } - let len = i32::from_be_bytes([ - data[offset], - data[offset + 1], - data[offset + 2], - data[offset + 3], - ]); - offset += 4; - if len < 0 { - if field_i == idx { - return None; - } - continue; - } - let len = len as usize; - if offset + len > data.len() { - return None; - } - if field_i == idx { - return Some(data[offset..offset + len].to_vec()); - } - offset += len; - } - None + /// An integer under a TIMESTAMP column is a row-encode error naming the + /// column — never a guessed epoch unit. + #[tokio::test] + async fn integer_under_timestamp_column_is_an_error_naming_the_column() { + let shaped = shaped_typed( + &["created_at"], + vec![DdlColType::Timestamp], + obj(&[("created_at", Value::Integer(1_583_402_400_000_000))]), + ); + let (response, _notice) = shaped_query_response(shaped, &[]); + let results = drain_results(query_of(response)).await; + let err = results + .into_iter() + .next() + .expect("one row") + .expect_err("an integer under a timestamp column must not encode"); + let PgWireError::UserError(info) = err else { + panic!("expected a UserError, got {err:?}"); + }; + assert!( + info.message + .contains("column \"created_at\" holds an integer where a timestamp is required"), + "message must name the column, got: {}", + info.message + ); } - fn shaped_typed(columns: &[&str], column_types: Vec, row: ShapedRow) -> ShapedRows { - ShapedRows { - columns: columns.iter().map(|s| s.to_string()).collect(), - column_types, - rows: vec![row], - notice: None, + /// Binary `timestamp` cells are the big-endian `i64` of microseconds + /// since the PostgreSQL epoch (2000-01-01), and the RowDescription + /// advertises `FieldFormat::Binary`. + #[tokio::test] + async fn binary_timestamp_encodes_pg_epoch_micros() { + let micros = 1_583_402_400_000_000i64; + let at = nodedb_types::NdbDateTime::from_micros(micros); + let shaped = shaped_typed( + &["ts", "tstz"], + vec![DdlColType::Timestamp, DdlColType::Timestamptz], + obj(&[ + ("ts", Value::NaiveDateTime(at)), + ("tstz", Value::DateTime(at)), + ]), + ); + let formats = vec![FieldFormat::Binary; 2]; + let (response, _notice) = shaped_query_response(shaped, &formats); + let qr = query_of(response); + for f in qr.row_schema.iter() { + assert_eq!(f.format(), FieldFormat::Binary); } + let rows = drain(qr).await; + let expected = (micros - 946_684_800_000_000).to_be_bytes().to_vec(); + assert_eq!(field_bytes(&rows[0], 0), Some(expected.clone())); + assert_eq!(field_bytes(&rows[0], 1), Some(expected)); + } + + #[tokio::test] + async fn notice_is_preserved_not_dropped() { + let mut shaped = make_shaped(&["a"], vec![obj(&[("a", text("x"))])]); + shaped.notice = Some("heads up".to_owned()); + let (_response, notice) = shaped_query_response(shaped, &[]); + assert_eq!(notice.as_deref(), Some("heads up")); } /// A binary-format request for the supported scalar types encodes each @@ -519,8 +426,6 @@ mod tests { /// RowDescription advertises `FieldFormat::Binary`. #[tokio::test] async fn binary_format_encodes_scalar_wire_bytes() { - use pgwire::api::results::FieldFormat; - let shaped = shaped_typed( &["i", "f", "b", "t"], vec![ @@ -538,9 +443,7 @@ mod tests { ); let formats = vec![FieldFormat::Binary; 4]; let (response, _notice) = shaped_query_response(shaped, &formats); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; + let qr = query_of(response); // RowDescription advertises Binary for every column. for f in qr.row_schema.iter() { assert_eq!(f.format(), FieldFormat::Binary); @@ -563,8 +466,6 @@ mod tests { /// binary-encoded; the rest stay text. Mirrors an `Individual` Bind. #[tokio::test] async fn mixed_formats_are_per_column() { - use pgwire::api::results::FieldFormat; - let shaped = shaped_typed( &["i", "j"], vec![DdlColType::Int8, DdlColType::Int8], @@ -572,10 +473,7 @@ mod tests { ); let formats = vec![FieldFormat::Binary, FieldFormat::Text]; let (response, _notice) = shaped_query_response(shaped, &formats); - let Response::Query(qr) = response else { - panic!("expected Query response"); - }; - let rows = drain(qr).await; + let rows = drain(query_of(response)).await; // Column 0 binary: 8 raw bytes. assert_eq!(field_bytes(&rows[0], 0), Some(7i64.to_be_bytes().to_vec())); // Column 1 text: ASCII "9". diff --git a/nodedb/src/control/server/pgwire/numeric_narrow.rs b/nodedb/src/control/server/pgwire/numeric_narrow.rs index 01f3d2aa8..dffa427ee 100644 --- a/nodedb/src/control/server/pgwire/numeric_narrow.rs +++ b/nodedb/src/control/server/pgwire/numeric_narrow.rs @@ -32,9 +32,8 @@ use pgwire::error::PgWireResult; /// Range-check an integer cell against the width its column advertises, before /// it is narrowed for transmission. /// -/// `Ok(None)` means the cell is absent or not an integer and encodes as SQL -/// NULL, matching the wider `Int8` arm. `Ok(Some(n))` guarantees `n` fits -/// `width`, so the caller's narrowing cast is lossless by construction. +/// `Ok(n)` guarantees `n` fits `width`, so the caller's narrowing cast is +/// lossless by construction. /// /// An out-of-range value is a hard error rather than a truncation: the /// column's `RowDescription` already told the client to read two or four @@ -49,14 +48,11 @@ use pgwire::error::PgWireResult; /// exist: rows written before a column's width was declared, and rows /// arriving over non-SQL ingest paths, are not covered by that check. pub(in crate::control::server::pgwire) fn checked_narrow( - v: &serde_json::Value, + n: i64, width: IntWidth, -) -> PgWireResult> { - let Some(n) = v.as_i64() else { - return Ok(None); - }; +) -> PgWireResult { if width.contains(n) { - return Ok(Some(n)); + return Ok(n); } Err(out_of_range(format!( "value {n} is out of range for type {}", @@ -67,9 +63,6 @@ pub(in crate::control::server::pgwire) fn checked_narrow( /// Narrow a float cell to the `f32` a `real` column transmits, refusing the /// one narrowing that is not value-preserving. /// -/// `Ok(None)` means the cell is absent or not a number and encodes as SQL -/// NULL, matching the wider `Float8` arm. -/// /// This is deliberately *not* the float mirror of [`checked_narrow`]'s range /// constraint — see the module docs. Rounding is correct and never an error; /// only a finite `f64` overflowing to infinity is refused, under the same @@ -83,12 +76,7 @@ pub(in crate::control::server::pgwire) fn checked_narrow( /// exist, layered underneath rather than replaced by it: rows written before a /// column's width was declared, and rows arriving over non-SQL ingest paths, /// are not covered by that check. -pub(in crate::control::server::pgwire) fn checked_narrow_f32( - v: &serde_json::Value, -) -> PgWireResult> { - let Some(f) = v.as_f64() else { - return Ok(None); - }; +pub(in crate::control::server::pgwire) fn checked_narrow_f32(f: f64) -> PgWireResult { let narrowed = f as f32; if f.is_finite() && !narrowed.is_finite() { return Err(out_of_range(format!( @@ -96,7 +84,7 @@ pub(in crate::control::server::pgwire) fn checked_narrow_f32( FloatWidth::F32.pg_type_name() ))); } - Ok(Some(narrowed)) + Ok(narrowed) } /// A pgwire `22003` (`numeric_value_out_of_range`) error — the SQLSTATE @@ -113,7 +101,6 @@ fn out_of_range(message: String) -> pgwire::error::PgWireError { #[cfg(test)] mod tests { use super::*; - use serde_json::json; /// The SQLSTATE a guard raised, or a panic naming what it raised instead. fn sqlstate_of(err: pgwire::error::PgWireError) -> String { @@ -126,46 +113,28 @@ mod tests { #[test] fn integer_inside_declared_width_passes_through() { assert_eq!( - checked_narrow(&json!(i16::MAX as i64), IntWidth::I16).expect("in range"), - Some(i16::MAX as i64) - ); - assert_eq!( - checked_narrow(&json!(-1), IntWidth::I32).expect("in range"), - Some(-1) + checked_narrow(i16::MAX as i64, IntWidth::I16).expect("in range"), + i16::MAX as i64 ); + assert_eq!(checked_narrow(-1, IntWidth::I32).expect("in range"), -1); } #[test] fn integer_outside_declared_width_is_rejected() { - let err = checked_narrow(&json!(i16::MAX as i64 + 1), IntWidth::I16) + let err = checked_narrow(i16::MAX as i64 + 1, IntWidth::I16) .expect_err("one past the boundary must be refused"); assert_eq!(sqlstate_of(err), "22003"); } - #[test] - fn non_integer_cell_encodes_as_null() { - assert_eq!( - checked_narrow(&json!(null), IntWidth::I16).expect("null is not an error"), - None - ); - assert_eq!( - checked_narrow(&json!("x"), IntWidth::I16).expect("text is not an error"), - None - ); - } - /// Narrowing rounds, and rounding is not an error: PostgreSQL's `real` /// stores `1.1` as `1.10000002`. Rejecting that would make `REAL` columns /// unreadable for almost every value they hold. #[test] fn float_narrowing_rounds_without_erroring() { - for v in [json!(1.1), json!(0.0), json!(-2.5), json!(3.4e38)] { - let narrowed = checked_narrow_f32(&v) - .expect("an in-range value must narrow without error") - .expect("a JSON number must narrow to Some"); + for v in [1.1, 0.0, -2.5, 3.4e38] { + let narrowed = checked_narrow_f32(v).expect("an in-range value must narrow"); assert_eq!( - narrowed, - v.as_f64().expect("test value is a number") as f32, + narrowed, v as f32, "{v} must narrow by the ordinary rounding cast" ); } @@ -175,9 +144,9 @@ mod tests { /// number silently replaced by infinity on the wire. #[test] fn finite_float_overflowing_f32_is_rejected() { - for v in [json!(1e39), json!(-1e39), json!(f64::MAX), json!(f64::MIN)] { - let err = checked_narrow_f32(&v) - .expect_err("a finite value beyond f32 range must be refused"); + for v in [1e39, -1e39, f64::MAX, f64::MIN] { + let err = + checked_narrow_f32(v).expect_err("a finite value beyond f32 range must be refused"); assert_eq!(sqlstate_of(err), "22003", "{v} must report 22003"); } } @@ -186,8 +155,7 @@ mod tests { /// `f64` nodedb actually stores. #[test] fn float_overflow_error_names_the_declared_type() { - let err = - checked_narrow_f32(&json!(1e300)).expect_err("1e300 must overflow single precision"); + let err = checked_narrow_f32(1e300).expect_err("1e300 must overflow single precision"); let pgwire::error::PgWireError::UserError(info) = err else { panic!("expected a UserError carrying a SQLSTATE"); }; @@ -198,17 +166,18 @@ mod tests { ); } - /// A cell that is not a number at all encodes as SQL NULL, exactly as the - /// wider `Float8` arm does — never an error. + /// A value already infinite or NaN is representable in both widths and + /// passes through as itself, never as an overflow. #[test] - fn non_numeric_float_cell_encodes_as_null() { - assert_eq!( - checked_narrow_f32(&json!(null)).expect("null is not an error"), - None + fn non_finite_float_passes_through() { + assert!( + checked_narrow_f32(f64::NAN) + .expect("NaN is not an overflow") + .is_nan() ); assert_eq!( - checked_narrow_f32(&json!("Infinity")).expect("string is not an error"), - None + checked_narrow_f32(f64::INFINITY).expect("inf is not an overflow"), + f32::INFINITY ); } } diff --git a/nodedb/src/control/server/response_shape/cell.rs b/nodedb/src/control/server/response_shape/cell.rs index 4d497ff9f..061c4f7b1 100644 --- a/nodedb/src/control/server/response_shape/cell.rs +++ b/nodedb/src/control/server/response_shape/cell.rs @@ -1,12 +1,19 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Row-level wrapper around the shared wire-JSON cell conversion. +//! Cell-level conversions a protocol entrypoint renders a shaped row through. //! -//! The scalar conversion itself, [`value_to_wire_json`], lives in +//! The scalar JSON conversion, [`value_to_wire_json`], lives in //! [`crate::util::wire_json`] — a neutral home so `control::security` //! (which `response_shape` depends on for redaction) never has to depend //! back on `control::server`. This module re-exports it and adds the -//! row-level helper, which depends on [`ShapedRow`]. +//! row-level helper, which depends on [`ShapedRow`], plus the two typed +//! readings pgwire renders a cell from: its PostgreSQL text form +//! ([`cell_text`]) and, for a timestamp column, the instant it denotes +//! ([`instant_of`]). + +use base64::Engine; +use nodedb_types::error::NodeDbError; +use nodedb_types::{NdbDateTime, Value}; use super::types::ShapedRow; @@ -18,3 +25,237 @@ pub fn row_to_wire_json(row: &ShapedRow) -> serde_json::Map Option { + match v { + Value::Null => None, + Value::Bool(b) => Some(if *b { "t" } else { "f" }.to_owned()), + Value::Integer(i) => Some(i.to_string()), + Value::Float(f) => serde_json::Number::from_f64(*f).map(|n| n.to_string()), + Value::String(s) => Some(s.clone()), + Value::Bytes(bytes) => Some(base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes)), + Value::DateTime(at) | Value::NaiveDateTime(at) => Some(at.to_iso8601()), + Value::Array(_) + | Value::Object(_) + | Value::Uuid(_) + | Value::Ulid(_) + | Value::Duration(_) + | Value::Decimal(_) + | Value::Geometry(_) + | Value::Set(_) + | Value::Regex(_) + | Value::Range { .. } + | Value::Record { .. } + | Value::ArrayCell(_) + | Value::Vector(_) => wire_json_text(&value_to_wire_json(v)), + // `Value` is `#[non_exhaustive]`: a variant this crate cannot name + // renders through its wire JSON, like the composite arm above. + _ => wire_json_text(&value_to_wire_json(v)), + } +} + +/// The PostgreSQL text form of a wire-JSON cell, or `None` for JSON null. +/// +/// A string is verbatim, a bool is `t`/`f`, and every other JSON value is +/// its `Display` text. +fn wire_json_text(v: &serde_json::Value) -> Option { + match v { + serde_json::Value::Null => None, + serde_json::Value::String(s) => Some(s.clone()), + serde_json::Value::Bool(b) => Some(if *b { "t" } else { "f" }.to_owned()), + other => Some(other.to_string()), + } +} + +/// The instant a cell under a timestamp column denotes. +/// +/// A typed instant is itself. A string is the instant it parses to as +/// ISO-8601. Any other shape, including an integer, is an error: an integer +/// carries no unit, so reading it as any epoch scale would be a guess. The +/// error names `column` and the shape found. +pub fn instant_of(v: &Value, column: &str) -> Result { + match v { + Value::DateTime(at) | Value::NaiveDateTime(at) => Ok(*at), + Value::String(s) => NdbDateTime::parse(s).ok_or_else(|| { + NodeDbError::serialization( + "cell", + format!("column \"{column}\" holds text that is not a timestamp: {s:?}"), + ) + }), + other => Err(shape_mismatch(column, "a timestamp", other)), + } +} + +/// The error for a cell whose shape is not the one its column requires: +/// `column "ts" holds an integer where a timestamp is required`. +pub fn shape_mismatch(column: &str, expected: &str, found: &Value) -> NodeDbError { + NodeDbError::serialization( + "cell", + format!( + "column \"{column}\" holds {} where {expected} is required", + shape_name(found) + ), + ) +} + +/// The article-prefixed shape name an error names a cell by. +fn shape_name(v: &Value) -> &'static str { + match v { + Value::Null => "null", + Value::Bool(_) => "a bool", + Value::Integer(_) => "an integer", + Value::Float(_) => "a float", + Value::String(_) => "text", + Value::Bytes(_) => "bytes", + Value::Array(_) => "an array", + Value::Object(_) => "an object", + Value::Uuid(_) => "a uuid", + Value::Ulid(_) => "a ulid", + Value::DateTime(_) | Value::NaiveDateTime(_) => "a timestamp", + Value::Duration(_) => "a duration", + Value::Decimal(_) => "a decimal", + Value::Geometry(_) => "a geometry", + Value::Set(_) => "a set", + Value::Regex(_) => "a regex", + Value::Range { .. } => "a range", + Value::Record { .. } => "a record", + Value::ArrayCell(_) => "an array cell", + Value::Vector(_) => "a vector", + // `Value` is `#[non_exhaustive]`: a variant this crate cannot name. + _ => "an unsupported value", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The one instant these tests use: 2020-03-05T10:00:00Z. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + /// Every scalar renders the same text its wire JSON renders to, so the + /// direct arms and the JSON edge cannot drift apart. + #[test] + fn scalar_text_matches_the_wire_json_text() { + let at = NdbDateTime::from_micros(EARLY_MICROS); + let scalars = [ + Value::Null, + Value::Bool(true), + Value::Bool(false), + Value::Integer(42), + Value::Integer(-7), + Value::Float(0.0), + Value::Float(1.5), + Value::Float(1e20), + Value::Float(f64::NAN), + Value::Float(f64::INFINITY), + Value::String("hello".into()), + Value::Bytes(vec![0, 255, 7]), + Value::DateTime(at), + Value::NaiveDateTime(at), + Value::Uuid("550e8400-e29b-41d4-a716-446655440000".into()), + Value::Decimal(rust_decimal::Decimal::new(110, 2)), + Value::Duration(nodedb_types::NdbDuration::from_micros(1_500_000)), + Value::Array(vec![Value::Integer(1), Value::Bool(true)]), + Value::Range { + start: None, + end: None, + inclusive: false, + }, + ]; + for v in scalars { + assert_eq!( + cell_text(&v), + wire_json_text(&value_to_wire_json(&v)), + "{v:?} must render the same text either way" + ); + } + } + + /// The pinned text of the scalars pgwire renders most. + #[test] + fn scalar_text_is_the_postgres_text_form() { + let at = NdbDateTime::from_micros(EARLY_MICROS); + assert_eq!(cell_text(&Value::Null), None); + assert_eq!(cell_text(&Value::Bool(true)).as_deref(), Some("t")); + assert_eq!(cell_text(&Value::Bool(false)).as_deref(), Some("f")); + assert_eq!(cell_text(&Value::Integer(42)).as_deref(), Some("42")); + assert_eq!(cell_text(&Value::Float(0.0)).as_deref(), Some("0.0")); + assert_eq!(cell_text(&Value::Float(f64::NAN)), None); + assert_eq!(cell_text(&Value::String("x".into())).as_deref(), Some("x")); + assert_eq!( + cell_text(&Value::Bytes(vec![0, 255, 7])).as_deref(), + Some("AP8H") + ); + assert_eq!( + cell_text(&Value::NaiveDateTime(at)).as_deref(), + Some("2020-03-05T10:00:00.000000Z") + ); + } + + /// A typed instant is itself, whichever kind it is. + #[test] + fn instant_of_reads_a_typed_instant() { + let at = NdbDateTime::from_micros(EARLY_MICROS); + assert_eq!(instant_of(&Value::DateTime(at), "ts").expect("instant"), at); + assert_eq!( + instant_of(&Value::NaiveDateTime(at), "ts").expect("instant"), + at + ); + } + + /// An ISO-8601 string, with or without the `T` separator, is the instant + /// it denotes. + #[test] + fn instant_of_parses_iso8601_text() { + let at = NdbDateTime::from_micros(EARLY_MICROS); + for text in [ + "2020-03-05 10:00:00", + "2020-03-05T10:00:00Z", + "2020-03-05T10:00:00.000000Z", + ] { + assert_eq!( + instant_of(&Value::String(text.into()), "ts").expect("parses"), + at, + "{text} must parse to the instant" + ); + } + } + + /// An integer under a timestamp column is refused, and the error names + /// the column and the shape. + #[test] + fn instant_of_refuses_an_integer() { + let err = instant_of(&Value::Integer(EARLY_MICROS), "created_at") + .expect_err("an integer carries no unit"); + assert!( + err.message() + .contains("column \"created_at\" holds an integer where a timestamp is required"), + "message must name the column and the shape, got: {}", + err.message() + ); + } + + /// Text that is not a timestamp is refused, and the error names it. + #[test] + fn instant_of_refuses_text_that_does_not_parse() { + let err = instant_of(&Value::String("yesterday".into()), "ts") + .expect_err("free text is not an instant"); + assert!( + err.message() + .contains("column \"ts\" holds text that is not a timestamp"), + "message must name the column, got: {}", + err.message() + ); + assert!(err.message().contains("yesterday")); + } +} diff --git a/nodedb/src/control/server/response_shape/project.rs b/nodedb/src/control/server/response_shape/project.rs index 4ee1e2dc1..b976cac52 100644 --- a/nodedb/src/control/server/response_shape/project.rs +++ b/nodedb/src/control/server/response_shape/project.rs @@ -5,9 +5,8 @@ //! These operate on decoded `nodedb_types::Value` rows — no pgwire wire //! types — so they are shared across any protocol-specific response shaper. //! Protocol-specific encode glue that turns these into wire rows (e.g. -//! pgwire's `DataRow`) lives in each protocol's own handler code. -//! `json_value_to_text` is the one JSON helper here: pgwire converts a cell -//! to JSON at its edge and renders that JSON as PostgreSQL text. +//! pgwire's `DataRow`) lives in each protocol's own handler code; the +//! per-cell text form lives in `response_shape::cell`. use std::collections::HashMap; @@ -15,22 +14,6 @@ use nodedb_types::Value; use super::types::ShapedRow; -/// Convert a JSON scalar value to its PostgreSQL text-format string. -/// -/// - `String` values are returned as-is (no extra quoting). -/// - `Bool` uses PostgreSQL text format: `t` for true, `f` for false. -/// - All other scalars (`Number`, `Array`, `Object`) use their JSON -/// `Display` representation; arrays/objects should not normally appear -/// as individual cell values but are rendered faithfully. -pub fn json_value_to_text(v: &serde_json::Value) -> String { - match v { - serde_json::Value::String(s) => s.clone(), - // PostgreSQL text format for boolean is `t`/`f`. - serde_json::Value::Bool(b) => if *b { "t" } else { "f" }.to_string(), - other => other.to_string(), - } -} - /// Flatten a decoded Data-Plane value into typed row objects. /// /// The envelope `id` is a rendered [`StorageKey`](crate::engine::document::store::StorageKey) diff --git a/nodedb/src/control/server/response_shape/returning.rs b/nodedb/src/control/server/response_shape/returning.rs index 72124f3db..e9450ebed 100644 --- a/nodedb/src/control/server/response_shape/returning.rs +++ b/nodedb/src/control/server/response_shape/returning.rs @@ -34,7 +34,7 @@ //! a schemaless row carries fields no catalog column declares — which is the //! same answer `SELECT *` gives for the same row. -use nodedb_types::{NativeCell, NodeDbError, Value}; +use nodedb_types::{NativeCell, NdbDateTime, NodeDbError, Value}; use crate::data::executor::response_codec::{RowsPayload, decode_payload_to_json}; @@ -168,22 +168,25 @@ fn project_onto_announced(schema: &OutputSchema, rows: &[ShapedRow]) -> ShapedRo /// (a schemaless row can hold `"42"` under a declared `INT`), and the /// RowDescription this response is held to announces the catalog type — a /// client that asked for BINARY result format is handed the scalar's wire -/// bytes, which have to come from a number or a bool, not from the digits -/// of its text form. +/// bytes, which have to come from a number, a bool or an instant, not from +/// the digits of its text form. /// -/// A cell that does not parse as its announced type is left as text, which -/// both encoders render verbatim. +/// A cell that does not parse as its announced type is left as text. The +/// numeric and bool encoders render such text verbatim; a timestamp column +/// refuses it, because a cell under a timestamp column renders only from an +/// instant. fn retype_cell(ct: DdlColType, cell: &mut Value) { let Value::String(text) = cell else { return; }; let retyped = match ct { - DdlColType::Int8 - | DdlColType::Int4 - | DdlColType::Int2 - // Epoch microseconds; the encoder formats the number as ISO-8601. - | DdlColType::Timestamp - | DdlColType::Timestamptz => text.parse::().ok().map(Value::Integer), + DdlColType::Int8 | DdlColType::Int4 | DdlColType::Int2 => { + text.parse::().ok().map(Value::Integer) + } + // ISO-8601 text becomes the instant it denotes; an integer string + // carries no unit and stays text. + DdlColType::Timestamp => NdbDateTime::parse(text).map(Value::NaiveDateTime), + DdlColType::Timestamptz => NdbDateTime::parse(text).map(Value::DateTime), // A float that parses but is not finite has no JSON form, so it stays // text. DdlColType::Float8 | DdlColType::Float4 => text @@ -238,7 +241,6 @@ mod tests { use super::*; use crate::control::server::response_shape::cell::value_to_wire_json; use crate::control::server::response_shape::schema::OutputColumn; - use nodedb_types::NdbDateTime; fn text(s: &str) -> Value { Value::String(s.to_string()) @@ -358,6 +360,31 @@ mod tests { assert_eq!(shaped.rows[0]["s"], text("42")); } + /// ISO-8601 text under a timestamp column becomes the instant it denotes, + /// naive under `TIMESTAMP` and UTC under `TIMESTAMPTZ`; an integer string + /// carries no unit and stays text. + #[test] + fn timestamp_text_is_retyped_to_an_instant() { + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + let bytes = payload( + &["ts", "tstz", "digits"], + &[&[ + Some("2020-03-05 10:00:00"), + Some("2020-03-05T10:00:00Z"), + Some("1583402400000000"), + ]], + ); + let schema = announced(&[ + ("ts", DdlColType::Timestamp), + ("tstz", DdlColType::Timestamptz), + ("digits", DdlColType::Timestamp), + ]); + let shaped = shape_returning_rows(&bytes, Some(&schema), None).expect("shape"); + assert_eq!(shaped.rows[0]["ts"], Value::NaiveDateTime(at)); + assert_eq!(shaped.rows[0]["tstz"], Value::DateTime(at)); + assert_eq!(shaped.rows[0]["digits"], text("1583402400000000")); + } + /// A typed cell passes through as itself: an integer stays a number under /// an INT column, an instant stays an instant (and renders at the edge as /// the ISO-8601 text a `SELECT` of the same column renders), and SQL NULL diff --git a/nodedb/src/util/wire_json.rs b/nodedb/src/util/wire_json.rs index fef164e06..989cf247f 100644 --- a/nodedb/src/util/wire_json.rs +++ b/nodedb/src/util/wire_json.rs @@ -2,7 +2,8 @@ //! Edge conversion of a typed cell to the JSON a text protocol emits. //! -//! pgwire and HTTP render JSON text, and `control::security` redacts a +//! HTTP renders JSON text, pgwire renders a composite cell's JSON text +//! (`response_shape::cell::cell_text`), and `control::security` redacts a //! typed cell before rendering it into a redaction preview; each converts //! through [`value_to_wire_json`], and nowhere else: the one place a byte //! cell picks its text form is here. This lives outside diff --git a/nodedb/tests/wire/cases/pgwire_extended_query.rs b/nodedb/tests/wire/cases/pgwire_extended_query.rs index e1080ac1f..612ee388c 100644 --- a/nodedb/tests/wire/cases/pgwire_extended_query.rs +++ b/nodedb/tests/wire/cases/pgwire_extended_query.rs @@ -555,12 +555,13 @@ async fn extended_query_binary_typed_columns_decode() { assert_eq!(name, "hello"); } -/// A `TIMESTAMP` column is feature-blocked for binary encoding, so it stays -/// text even when the client requests binary. The extended query still -/// succeeds and its binary-capable sibling column decodes; the same timestamp -/// is retrievable as text over the simple-query path. +/// A `TIMESTAMP` column honours a binary result request: the cell arrives as +/// PostgreSQL binary `timestamp` (microseconds since 2000-01-01) and decodes +/// through the driver's `SystemTime` reader to the stored instant, next to +/// its integer sibling. The same timestamp renders as ISO-8601 text over the +/// simple-query path. #[tokio::test] -async fn extended_query_timestamp_text_fallback_with_binary_sibling() { +async fn extended_query_timestamp_decodes_from_binary_with_sibling() { let server = TestServer::start().await; server .exec( @@ -574,9 +575,9 @@ async fn extended_query_timestamp_text_fallback_with_binary_sibling() { .await .unwrap(); - // Extended path: the query carrying a timestamp column must succeed, and - // the integer sibling decodes from binary. `n` is declared INT, so it - // advertises OID 23 and decodes as i32. + // Extended path: `tokio-postgres` requests binary for every result + // column. `n` is declared INT, so it advertises OID 23 and decodes as + // i32; `ts` decodes as the stored instant, 2024-01-01T00:00:00Z. let rows = server .client .query("SELECT n, ts FROM ev WHERE id = $1", &[&"a"]) @@ -585,18 +586,19 @@ async fn extended_query_timestamp_text_fallback_with_binary_sibling() { assert_eq!(rows.len(), 1); let n: i32 = rows[0].get("n"); assert_eq!(n, 7); + let ts: std::time::SystemTime = rows[0].get("ts"); + assert_eq!( + ts, + std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_704_067_200), + "binary timestamp must decode to the stored instant" + ); - // Simple-query path returns every column as text, including the timestamp. + // Simple-query path renders the timestamp as ISO-8601 text. let text_rows = server .query_text("SELECT ts FROM ev WHERE id = 'a'") .await .expect("simple-query text select should succeed"); - assert_eq!(text_rows.len(), 1); - assert!( - !text_rows[0].is_empty(), - "timestamp must be present as text on the simple-query path, got {:?}", - text_rows[0] - ); + assert_eq!(text_rows, vec!["2024-01-01T00:00:00.000000Z".to_string()]); } /// Regression lock: when the client declares a parameter's type (via diff --git a/nodedb/tests/wire/cases/strict_typed_column_rendering.rs b/nodedb/tests/wire/cases/strict_typed_column_rendering.rs index 4063e78bd..3dfa4c5b8 100644 --- a/nodedb/tests/wire/cases/strict_typed_column_rendering.rs +++ b/nodedb/tests/wire/cases/strict_typed_column_rendering.rs @@ -1,26 +1,29 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A `document_strict` `TIMESTAMP` column renders the stored instant the same -//! way whichever route reads it, and the same way a timeseries time key does. -//! A `columnar` `TIMESTAMP` column renders the same instant from the live -//! memtable and from a flushed segment. +//! A `document_strict` `TIMESTAMP` or `TIMESTAMPTZ` column renders the +//! stored instant as ISO-8601 whichever route reads it, and the same way a +//! timeseries time key does. A `columnar` `TIMESTAMP` column renders the +//! same instant from the live memtable and from a flushed segment. An +//! integer literal written into a `TIMESTAMP` column is epoch milliseconds +//! on every engine, and a literal that carries no instant is refused. use crate::harness::TestServer; /// The instant every test in this file stores. const EARLY: &str = "2020-03-05 10:00:00"; -/// `EARLY` as a declared `TIMESTAMP` column renders it. The engine stores -/// 1583402400000 epoch milliseconds; a `TIMESTAMP` cell carries epoch -/// microseconds, which the pgwire encoder writes as ISO-8601 UTC. +/// `EARLY` as a declared timestamp column renders it: the engine stores the +/// instant as epoch microseconds and hands the encoder a typed instant, +/// which renders as ISO-8601 UTC. const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; -/// `EARLY` as epoch microseconds — 1583402400000 milliseconds times 1000. -/// A projection that announces no catalog type leaves its cells this number. -const EARLY_MICROS: &str = "1583402400000000"; +/// `EARLY` as seconds since the Unix epoch. +const EARLY_UNIX_SECS: u64 = 1_583_402_400; +/// `EARLY` as milliseconds since the Unix epoch, the unit an integer literal +/// written into a timestamp column denotes. +const EARLY_UNIX_MILLIS: i64 = 1_583_402_400_000; /// A strict `document_strict` collection carrying a `TIMESTAMP` column, read -/// back with a direct `SELECT`, denotes the stored instant. Epoch -/// milliseconds — 1583402400000 — read as microseconds denote 1970-01-19, so -/// a millisecond value fails both arms. +/// back with a direct `SELECT`, renders the stored instant as ISO-8601 — +/// never as an epoch integer. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_strict_timestamp_column_renders_the_stored_instant() { let server = TestServer::start().await; @@ -43,12 +46,83 @@ async fn a_strict_timestamp_column_renders_the_stored_instant() { .query_text("SELECT created_at FROM strict_ts_direct WHERE id = 'r1'") .await .expect("SELECT of a strict TIMESTAMP column must succeed"); - assert_eq!(rows.len(), 1, "one stored row: {rows:?}"); + assert_eq!( + rows, + vec![EARLY_ISO.to_string()], + "a strict TIMESTAMP column must render {EARLY} as {EARLY_ISO}" + ); +} - assert!( - rows[0] == EARLY_ISO || rows[0] == EARLY_MICROS, - "a strict TIMESTAMP column must denote {EARLY}: expected {EARLY_ISO} \ - or {EARLY_MICROS}, got {rows:?}" +/// The `TIMESTAMPTZ` sibling renders the same instant the same way. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_strict_timestamptz_column_renders_the_stored_instant() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION strict_tstz_direct \ + (id TEXT PRIMARY KEY, created_at TIMESTAMPTZ) \ + WITH (engine='document_strict')", + ) + .await + .expect("create strict_tstz_direct"); + server + .exec(&format!( + "INSERT INTO strict_tstz_direct (id, created_at) VALUES ('r1', '{EARLY}')" + )) + .await + .expect("insert into strict_tstz_direct"); + + let rows = server + .query_text("SELECT created_at FROM strict_tstz_direct WHERE id = 'r1'") + .await + .expect("SELECT of a strict TIMESTAMPTZ column must succeed"); + assert_eq!( + rows, + vec![EARLY_ISO.to_string()], + "a strict TIMESTAMPTZ column must render {EARLY} as {EARLY_ISO}" + ); +} + +/// Over the extended protocol the driver requests binary results, and a +/// timestamp column honours that: both `TIMESTAMP` and `TIMESTAMPTZ` decode +/// through the driver's `SystemTime` reader to the stored instant. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_strict_timestamp_column_decodes_from_binary() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION strict_ts_binary \ + (id TEXT PRIMARY KEY, at TIMESTAMP, at_tz TIMESTAMPTZ) \ + WITH (engine='document_strict')", + ) + .await + .expect("create strict_ts_binary"); + server + .exec(&format!( + "INSERT INTO strict_ts_binary (id, at, at_tz) VALUES ('r1', '{EARLY}', '{EARLY}')" + )) + .await + .expect("insert into strict_ts_binary"); + + let rows = server + .client + .query( + "SELECT at, at_tz FROM strict_ts_binary WHERE id = $1", + &[&"r1"], + ) + .await + .expect("extended-protocol SELECT of timestamp columns must succeed"); + assert_eq!(rows.len(), 1, "one stored row"); + let expected = std::time::UNIX_EPOCH + std::time::Duration::from_secs(EARLY_UNIX_SECS); + assert_eq!( + rows[0].get::<_, std::time::SystemTime>("at"), + expected, + "binary TIMESTAMP must decode to {EARLY}" + ); + assert_eq!( + rows[0].get::<_, std::time::SystemTime>("at_tz"), + expected, + "binary TIMESTAMPTZ must decode to {EARLY}" ); } @@ -75,12 +149,10 @@ async fn a_strict_timestamp_column_returned_by_insert_renders_the_stored_instant )) .await .expect("INSERT ... RETURNING of a strict TIMESTAMP column must succeed"); - assert_eq!(rows.len(), 1, "one inserted row: {rows:?}"); - - assert!( - rows[0] == EARLY_ISO || rows[0] == EARLY_MICROS, - "a strict TIMESTAMP column returned by INSERT must denote {EARLY}: expected \ - {EARLY_ISO} or {EARLY_MICROS}, got {rows:?}" + assert_eq!( + rows, + vec![EARLY_ISO.to_string()], + "a strict TIMESTAMP column returned by INSERT must render {EARLY} as {EARLY_ISO}" ); } @@ -236,3 +308,90 @@ async fn a_columnar_timestamp_column_renders_the_same_before_and_after_flush() { instant identically: before={before_flush:?} after={after_flush:?}" ); } + +/// An integer literal written into a `TIMESTAMP` column is epoch +/// milliseconds, resolved once in the planner, so the four engines that +/// store a declared column — strict, key-value, columnar, and schemaless +/// document — all render the one instant it denotes. No engine stores the +/// bare integer, and no read path picks a unit for it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_integer_literal_into_a_timestamp_column_is_epoch_milliseconds() { + let server = TestServer::start().await; + let collections = [ + ("int_ts_strict", "id", "document_strict"), + ("int_ts_kv", "key", "kv"), + ("int_ts_columnar", "id", "columnar"), + ("int_ts_document", "id", "document_schemaless"), + ]; + for (name, key, engine) in collections { + server + .exec(&format!( + "CREATE COLLECTION {name} \ + ({key} TEXT PRIMARY KEY, created_at TIMESTAMP) \ + WITH (engine='{engine}')" + )) + .await + .unwrap_or_else(|e| panic!("create {name} on {engine}: {e}")); + server + .exec(&format!( + "INSERT INTO {name} ({key}, created_at) VALUES ('r1', {EARLY_UNIX_MILLIS})" + )) + .await + .unwrap_or_else(|e| panic!("insert an integer literal into {name}: {e}")); + + let rows = server + .query_text(&format!("SELECT created_at FROM {name} WHERE {key} = 'r1'")) + .await + .unwrap_or_else(|e| panic!("SELECT of {name}.created_at: {e}")); + assert_eq!( + rows, + vec![EARLY_ISO.to_string()], + "{engine}: an integer literal into a TIMESTAMP column must render \ + {EARLY_UNIX_MILLIS} epoch milliseconds as {EARLY_ISO}" + ); + } +} + +/// A literal that carries no instant is refused at the statement, naming +/// the column, rather than stored under the `TIMESTAMP` column for a read +/// to fail on later. Text that spells no date and a boolean are both +/// refused, on an engine that persists the planner's value verbatim. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_non_datetime_literal_into_a_timestamp_column_is_refused() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION refused_ts \ + (id TEXT PRIMARY KEY, created_at TIMESTAMP) \ + WITH (engine='document_schemaless')", + ) + .await + .expect("create refused_ts"); + + let text_error = server + .exec("INSERT INTO refused_ts (id, created_at) VALUES ('r2', 'not a date')") + .await + .expect_err("text that spells no date must be refused"); + assert!( + text_error.contains("created_at"), + "the refusal must name the column: {text_error}" + ); + + let bool_error = server + .exec("INSERT INTO refused_ts (id, created_at) VALUES ('r3', true)") + .await + .expect_err("a boolean carries no instant and must be refused"); + assert!( + bool_error.contains("created_at"), + "the refusal must name the column: {bool_error}" + ); + + let rows = server + .query_text("SELECT id FROM refused_ts") + .await + .expect("SELECT from refused_ts must succeed"); + assert!( + rows.is_empty(), + "a refused INSERT must store nothing: {rows:?}" + ); +} From 896bd804062ea8ad648ddd02a8dfd61fc8713d74 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 17 Sep 2026 11:12:05 +0800 Subject: [PATCH 14/21] fix(types): make Value's coerced comparison partial Value::cmp_coerced silently ordered any unordered pair (an integer against an instant, text against a number, a NaN) as Equal, so a range predicate over such a pair matched every row instead of none. Replace it with partial_cmp_coerced, returning None for a pair with no defined order, the row-level counterpart of PostgreSQL refusing to compare two incompatible types. cmp_coerced remains for ORDER BY / MIN / MAX, where an unordered pair now maps explicitly to Equal so a stable sort keeps it in place. Also extends eq_coerced/ordering to compare a Decimal literal against a Float or Integer through the same numeric path as the existing Integer/Float/String coercions. --- nodedb-types/src/value/coerce.rs | 156 ++++++++++++++++++++++++------- 1 file changed, 121 insertions(+), 35 deletions(-) diff --git a/nodedb-types/src/value/coerce.rs b/nodedb-types/src/value/coerce.rs index 78a86aa3b..8ad93e798 100644 --- a/nodedb-types/src/value/coerce.rs +++ b/nodedb-types/src/value/coerce.rs @@ -39,6 +39,15 @@ impl Value { (Value::String(s), Value::Float(b)) => s.parse::().is_ok_and(|n| n == *b), // Structural equality on ND cells: same coords and same attrs. (Value::ArrayCell(a), Value::ArrayCell(b)) => a == b, + // Two exact decimals compare exactly; a decimal against any other + // number compares through f64 like the arms above. + (Value::Decimal(a), Value::Decimal(b)) => a == b, + (Value::Decimal(_), _) | (_, Value::Decimal(_)) => { + match (numeric_f64(self), numeric_f64(other)) { + (Some(x), Some(y)) => x == y, + _ => false, + } + } (a, b) => match (datetime_micros(a), datetime_micros(b)) { (Some(x), Some(y)) => x == y, _ => false, @@ -46,59 +55,71 @@ impl Value { } } - /// Coerced ordering: `Value` vs `Value` with numeric/string coercion. + /// Coerced partial ordering for predicate evaluation. /// - /// Single source of truth for ordering in filter/sort evaluation. - pub fn cmp_coerced(&self, other: &Value) -> std::cmp::Ordering { + /// Two numbers (or numeric strings) order numerically, two instants (or + /// ISO-8601 strings) by epoch microseconds, two other strings + /// lexicographically, and two ND cells coordinate-major. A pair with no + /// defined order — an integer against an instant, text against a number, + /// a NaN — is `None`, so a range predicate over it matches nothing rather + /// than every row: the row-level counterpart of PostgreSQL refusing to + /// compare the two types. + pub fn partial_cmp_coerced(&self, other: &Value) -> Option { use std::cmp::Ordering; - // ND cells: lexicographic on coords, then attrs. Matches array - // engine cell ordering (coordinate-major). if let (Value::ArrayCell(a), Value::ArrayCell(b)) = (self, other) { for (x, y) in a.coords.iter().zip(b.coords.iter()) { - match x.cmp_coerced(y) { + match x.partial_cmp_coerced(y)? { Ordering::Equal => continue, - non_eq => return non_eq, + non_eq => return Some(non_eq), } } match a.coords.len().cmp(&b.coords.len()) { Ordering::Equal => {} - non_eq => return non_eq, + non_eq => return Some(non_eq), } for (x, y) in a.attrs.iter().zip(b.attrs.iter()) { - match x.cmp_coerced(y) { + match x.partial_cmp_coerced(y)? { Ordering::Equal => continue, - non_eq => return non_eq, + non_eq => return Some(non_eq), } } - return a.attrs.len().cmp(&b.attrs.len()); + return Some(a.attrs.len().cmp(&b.attrs.len())); } - let self_f64 = match self { - Value::Integer(i) => Some(*i as f64), - Value::Float(f) => Some(*f), - Value::String(s) => s.parse::().ok(), - _ => None, - }; - let other_f64 = match other { - Value::Integer(i) => Some(*i as f64), - Value::Float(f) => Some(*f), - Value::String(s) => s.parse::().ok(), - _ => None, - }; - if let (Some(a), Some(b)) = (self_f64, other_f64) { - return a.partial_cmp(&b).unwrap_or(Ordering::Equal); + if let (Some(a), Some(b)) = (numeric_f64(self), numeric_f64(other)) { + return a.partial_cmp(&b); } if let (Some(a), Some(b)) = (datetime_micros(self), datetime_micros(other)) { - return a.cmp(&b); + return Some(a.cmp(&b)); + } + match (self, other) { + (Value::String(a), Value::String(b)) => Some(a.cmp(b)), + _ => None, } - let a_str = match self { - Value::String(s) => s.as_str(), - _ => return Ordering::Equal, - }; - let b_str = match other { - Value::String(s) => s.as_str(), - _ => return Ordering::Equal, - }; - a_str.cmp(b_str) + } + + /// Coerced total ordering for sorting. + /// + /// [`Value::partial_cmp_coerced`] with an unordered pair placed as + /// `Equal`, so a stable sort keeps such rows in their input order. Only + /// for ORDER BY / MIN / MAX style paths that need an `Ordering` for every + /// pair; a predicate uses `partial_cmp_coerced` so an unordered pair + /// matches nothing. + pub fn cmp_coerced(&self, other: &Value) -> std::cmp::Ordering { + self.partial_cmp_coerced(other) + .unwrap_or(std::cmp::Ordering::Equal) + } +} + +/// The number a value denotes for coerced ordering: an integer, a float, or +/// a string that parses as one. `None` for anything else. +fn numeric_f64(v: &Value) -> Option { + use rust_decimal::prelude::ToPrimitive; + match v { + Value::Integer(i) => Some(*i as f64), + Value::Float(f) => Some(*f), + Value::Decimal(d) => d.to_f64(), + Value::String(s) => s.parse::().ok(), + _ => None, } } @@ -117,6 +138,18 @@ fn datetime_micros(v: &Value) -> Option { mod tests { use super::*; + #[test] + fn a_decimal_literal_equals_the_float_it_denotes() { + let d = Value::Decimal(rust_decimal::Decimal::from_str_exact("2.5").expect("decimal")); + assert!(d.eq_coerced(&Value::Float(2.5))); + assert!(Value::Float(2.5).eq_coerced(&d)); + assert!(!d.eq_coerced(&Value::Float(2.25))); + assert_eq!( + d.partial_cmp_coerced(&Value::Integer(3)), + Some(std::cmp::Ordering::Less) + ); + } + #[test] fn eq_coerced_same_type() { assert!(Value::Null.eq_coerced(&Value::Null)); @@ -219,6 +252,59 @@ mod tests { ); } + #[test] + fn partial_cmp_coerced_orders_instants_against_instants_and_iso_text() { + use std::cmp::Ordering; + let earlier = Value::NaiveDateTime(crate::NdbDateTime::from_micros(1_583_402_400_000_000)); + let later = Value::DateTime(crate::NdbDateTime::from_micros(1_583_406_000_000_000)); + assert_eq!(earlier.partial_cmp_coerced(&later), Some(Ordering::Less)); + assert_eq!( + later.partial_cmp_coerced(&Value::String("2020-03-05 10:00:00".into())), + Some(Ordering::Greater) + ); + assert_eq!( + earlier.partial_cmp_coerced(&Value::String("2020-03-05T10:00:00Z".into())), + Some(Ordering::Equal) + ); + } + + /// An integer carries no unit, so it has no order against an instant: + /// `WHERE at >= 5` over an instant column matches nothing, in both + /// orientations. The same holds for text against a number and for NaN. + #[test] + fn partial_cmp_coerced_is_none_for_an_unordered_pair() { + let instant = Value::NaiveDateTime(crate::NdbDateTime::from_micros(1_583_402_400_000_000)); + assert_eq!(instant.partial_cmp_coerced(&Value::Integer(5)), None); + assert_eq!(Value::Integer(5).partial_cmp_coerced(&instant), None); + assert_eq!( + Value::Integer(5).partial_cmp_coerced(&Value::String("abc".into())), + None + ); + assert_eq!( + Value::Bool(true).partial_cmp_coerced(&Value::Integer(1)), + None + ); + assert_eq!(Value::Null.partial_cmp_coerced(&Value::Integer(0)), None); + assert_eq!( + Value::Float(f64::NAN).partial_cmp_coerced(&Value::Float(1.0)), + None + ); + } + + /// The sort order places an unordered pair as `Equal` so a stable sort + /// keeps its input order; an ordered pair sorts as the predicate orders + /// it. + #[test] + fn cmp_coerced_places_an_unordered_pair_as_equal_for_sorting() { + use std::cmp::Ordering; + let instant = Value::NaiveDateTime(crate::NdbDateTime::from_micros(1_583_402_400_000_000)); + assert_eq!(instant.cmp_coerced(&Value::Integer(5)), Ordering::Equal); + assert_eq!( + Value::Integer(5).cmp_coerced(&Value::Integer(7)), + Ordering::Less + ); + } + #[test] fn eq_coerced_symmetry() { let cases = [ From e703e3ff14c651f514dc07abb60781ca3bb86be8 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 17 Sep 2026 11:12:12 +0800 Subject: [PATCH 15/21] refactor(planner): split sql_plan_convert/filter.rs by concern Splits the 500+ line file into a directory: expr_lower.rs (raw WHERE SqlExpr -> ScanFilter reduction) and serialize.rs (Filter tree -> ScanFilter msgpack encoding), with mod.rs re-exporting the entry points. No behavior change. --- .../{filter.rs => filter/expr_lower.rs} | 181 +++--------------- .../planner/sql_plan_convert/filter/mod.rs | 17 ++ .../sql_plan_convert/filter/serialize.rs | 154 +++++++++++++++ 3 files changed, 193 insertions(+), 159 deletions(-) rename nodedb/src/control/planner/sql_plan_convert/{filter.rs => filter/expr_lower.rs} (68%) create mode 100644 nodedb/src/control/planner/sql_plan_convert/filter/mod.rs create mode 100644 nodedb/src/control/planner/sql_plan_convert/filter/serialize.rs diff --git a/nodedb/src/control/planner/sql_plan_convert/filter.rs b/nodedb/src/control/planner/sql_plan_convert/filter/expr_lower.rs similarity index 68% rename from nodedb/src/control/planner/sql_plan_convert/filter.rs rename to nodedb/src/control/planner/sql_plan_convert/filter/expr_lower.rs index 6cef153d8..cb325c4cf 100644 --- a/nodedb/src/control/planner/sql_plan_convert/filter.rs +++ b/nodedb/src/control/planner/sql_plan_convert/filter/expr_lower.rs @@ -1,168 +1,27 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Filter serialization: SqlPlan filters → ScanFilter msgpack bytes. +//! Raw WHERE `SqlExpr` → `ScanFilter` list. //! -//! This is the boundary between the Control Plane planner and the Data Plane -//! scan evaluator. Filter expressions the planner can reduce to simple -//! `(field, op, value)` triples travel as native `ScanFilter` records; any -//! expression the planner cannot reduce — scalar functions in WHERE, -//! non-literal BETWEEN bounds, column arithmetic, `NOT(...)`, IN with -//! computed elements — is shipped verbatim as a `FilterOp::Expr` carrying -//! a `nodedb_query::expr::SqlExpr`. The Data Plane evaluates that against -//! each candidate row via the shared evaluator. +//! Filter expressions the planner can reduce to simple `(field, op, value)` +//! triples travel as native `ScanFilter` records; any expression the planner +//! cannot reduce — scalar functions in WHERE, non-literal BETWEEN bounds, +//! column arithmetic, `NOT(...)`, IN with computed elements — is shipped +//! verbatim as a `FilterOp::Expr` carrying a `nodedb_query::expr::SqlExpr`. +//! The Data Plane evaluates that against each candidate row via the shared +//! evaluator. use nodedb_sql::planner::qualified_name; -use nodedb_sql::types::{Filter, FilterExpr, SqlExpr, SqlValue}; +use nodedb_sql::types::{SqlExpr, SqlValue}; -use super::expr::sql_expr_to_bridge_expr; -use super::value::sql_value_to_nodedb_value; - -/// Convert SqlPlan filters to ScanFilter msgpack bytes. -pub(super) fn serialize_filters(filters: &[Filter]) -> crate::Result> { - if filters.is_empty() { - return Ok(Vec::new()); - } - let scan_filters: Vec = filters - .iter() - .flat_map(|f| filter_to_scan_filters(&f.expr)) - .collect(); - if scan_filters.is_empty() { - return Ok(Vec::new()); - } - encode_scan_filters(&scan_filters) -} - -/// Serialize post-join WHERE filters, preserving table qualifiers inside full -/// expression predicates so they resolve against alias-prefixed merged rows. -pub(super) fn serialize_join_post_filters(filters: &[Filter]) -> crate::Result> { - if filters.is_empty() { - return Ok(Vec::new()); - } - let scan_filters = filters - .iter() - .flat_map(|filter| filter_to_join_scan_filters(&filter.expr)) - .collect::>(); - encode_scan_filters(&scan_filters) -} - -pub(super) fn encode_scan_filters( - filters: &Vec, -) -> crate::Result> { - if filters.is_empty() { - return Ok(Vec::new()); - } - zerompk::to_msgpack_vec(filters).map_err(|e| crate::Error::Serialization { - format: "msgpack".into(), - detail: format!("filter serialization: {e}"), - }) -} - -fn filter_to_join_scan_filters(expr: &FilterExpr) -> Vec { - use nodedb_query::scan_filter::{FilterOp, ScanFilter}; - - match expr { - FilterExpr::And(filters) => filters - .iter() - .flat_map(|filter| filter_to_join_scan_filters(&filter.expr)) - .collect(), - FilterExpr::Or(filters) => vec![ScanFilter { - field: String::new(), - op: FilterOp::Or, - value: nodedb_types::Value::Null, - clauses: filters - .iter() - .map(|filter| filter_to_join_scan_filters(&filter.expr)) - .collect(), - expr: None, - }], - FilterExpr::Expr(sql_expr) => sql_expr_to_join_scan_filters(sql_expr), - _ => filter_to_scan_filters(expr), - } -} - -pub(super) fn filter_to_scan_filters( - expr: &FilterExpr, -) -> Vec { - use nodedb_query::scan_filter::{FilterOp, ScanFilter}; - - match expr { - FilterExpr::Comparison { field, op, value } => { - let filter_op = match op { - nodedb_sql::types::CompareOp::Eq => FilterOp::Eq, - nodedb_sql::types::CompareOp::Ne => FilterOp::Ne, - nodedb_sql::types::CompareOp::Gt => FilterOp::Gt, - nodedb_sql::types::CompareOp::Ge => FilterOp::Gte, - nodedb_sql::types::CompareOp::Lt => FilterOp::Lt, - nodedb_sql::types::CompareOp::Le => FilterOp::Lte, - }; - vec![ScanFilter { - field: field.clone(), - op: filter_op, - value: sql_value_to_nodedb_value(value), - clauses: Vec::new(), - expr: None, - }] - } - FilterExpr::InList { field, values } => { - let arr = values.iter().map(sql_value_to_nodedb_value).collect(); - vec![ScanFilter { - field: field.clone(), - op: FilterOp::In, - value: nodedb_types::Value::Array(arr), - clauses: Vec::new(), - expr: None, - }] - } - FilterExpr::IsNull { field } => { - vec![ScanFilter { - field: field.clone(), - op: FilterOp::IsNull, - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }] - } - FilterExpr::IsNotNull { field } => { - vec![ScanFilter { - field: field.clone(), - op: FilterOp::IsNotNull, - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }] - } - FilterExpr::And(filters) => filters - .iter() - .flat_map(|f| filter_to_scan_filters(&f.expr)) - .collect(), - FilterExpr::Or(filters) => { - let clauses: Vec> = filters - .iter() - .map(|f| filter_to_scan_filters(&f.expr)) - .collect(); - vec![ScanFilter { - field: String::new(), - op: FilterOp::Or, - value: nodedb_types::Value::Null, - clauses, - expr: None, - }] - } - FilterExpr::Expr(sql_expr) => sql_expr_to_scan_filters(sql_expr), - _ => vec![ScanFilter { - field: String::new(), - op: FilterOp::MatchAll, - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }], - } -} +use crate::control::planner::sql_plan_convert::expr::{ + sql_expr_to_bridge_expr, sql_expr_to_bridge_expr_qualified, +}; +use crate::control::planner::sql_plan_convert::value::sql_value_to_nodedb_value; /// Build a `ScanFilter` carrying a full expression predicate. Used whenever /// the planner cannot reduce the WHERE expression to a simple /// `(field, op, value)` tuple. -pub(super) fn expr_filter(expr: &SqlExpr) -> nodedb_query::scan_filter::ScanFilter { +pub(crate) fn expr_filter(expr: &SqlExpr) -> nodedb_query::scan_filter::ScanFilter { nodedb_query::scan_filter::ScanFilter { field: String::new(), op: nodedb_query::scan_filter::FilterOp::Expr, @@ -174,13 +33,13 @@ pub(super) fn expr_filter(expr: &SqlExpr) -> nodedb_query::scan_filter::ScanFilt /// Like [`expr_filter`] but qualifies column references with table names /// for evaluation against join-merged documents. -pub(super) fn expr_filter_qualified(expr: &SqlExpr) -> nodedb_query::scan_filter::ScanFilter { +pub(crate) fn expr_filter_qualified(expr: &SqlExpr) -> nodedb_query::scan_filter::ScanFilter { nodedb_query::scan_filter::ScanFilter { field: String::new(), op: nodedb_query::scan_filter::FilterOp::Expr, value: nodedb_types::Value::Null, clauses: Vec::new(), - expr: Some(super::expr::sql_expr_to_bridge_expr_qualified(expr)), + expr: Some(sql_expr_to_bridge_expr_qualified(expr)), } } @@ -191,7 +50,9 @@ pub(super) fn expr_filter_qualified(expr: &SqlExpr) -> nodedb_query::scan_filter /// use its fast pre-filtered path. Anything that doesn't fit — scalar /// functions on the LHS, arithmetic, NOT, non-literal bounds — is shipped /// as a single `FilterOp::Expr` carrying the whole expression tree. -fn sql_expr_to_join_scan_filters(root: &SqlExpr) -> Vec { +pub(super) fn sql_expr_to_join_scan_filters( + root: &SqlExpr, +) -> Vec { use nodedb_query::scan_filter::{FilterOp, ScanFilter}; match root { @@ -229,7 +90,9 @@ fn sql_expr_to_join_scan_filters(root: &SqlExpr) -> Vec Vec { +pub(super) fn sql_expr_to_scan_filters( + root: &SqlExpr, +) -> Vec { use nodedb_query::scan_filter::{FilterOp, ScanFilter}; match root { diff --git a/nodedb/src/control/planner/sql_plan_convert/filter/mod.rs b/nodedb/src/control/planner/sql_plan_convert/filter/mod.rs new file mode 100644 index 000000000..903fd27a1 --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/filter/mod.rs @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Filter serialization: SqlPlan filters → ScanFilter msgpack bytes. +//! +//! The boundary between the Control Plane planner and the Data Plane scan +//! evaluator. `serialize` walks the planner's `Filter` tree and encodes the +//! `ScanFilter` list; `expr_lower` reduces a raw WHERE `SqlExpr` to native +//! `(field, op, value)` triples where it can and ships anything else verbatim +//! as a `FilterOp::Expr`. + +mod expr_lower; +mod serialize; + +pub(super) use expr_lower::{expr_filter, expr_filter_qualified}; +pub(super) use serialize::{ + encode_scan_filters, filter_to_scan_filters, serialize_filters, serialize_join_post_filters, +}; diff --git a/nodedb/src/control/planner/sql_plan_convert/filter/serialize.rs b/nodedb/src/control/planner/sql_plan_convert/filter/serialize.rs new file mode 100644 index 000000000..b9d974a0b --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/filter/serialize.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `Filter` tree → `ScanFilter` list → msgpack bytes. +//! +//! A `FilterExpr` the planner already reduced to a field/op/value triple maps +//! one-to-one onto a `ScanFilter`; a `FilterExpr::Expr` is handed to +//! [`super::expr_lower`] for reduction. + +use nodedb_sql::types::{Filter, FilterExpr}; + +use super::expr_lower::{sql_expr_to_join_scan_filters, sql_expr_to_scan_filters}; +use crate::control::planner::sql_plan_convert::value::sql_value_to_nodedb_value; + +/// Convert SqlPlan filters to ScanFilter msgpack bytes. +pub(crate) fn serialize_filters(filters: &[Filter]) -> crate::Result> { + if filters.is_empty() { + return Ok(Vec::new()); + } + let scan_filters: Vec = filters + .iter() + .flat_map(|f| filter_to_scan_filters(&f.expr)) + .collect(); + if scan_filters.is_empty() { + return Ok(Vec::new()); + } + encode_scan_filters(&scan_filters) +} + +/// Serialize post-join WHERE filters, preserving table qualifiers inside full +/// expression predicates so they resolve against alias-prefixed merged rows. +pub(crate) fn serialize_join_post_filters(filters: &[Filter]) -> crate::Result> { + if filters.is_empty() { + return Ok(Vec::new()); + } + let scan_filters = filters + .iter() + .flat_map(|filter| filter_to_join_scan_filters(&filter.expr)) + .collect::>(); + encode_scan_filters(&scan_filters) +} + +pub(crate) fn encode_scan_filters( + filters: &Vec, +) -> crate::Result> { + if filters.is_empty() { + return Ok(Vec::new()); + } + zerompk::to_msgpack_vec(filters).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("filter serialization: {e}"), + }) +} + +fn filter_to_join_scan_filters(expr: &FilterExpr) -> Vec { + use nodedb_query::scan_filter::{FilterOp, ScanFilter}; + + match expr { + FilterExpr::And(filters) => filters + .iter() + .flat_map(|filter| filter_to_join_scan_filters(&filter.expr)) + .collect(), + FilterExpr::Or(filters) => vec![ScanFilter { + field: String::new(), + op: FilterOp::Or, + value: nodedb_types::Value::Null, + clauses: filters + .iter() + .map(|filter| filter_to_join_scan_filters(&filter.expr)) + .collect(), + expr: None, + }], + FilterExpr::Expr(sql_expr) => sql_expr_to_join_scan_filters(sql_expr), + _ => filter_to_scan_filters(expr), + } +} + +pub(crate) fn filter_to_scan_filters( + expr: &FilterExpr, +) -> Vec { + use nodedb_query::scan_filter::{FilterOp, ScanFilter}; + + match expr { + FilterExpr::Comparison { field, op, value } => { + let filter_op = match op { + nodedb_sql::types::CompareOp::Eq => FilterOp::Eq, + nodedb_sql::types::CompareOp::Ne => FilterOp::Ne, + nodedb_sql::types::CompareOp::Gt => FilterOp::Gt, + nodedb_sql::types::CompareOp::Ge => FilterOp::Gte, + nodedb_sql::types::CompareOp::Lt => FilterOp::Lt, + nodedb_sql::types::CompareOp::Le => FilterOp::Lte, + }; + vec![ScanFilter { + field: field.clone(), + op: filter_op, + value: sql_value_to_nodedb_value(value), + clauses: Vec::new(), + expr: None, + }] + } + FilterExpr::InList { field, values } => { + let arr = values.iter().map(sql_value_to_nodedb_value).collect(); + vec![ScanFilter { + field: field.clone(), + op: FilterOp::In, + value: nodedb_types::Value::Array(arr), + clauses: Vec::new(), + expr: None, + }] + } + FilterExpr::IsNull { field } => { + vec![ScanFilter { + field: field.clone(), + op: FilterOp::IsNull, + value: nodedb_types::Value::Null, + clauses: Vec::new(), + expr: None, + }] + } + FilterExpr::IsNotNull { field } => { + vec![ScanFilter { + field: field.clone(), + op: FilterOp::IsNotNull, + value: nodedb_types::Value::Null, + clauses: Vec::new(), + expr: None, + }] + } + FilterExpr::And(filters) => filters + .iter() + .flat_map(|f| filter_to_scan_filters(&f.expr)) + .collect(), + FilterExpr::Or(filters) => { + let clauses: Vec> = filters + .iter() + .map(|f| filter_to_scan_filters(&f.expr)) + .collect(); + vec![ScanFilter { + field: String::new(), + op: FilterOp::Or, + value: nodedb_types::Value::Null, + clauses, + expr: None, + }] + } + FilterExpr::Expr(sql_expr) => sql_expr_to_scan_filters(sql_expr), + _ => vec![ScanFilter { + field: String::new(), + op: FilterOp::MatchAll, + value: nodedb_types::Value::Null, + clauses: Vec::new(), + expr: None, + }], + } +} From 1ea3dc7515adfeaa41e1b77d193616598c3fdd50 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 17 Sep 2026 11:12:24 +0800 Subject: [PATCH 16/21] feat(sql): coerce predicate literals against declared instant columns A literal in WHERE, ON, or WHEN compared against a declared TIMESTAMP / TIMESTAMPTZ column reached the planner exactly as written: a numeric literal stayed an untyped integer, and an instant column has no defined order or equality against an integer, so `=` never matched and `>=` matched nothing or matched by accident depending on the evaluator. predicate_coerce::coerce_predicate_literals walks the predicate tree (column literal in either orientation, BETWEEN bounds, IN lists, through AND/OR/NOT) and resolves each literal against the column's declared type through the same coerce_value the write side already uses, so an instant column is always compared against an instant. A literal with no representable instant is refused at the statement, naming the column and the literal, instead of silently matching nothing. The primary-key column and every non-instant declared type are left untouched: the primary key derives identity from the literal's own rendering on both sides, and every other type is already covered by the Data Plane's coerced comparison. Wired into convert_where_to_filters (the one choke point for SELECT, UPDATE/DELETE, HAVING, LATERAL, and MERGE WHEN predicates) and into join ON post-filters. declared_type_coerce::coerce_value is exposed to the new module, and its type-mismatch messages are reworded to state what a literal is, not what a column can't store. --- .../src/planner/declared_type_coerce.rs | 15 +- nodedb-sql/src/planner/join/constraint.rs | 5 + nodedb-sql/src/planner/mod.rs | 1 + nodedb-sql/src/planner/predicate_coerce.rs | 459 ++++++++++++++++++ nodedb-sql/src/planner/select/helpers.rs | 10 +- 5 files changed, 486 insertions(+), 4 deletions(-) create mode 100644 nodedb-sql/src/planner/predicate_coerce.rs diff --git a/nodedb-sql/src/planner/declared_type_coerce.rs b/nodedb-sql/src/planner/declared_type_coerce.rs index 0170ac218..dcc4955f5 100644 --- a/nodedb-sql/src/planner/declared_type_coerce.rs +++ b/nodedb-sql/src/planner/declared_type_coerce.rs @@ -156,7 +156,16 @@ fn is_exempt(exempt_column: Option<&str>, column: &str) -> bool { /// Coerce one literal to `declared`, returning it unchanged when the declared /// type imposes no representation of its own. -fn coerce_value(column: &str, value: SqlValue, declared: &SqlDataType) -> Result { +/// +/// The one rule for a literal bound for a declared column, on the write side +/// (`VALUES`, `SET`) and on the read side (`predicate_coerce`, for a literal +/// compared against the column), so a row is found by the same literal that +/// stored it. +pub(crate) fn coerce_value( + column: &str, + value: SqlValue, + declared: &SqlDataType, +) -> Result { match declared { SqlDataType::Int64 => coerce_to_int(column, value), SqlDataType::Float64 => coerce_to_float(column, value), @@ -270,7 +279,7 @@ fn coerce_to_instant(column: &str, value: SqlValue, declared: &SqlDataType) -> R SqlValue::Bool(_) | SqlValue::Bytes(_) | SqlValue::Array(_) => { Err(SqlError::TypeMismatch { detail: format!( - "column '{column}': cannot store {} as {declared_name}", + "column '{column}': {} is not representable as {declared_name}", literal_kind(&value) ), }) @@ -319,7 +328,7 @@ fn whole_f64(f: f64) -> Option { fn not_representable(column: &str, value: &str, declared: &str) -> SqlError { SqlError::TypeMismatch { - detail: format!("column '{column}': cannot store '{value}' as {declared}"), + detail: format!("column '{column}': '{value}' is not representable as {declared}"), } } diff --git a/nodedb-sql/src/planner/join/constraint.rs b/nodedb-sql/src/planner/join/constraint.rs index 7b7c9a999..df26a5da2 100644 --- a/nodedb-sql/src/planner/join/constraint.rs +++ b/nodedb-sql/src/planner/join/constraint.rs @@ -6,6 +6,7 @@ use sqlparser::ast; use crate::error::{Result, SqlError}; use crate::parser::normalize::normalize_ident; +use crate::planner::predicate_coerce::coerce_predicate_literals; use crate::resolver::ColumnScope; use crate::resolver::columns::TableScope; use crate::resolver::expr::convert_expr; @@ -74,6 +75,10 @@ fn extract_join_constraint( right: Box::new(convert_expr(pred, &ColumnScope::Relations(scope))?), }; } + // A non-equi ON predicate is evaluated per candidate pair by + // the same comparison the WHERE path uses, so its literals + // follow the same declared-instant rule. + coerce_predicate_literals(&mut combined, scope)?; Some(combined) }; Ok((keys, cond)) diff --git a/nodedb-sql/src/planner/mod.rs b/nodedb-sql/src/planner/mod.rs index 6c6062d5e..e222cbc53 100644 --- a/nodedb-sql/src/planner/mod.rs +++ b/nodedb-sql/src/planner/mod.rs @@ -27,6 +27,7 @@ pub mod index_ddl; pub mod join; pub mod lateral; pub mod merge; +pub mod predicate_coerce; pub mod select; pub use select::qualified_name; diff --git a/nodedb-sql/src/planner/predicate_coerce.rs b/nodedb-sql/src/planner/predicate_coerce.rs new file mode 100644 index 000000000..085b58b6e --- /dev/null +++ b/nodedb-sql/src/planner/predicate_coerce.rs @@ -0,0 +1,459 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Coerce the literals a predicate compares against a declared `TIMESTAMP` / +//! `TIMESTAMPTZ` column into typed instants, once, at plan time. +//! +//! # Why this exists +//! +//! A time literal in a `WHERE`, `ON`, or `WHEN` clause reaches the planner as +//! whatever the user typed: `at > 1583402400000`, `at > '2020-03-05 +//! 10:00:00'`, `at > TIMESTAMP '...'`. The write side already resolves the +//! same three spellings to one typed instant through +//! [`super::declared_type_coerce::coerce_value`], so every engine stores an +//! instant. Left as typed, a numeric read literal is an `Integer` the Data +//! Plane compares against an instant: the pair has no defined equality or +//! order, so `=` never matches and `>=` either matches nothing or, on a path +//! that guessed a unit, matches by accident. +//! +//! Resolving the literal here, against the declared column type the scope +//! already carries, means the Data Plane only ever compares instants with +//! instants. It also puts the type error where PostgreSQL puts it: `WHERE at +//! = true` fails the statement naming the column, instead of scanning every +//! row to match none. +//! +//! # Scope +//! +//! Only `TIMESTAMP` / `TIMESTAMPTZ` columns are coerced. Every other declared +//! type is left as written: the Data Plane's coerced comparison already +//! reads `"5"` against `5` and `1.5` against `1`, so no read literal for +//! those types is lost the way a bare integer against an instant is. +//! +//! The primary-key column is exempt, mirroring the write side: the engines +//! derive a row's identity from the literal's own rendering on both the +//! write and the read side, and re-typing one side would make the row +//! unfindable by the other. +//! +//! A column the scope does not declare (a schemaless field, a computed +//! alias, an aggregate output) is left untouched: there is no declared type +//! to coerce to. + +use crate::error::Result; +use crate::resolver::columns::TableScope; +use crate::types::{BinaryOp, ColumnInfo, SqlDataType, SqlExpr, SqlValue, UnaryOp}; + +use super::declared_type_coerce::coerce_value; + +/// Coerce one literal compared against `column` by the read-side rule. +/// +/// The one rule for a literal a predicate compares against a declared +/// column: an instant column (`TIMESTAMP` / `TIMESTAMPTZ`) types the literal +/// through [`coerce_value`], the primary-key column and every other declared +/// type keep the literal as written. `WHERE` clauses reach it through +/// [`coerce_predicate_literals`]; row-level-security policy compilation calls +/// it directly, so a policy literal and a query literal on the same column +/// resolve to the same typed instant. +/// +/// Errors name the column and the literal when the literal carries no +/// instant. +pub fn coerce_read_literal(column: &ColumnInfo, value: SqlValue) -> Result { + if column.is_primary_key || !is_instant(&column.data_type) { + return Ok(value); + } + coerce_value(&column.name, value, &column.data_type) +} + +/// Coerce, in place, every literal `expr` compares against a declared +/// instant column. +/// +/// Walks `AND` / `OR` / `NOT` down to each predicate and handles the shapes +/// that pair one column with literals: `column literal` (either +/// orientation), `column BETWEEN literal AND literal`, and `column IN +/// (literal, …)`. A literal of a kind that carries no instant (a boolean, +/// bytes, an array) is a typed error naming the column. +pub(crate) fn coerce_predicate_literals(expr: &mut SqlExpr, scope: &TableScope) -> Result<()> { + match expr { + SqlExpr::BinaryOp { left, op, right } => match op { + BinaryOp::And | BinaryOp::Or => { + coerce_predicate_literals(left, scope)?; + coerce_predicate_literals(right, scope) + } + BinaryOp::Eq + | BinaryOp::Ne + | BinaryOp::Gt + | BinaryOp::Ge + | BinaryOp::Lt + | BinaryOp::Le => match (left.as_mut(), right.as_mut()) { + (SqlExpr::Column { table, name }, SqlExpr::Literal(value)) + | (SqlExpr::Literal(value), SqlExpr::Column { table, name }) => { + coerce_literal(scope, table.as_deref(), name, value) + } + // Column-vs-column, computed operands, subqueries: no literal + // to coerce. + _ => Ok(()), + }, + BinaryOp::Add + | BinaryOp::Sub + | BinaryOp::Mul + | BinaryOp::Div + | BinaryOp::Mod + | BinaryOp::Concat => Ok(()), + }, + SqlExpr::UnaryOp { + op: UnaryOp::Not, + expr, + } => coerce_predicate_literals(expr, scope), + SqlExpr::Between { + expr, low, high, .. + } => { + let SqlExpr::Column { table, name } = expr.as_ref() else { + return Ok(()); + }; + for bound in [low.as_mut(), high.as_mut()] { + if let SqlExpr::Literal(value) = bound { + coerce_literal(scope, table.as_deref(), name, value)?; + } + } + Ok(()) + } + SqlExpr::InList { expr, list, .. } => { + let SqlExpr::Column { table, name } = expr.as_ref() else { + return Ok(()); + }; + for element in list.iter_mut() { + if let SqlExpr::Literal(value) = element { + coerce_literal(scope, table.as_deref(), name, value)?; + } + } + Ok(()) + } + SqlExpr::UnaryOp { + op: UnaryOp::Neg, .. + } + | SqlExpr::Column { .. } + | SqlExpr::Literal(_) + | SqlExpr::Function { .. } + | SqlExpr::Case { .. } + | SqlExpr::Cast { .. } + | SqlExpr::Subquery(_) + | SqlExpr::Wildcard + | SqlExpr::IsNull { .. } + | SqlExpr::Like { .. } + | SqlExpr::ArrayLiteral(_) => Ok(()), + } +} + +/// Coerce one literal compared against the column `table.name` names, when +/// the scope declares that column as an instant. +fn coerce_literal( + scope: &TableScope, + table: Option<&str>, + name: &str, + value: &mut SqlValue, +) -> Result<()> { + let Some(column) = scope.declared_column(table, name) else { + return Ok(()); + }; + let taken = std::mem::replace(value, SqlValue::Null); + *value = coerce_read_literal(column, taken)?; + Ok(()) +} + +/// Whether a declared type stores a typed instant. +fn is_instant(declared: &SqlDataType) -> bool { + match declared { + SqlDataType::Timestamp | SqlDataType::Timestamptz => true, + SqlDataType::Int64 + | SqlDataType::Float64 + | SqlDataType::String + | SqlDataType::Bool + | SqlDataType::Bytes + | SqlDataType::Decimal + | SqlDataType::Uuid + | SqlDataType::Vector(_) + | SqlDataType::Geometry + | SqlDataType::Unknown => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::resolver::columns::ResolvedTable; + use crate::types::{CollectionInfo, ColumnInfo, EngineType}; + use nodedb_types::datetime::NdbDateTime; + + /// `2020-03-05T10:00:00Z` as epoch milliseconds. + const EARLY_MS: i64 = 1_583_402_400_000; + + fn early() -> NdbDateTime { + NdbDateTime::from_micros(EARLY_MS * 1_000) + } + + fn column(name: &str, data_type: SqlDataType, is_primary_key: bool) -> ColumnInfo { + ColumnInfo { + name: name.to_string(), + data_type, + nullable: true, + is_primary_key, + default: None, + raw_type: None, + int_width: None, + float_width: None, + } + } + + /// `t (id TEXT PRIMARY KEY, at TIMESTAMP, at_tz TIMESTAMPTZ, n BIGINT, + /// pk_at TIMESTAMP PRIMARY KEY)` under alias `a`. + fn scope() -> TableScope { + let info = CollectionInfo { + name: "t".into(), + engine: EngineType::DocumentStrict, + columns: vec![ + column("id", SqlDataType::String, true), + column("at", SqlDataType::Timestamp, false), + column("at_tz", SqlDataType::Timestamptz, false), + column("n", SqlDataType::Int64, false), + column("pk_at", SqlDataType::Timestamp, true), + ], + primary_key: Some("id".into()), + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentStrict), + }; + TableScope::single(ResolvedTable { + name: info.name.clone(), + alias: Some("a".into()), + info, + }) + .expect("single-relation scope") + } + + fn col(table: Option<&str>, name: &str) -> SqlExpr { + SqlExpr::Column { + table: table.map(str::to_string), + name: name.into(), + } + } + + fn lit(value: SqlValue) -> SqlExpr { + SqlExpr::Literal(value) + } + + fn cmp(left: SqlExpr, op: BinaryOp, right: SqlExpr) -> SqlExpr { + SqlExpr::BinaryOp { + left: Box::new(left), + op, + right: Box::new(right), + } + } + + fn literal_of(expr: &SqlExpr) -> &SqlValue { + match expr { + SqlExpr::Literal(v) => v, + other => panic!("expected a literal, got {other:?}"), + } + } + + fn coerced(mut expr: SqlExpr) -> SqlExpr { + coerce_predicate_literals(&mut expr, &scope()).expect("predicate coerces"); + expr + } + + #[test] + fn column_op_literal_coerces_a_numeric_literal_to_the_declared_instant() { + let expr = coerced(cmp( + col(None, "at"), + BinaryOp::Gt, + lit(SqlValue::Int(EARLY_MS)), + )); + let SqlExpr::BinaryOp { right, .. } = &expr else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(right), &SqlValue::Timestamp(early())); + } + + #[test] + fn literal_op_column_is_coerced_in_the_mirrored_orientation() { + let expr = coerced(cmp( + lit(SqlValue::String("2020-03-05 10:00:00".into())), + BinaryOp::Lt, + col(Some("a"), "at_tz"), + )); + let SqlExpr::BinaryOp { left, .. } = &expr else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(left), &SqlValue::Timestamptz(early())); + } + + #[test] + fn between_coerces_both_bounds() { + let expr = coerced(SqlExpr::Between { + expr: Box::new(col(None, "at")), + low: Box::new(lit(SqlValue::Int(EARLY_MS))), + high: Box::new(lit(SqlValue::String("2020-03-05T11:00:00Z".into()))), + negated: false, + }); + let SqlExpr::Between { low, high, .. } = &expr else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(low), &SqlValue::Timestamp(early())); + assert_eq!( + literal_of(high), + &SqlValue::Timestamp(NdbDateTime::from_micros((EARLY_MS + 3_600_000) * 1_000)) + ); + } + + #[test] + fn in_list_coerces_every_literal_element() { + let expr = coerced(SqlExpr::InList { + expr: Box::new(col(None, "at")), + list: vec![ + lit(SqlValue::Int(EARLY_MS)), + lit(SqlValue::String("2020-03-05 10:00:00".into())), + ], + negated: true, + }); + let SqlExpr::InList { list, .. } = &expr else { + panic!("shape preserved"); + }; + for element in list { + assert_eq!(literal_of(element), &SqlValue::Timestamp(early())); + } + } + + #[test] + fn nested_and_or_not_reach_every_predicate() { + let expr = coerced(SqlExpr::UnaryOp { + op: UnaryOp::Not, + expr: Box::new(cmp( + cmp(col(None, "at"), BinaryOp::Ge, lit(SqlValue::Int(EARLY_MS))), + BinaryOp::Or, + cmp( + cmp(col(None, "n"), BinaryOp::Eq, lit(SqlValue::Int(1))), + BinaryOp::And, + cmp( + col(None, "at_tz"), + BinaryOp::Le, + lit(SqlValue::Int(EARLY_MS)), + ), + ), + )), + }); + let SqlExpr::UnaryOp { expr, .. } = &expr else { + panic!("shape preserved"); + }; + let SqlExpr::BinaryOp { left, right, .. } = expr.as_ref() else { + panic!("shape preserved"); + }; + let SqlExpr::BinaryOp { right: at_lit, .. } = left.as_ref() else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(at_lit), &SqlValue::Timestamp(early())); + let SqlExpr::BinaryOp { + left: n_pred, + right: tz_pred, + .. + } = right.as_ref() + else { + panic!("shape preserved"); + }; + let SqlExpr::BinaryOp { right: n_lit, .. } = n_pred.as_ref() else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(n_lit), &SqlValue::Int(1), "BIGINT is untouched"); + let SqlExpr::BinaryOp { right: tz_lit, .. } = tz_pred.as_ref() else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(tz_lit), &SqlValue::Timestamptz(early())); + } + + #[test] + fn a_column_the_scope_does_not_declare_is_left_untouched() { + let expr = coerced(cmp( + col(None, "created"), + BinaryOp::Eq, + lit(SqlValue::Int(EARLY_MS)), + )); + let SqlExpr::BinaryOp { right, .. } = &expr else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(right), &SqlValue::Int(EARLY_MS)); + } + + #[test] + fn a_qualifier_naming_another_relation_is_left_untouched() { + let expr = coerced(cmp( + col(Some("other"), "at"), + BinaryOp::Eq, + lit(SqlValue::Int(EARLY_MS)), + )); + let SqlExpr::BinaryOp { right, .. } = &expr else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(right), &SqlValue::Int(EARLY_MS)); + } + + #[test] + fn the_primary_key_column_is_exempt() { + let expr = coerced(cmp( + col(None, "pk_at"), + BinaryOp::Eq, + lit(SqlValue::Int(EARLY_MS)), + )); + let SqlExpr::BinaryOp { right, .. } = &expr else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(right), &SqlValue::Int(EARLY_MS)); + } + + #[test] + fn column_vs_column_and_null_are_left_alone() { + let expr = coerced(cmp( + cmp(col(None, "at"), BinaryOp::Lt, col(None, "at_tz")), + BinaryOp::And, + cmp(col(None, "at"), BinaryOp::Eq, lit(SqlValue::Null)), + )); + let SqlExpr::BinaryOp { right, .. } = &expr else { + panic!("shape preserved"); + }; + let SqlExpr::BinaryOp { + right: null_lit, .. + } = right.as_ref() + else { + panic!("shape preserved"); + }; + assert_eq!(literal_of(null_lit), &SqlValue::Null); + } + + #[test] + fn a_boolean_literal_is_a_typed_error_naming_the_column() { + let mut expr = cmp(col(None, "at"), BinaryOp::Eq, lit(SqlValue::Bool(true))); + let err = coerce_predicate_literals(&mut expr, &scope()) + .expect_err("a boolean carries no instant"); + let detail = err.to_string(); + assert!( + detail.contains("at") && detail.contains("a boolean"), + "error must name the column and the kind: {detail}" + ); + } + + #[test] + fn unparseable_text_is_a_typed_error_naming_the_column_and_literal() { + let mut expr = SqlExpr::InList { + expr: Box::new(col(None, "at")), + list: vec![lit(SqlValue::String("not a date".into()))], + negated: false, + }; + let err = coerce_predicate_literals(&mut expr, &scope()) + .expect_err("text that spells no instant is refused"); + let detail = err.to_string(); + assert!( + detail.contains("at") && detail.contains("not a date"), + "error must name the column and the literal: {detail}" + ); + } +} diff --git a/nodedb-sql/src/planner/select/helpers.rs b/nodedb-sql/src/planner/select/helpers.rs index d8bcc2f2c..c46421920 100644 --- a/nodedb-sql/src/planner/select/helpers.rs +++ b/nodedb-sql/src/planner/select/helpers.rs @@ -7,6 +7,7 @@ use sqlparser::ast; use crate::error::{Result, SqlError}; use crate::parser::normalize::{SCHEMA_QUALIFIED_MSG, normalize_ident}; +use crate::planner::predicate_coerce::coerce_predicate_literals; use crate::resolver::ColumnScope; use crate::resolver::columns::TableScope; use crate::resolver::expr::convert_expr; @@ -95,8 +96,15 @@ pub fn qualified_name(table: Option<&str>, name: &str) -> String { } /// Convert a WHERE expression into a list of Filter. +/// +/// The one choke point every predicate passes through: `SELECT ... WHERE`, +/// `UPDATE` / `DELETE ... WHERE`, a join's post-filter, `HAVING`, a LATERAL +/// body, and a MERGE `WHEN ... AND` predicate. Literals compared against a +/// declared instant column are coerced here, so no downstream path sees a +/// bare integer where an instant is stored. pub fn convert_where_to_filters(expr: &ast::Expr, scope: &TableScope) -> Result> { - let sql_expr = canonicalize_predicate(convert_expr(expr, &ColumnScope::Relations(scope))?); + let mut sql_expr = canonicalize_predicate(convert_expr(expr, &ColumnScope::Relations(scope))?); + coerce_predicate_literals(&mut sql_expr, scope)?; Ok(vec![Filter { expr: FilterExpr::Expr(sql_expr), }]) From 83442e4f3486206565738c899620123f2d0ad8fa Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 17 Sep 2026 11:12:39 +0800 Subject: [PATCH 17/21] feat(query): decode filter operators fallibly, lower time literals by kind FilterOp::parse_op (and the From<&str>/From impls built on it) silently mapped any unrecognized wire tag to MatchAll, so a typo or a future operator name decoded as "match everything" instead of failing. Replace it with a fallible FilterOp::parse returning UnknownFilterOp, wired through serde deserialization and every msgpack/wire decode path. FilterOp also drops its Default impl (MatchAll is no longer a sentinel default), and value_ops::compare_values becomes partial_compare_values, returning None for a pair with no defined order per the new Value::partial_cmp_coerced, propagated through binary-op evaluation and graph pattern predicate checks. Adds TimeKind::literal_ms / cell_value (columnar_memtable::time_literal) as the one place a time column's stored millisecond count is lowered from, and read back to, the value its kind denotes: an Instant column reads a typed datetime or datetime text, a Millis column reads an integer, a truncated finite float, or datetime text. This replaces the removed scan_filter::value_as_timestamp_ms, which guessed a single unit for every time column regardless of declared kind, across the columnar filter evaluator, the timeseries time-range narrowing prefilter, grouped bitmask filtering, and scan materialization. Every ScanFilter constructed with a string op literal (`op: "eq".into()`) across the executor handlers, storage cold-filter pruning, and tests is updated to the typed FilterOp variant, and cold_filter's row-group pruning now matches on FilterOp directly with an exhaustive arm per operator instead of a serde_json::Value round-trip. --- nodedb-query/src/expr/binary.rs | 37 +- nodedb-query/src/msgpack_scan/filter.rs | 112 ++++-- nodedb-query/src/scan_filter/mod.rs | 4 +- nodedb-query/src/scan_filter/op.rs | 118 +++++- nodedb-query/src/scan_filter/parse.rs | 20 +- nodedb-query/src/scan_filter/timestamp.rs | 85 ---- nodedb-query/src/scan_filter/types.rs | 303 ++++++++++++-- nodedb-query/src/value_ops.rs | 42 +- .../src/control/planner/calvin/predicate.rs | 33 +- .../src/data/executor/dispatch/timeseries.rs | 4 +- .../data/executor/handlers/columnar_agg.rs | 2 +- .../executor/handlers/columnar_filter/eval.rs | 12 +- .../columnar_filter/memtable_source.rs | 16 +- .../columnar_filter/partition_source.rs | 18 +- .../columnar_read/materialize_scan_ts.rs | 17 +- .../handlers/columnar_read/scan/execute.rs | 2 +- .../executor/handlers/graph_edge_write.rs | 2 +- .../data/executor/handlers/returning_rows.rs | 4 +- nodedb/src/data/executor/handlers/rls_eval.rs | 33 +- .../data/executor/handlers/rls_write_gate.rs | 2 +- .../executor/handlers/timeseries/rls_gate.rs | 6 +- .../data/executor/handlers/timeseries/scan.rs | 3 +- .../handlers/timeseries/time_range.rs | 63 ++- .../transaction/overlay/timeseries_merge.rs | 14 +- .../stage_write/stage_timeseries.rs | 25 +- .../graph/pattern/executor/predicates.rs | 18 +- .../timeseries/columnar_memtable/mod.rs | 1 + .../columnar_memtable/time_literal.rs | 209 ++++++++++ .../src/engine/timeseries/grouped_filter.rs | 111 ++--- nodedb/src/storage/cold_filter.rs | 53 ++- .../cases/executor_tests/test_array_ops.rs | 14 +- .../executor_tests/test_conditional_update.rs | 40 +- .../test_cross_engine_validation.rs | 2 +- .../inproc/cases/executor_tests/test_facet.rs | 10 +- .../executor_tests/test_ollp_verification.rs | 2 +- .../cases/executor_tests/test_timeseries.rs | 2 +- nodedb/tests/wire/cases/mod.rs | 1 + .../wire/cases/sql_where_instant_literals.rs | 380 ++++++++++++++++++ 38 files changed, 1409 insertions(+), 411 deletions(-) delete mode 100644 nodedb-query/src/scan_filter/timestamp.rs create mode 100644 nodedb/src/engine/timeseries/columnar_memtable/time_literal.rs create mode 100644 nodedb/tests/wire/cases/sql_where_instant_literals.rs diff --git a/nodedb-query/src/expr/binary.rs b/nodedb-query/src/expr/binary.rs index 281d96f10..91b55c9d8 100644 --- a/nodedb-query/src/expr/binary.rs +++ b/nodedb-query/src/expr/binary.rs @@ -7,7 +7,8 @@ use rust_decimal::Decimal; use nodedb_types::Value; use crate::value_ops::{ - coerced_eq, compare_values, is_truthy, to_value_number, value_to_display_string, value_to_f64, + coerced_eq, is_truthy, partial_compare_values, to_value_number, value_to_display_string, + value_to_f64, }; use super::eval::EvalError; @@ -123,24 +124,22 @@ pub(super) fn eval_binary_op( } BinaryOp::Eq => Ok(Value::Bool(coerced_eq(left, right))), BinaryOp::NotEq => Ok(Value::Bool(!coerced_eq(left, right))), - BinaryOp::Gt => Ok(Value::Bool( - compare_values(left, right) == std::cmp::Ordering::Greater, - )), - BinaryOp::GtEq => { - let c = compare_values(left, right); - Ok(Value::Bool( - c == std::cmp::Ordering::Greater || c == std::cmp::Ordering::Equal, - )) - } - BinaryOp::Lt => Ok(Value::Bool( - compare_values(left, right) == std::cmp::Ordering::Less, - )), - BinaryOp::LtEq => { - let c = compare_values(left, right); - Ok(Value::Bool( - c == std::cmp::Ordering::Less || c == std::cmp::Ordering::Equal, - )) - } + BinaryOp::Gt => Ok(Value::Bool(matches!( + partial_compare_values(left, right), + Some(std::cmp::Ordering::Greater) + ))), + BinaryOp::GtEq => Ok(Value::Bool(matches!( + partial_compare_values(left, right), + Some(std::cmp::Ordering::Greater | std::cmp::Ordering::Equal) + ))), + BinaryOp::Lt => Ok(Value::Bool(matches!( + partial_compare_values(left, right), + Some(std::cmp::Ordering::Less) + ))), + BinaryOp::LtEq => Ok(Value::Bool(matches!( + partial_compare_values(left, right), + Some(std::cmp::Ordering::Less | std::cmp::Ordering::Equal) + ))), BinaryOp::And => Ok(Value::Bool(is_truthy(left) && is_truthy(right))), BinaryOp::Or => Ok(Value::Bool(is_truthy(left) || is_truthy(right))), } diff --git a/nodedb-query/src/msgpack_scan/filter.rs b/nodedb-query/src/msgpack_scan/filter.rs index 990a2cc57..9ec23d83a 100644 --- a/nodedb-query/src/msgpack_scan/filter.rs +++ b/nodedb-query/src/msgpack_scan/filter.rs @@ -4,7 +4,7 @@ //! //! `ScanFilter::matches_binary(doc: &[u8])` evaluates a filter predicate //! directly on msgpack bytes without decoding to `serde_json::Value`. -//! Uses `Value::eq_coerced`/`cmp_coerced` for type coercion — single +//! Uses `Value::eq_coerced`/`partial_cmp_coerced` for type coercion — single //! source of truth shared with the JSON filter path. use std::cmp::Ordering; @@ -16,6 +16,7 @@ use crate::msgpack_scan::reader::{ array_header, map_header, read_null, read_str, read_value, skip_value, }; use crate::scan_filter::like::sql_like_match; +use crate::scan_filter::types::column_compare; use crate::scan_filter::{FilterOp, ScanFilter}; impl ScanFilter { @@ -79,7 +80,30 @@ impl ScanFilter { _ => Ok(false), }; } - _ => {} + FilterOp::Eq + | FilterOp::Ne + | FilterOp::Gt + | FilterOp::Gte + | FilterOp::Lt + | FilterOp::Lte + | FilterOp::Contains + | FilterOp::Like + | FilterOp::NotLike + | FilterOp::Ilike + | FilterOp::NotIlike + | FilterOp::In + | FilterOp::NotIn + | FilterOp::IsNull + | FilterOp::IsNotNull + | FilterOp::ArrayContains + | FilterOp::ArrayContainsAll + | FilterOp::ArrayOverlap + | FilterOp::GtColumn + | FilterOp::GteColumn + | FilterOp::LtColumn + | FilterOp::LteColumn + | FilterOp::EqColumn + | FilterOp::NeColumn => {} } let (start, end) = match extract_field(doc, 0, &self.field) { @@ -117,7 +141,30 @@ impl ScanFilter { _ => Ok(false), }; } - _ => {} + FilterOp::Eq + | FilterOp::Ne + | FilterOp::Gt + | FilterOp::Gte + | FilterOp::Lt + | FilterOp::Lte + | FilterOp::Contains + | FilterOp::Like + | FilterOp::NotLike + | FilterOp::Ilike + | FilterOp::NotIlike + | FilterOp::In + | FilterOp::NotIn + | FilterOp::IsNull + | FilterOp::IsNotNull + | FilterOp::ArrayContains + | FilterOp::ArrayContainsAll + | FilterOp::ArrayOverlap + | FilterOp::GtColumn + | FilterOp::GteColumn + | FilterOp::LtColumn + | FilterOp::LteColumn + | FilterOp::EqColumn + | FilterOp::NeColumn => {} } let (start, end) = match idx.get(&self.field) { @@ -136,16 +183,16 @@ fn eval_op(filter: &ScanFilter, doc: &[u8], start: usize, _end: usize) -> bool { FilterOp::IsNotNull => !read_null(doc, start), FilterOp::Eq => eq_value(doc, start, &filter.value), FilterOp::Ne => !eq_value(doc, start, &filter.value), - FilterOp::Gt => cmp_value(doc, start, &filter.value) == Ordering::Greater, - FilterOp::Gte => { - let c = cmp_value(doc, start, &filter.value); - c == Ordering::Greater || c == Ordering::Equal - } - FilterOp::Lt => cmp_value(doc, start, &filter.value) == Ordering::Less, - FilterOp::Lte => { - let c = cmp_value(doc, start, &filter.value); - c == Ordering::Less || c == Ordering::Equal - } + FilterOp::Gt => cmp_value(doc, start, &filter.value) == Some(Ordering::Greater), + FilterOp::Gte => matches!( + cmp_value(doc, start, &filter.value), + Some(Ordering::Greater | Ordering::Equal) + ), + FilterOp::Lt => cmp_value(doc, start, &filter.value) == Some(Ordering::Less), + FilterOp::Lte => matches!( + cmp_value(doc, start, &filter.value), + Some(Ordering::Less | Ordering::Equal) + ), FilterOp::Contains => { if let (Some(s), Some(pattern)) = (read_str(doc, start), filter.value.as_str()) { s.contains(pattern) @@ -214,17 +261,11 @@ fn eval_op(filter: &ScanFilter, doc: &[u8], start: usize, _end: usize) -> bool { // Read both sides as Value and compare. let left = read_value(doc, start).unwrap_or(nodedb_types::Value::Null); let right = read_value(doc, other_start).unwrap_or(nodedb_types::Value::Null); - match filter.op { - FilterOp::GtColumn => left.cmp_coerced(&right) == Ordering::Greater, - FilterOp::GteColumn => left.cmp_coerced(&right) != Ordering::Less, - FilterOp::LtColumn => left.cmp_coerced(&right) == Ordering::Less, - FilterOp::LteColumn => left.cmp_coerced(&right) != Ordering::Greater, - FilterOp::EqColumn => left.eq_coerced(&right), - FilterOp::NeColumn => !left.eq_coerced(&right), - _ => false, - } + column_compare(filter.op, &left, &right) } - _ => false, + // Handled before field extraction by both `matches_binary` paths. + FilterOp::MatchAll | FilterOp::Exists | FilterOp::NotExists => true, + FilterOp::Or | FilterOp::Expr => false, } } @@ -262,15 +303,14 @@ fn eq_value(buf: &[u8], offset: usize, filter_val: &nodedb_types::Value) -> bool } /// Coerced ordering: read msgpack value at offset → compare with `Value`. -/// Uses `Value::cmp_coerced` — single source of truth for ordering. +/// Uses `Value::partial_cmp_coerced` — single source of truth for ordering. /// -/// Returns ordering of field_val relative to filter_val (field <=> filter). +/// Returns ordering of field_val relative to filter_val (field <=> filter), +/// or `None` when the cell cannot be read or the pair has no defined order, +/// so a range predicate over it matches nothing. #[inline] -fn cmp_value(buf: &[u8], offset: usize, filter_val: &nodedb_types::Value) -> Ordering { - match read_value(buf, offset) { - Some(field_val) => field_val.cmp_coerced(filter_val), - None => Ordering::Equal, - } +fn cmp_value(buf: &[u8], offset: usize, filter_val: &nodedb_types::Value) -> Option { + read_value(buf, offset)?.partial_cmp_coerced(filter_val) } /// LIKE/ILIKE/NOT LIKE/NOT ILIKE helper. @@ -319,7 +359,7 @@ mod tests { fn filter(field: &str, op: &str, value: nodedb_types::Value) -> ScanFilter { ScanFilter { field: field.into(), - op: op.into(), + op: FilterOp::parse(op).expect(op), value, clauses: vec![], expr: None, @@ -585,7 +625,7 @@ mod tests { assert!( ScanFilter { field: "status".into(), - op: "in".into(), + op: FilterOp::In, value: vals.clone(), clauses: vec![], expr: None @@ -598,7 +638,7 @@ mod tests { assert!( ScanFilter { field: "status".into(), - op: "not_in".into(), + op: FilterOp::NotIn, value: vals, clauses: vec![], expr: None @@ -641,7 +681,7 @@ mod tests { assert!( ScanFilter { field: "tags".into(), - op: "array_contains_all".into(), + op: FilterOp::ArrayContainsAll, value: needles, clauses: vec![], expr: None @@ -661,7 +701,7 @@ mod tests { assert!( ScanFilter { field: "tags".into(), - op: "array_overlap".into(), + op: FilterOp::ArrayOverlap, value: needles, clauses: vec![], expr: None @@ -676,7 +716,7 @@ mod tests { let doc = encode(&json!({"x": 5})); let f = ScanFilter { field: String::new(), - op: "or".into(), + op: FilterOp::Or, value: nodedb_types::Value::Null, clauses: vec![ vec![filter("x", "eq", nodedb_types::Value::Integer(10))], diff --git a/nodedb-query/src/scan_filter/mod.rs b/nodedb-query/src/scan_filter/mod.rs index 39c261f14..066462534 100644 --- a/nodedb-query/src/scan_filter/mod.rs +++ b/nodedb-query/src/scan_filter/mod.rs @@ -9,11 +9,9 @@ pub mod like; pub mod op; pub mod parse; -pub mod timestamp; pub mod types; pub use like::sql_like_match; -pub use op::FilterOp; +pub use op::{FilterOp, UnknownFilterOp}; pub use parse::parse_simple_predicates; -pub use timestamp::value_as_timestamp_ms; pub use types::ScanFilter; diff --git a/nodedb-query/src/scan_filter/op.rs b/nodedb-query/src/scan_filter/op.rs index 6f79bec41..3c4db3400 100644 --- a/nodedb-query/src/scan_filter/op.rs +++ b/nodedb-query/src/scan_filter/op.rs @@ -4,10 +4,25 @@ //! //! `FilterOp` is an O(1)-dispatch discriminant used by the scan filter //! evaluator. On-wire it travels as a lowercase string tag so physical -//! plans remain debuggable by hand. +//! plans remain debuggable by hand. Decoding a tag is fallible: a name +//! `FilterOp::parse` does not know is an error, never a default operator. + +/// An operator name no `FilterOp` variant carries. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("unknown filter operator `{op}`")] +pub struct UnknownFilterOp { + /// The operator text as it arrived. + pub op: String, +} + +impl From for zerompk::Error { + fn from(e: UnknownFilterOp) -> Self { + zerompk::Error::IoError(std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + } +} /// Filter operator enum for O(1) dispatch instead of string comparison. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FilterOp { Eq, Ne, @@ -27,7 +42,8 @@ pub enum FilterOp { ArrayContains, ArrayContainsAll, ArrayOverlap, - #[default] + /// Every row passes. Emitted by policy lowering for a predicate decided + /// true at plan time and by the planner for a filter it cannot lower. MatchAll, Exists, NotExists, @@ -50,8 +66,10 @@ pub enum FilterOp { } impl FilterOp { - pub fn parse_op(s: &str) -> Self { - match s { + /// Parse a wire operator name. Every name `as_str` emits parses back to + /// the same variant. `ne`/`neq`, `gte`/`ge`, and `lte`/`le` are aliases. + pub fn parse(name: &str) -> Result { + Ok(match name { "eq" => Self::Eq, "ne" | "neq" => Self::Ne, "gt" => Self::Gt, @@ -81,8 +99,12 @@ impl FilterOp { "lte_col" => Self::LteColumn, "eq_col" => Self::EqColumn, "ne_col" => Self::NeColumn, - _ => Self::MatchAll, - } + other => { + return Err(UnknownFilterOp { + op: other.to_string(), + }); + } + }) } pub fn as_str(&self) -> &'static str { @@ -120,18 +142,6 @@ impl FilterOp { } } -impl From<&str> for FilterOp { - fn from(s: &str) -> Self { - Self::parse_op(s) - } -} - -impl From for FilterOp { - fn from(s: String) -> Self { - Self::parse_op(&s) - } -} - impl serde::Serialize for FilterOp { fn serialize(&self, serializer: S) -> Result { serializer.serialize_str(self.as_str()) @@ -141,6 +151,74 @@ impl serde::Serialize for FilterOp { impl<'de> serde::Deserialize<'de> for FilterOp { fn deserialize>(deserializer: D) -> Result { let s = String::deserialize(deserializer)?; - Ok(FilterOp::parse_op(&s)) + FilterOp::parse(&s).map_err(serde::de::Error::custom) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ALL: [FilterOp; 29] = [ + FilterOp::Eq, + FilterOp::Ne, + FilterOp::Gt, + FilterOp::Gte, + FilterOp::Lt, + FilterOp::Lte, + FilterOp::Contains, + FilterOp::Like, + FilterOp::NotLike, + FilterOp::Ilike, + FilterOp::NotIlike, + FilterOp::In, + FilterOp::NotIn, + FilterOp::IsNull, + FilterOp::IsNotNull, + FilterOp::ArrayContains, + FilterOp::ArrayContainsAll, + FilterOp::ArrayOverlap, + FilterOp::MatchAll, + FilterOp::Exists, + FilterOp::NotExists, + FilterOp::Or, + FilterOp::Expr, + FilterOp::GtColumn, + FilterOp::GteColumn, + FilterOp::LtColumn, + FilterOp::LteColumn, + FilterOp::EqColumn, + FilterOp::NeColumn, + ]; + + #[test] + fn every_name_round_trips() { + for op in ALL { + assert_eq!(FilterOp::parse(op.as_str()), Ok(op), "{op:?}"); + } + } + + #[test] + fn aliases_parse_to_their_canonical_variant() { + assert_eq!(FilterOp::parse("neq"), Ok(FilterOp::Ne)); + assert_eq!(FilterOp::parse("ge"), Ok(FilterOp::Gte)); + assert_eq!(FilterOp::parse("le"), Ok(FilterOp::Lte)); + } + + #[test] + fn unknown_name_is_an_error_naming_the_text() { + let err = FilterOp::parse("any_in").expect_err("unknown"); + assert_eq!(err.op, "any_in"); + assert_eq!(err.to_string(), "unknown filter operator `any_in`"); + assert!(FilterOp::parse("").is_err()); + assert!(FilterOp::parse("EQ").is_err()); + } + + #[test] + fn serde_rejects_an_unknown_name() { + let ok: FilterOp = serde_json::from_str("\"array_overlap\"").expect("known"); + assert_eq!(ok, FilterOp::ArrayOverlap); + let err = serde_json::from_str::("\"any_in\"").expect_err("unknown"); + assert!(err.to_string().contains("any_in"), "{err}"); } } diff --git a/nodedb-query/src/scan_filter/parse.rs b/nodedb-query/src/scan_filter/parse.rs index 9020e442c..238e9822e 100644 --- a/nodedb-query/src/scan_filter/parse.rs +++ b/nodedb-query/src/scan_filter/parse.rs @@ -2,7 +2,7 @@ use nodedb_types::find_ascii_case_insensitive; -use super::ScanFilter; +use super::{FilterOp, ScanFilter}; /// Parse simple SQL predicates into `ScanFilter` values. /// @@ -33,17 +33,17 @@ fn parse_single_predicate(clause: &str) -> Option { let field = clause[..pos].trim().to_string(); let raw_value = clause[pos + op_str.len()..].trim(); let op = match *op_str { - "=" => "eq", - "!=" | "<>" => "ne", - ">" => "gt", - ">=" => "gte", - "<" => "lt", - "<=" => "lte", + "=" => FilterOp::Eq, + "!=" | "<>" => FilterOp::Ne, + ">" => FilterOp::Gt, + ">=" => FilterOp::Gte, + "<" => FilterOp::Lt, + "<=" => FilterOp::Lte, _ => return None, }; return Some(ScanFilter { field, - op: super::FilterOp::parse_op(op), + op, value: nodedb_types::Value::from(parse_predicate_value(raw_value)), clauses: Vec::new(), expr: None, @@ -56,7 +56,7 @@ fn parse_single_predicate(clause: &str) -> Option { let raw_value = clause[pos + 6..].trim(); return Some(ScanFilter { field, - op: super::FilterOp::Like, + op: FilterOp::Like, value: nodedb_types::Value::from(parse_predicate_value(raw_value)), clauses: Vec::new(), expr: None, @@ -67,7 +67,7 @@ fn parse_single_predicate(clause: &str) -> Option { let raw_value = clause[pos + 7..].trim(); return Some(ScanFilter { field, - op: super::FilterOp::Ilike, + op: FilterOp::Ilike, value: nodedb_types::Value::from(parse_predicate_value(raw_value)), clauses: Vec::new(), expr: None, diff --git a/nodedb-query/src/scan_filter/timestamp.rs b/nodedb-query/src/scan_filter/timestamp.rs deleted file mode 100644 index 58d875a1f..000000000 --- a/nodedb-query/src/scan_filter/timestamp.rs +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Coercion of a filter's comparison value to epoch milliseconds. -//! -//! A predicate against a timestamp column can arrive carrying any of the -//! shapes SQL accepts for an instant: a datetime literal (`ts < '2020-03-05 -//! 10:00:00'`), an epoch-millisecond integer (`ts < 1583402400000`), or an -//! already-typed datetime value. Timestamp columns store epoch milliseconds, -//! so every one of those has to reduce to the same scalar before comparison — -//! otherwise a perfectly ordinary predicate silently matches nothing. - -use nodedb_types::Value; - -/// Reduce a filter value to epoch milliseconds, or `None` when it cannot -/// denote an instant. -pub fn value_as_timestamp_ms(value: &Value) -> Option { - match value { - Value::Integer(ms) => Some(*ms), - Value::Float(ms) => Some(*ms as i64), - Value::DateTime(dt) | Value::NaiveDateTime(dt) => Some(dt.unix_millis()), - Value::String(text) => nodedb_types::datetime::NdbDateTime::parse(text) - .map(|dt| dt.unix_millis()) - .or_else(|| text.trim().parse::().ok()), - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// 2020-03-05T10:00:00Z in epoch milliseconds. - const MARCH_5_2020: i64 = 1_583_402_400_000; - - #[test] - fn epoch_millis_pass_through() { - assert_eq!( - value_as_timestamp_ms(&Value::Integer(MARCH_5_2020)), - Some(MARCH_5_2020) - ); - } - - #[test] - fn datetime_literals_are_parsed() { - assert_eq!( - value_as_timestamp_ms(&Value::String("2020-03-05 10:00:00".into())), - Some(MARCH_5_2020) - ); - assert_eq!( - value_as_timestamp_ms(&Value::String("2020-03-05T10:00:00Z".into())), - Some(MARCH_5_2020) - ); - } - - #[test] - fn numeric_strings_are_read_as_epoch_millis() { - assert_eq!( - value_as_timestamp_ms(&Value::String(MARCH_5_2020.to_string())), - Some(MARCH_5_2020) - ); - } - - #[test] - fn typed_datetimes_are_accepted() { - let dt = nodedb_types::NdbDateTime::from_millis(MARCH_5_2020).unwrap(); - assert_eq!( - value_as_timestamp_ms(&Value::NaiveDateTime(dt)), - Some(MARCH_5_2020) - ); - assert_eq!( - value_as_timestamp_ms(&Value::DateTime(dt)), - Some(MARCH_5_2020) - ); - } - - #[test] - fn non_instants_are_rejected() { - assert_eq!(value_as_timestamp_ms(&Value::Null), None); - assert_eq!(value_as_timestamp_ms(&Value::Bool(true)), None); - assert_eq!( - value_as_timestamp_ms(&Value::String("not a date".into())), - None - ); - } -} diff --git a/nodedb-query/src/scan_filter/types.rs b/nodedb-query/src/scan_filter/types.rs index 29fb1aec8..60432e81a 100644 --- a/nodedb-query/src/scan_filter/types.rs +++ b/nodedb-query/src/scan_filter/types.rs @@ -19,7 +19,15 @@ use super::op::FilterOp; /// OR representation: `{"op": "or", "clauses": [[filter1, filter2], [filter3]]}` /// means `(filter1 AND filter2) OR filter3`. Each clause is an AND-group; /// the document matches if ANY clause group fully matches. -#[derive(Clone, serde::Serialize, serde::Deserialize, Default)] +/// +/// Wire form (zerompk): `[field, op_name, value, clauses, expr]`. `op_name` +/// is the `FilterOp::as_str` tag and decodes through `FilterOp::parse`, so +/// an unknown name is a decode error. `value` is the tagged +/// `nodedb_types::Value` encoding, the same form `SqlExpr::Literal` uses: +/// every variant, including `DateTime` / `NaiveDateTime` / `Bytes` / +/// `Decimal`, arrives typed. The serde derive is a JSON rendering; it never +/// crosses the bridge. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct ScanFilter { #[serde(default)] pub field: String, @@ -41,9 +49,7 @@ impl zerompk::ToMessagePack for ScanFilter { writer.write_array_len(5)?; self.field.write(writer)?; writer.write_string(self.op.as_str())?; - // Convert nodedb_types::Value → serde_json::Value for wire compat. - let json_val: serde_json::Value = self.value.clone().into(); - nodedb_types::JsonValue(json_val).write(writer)?; + self.value.write(writer)?; self.clauses.write(writer)?; self.expr.write(writer) } @@ -53,15 +59,14 @@ impl<'a> zerompk::FromMessagePack<'a> for ScanFilter { fn read>(reader: &mut R) -> zerompk::Result { reader.check_array_len(5)?; let field = String::read(reader)?; - let op_str = String::read(reader)?; - let jv = nodedb_types::JsonValue::read(reader)?; + let op = FilterOp::parse(&reader.read_string()?)?; + let value = nodedb_types::Value::read(reader)?; let clauses = Vec::>::read(reader)?; let expr = Option::::read(reader)?; Ok(Self { field, - op: FilterOp::parse_op(&op_str), - // Convert serde_json::Value → nodedb_types::Value at wire boundary. - value: nodedb_types::Value::from(jv.0), + op, + value, clauses, expr, }) @@ -117,7 +122,30 @@ impl ScanFilter { None => Ok(false), }; } - _ => {} + FilterOp::Eq + | FilterOp::Ne + | FilterOp::Gt + | FilterOp::Gte + | FilterOp::Lt + | FilterOp::Lte + | FilterOp::Contains + | FilterOp::Like + | FilterOp::NotLike + | FilterOp::Ilike + | FilterOp::NotIlike + | FilterOp::In + | FilterOp::NotIn + | FilterOp::IsNull + | FilterOp::IsNotNull + | FilterOp::ArrayContains + | FilterOp::ArrayContainsAll + | FilterOp::ArrayOverlap + | FilterOp::GtColumn + | FilterOp::GteColumn + | FilterOp::LtColumn + | FilterOp::LteColumn + | FilterOp::EqColumn + | FilterOp::NeColumn => {} } let field_val = match doc.get(&self.field) { @@ -126,18 +154,24 @@ impl ScanFilter { }; Ok(match self.op { + FilterOp::MatchAll | FilterOp::Exists | FilterOp::NotExists => true, + FilterOp::Or | FilterOp::Expr => false, FilterOp::Eq => self.value.eq_coerced(field_val), FilterOp::Ne => !self.value.eq_coerced(field_val), - FilterOp::Gt => self.value.cmp_coerced(field_val) == std::cmp::Ordering::Less, - FilterOp::Gte => { - let cmp = self.value.cmp_coerced(field_val); - cmp == std::cmp::Ordering::Less || cmp == std::cmp::Ordering::Equal + FilterOp::Gt => { + field_val.partial_cmp_coerced(&self.value) == Some(std::cmp::Ordering::Greater) } - FilterOp::Lt => self.value.cmp_coerced(field_val) == std::cmp::Ordering::Greater, - FilterOp::Lte => { - let cmp = self.value.cmp_coerced(field_val); - cmp == std::cmp::Ordering::Greater || cmp == std::cmp::Ordering::Equal + FilterOp::Gte => matches!( + field_val.partial_cmp_coerced(&self.value), + Some(std::cmp::Ordering::Greater | std::cmp::Ordering::Equal) + ), + FilterOp::Lt => { + field_val.partial_cmp_coerced(&self.value) == Some(std::cmp::Ordering::Less) } + FilterOp::Lte => matches!( + field_val.partial_cmp_coerced(&self.value), + Some(std::cmp::Ordering::Less | std::cmp::Ordering::Equal) + ), FilterOp::Contains => { if let (Some(s), Some(pattern)) = (field_val.as_str(), self.value.as_str()) { s.contains(pattern) @@ -228,25 +262,222 @@ impl ScanFilter { Some(v) => v, None => return Ok(false), }; - match self.op { - FilterOp::GtColumn => { - field_val.cmp_coerced(other_val) == std::cmp::Ordering::Greater - } - FilterOp::GteColumn => { - field_val.cmp_coerced(other_val) != std::cmp::Ordering::Less - } - FilterOp::LtColumn => { - field_val.cmp_coerced(other_val) == std::cmp::Ordering::Less - } - FilterOp::LteColumn => { - field_val.cmp_coerced(other_val) != std::cmp::Ordering::Greater - } - FilterOp::EqColumn => field_val.eq_coerced(other_val), - FilterOp::NeColumn => !field_val.eq_coerced(other_val), - _ => false, - } + column_compare(self.op, field_val, other_val) } - _ => false, }) } } + +/// Evaluate a column-vs-column operator over two cells of the same row. +/// Any other operator has no column form and matches nothing. +pub(crate) fn column_compare( + op: FilterOp, + left: &nodedb_types::Value, + right: &nodedb_types::Value, +) -> bool { + use std::cmp::Ordering; + match op { + FilterOp::GtColumn => left.partial_cmp_coerced(right) == Some(Ordering::Greater), + FilterOp::GteColumn => matches!( + left.partial_cmp_coerced(right), + Some(Ordering::Greater | Ordering::Equal) + ), + FilterOp::LtColumn => left.partial_cmp_coerced(right) == Some(Ordering::Less), + FilterOp::LteColumn => matches!( + left.partial_cmp_coerced(right), + Some(Ordering::Less | Ordering::Equal) + ), + FilterOp::EqColumn => left.eq_coerced(right), + FilterOp::NeColumn => !left.eq_coerced(right), + FilterOp::Eq + | FilterOp::Ne + | FilterOp::Gt + | FilterOp::Gte + | FilterOp::Lt + | FilterOp::Lte + | FilterOp::Contains + | FilterOp::Like + | FilterOp::NotLike + | FilterOp::Ilike + | FilterOp::NotIlike + | FilterOp::In + | FilterOp::NotIn + | FilterOp::IsNull + | FilterOp::IsNotNull + | FilterOp::ArrayContains + | FilterOp::ArrayContainsAll + | FilterOp::ArrayOverlap + | FilterOp::MatchAll + | FilterOp::Exists + | FilterOp::NotExists + | FilterOp::Or + | FilterOp::Expr => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_types::Value; + use nodedb_types::datetime::NdbDateTime; + + fn round_trip(filter: &ScanFilter) -> ScanFilter { + let bytes = zerompk::to_msgpack_vec(filter).expect("encode"); + zerompk::from_msgpack(&bytes).expect("decode") + } + + fn compare(op: FilterOp, value: Value) -> ScanFilter { + ScanFilter { + field: "captured_at".into(), + op, + value, + clauses: Vec::new(), + expr: None, + } + } + + #[test] + fn naive_instant_crosses_the_bridge_typed() { + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + let back = round_trip(&compare(FilterOp::Gte, Value::NaiveDateTime(at))); + assert_eq!(back.op, FilterOp::Gte); + assert_eq!(back.field, "captured_at"); + assert_eq!(back.value, Value::NaiveDateTime(at)); + } + + #[test] + fn utc_instant_crosses_the_bridge_typed() { + let at = NdbDateTime::from_micros(-86_400_000_000); + let back = round_trip(&compare(FilterOp::Lt, Value::DateTime(at))); + assert_eq!(back.value, Value::DateTime(at)); + } + + #[test] + fn bytes_cross_the_bridge_as_bytes() { + let back = round_trip(&compare(FilterOp::Eq, Value::Bytes(vec![0, 159, 146, 150]))); + assert_eq!(back.value, Value::Bytes(vec![0, 159, 146, 150])); + } + + #[test] + fn nested_clauses_keep_typed_values() { + let at = NdbDateTime::from_micros(1_000_000); + let filter = ScanFilter { + field: String::new(), + op: FilterOp::Or, + value: Value::Null, + clauses: vec![ + vec![compare(FilterOp::Gt, Value::NaiveDateTime(at))], + vec![compare( + FilterOp::In, + Value::Array(vec![Value::DateTime(at)]), + )], + ], + expr: None, + }; + let back = round_trip(&filter); + assert_eq!(back.op, FilterOp::Or); + assert_eq!(back.clauses[0][0].value, Value::NaiveDateTime(at)); + assert_eq!( + back.clauses[1][0].value, + Value::Array(vec![Value::DateTime(at)]) + ); + } + + #[test] + fn every_operator_name_round_trips_through_the_wire() { + for name in [ + "eq", + "ne", + "gt", + "gte", + "lt", + "lte", + "contains", + "like", + "not_like", + "ilike", + "not_ilike", + "in", + "not_in", + "is_null", + "is_not_null", + "array_contains", + "array_contains_all", + "array_overlap", + "match_all", + "exists", + "not_exists", + "or", + "expr", + "gt_col", + "gte_col", + "lt_col", + "lte_col", + "eq_col", + "ne_col", + ] { + let op = FilterOp::parse(name).expect(name); + let back = round_trip(&compare(op, Value::Null)); + assert_eq!(back.op, op, "{name}"); + assert_eq!(back.op.as_str(), name); + } + } + + /// Msgpack `fixarray` header for `len` elements (`len < 16`). + fn fixarray(len: u8) -> u8 { + 0x90 | len + } + + /// The wire bytes of a filter carrying operator name `op`, built from + /// the same fragments `ScanFilter::write` emits so the layout matches. + fn wire_with_op(op: &str) -> Vec { + let mut buf = vec![fixarray(5)]; + buf.extend(zerompk::to_msgpack_vec(&"tags".to_string()).expect("field")); + buf.extend(zerompk::to_msgpack_vec(&op.to_string()).expect("op")); + buf.extend(zerompk::to_msgpack_vec(&Value::Null).expect("value")); + buf.push(fixarray(0)); + buf.extend(zerompk::to_msgpack_vec(&None::).expect("expr")); + buf + } + + #[test] + fn unknown_operator_fails_the_decode_naming_the_operator() { + let err = + zerompk::from_msgpack::(&wire_with_op("any_in")).expect_err("unknown"); + let text = err.to_string(); + assert!(text.contains("any_in"), "{text}"); + assert!(text.contains("unknown filter operator"), "{text}"); + + let ok: ScanFilter = zerompk::from_msgpack(&wire_with_op("array_overlap")).expect("known"); + assert_eq!(ok.op, FilterOp::ArrayOverlap); + } + + #[test] + fn unknown_operator_inside_a_clause_fails_the_whole_set() { + let mut buf = vec![fixarray(1), fixarray(5)]; + buf.extend(zerompk::to_msgpack_vec(&String::new()).expect("field")); + buf.extend(zerompk::to_msgpack_vec(&"or".to_string()).expect("op")); + buf.extend(zerompk::to_msgpack_vec(&Value::Null).expect("value")); + buf.push(fixarray(1)); + buf.push(fixarray(1)); + buf.extend_from_slice(&wire_with_op("any_in")); + buf.extend(zerompk::to_msgpack_vec(&None::).expect("expr")); + let err = zerompk::from_msgpack::>(&buf).expect_err("unknown"); + assert!(err.to_string().contains("any_in"), "{err}"); + } + + /// The hand-built bytes decode when the operator is known, which pins + /// the layout the malformed cases rely on. + #[test] + fn hand_built_wire_matches_the_codec_layout() { + let filter = ScanFilter { + field: "tags".into(), + op: FilterOp::Eq, + value: Value::Null, + clauses: Vec::new(), + expr: None, + }; + let encoded = zerompk::to_msgpack_vec(&filter).expect("encode"); + assert_eq!(encoded, wire_with_op("eq")); + } +} diff --git a/nodedb-query/src/value_ops.rs b/nodedb-query/src/value_ops.rs index 3ab6950ca..4d2226e58 100644 --- a/nodedb-query/src/value_ops.rs +++ b/nodedb-query/src/value_ops.rs @@ -31,27 +31,39 @@ pub fn value_to_f64(v: &Value, coerce_bool: bool) -> Option { } /// Whether either side is a typed instant, so the pair compares by epoch -/// microseconds through `Value::cmp_coerced` (an ISO string on the other -/// side is parsed). +/// microseconds through `Value::partial_cmp_coerced` (an ISO string on the +/// other side is parsed). fn involves_instant(a: &Value, b: &Value) -> bool { a.as_instant().is_some() || b.as_instant().is_some() } -/// Compare two Values with type coercion. +/// Compare two Values with type coercion, for a predicate. /// /// A typed instant compares by epoch microseconds against another instant -/// or an ISO-8601 string. Otherwise numeric comparison first (with bool -/// coercion), then string comparison. -pub fn compare_values(a: &Value, b: &Value) -> Ordering { +/// or an ISO-8601 string, and against anything else has no order: `None`, +/// so `WHERE at >= 5` over an instant matches nothing. Otherwise numeric +/// comparison first (with bool coercion; `None` for a NaN), then string +/// comparison of the display forms. +pub fn partial_compare_values(a: &Value, b: &Value) -> Option { if involves_instant(a, b) { - return a.cmp_coerced(b); + return a.partial_cmp_coerced(b); } if let (Some(na), Some(nb)) = (value_to_f64(a, true), value_to_f64(b, true)) { - return na.partial_cmp(&nb).unwrap_or(Ordering::Equal); + return na.partial_cmp(&nb); } let sa = value_to_display_string(a); let sb = value_to_display_string(b); - sa.cmp(&sb) + Some(sa.cmp(&sb)) +} + +/// Compare two Values with type coercion, for sorting and extremum +/// selection (ORDER BY, MIN / MAX, GREATEST / LEAST, window frames). +/// +/// [`partial_compare_values`] with an unordered pair placed as `Equal`, so a +/// stable sort keeps such rows in their input order. A predicate uses +/// `partial_compare_values` instead. +pub fn compare_values(a: &Value, b: &Value) -> Ordering { + partial_compare_values(a, b).unwrap_or(Ordering::Equal) } /// Check equality with type coercion. @@ -173,6 +185,18 @@ mod tests { assert!(!coerced_eq(&earlier, &later)); } + /// An instant against a bare integer has no order on the predicate path + /// and sorts as equal on the extremum path. + #[test] + fn an_instant_against_an_integer_is_unordered_for_predicates() { + let at = Value::NaiveDateTime(nodedb_types::NdbDateTime::from_micros( + 1_583_402_400_000_000, + )); + assert_eq!(partial_compare_values(&at, &Value::Integer(5)), None); + assert_eq!(partial_compare_values(&Value::Integer(5), &at), None); + assert_eq!(compare_values(&at, &Value::Integer(5)), Ordering::Equal); + } + #[test] fn truthiness() { assert!(is_truthy(&Value::Bool(true))); diff --git a/nodedb/src/control/planner/calvin/predicate.rs b/nodedb/src/control/planner/calvin/predicate.rs index 6f9f56c0b..52685298f 100644 --- a/nodedb/src/control/planner/calvin/predicate.rs +++ b/nodedb/src/control/planner/calvin/predicate.rs @@ -268,13 +268,15 @@ mod tests { field: "balance".into(), op: FilterOp::Gt, value: Value::Integer(1000), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let f2 = vec![ScanFilter { field: "balance".into(), op: FilterOp::Gt, value: Value::Integer(9999), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let h1 = predicate_class_for_filters(&encode_filters(&f1), "accounts"); let h2 = predicate_class_for_filters(&encode_filters(&f2), "accounts"); @@ -290,13 +292,15 @@ mod tests { field: "name".into(), op: FilterOp::Eq, value: Value::String("alice".into()), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let f2 = vec![ScanFilter { field: "name".into(), op: FilterOp::Eq, value: Value::String("bob".into()), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let h1 = predicate_class_for_filters(&encode_filters(&f1), "users"); let h2 = predicate_class_for_filters(&encode_filters(&f2), "users"); @@ -312,13 +316,15 @@ mod tests { field: "balance".into(), op: FilterOp::Gt, value: Value::Integer(100), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let f2 = vec![ScanFilter { field: "age".into(), op: FilterOp::Gt, value: Value::Integer(100), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let h1 = predicate_class_for_filters(&encode_filters(&f1), "accounts"); let h2 = predicate_class_for_filters(&encode_filters(&f2), "accounts"); @@ -331,13 +337,15 @@ mod tests { field: "score".into(), op: FilterOp::Gt, value: Value::Integer(5), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let f2 = vec![ScanFilter { field: "score".into(), op: FilterOp::Lt, value: Value::Integer(5), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let h1 = predicate_class_for_filters(&encode_filters(&f1), "items"); let h2 = predicate_class_for_filters(&encode_filters(&f2), "items"); @@ -350,7 +358,8 @@ mod tests { field: "x".into(), op: FilterOp::Eq, value: Value::Integer(1), - ..Default::default() + clauses: Vec::new(), + expr: None, }]; let bytes = encode_filters(&filters); let h1 = predicate_class_for_filters(&bytes, "col_a"); @@ -369,13 +378,15 @@ mod tests { field: "status".into(), op: FilterOp::Eq, value: Value::String("active".into()), - ..Default::default() + clauses: Vec::new(), + expr: None, }], vec![ScanFilter { field: "status".into(), op: FilterOp::Eq, value: Value::String("pending".into()), - ..Default::default() + clauses: Vec::new(), + expr: None, }], ], expr: None, diff --git a/nodedb/src/data/executor/dispatch/timeseries.rs b/nodedb/src/data/executor/dispatch/timeseries.rs index e20afac92..5d1e0eb80 100644 --- a/nodedb/src/data/executor/dispatch/timeseries.rs +++ b/nodedb/src/data/executor/dispatch/timeseries.rs @@ -186,7 +186,7 @@ mod tests { fn policy_owner_eq(owner: &str) -> Vec { let filter = crate::bridge::scan_filter::ScanFilter { field: "owner".into(), - op: "eq".into(), + op: crate::bridge::scan_filter::FilterOp::Eq, value: nodedb_types::Value::String(owner.into()), clauses: vec![], expr: None, @@ -340,7 +340,7 @@ mod tests { let filter = crate::bridge::scan_filter::ScanFilter { field: "owner".into(), - op: "like".into(), + op: crate::bridge::scan_filter::FilterOp::Like, value: nodedb_types::Value::String("mi%".into()), clauses: vec![], expr: None, diff --git a/nodedb/src/data/executor/handlers/columnar_agg.rs b/nodedb/src/data/executor/handlers/columnar_agg.rs index c2e5c8a8b..a9f32fcfd 100644 --- a/nodedb/src/data/executor/handlers/columnar_agg.rs +++ b/nodedb/src/data/executor/handlers/columnar_agg.rs @@ -542,7 +542,7 @@ mod tests { let mt = make_test_memtable(); let filter = crate::bridge::scan_filter::ScanFilter { field: "value".into(), - op: "gt".into(), + op: crate::bridge::scan_filter::FilterOp::Gt, value: nodedb_types::Value::Float(5000.0), clauses: vec![], expr: None, diff --git a/nodedb/src/data/executor/handlers/columnar_filter/eval.rs b/nodedb/src/data/executor/handlers/columnar_filter/eval.rs index 4d02773e0..9cde152fe 100644 --- a/nodedb/src/data/executor/handlers/columnar_filter/eval.rs +++ b/nodedb/src/data/executor/handlers/columnar_filter/eval.rs @@ -68,8 +68,8 @@ pub(crate) fn eval_filters_sparse( } } } - ColumnType::Timestamp(_) => { - let fv = nodedb_query::scan_filter::value_as_timestamp_ms(&f.value)?; + ColumnType::Timestamp(kind) => { + let fv = kind.literal_ms(&f.value)?; if let ColumnData::Timestamp(vals) = col_data { for (mi, &idx) in indices.iter().enumerate() { if !mask[mi] { @@ -142,8 +142,8 @@ pub(crate) fn eval_filters_dense( } } } - ColumnType::Timestamp(_) => { - let fv = nodedb_query::scan_filter::value_as_timestamp_ms(&f.value)?; + ColumnType::Timestamp(kind) => { + let fv = kind.literal_ms(&f.value)?; if let ColumnData::Timestamp(vals) = col_data { for i in 0..row_count { if !mask[i] { @@ -273,8 +273,8 @@ pub(crate) fn eval_filters_bitmask( _ => return None, } } - ColumnType::Timestamp(_) => { - let fv = nodedb_query::scan_filter::value_as_timestamp_ms(&f.value)?; + ColumnType::Timestamp(kind) => { + let fv = kind.literal_ms(&f.value)?; let ColumnData::Timestamp(vals) = col_data else { return None; }; diff --git a/nodedb/src/data/executor/handlers/columnar_filter/memtable_source.rs b/nodedb/src/data/executor/handlers/columnar_filter/memtable_source.rs index 6f6019007..63e320566 100644 --- a/nodedb/src/data/executor/handlers/columnar_filter/memtable_source.rs +++ b/nodedb/src/data/executor/handlers/columnar_filter/memtable_source.rs @@ -22,7 +22,7 @@ impl ColumnarSource for ColumnarMemtable { #[cfg(test)] mod tests { use super::*; - use crate::bridge::scan_filter::ScanFilter; + use crate::bridge::scan_filter::{FilterOp, ScanFilter}; use crate::engine::timeseries::columnar_memtable::{ ColumnValue, ColumnarMemtable, ColumnarMemtableConfig, ColumnarSchema, TimeKind, }; @@ -62,7 +62,7 @@ mod tests { let mt = make_test_mt(); let f = ScanFilter { field: "value".into(), - op: "gt".into(), + op: FilterOp::Gt, value: nodedb_types::Value::Float(200.0), clauses: vec![], expr: None, @@ -78,7 +78,7 @@ mod tests { let indices: Vec = (0..30).collect(); let f = ScanFilter { field: "host".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("db-1".into()), clauses: vec![], expr: None, @@ -94,7 +94,7 @@ mod tests { let indices: Vec = (0..30).collect(); let f = ScanFilter { field: "host".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("nonexistent".into()), clauses: vec![], expr: None, @@ -111,14 +111,14 @@ mod tests { let filters = vec![ ScanFilter { field: "value".into(), - op: "gte".into(), + op: FilterOp::Gte, value: nodedb_types::Value::Float(100.0), clauses: vec![], expr: None, }, ScanFilter { field: "host".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("web-1".into()), clauses: vec![], expr: None, @@ -134,11 +134,11 @@ mod tests { let mt = make_test_mt(); let f = ScanFilter { field: "value".into(), - op: "or".into(), + op: FilterOp::Or, value: nodedb_types::Value::Null, clauses: vec![vec![ScanFilter { field: "value".into(), - op: "gt".into(), + op: FilterOp::Gt, value: nodedb_types::Value::Float(100.0), clauses: vec![], expr: None, diff --git a/nodedb/src/data/executor/handlers/columnar_filter/partition_source.rs b/nodedb/src/data/executor/handlers/columnar_filter/partition_source.rs index 67f5969d5..b84bea1ca 100644 --- a/nodedb/src/data/executor/handlers/columnar_filter/partition_source.rs +++ b/nodedb/src/data/executor/handlers/columnar_filter/partition_source.rs @@ -28,7 +28,7 @@ impl ColumnarSource for PartitionColumns<'_> { #[cfg(test)] mod tests { use super::*; - use crate::bridge::scan_filter::ScanFilter; + use crate::bridge::scan_filter::{FilterOp, ScanFilter}; use nodedb_types::timeseries::SymbolDictionary; use super::super::{eval_filters_bitmask, eval_filters_dense, eval_filters_sparse}; @@ -60,7 +60,7 @@ mod tests { let f = ScanFilter { field: "host".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("web-1".into()), clauses: vec![], expr: None, @@ -96,7 +96,7 @@ mod tests { let f = ScanFilter { field: "col".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("z".into()), clauses: vec![], expr: None, @@ -130,7 +130,7 @@ mod tests { let f = ScanFilter { field: "host".into(), - op: "contains".into(), + op: FilterOp::Contains, value: nodedb_types::Value::String("web".into()), clauses: vec![], expr: None, @@ -165,7 +165,7 @@ mod tests { let f = ScanFilter { field: "tag".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("alpha".into()), clauses: vec![], expr: None, @@ -200,7 +200,7 @@ mod tests { let f = ScanFilter { field: "tag".into(), - op: "ne".into(), + op: FilterOp::Ne, value: nodedb_types::Value::String("y".into()), clauses: vec![], expr: None, @@ -234,7 +234,7 @@ mod tests { let f = ScanFilter { field: "col".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("a".into()), clauses: vec![], expr: None, @@ -284,7 +284,7 @@ mod tests { // Filter: value > 500 let f = ScanFilter { field: "value".into(), - op: "gt".into(), + op: FilterOp::Gt, value: nodedb_types::Value::Float(500.0), clauses: vec![], expr: None, @@ -296,7 +296,7 @@ mod tests { // Filter: host = 'alpha' let f2 = ScanFilter { field: "host".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String("alpha".into()), clauses: vec![], expr: None, diff --git a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs index 7829aa2d9..eca7e34a9 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan_ts.rs @@ -51,7 +51,7 @@ use nodedb_types::value::Value; use super::materialize_scan::{build_response, encode_cursor}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; -use crate::engine::timeseries::columnar_memtable::{ColumnData, ColumnType, TimeKind}; +use crate::engine::timeseries::columnar_memtable::{ColumnData, ColumnType}; use crate::engine::timeseries::columnar_segment::ColumnarSegmentReader; impl CoreLoop { @@ -352,17 +352,6 @@ fn instant_read_error(column: &str, e: nodedb_types::NdbDateTimeError) -> crate: // Column-to-Value converters // --------------------------------------------------------------------------- -/// Read a stored millisecond time cell as the value its kind denotes. -/// -/// An instant column yields a typed instant, a `Millis` column the integer -/// stored. `Err` when the milliseconds overflow the microsecond range. -fn time_cell_value(kind: TimeKind, millis: i64) -> Result { - match kind { - TimeKind::Instant(k) => k.from_millis(millis), - TimeKind::Millis => Ok(Value::Integer(millis)), - } -} - /// Convert a memtable column entry to `nodedb_types::Value`. fn memtable_col_to_value( col_data: &ColumnData, @@ -373,7 +362,7 @@ fn memtable_col_to_value( ) -> Result { let value = match col_type { ColumnType::Timestamp(kind) => { - return time_cell_value(*kind, col_data.as_timestamps()[row_idx]); + return kind.cell_value(col_data.as_timestamps()[row_idx]); } ColumnType::Float64 => { let v = col_data.as_f64()[row_idx]; @@ -405,7 +394,7 @@ fn partition_col_to_value( ) -> Result { let value = match col_type { ColumnType::Timestamp(kind) => { - return time_cell_value(*kind, data.as_timestamps()[row_idx]); + return kind.cell_value(data.as_timestamps()[row_idx]); } ColumnType::Float64 => { let v = data.as_f64()[row_idx]; diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs index 78cc2de69..9787d637f 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs @@ -599,7 +599,7 @@ mod tests { fn policy_name_eq(name: &str) -> Vec { let filter = crate::bridge::scan_filter::ScanFilter { field: "name".into(), - op: "eq".into(), + op: crate::bridge::scan_filter::FilterOp::Eq, value: Value::String(name.into()), clauses: vec![], expr: None, diff --git a/nodedb/src/data/executor/handlers/graph_edge_write.rs b/nodedb/src/data/executor/handlers/graph_edge_write.rs index a2a75ea80..7ef48c371 100644 --- a/nodedb/src/data/executor/handlers/graph_edge_write.rs +++ b/nodedb/src/data/executor/handlers/graph_edge_write.rs @@ -654,7 +654,7 @@ mod tests { fn owner_write_check(owner: &str) -> Vec { let filter = crate::bridge::scan_filter::ScanFilter { field: "owner".into(), - op: "eq".into(), + op: crate::bridge::scan_filter::FilterOp::Eq, value: nodedb_types::Value::String(owner.into()), clauses: Vec::new(), expr: None, diff --git a/nodedb/src/data/executor/handlers/returning_rows.rs b/nodedb/src/data/executor/handlers/returning_rows.rs index c3d9e1441..2a446c05f 100644 --- a/nodedb/src/data/executor/handlers/returning_rows.rs +++ b/nodedb/src/data/executor/handlers/returning_rows.rs @@ -284,7 +284,7 @@ fn project_row(doc: &Value, source_names: &[String]) -> Vec { #[cfg(test)] mod tests { use super::*; - use crate::bridge::scan_filter::ScanFilter; + use crate::bridge::scan_filter::{FilterOp, ScanFilter}; use nodedb_physical::physical_plan::ReturningItem; use serde_json::json; @@ -292,7 +292,7 @@ mod tests { fn owner_policy(value: &str) -> Vec { let filter = ScanFilter { field: "owner".into(), - op: "eq".into(), + op: FilterOp::Eq, value: nodedb_types::Value::String(value.into()), clauses: Vec::new(), expr: None, diff --git a/nodedb/src/data/executor/handlers/rls_eval.rs b/nodedb/src/data/executor/handlers/rls_eval.rs index 844624e4a..5bdbb985e 100644 --- a/nodedb/src/data/executor/handlers/rls_eval.rs +++ b/nodedb/src/data/executor/handlers/rls_eval.rs @@ -83,12 +83,13 @@ fn check_encoded(rls_filters: &[u8], msgpack: &[u8]) -> bool { #[cfg(test)] mod tests { use super::*; + use crate::bridge::scan_filter::FilterOp; use serde_json::json; - fn make_rls_bytes(field: &str, op: &str, value: nodedb_types::Value) -> Vec { + fn make_rls_bytes(field: &str, op: FilterOp, value: nodedb_types::Value) -> Vec { let filter = ScanFilter { field: field.into(), - op: op.into(), + op, value, clauses: Vec::new(), expr: None, @@ -104,28 +105,44 @@ mod tests { #[test] fn matching_filter_allows() { - let rls = make_rls_bytes("user_id", "eq", nodedb_types::Value::String("42".into())); + let rls = make_rls_bytes( + "user_id", + FilterOp::Eq, + nodedb_types::Value::String("42".into()), + ); let doc = json!({"user_id": "42", "name": "alice"}); assert!(rls_check_document(&rls, &doc)); } #[test] fn non_matching_filter_denies() { - let rls = make_rls_bytes("user_id", "eq", nodedb_types::Value::String("42".into())); + let rls = make_rls_bytes( + "user_id", + FilterOp::Eq, + nodedb_types::Value::String("42".into()), + ); let doc = json!({"user_id": "99", "name": "bob"}); assert!(!rls_check_document(&rls, &doc)); } #[test] fn missing_field_denies() { - let rls = make_rls_bytes("user_id", "eq", nodedb_types::Value::String("42".into())); + let rls = make_rls_bytes( + "user_id", + FilterOp::Eq, + nodedb_types::Value::String("42".into()), + ); let doc = json!({"name": "alice"}); assert!(!rls_check_document(&rls, &doc)); } #[test] fn typed_documents_evaluate_the_same_filters() { - let rls = make_rls_bytes("user_id", "eq", nodedb_types::Value::String("42".into())); + let rls = make_rls_bytes( + "user_id", + FilterOp::Eq, + nodedb_types::Value::String("42".into()), + ); let ok = Value::from(json!({"user_id": "42"})); let bad = Value::from(json!({"user_id": "99"})); assert!(rls_check_value(&rls, &ok)); @@ -145,14 +162,14 @@ mod tests { let filters = vec![ ScanFilter { field: "user_id".into(), - op: "eq".into(), + op: crate::bridge::scan_filter::FilterOp::Eq, value: nodedb_types::Value::String("42".into()), clauses: Vec::new(), expr: None, }, ScanFilter { field: "status".into(), - op: "eq".into(), + op: crate::bridge::scan_filter::FilterOp::Eq, value: nodedb_types::Value::String("active".into()), clauses: Vec::new(), expr: None, diff --git a/nodedb/src/data/executor/handlers/rls_write_gate.rs b/nodedb/src/data/executor/handlers/rls_write_gate.rs index 3e985a81d..211369b15 100644 --- a/nodedb/src/data/executor/handlers/rls_write_gate.rs +++ b/nodedb/src/data/executor/handlers/rls_write_gate.rs @@ -243,7 +243,7 @@ mod tests { fn owner_policy(value: &str) -> Vec { let filter = ScanFilter { field: "owner".into(), - op: "eq".into(), + op: crate::bridge::scan_filter::FilterOp::Eq, value: nodedb_types::Value::String(value.into()), clauses: Vec::new(), expr: None, diff --git a/nodedb/src/data/executor/handlers/timeseries/rls_gate.rs b/nodedb/src/data/executor/handlers/timeseries/rls_gate.rs index 030ccbfcd..f28c8424e 100644 --- a/nodedb/src/data/executor/handlers/timeseries/rls_gate.rs +++ b/nodedb/src/data/executor/handlers/timeseries/rls_gate.rs @@ -114,12 +114,12 @@ fn field_value(value: &FieldValue<'_>) -> Value { #[cfg(test)] mod tests { use super::*; - use crate::bridge::scan_filter::ScanFilter; + use crate::bridge::scan_filter::{FilterOp, ScanFilter}; fn owner_policy(owner: &str) -> Vec { let filter = ScanFilter { field: "owner".into(), - op: "eq".into(), + op: FilterOp::Eq, value: Value::String(owner.into()), clauses: Vec::new(), expr: None, @@ -242,7 +242,7 @@ mod tests { fn a_msgpack_row_is_decided_on_its_normalized_values() { let filter = ScanFilter { field: "reading".into(), - op: "eq".into(), + op: FilterOp::Eq, value: Value::Float(1.5), clauses: Vec::new(), expr: None, diff --git a/nodedb/src/data/executor/handlers/timeseries/scan.rs b/nodedb/src/data/executor/handlers/timeseries/scan.rs index 5b2851975..ccbb8b3e1 100644 --- a/nodedb/src/data/executor/handlers/timeseries/scan.rs +++ b/nodedb/src/data/executor/handlers/timeseries/scan.rs @@ -82,6 +82,7 @@ impl CoreLoop { // projection pushdown. Resolved once here and threaded through both // branches; nothing downstream guesses it from a column name. let time_key = self.ts_time_column(task.request.database_id, tid, collection); + let time_key_kind = self.ts_time_key_kind(task.request.database_id, tid, collection); // Lazy-load partition registry from disk if not yet loaded. if let Err(e) = self.ensure_ts_registry(tid, task.request.database_id, collection) { @@ -142,7 +143,7 @@ impl CoreLoop { let time_range = super::time_range::narrow_time_range( time_range, &filter_predicates, - Some(time_key.as_str()), + Some((time_key.as_str(), time_key_kind)), ); let has_filters = !filter_predicates.is_empty(); diff --git a/nodedb/src/data/executor/handlers/timeseries/time_range.rs b/nodedb/src/data/executor/handlers/timeseries/time_range.rs index ac14e16a6..35dbf67b7 100644 --- a/nodedb/src/data/executor/handlers/timeseries/time_range.rs +++ b/nodedb/src/data/executor/handlers/timeseries/time_range.rs @@ -17,20 +17,22 @@ //! is still evaluated per row afterwards. A too-wide envelope costs I/O; a //! too-narrow one loses rows. -use nodedb_query::scan_filter::value_as_timestamp_ms; - use crate::bridge::scan_filter::{FilterOp, ScanFilter}; +use crate::engine::timeseries::columnar_memtable::TimeKind; -/// Narrow `plan_range` with every bound the query places on `time_key`. +/// Narrow `plan_range` with every bound the query places on the time key +/// named by `time_key`, lowering each literal to stored milliseconds by the +/// key's kind. /// /// Returns `plan_range` unchanged when the collection has no declared time -/// key, or when no predicate references it. +/// key, or when no predicate references it. A literal the kind cannot lower +/// narrows nothing: the exact predicate still decides each row afterwards. pub(in crate::data::executor) fn narrow_time_range( plan_range: (i64, i64), filters: &[ScanFilter], - time_key: Option<&str>, + time_key: Option<(&str, TimeKind)>, ) -> (i64, i64) { - let Some(time_key) = time_key else { + let Some((time_key, kind)) = time_key else { return plan_range; }; let (mut min_ts, mut max_ts) = plan_range; @@ -38,7 +40,7 @@ pub(in crate::data::executor) fn narrow_time_range( if !filter.field.eq_ignore_ascii_case(time_key) { continue; } - let Some(ms) = value_as_timestamp_ms(&filter.value) else { + let Some(ms) = kind.literal_ms(&filter.value) else { continue; }; match filter.op { @@ -69,9 +71,11 @@ pub(in crate::data::executor) fn narrow_time_range( #[cfg(test)] mod tests { use super::*; - use nodedb_types::Value; + use nodedb_types::{InstantKind, Value}; const UNBOUNDED: (i64, i64) = (i64::MIN, i64::MAX); + const MILLIS: TimeKind = TimeKind::Millis; + const NAIVE: TimeKind = TimeKind::Instant(InstantKind::Naive); fn filter(field: &str, op: FilterOp, value: Value) -> ScanFilter { ScanFilter { @@ -96,21 +100,38 @@ mod tests { filter("captured_at", FilterOp::Lt, Value::Integer(900)), ]; assert_eq!( - narrow_time_range(UNBOUNDED, &filters, Some("captured_at")), + narrow_time_range(UNBOUNDED, &filters, Some(("captured_at", MILLIS))), (100, 900) ); } + /// A declared instant key lowers a typed instant, and text that spells + /// one, to stored milliseconds. #[test] - fn a_datetime_literal_bound_is_understood() { - let filters = vec![filter( - "captured_at", - FilterOp::Lt, - Value::String("2020-03-05 10:00:00".into()), - )]; + fn an_instant_key_lowers_instant_literals() { + let at = nodedb_types::NdbDateTime::from_micros(1_583_402_400_000_000); + let filters = vec![ + filter("captured_at", FilterOp::Gte, Value::NaiveDateTime(at)), + filter( + "captured_at", + FilterOp::Lt, + Value::String("2020-03-05 11:00:00".into()), + ), + ]; assert_eq!( - narrow_time_range(UNBOUNDED, &filters, Some("captured_at")), - (i64::MIN, 1_583_402_400_000) + narrow_time_range(UNBOUNDED, &filters, Some(("captured_at", NAIVE))), + (1_583_402_400_000, 1_583_406_000_000) + ); + } + + /// A bare integer against an instant key carries no unit, so it narrows + /// nothing; the per-row predicate still decides. + #[test] + fn an_integer_against_an_instant_key_narrows_nothing() { + let filters = vec![filter("captured_at", FilterOp::Gt, Value::Integer(100))]; + assert_eq!( + narrow_time_range(UNBOUNDED, &filters, Some(("captured_at", NAIVE))), + UNBOUNDED ); } @@ -123,7 +144,7 @@ mod tests { filter("value", FilterOp::Lt, Value::Float(1.0)), ]; assert_eq!( - narrow_time_range(UNBOUNDED, &filters, Some("captured_at")), + narrow_time_range(UNBOUNDED, &filters, Some(("captured_at", MILLIS))), UNBOUNDED ); } @@ -132,7 +153,7 @@ mod tests { fn equality_pins_both_ends() { let filters = vec![filter("ts", FilterOp::Eq, Value::Integer(4_200))]; assert_eq!( - narrow_time_range(UNBOUNDED, &filters, Some("ts")), + narrow_time_range(UNBOUNDED, &filters, Some(("ts", MILLIS))), (4_200, 4_200) ); } @@ -144,7 +165,7 @@ mod tests { filter("ts", FilterOp::Lte, Value::Integer(10_000)), ]; assert_eq!( - narrow_time_range((100, 900), &filters, Some("ts")), + narrow_time_range((100, 900), &filters, Some(("ts", MILLIS))), (100, 900) ); } @@ -153,7 +174,7 @@ mod tests { fn case_differences_in_the_predicate_still_match_the_key() { let filters = vec![filter("TS", FilterOp::Gt, Value::Integer(7))]; assert_eq!( - narrow_time_range(UNBOUNDED, &filters, Some("ts")), + narrow_time_range(UNBOUNDED, &filters, Some(("ts", MILLIS))), (7, i64::MAX) ); } diff --git a/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs b/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs index 5e9a7c05b..29bf7095e 100644 --- a/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs +++ b/nodedb/src/data/executor/handlers/transaction/overlay/timeseries_merge.rs @@ -26,6 +26,7 @@ use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::handlers::columnar_read::filter::value_matches_filters; use crate::data::executor::handlers::transaction::overlay::Staged; +use crate::engine::timeseries::columnar_memtable::TimeKind; use crate::types::{DatabaseId, TenantId, TxnId}; use crate::util::rmpv_value::value_to_rmpv; @@ -62,15 +63,17 @@ fn decode_staged_row(body: &[u8]) -> Option { /// /// A staged row keeps the INSERT's own column names, so the lookup is by the /// collection's declared time column — the same name the base scan prunes on. -/// A row that carries no readable instant under that name falls outside every -/// bounded range and is treated as non-matching by the caller. -fn row_timestamp_ms(row: &Value, time_column: &str) -> Option { +/// The cell was staged typed by the column's kind, so it lowers to stored +/// milliseconds by that same kind. A row that carries no such cell under +/// that name falls outside every bounded range and is treated as +/// non-matching by the caller. +fn row_timestamp_ms(row: &Value, time_column: &str, kind: TimeKind) -> Option { let Value::Object(map) = row else { return None; }; map.iter() .find(|(key, _)| key.eq_ignore_ascii_case(time_column)) - .and_then(|(_, value)| nodedb_query::scan_filter::value_as_timestamp_ms(value)) + .and_then(|(_, value)| kind.literal_ms(value)) } /// Convert a decoded staged row (`Value::Object`) into the `rmpv::Value::Map` @@ -104,6 +107,7 @@ impl CoreLoop { } = params; let time_column = self.ts_time_column(coll_key.0, coll_key.1, &coll_key.2); + let time_kind = self.ts_time_key_kind(coll_key.0, coll_key.1, &coll_key.2); // Read-your-own-writes refreshes the lease (see the reaper). self.touch_overlay(txn_id); @@ -127,7 +131,7 @@ impl CoreLoop { // Time-range prune, mirroring the base memtable scan's // `timestamp_range_filter` (inclusive bounds). - match row_timestamp_ms(&row, &time_column) { + match row_timestamp_ms(&row, &time_column, time_kind) { Some(ts) if ts >= time_range.0 && ts <= time_range.1 => {} _ => continue, } diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs index 5fb54863f..95731299a 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs @@ -338,12 +338,19 @@ impl CoreLoop { // A staged row is read back by name, so the line's timestamp must be // keyed under the collection's own time column — the same name the - // base scan and the overlay merge resolve. + // base scan and the overlay merge resolve — and typed by that + // column's kind, so a predicate literal coerced for the column + // compares against the staged cell exactly as against a base cell. let time_column = self.ts_time_column( task.request.database_id, crate::types::TenantId::new(tid), collection, ); + let time_kind = self.ts_time_key_kind( + task.request.database_id, + crate::types::TenantId::new(tid), + collection, + ); // Encode every row before mutating an overlay. let mut images = Vec::with_capacity(lines.len()); @@ -374,7 +381,21 @@ impl CoreLoop { object.insert(name.to_string(), value); } if let Some(timestamp) = row.timestamp_ns { - object.insert(time_column.clone(), Value::Integer(timestamp / 1_000_000)); + let cell = match time_kind.cell_value(timestamp / 1_000_000) { + Ok(cell) => cell, + Err(error) => { + return self.response_error( + task, + ErrorCode::Internal { + detail: format!( + "canonical ILP overlay row time cell {}: {error}", + timestamp / 1_000_000 + ), + }, + ); + } + }; + object.insert(time_column.clone(), cell); } images.push(Value::Object(object)); } diff --git a/nodedb/src/engine/graph/pattern/executor/predicates.rs b/nodedb/src/engine/graph/pattern/executor/predicates.rs index 89a1c6b27..7c1f39030 100644 --- a/nodedb/src/engine/graph/pattern/executor/predicates.rs +++ b/nodedb/src/engine/graph/pattern/executor/predicates.rs @@ -293,7 +293,7 @@ fn check_property( op: &ComparisonOp, expected_value: &nodedb_types::Value, ) -> Result { - use nodedb_query::value_ops::{coerced_eq, compare_values}; + use nodedb_query::value_ops::{coerced_eq, partial_compare_values}; use std::cmp::Ordering; let collection = props.collection.ok_or_else(|| crate::Error::BadRequest { @@ -323,18 +323,22 @@ fn check_property( let result = match op { ComparisonOp::Eq => coerced_eq(field_value, expected_value), ComparisonOp::Neq => !coerced_eq(field_value, expected_value), - ComparisonOp::Lt => compare_values(field_value, expected_value) == Ordering::Less, + ComparisonOp::Lt => { + partial_compare_values(field_value, expected_value) == Some(Ordering::Less) + } ComparisonOp::Lte => { matches!( - compare_values(field_value, expected_value), - Ordering::Less | Ordering::Equal + partial_compare_values(field_value, expected_value), + Some(Ordering::Less | Ordering::Equal) ) } - ComparisonOp::Gt => compare_values(field_value, expected_value) == Ordering::Greater, + ComparisonOp::Gt => { + partial_compare_values(field_value, expected_value) == Some(Ordering::Greater) + } ComparisonOp::Gte => { matches!( - compare_values(field_value, expected_value), - Ordering::Greater | Ordering::Equal + partial_compare_values(field_value, expected_value), + Some(Ordering::Greater | Ordering::Equal) ) } }; diff --git a/nodedb/src/engine/timeseries/columnar_memtable/mod.rs b/nodedb/src/engine/timeseries/columnar_memtable/mod.rs index e990d80ba..20323f4e6 100644 --- a/nodedb/src/engine/timeseries/columnar_memtable/mod.rs +++ b/nodedb/src/engine/timeseries/columnar_memtable/mod.rs @@ -2,6 +2,7 @@ mod memtable; mod snapshot; +mod time_literal; mod types; pub use memtable::ColumnarMemtable; diff --git a/nodedb/src/engine/timeseries/columnar_memtable/time_literal.rs b/nodedb/src/engine/timeseries/columnar_memtable/time_literal.rs new file mode 100644 index 000000000..50262fc27 --- /dev/null +++ b/nodedb/src/engine/timeseries/columnar_memtable/time_literal.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Lowering between a time column's stored milliseconds and the value its +//! kind denotes, in both directions. +//! +//! A time column stores an `i64` millisecond count whatever its kind. The +//! kind decides what that count means to a reader: an `Instant` column's +//! cell is a typed datetime, a `Millis` column's cell is the integer stored. +//! A predicate literal against the column arrives already typed for that +//! kind (the planner coerces a literal against a declared `TIMESTAMP` / +//! `TIMESTAMPTZ` column to an instant), so lowering it to the stored unit is +//! a kind dispatch with no unit guessing: an instant yields its milliseconds, +//! an integer yields itself, and a value of the wrong kind yields nothing. + +use nodedb_types::datetime::{NdbDateTime, NdbDateTimeError}; +use nodedb_types::value::Value; + +use super::types::TimeKind; + +impl TimeKind { + /// The stored millisecond count a predicate literal (or a cell of the + /// same kind) denotes on a column of this kind, or `None` when the value + /// is not of the kind the column holds. + /// + /// `Instant`: a typed datetime, or text that parses as one (a literal + /// that reached the engine as text). `Millis`: an integer, a finite + /// float truncated toward zero, or text that parses as a datetime — a + /// `BIGINT TIME_KEY` is still a point in time, so a datetime literal + /// names its millisecond count. + pub fn literal_ms(self, value: &Value) -> Option { + match self { + TimeKind::Instant(_) => match value { + Value::DateTime(at) | Value::NaiveDateTime(at) => Some(at.unix_millis()), + Value::String(text) => NdbDateTime::parse(text).map(|at| at.unix_millis()), + Value::Null + | Value::Bool(_) + | Value::Integer(_) + | Value::Float(_) + | Value::Bytes(_) + | Value::Array(_) + | Value::Object(_) + | Value::Uuid(_) + | Value::Ulid(_) + | Value::Duration(_) + | Value::Decimal(_) + | Value::Geometry(_) + | Value::Set(_) + | Value::Regex(_) + | Value::ArrayCell(_) + | Value::Vector(_) => None, + // `Value` is `#[non_exhaustive]`: a kind added later carries + // no instant either. + _ => None, + }, + TimeKind::Millis => match value { + Value::Integer(ms) => Some(*ms), + Value::Float(ms) => ms.is_finite().then(|| ms.trunc() as i64), + Value::String(text) => NdbDateTime::parse(text).map(|at| at.unix_millis()), + Value::Null + | Value::Bool(_) + | Value::Bytes(_) + | Value::Array(_) + | Value::Object(_) + | Value::Uuid(_) + | Value::Ulid(_) + | Value::DateTime(_) + | Value::NaiveDateTime(_) + | Value::Duration(_) + | Value::Decimal(_) + | Value::Geometry(_) + | Value::Set(_) + | Value::Regex(_) + | Value::ArrayCell(_) + | Value::Vector(_) => None, + // `Value` is `#[non_exhaustive]`: a kind added later is not a + // millisecond count either. + _ => None, + }, + } + } + + /// The value a stored millisecond cell denotes on a column of this kind. + /// + /// An `Instant` column yields a typed instant, a `Millis` column the + /// integer stored. `Err` when the milliseconds overflow the microsecond + /// range an instant carries. + pub fn cell_value(self, millis: i64) -> Result { + match self { + TimeKind::Instant(kind) => kind.from_millis(millis), + TimeKind::Millis => Ok(Value::Integer(millis)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_types::InstantKind; + + /// `2020-03-05T10:00:00Z` in epoch milliseconds. + const EARLY_MS: i64 = 1_583_402_400_000; + + const NAIVE: TimeKind = TimeKind::Instant(InstantKind::Naive); + const UTC: TimeKind = TimeKind::Instant(InstantKind::Utc); + + fn early() -> NdbDateTime { + NdbDateTime::from_micros(EARLY_MS * 1_000) + } + + #[test] + fn an_instant_column_lowers_typed_instants_of_either_tag() { + for kind in [NAIVE, UTC] { + assert_eq!( + kind.literal_ms(&Value::NaiveDateTime(early())), + Some(EARLY_MS) + ); + assert_eq!(kind.literal_ms(&Value::DateTime(early())), Some(EARLY_MS)); + } + } + + #[test] + fn an_instant_column_parses_datetime_text() { + assert_eq!( + NAIVE.literal_ms(&Value::String("2020-03-05 10:00:00".into())), + Some(EARLY_MS) + ); + assert_eq!( + UTC.literal_ms(&Value::String("2020-03-05T10:00:00Z".into())), + Some(EARLY_MS) + ); + assert_eq!(NAIVE.literal_ms(&Value::String("not a date".into())), None); + } + + /// A bare number carries no unit, so it is not an instant literal. + #[test] + fn an_instant_column_refuses_numbers_and_other_kinds() { + for value in [ + Value::Integer(EARLY_MS), + Value::Float(EARLY_MS as f64), + Value::String(EARLY_MS.to_string()), + Value::Null, + Value::Bool(true), + Value::Array(vec![Value::Integer(EARLY_MS)]), + ] { + assert_eq!(NAIVE.literal_ms(&value), None, "{value:?}"); + } + } + + #[test] + fn a_millis_column_lowers_integers_and_truncates_finite_floats() { + assert_eq!(TimeKind::Millis.literal_ms(&Value::Integer(42)), Some(42)); + assert_eq!(TimeKind::Millis.literal_ms(&Value::Float(42.9)), Some(42)); + assert_eq!(TimeKind::Millis.literal_ms(&Value::Float(-1.5)), Some(-1)); + assert_eq!(TimeKind::Millis.literal_ms(&Value::Float(f64::NAN)), None); + assert_eq!( + TimeKind::Millis.literal_ms(&Value::Float(f64::INFINITY)), + None + ); + } + + #[test] + fn a_millis_column_refuses_instants_and_other_kinds() { + for value in [ + Value::NaiveDateTime(early()), + Value::DateTime(early()), + Value::String("42".into()), + Value::Null, + Value::Bool(true), + ] { + assert_eq!(TimeKind::Millis.literal_ms(&value), None, "{value:?}"); + } + } + + #[test] + fn a_millis_column_parses_datetime_text_to_its_millisecond_count() { + let value = Value::String("2020-03-05 10:00:00".into()); + assert_eq!( + TimeKind::Millis.literal_ms(&value), + Some(early().unix_millis()) + ); + } + + #[test] + fn a_cell_reads_back_as_its_kind() { + assert_eq!( + NAIVE.cell_value(EARLY_MS).expect("in range"), + Value::NaiveDateTime(early()) + ); + assert_eq!( + UTC.cell_value(EARLY_MS).expect("in range"), + Value::DateTime(early()) + ); + assert_eq!( + TimeKind::Millis.cell_value(EARLY_MS).expect("an integer"), + Value::Integer(EARLY_MS) + ); + assert!(NAIVE.cell_value(i64::MAX).is_err()); + } + + /// A cell written by `cell_value` lowers back to the milliseconds it was + /// written from, on every kind. + #[test] + fn cell_value_and_literal_ms_round_trip() { + for kind in [NAIVE, UTC, TimeKind::Millis] { + let cell = kind.cell_value(EARLY_MS).expect("in range"); + assert_eq!(kind.literal_ms(&cell), Some(EARLY_MS), "{kind:?}"); + } + } +} diff --git a/nodedb/src/engine/timeseries/grouped_filter.rs b/nodedb/src/engine/timeseries/grouped_filter.rs index 6c94a5af4..de8528403 100644 --- a/nodedb/src/engine/timeseries/grouped_filter.rs +++ b/nodedb/src/engine/timeseries/grouped_filter.rs @@ -6,7 +6,7 @@ //! returning packed `Vec` bitmasks. Uses SIMD kernels from //! `nodedb_query::simd_filter` for numeric and symbol comparisons. -use nodedb_query::scan_filter::value_as_timestamp_ms; +use nodedb_query::scan_filter::FilterOp; use nodedb_query::simd_filter; use super::columnar_memtable::{ColumnData, ColumnType}; @@ -54,7 +54,7 @@ pub fn eval_filters_to_bitmask<'a>( let mut mask = simd_filter::bitmask_all(row_count); for f in filters { - if f.op == nodedb_query::scan_filter::FilterOp::MatchAll { + if f.op == FilterOp::MatchAll { continue; } if !f.clauses.is_empty() { @@ -80,17 +80,17 @@ pub fn eval_filters_to_bitmask<'a>( .ok_or(UnsupportedPredicate::new(f, "literal is not a float"))?; let vals = col_data.as_f64(); let slice = &vals[..row_count.min(vals.len())]; - match f.op.as_str() { - "gt" => (rt.gt_f64)(slice, fv), - "gte" => (rt.gte_f64)(slice, fv), - "lt" => (rt.lt_f64)(slice, fv), - "lte" => (rt.lte_f64)(slice, fv), - "eq" => { + match f.op { + FilterOp::Gt => (rt.gt_f64)(slice, fv), + FilterOp::Gte => (rt.gte_f64)(slice, fv), + FilterOp::Lt => (rt.lt_f64)(slice, fv), + FilterOp::Lte => (rt.lte_f64)(slice, fv), + FilterOp::Eq => { let a = (rt.gte_f64)(slice, fv - f64::EPSILON); let b = (rt.lte_f64)(slice, fv + f64::EPSILON); simd_filter::bitmask_and(&a, &b) } - "ne" => { + FilterOp::Ne => { let a = (rt.gte_f64)(slice, fv - f64::EPSILON); let b = (rt.lte_f64)(slice, fv + f64::EPSILON); simd_filter::bitmask_not(&simd_filter::bitmask_and(&a, &b), row_count) @@ -100,44 +100,21 @@ pub fn eval_filters_to_bitmask<'a>( } } } - ColumnType::Int64 | ColumnType::Timestamp(_) => { - // A time column stores epoch milliseconds, so its literal is - // read as an instant: an integer, a datetime, or a datetime - // string all lower to the stored form. - let (fv, vals) = if *col_type == ColumnType::Int64 { - let fv = f - .value - .as_i64() - .ok_or(UnsupportedPredicate::new(f, "literal is not an integer"))?; - (fv, col_data.as_i64()) - } else { - let fv = value_as_timestamp_ms(&f.value) - .ok_or(UnsupportedPredicate::new(f, "literal is not an instant"))?; - (fv, col_data.as_timestamps()) - }; - let slice = &vals[..row_count.min(vals.len())]; - match f.op.as_str() { - "gt" => (rt.gt_i64)(slice, fv), - "gte" => (rt.gte_i64)(slice, fv), - "lt" => (rt.lt_i64)(slice, fv), - "lte" => (rt.lte_i64)(slice, fv), - "eq" => { - let a = (rt.gte_i64)(slice, fv); - let b = (rt.lte_i64)(slice, fv); - simd_filter::bitmask_and(&a, &b) - } - "ne" => { - let a = (rt.gte_i64)(slice, fv); - let b = (rt.lte_i64)(slice, fv); - simd_filter::bitmask_not(&simd_filter::bitmask_and(&a, &b), row_count) - } - _ => { - return Err(UnsupportedPredicate::new( - f, - "operator on an integer or time column", - )); - } - } + ColumnType::Int64 => { + let fv = f + .value + .as_i64() + .ok_or(UnsupportedPredicate::new(f, "literal is not an integer"))?; + i64_column_mask(rt, f, col_data.as_i64(), fv, row_count)? + } + ColumnType::Timestamp(kind) => { + // A time column stores epoch milliseconds; its literal lowers + // to that unit by the column's kind (an instant for a declared + // TIMESTAMP, an integer for a millisecond column). + let fv = kind + .literal_ms(&f.value) + .ok_or(UnsupportedPredicate::new(f, "literal is not an instant"))?; + i64_column_mask(rt, f, col_data.as_timestamps(), fv, row_count)? } ColumnType::Symbol => { let filter_str = f @@ -148,15 +125,15 @@ pub fn eval_filters_to_bitmask<'a>( .ok_or(UnsupportedPredicate::new(f, "symbol dictionary not loaded"))?; let sym_ids = col_data.as_symbols(); let slice = &sym_ids[..row_count.min(sym_ids.len())]; - match f.op.as_str() { - "eq" => { + match f.op { + FilterOp::Eq => { if let Some(target_id) = dict.get_id(filter_str) { (rt.eq_u32)(slice, target_id) } else { vec![0u64; simd_filter::words_for(row_count)] } } - "ne" => { + FilterOp::Ne => { if let Some(target_id) = dict.get_id(filter_str) { (rt.ne_u32)(slice, target_id) } else { @@ -215,3 +192,37 @@ pub fn apply_sparse_skip( } } } + +/// The bitmask of rows in an `i64` column (integer or time) that satisfy +/// `f.op` against the lowered literal `fv`. +fn i64_column_mask( + rt: &simd_filter::FilterSimdRuntime, + f: &ScanFilter, + vals: &[i64], + fv: i64, + row_count: usize, +) -> Result, UnsupportedPredicate> { + let slice = &vals[..row_count.min(vals.len())]; + Ok(match f.op { + FilterOp::Gt => (rt.gt_i64)(slice, fv), + FilterOp::Gte => (rt.gte_i64)(slice, fv), + FilterOp::Lt => (rt.lt_i64)(slice, fv), + FilterOp::Lte => (rt.lte_i64)(slice, fv), + FilterOp::Eq => { + let a = (rt.gte_i64)(slice, fv); + let b = (rt.lte_i64)(slice, fv); + simd_filter::bitmask_and(&a, &b) + } + FilterOp::Ne => { + let a = (rt.gte_i64)(slice, fv); + let b = (rt.lte_i64)(slice, fv); + simd_filter::bitmask_not(&simd_filter::bitmask_and(&a, &b), row_count) + } + _ => { + return Err(UnsupportedPredicate::new( + f, + "operator on an integer or time column", + )); + } + }) +} diff --git a/nodedb/src/storage/cold_filter.rs b/nodedb/src/storage/cold_filter.rs index 64bcdda69..42bd5d20d 100644 --- a/nodedb/src/storage/cold_filter.rs +++ b/nodedb/src/storage/cold_filter.rs @@ -22,13 +22,14 @@ use std::sync::{Arc, Mutex}; use arrow::array::{Array, Float64Array, Int64Array, RecordBatch, StringArray}; use arrow::datatypes::DataType; use bytes::Bytes; +use nodedb_types::Value; use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::{ArrowPredicateFn, ParquetRecordBatchReaderBuilder, RowFilter}; use parquet::file::metadata::RowGroupMetaData; use parquet::file::statistics::Statistics; use tracing::debug; -use crate::bridge::scan_filter::ScanFilter; +use crate::bridge::scan_filter::{FilterOp, ScanFilter}; /// Read a Parquet file with both row-group pruning and row-level filtering. /// @@ -221,19 +222,41 @@ fn row_group_might_match(rg: &RowGroupMetaData, filter: &ScanFilter) -> bool { return true; // No statistics — can't prune. }; - // Use min/max statistics to prune. - use nodedb_query::scan_filter::FilterOp; - let json_val: serde_json::Value = filter.value.clone().into(); - match &filter.op { - FilterOp::Eq => stat_might_contain(stats, &json_val), - FilterOp::Gt | FilterOp::Gte => stat_max_gte(stats, &json_val), - FilterOp::Lt | FilterOp::Lte => stat_min_lte(stats, &json_val), - _ => true, // Complex operators (contains, like, etc.) — can't prune. + // Use min/max statistics to prune. Only a scalar range predicate has a + // min/max form; every other operator keeps the row group. + match filter.op { + FilterOp::Eq => stat_might_contain(stats, &filter.value), + FilterOp::Gt | FilterOp::Gte => stat_max_gte(stats, &filter.value), + FilterOp::Lt | FilterOp::Lte => stat_min_lte(stats, &filter.value), + FilterOp::Ne + | FilterOp::Contains + | FilterOp::Like + | FilterOp::NotLike + | FilterOp::Ilike + | FilterOp::NotIlike + | FilterOp::In + | FilterOp::NotIn + | FilterOp::IsNull + | FilterOp::IsNotNull + | FilterOp::ArrayContains + | FilterOp::ArrayContainsAll + | FilterOp::ArrayOverlap + | FilterOp::MatchAll + | FilterOp::Exists + | FilterOp::NotExists + | FilterOp::Or + | FilterOp::Expr + | FilterOp::GtColumn + | FilterOp::GteColumn + | FilterOp::LtColumn + | FilterOp::LteColumn + | FilterOp::EqColumn + | FilterOp::NeColumn => true, } } /// Check if column statistics allow an equality match. -fn stat_might_contain(stats: &Statistics, value: &serde_json::Value) -> bool { +fn stat_might_contain(stats: &Statistics, value: &Value) -> bool { match stats { Statistics::Int64(s) => { let (Some(min), Some(max)) = (s.min_opt(), s.max_opt()) else { @@ -272,7 +295,7 @@ fn stat_might_contain(stats: &Statistics, value: &serde_json::Value) -> bool { } /// Check if the column max >= value (for gt/gte predicates). -fn stat_max_gte(stats: &Statistics, value: &serde_json::Value) -> bool { +fn stat_max_gte(stats: &Statistics, value: &Value) -> bool { match stats { Statistics::Int64(s) => { let Some(max) = s.max_opt() else { return true }; @@ -287,7 +310,7 @@ fn stat_max_gte(stats: &Statistics, value: &serde_json::Value) -> bool { } /// Check if the column min <= value (for lt/lte predicates). -fn stat_min_lte(stats: &Statistics, value: &serde_json::Value) -> bool { +fn stat_min_lte(stats: &Statistics, value: &Value) -> bool { match stats { Statistics::Int64(s) => { let Some(min) = s.min_opt() else { return true }; @@ -481,7 +504,7 @@ mod tests { let filters = vec![ScanFilter { field: "age".into(), - op: "gt".into(), + op: crate::bridge::scan_filter::FilterOp::Gt, value: nodedb_types::Value::Integer(25), clauses: vec![], expr: None, @@ -509,7 +532,7 @@ mod tests { let filters = vec![ScanFilter { field: String::new(), - op: "expr".into(), + op: crate::bridge::scan_filter::FilterOp::Expr, value: nodedb_types::Value::Null, clauses: vec![], expr: Some(SqlExpr::BinaryOp { @@ -538,7 +561,7 @@ mod tests { // `10 / age > 1` is true only for d1 (10/2 = 5 > 1); d2 (10/30 = 0) fails. let filters = vec![ScanFilter { field: String::new(), - op: "expr".into(), + op: crate::bridge::scan_filter::FilterOp::Expr, value: nodedb_types::Value::Null, clauses: vec![], expr: Some(SqlExpr::BinaryOp { diff --git a/nodedb/tests/inproc/cases/executor_tests/test_array_ops.rs b/nodedb/tests/inproc/cases/executor_tests/test_array_ops.rs index 66cf51961..76480c495 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_array_ops.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_array_ops.rs @@ -2,7 +2,7 @@ //! Integration tests for array operators and array aggregate functions. -use nodedb::bridge::scan_filter::ScanFilter; +use nodedb::bridge::scan_filter::{FilterOp, ScanFilter}; use nodedb_physical::physical_plan::{ AggregateSpec, DocumentOp, GroupKeySpec, PhysicalPlan, QueryOp, }; @@ -45,10 +45,10 @@ fn insert_product( ); } -fn filter(field: &str, op: &str, value: nodedb_types::Value) -> ScanFilter { +fn filter(field: &str, op: FilterOp, value: nodedb_types::Value) -> ScanFilter { ScanFilter { field: field.into(), - op: op.into(), + op, value, clauses: Vec::new(), expr: None, @@ -106,7 +106,7 @@ fn array_contains_filter() { // Products where tags contains "sale". let filters = vec![filter( "tags", - "array_contains", + FilterOp::ArrayContains, nodedb_types::Value::String("sale".into()), )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); @@ -145,7 +145,7 @@ fn array_contains_all_filter() { // Products where sizes contains ALL of ["S", "M"]. let filters = vec![filter( "sizes", - "array_contains_all", + FilterOp::ArrayContainsAll, nodedb_types::Value::Array(vec![ nodedb_types::Value::String("S".into()), nodedb_types::Value::String("M".into()), @@ -187,7 +187,7 @@ fn array_overlap_filter() { // Products where tags overlaps ["sale", "premium"]. let filters = vec![filter( "tags", - "array_overlap", + FilterOp::ArrayOverlap, nodedb_types::Value::Array(vec![ nodedb_types::Value::String("sale".into()), nodedb_types::Value::String("premium".into()), @@ -322,7 +322,7 @@ fn no_match_returns_zero() { // No product has tag "nonexistent". let filters = vec![filter( "tags", - "array_contains", + FilterOp::ArrayContains, nodedb_types::Value::String("nonexistent".into()), )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); diff --git a/nodedb/tests/inproc/cases/executor_tests/test_conditional_update.rs b/nodedb/tests/inproc/cases/executor_tests/test_conditional_update.rs index ed824abee..bc81c1b8c 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_conditional_update.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_conditional_update.rs @@ -10,15 +10,15 @@ //! - PointUpdate returns affected count use nodedb::bridge::envelope::Status; -use nodedb::bridge::scan_filter::ScanFilter; +use nodedb::bridge::scan_filter::{FilterOp, ScanFilter}; use nodedb_physical::physical_plan::{DocumentOp, MetaOp, PhysicalPlan, UpdateValue}; use super::helpers::*; -fn filter(field: &str, op: &str, value: nodedb_types::Value) -> ScanFilter { +fn filter(field: &str, op: FilterOp, value: nodedb_types::Value) -> ScanFilter { ScanFilter { field: field.into(), - op: op.into(), + op, value, clauses: Vec::new(), expr: None, @@ -107,7 +107,11 @@ fn bulk_update_returns_affected_count() { insert_product(&mut core, &mut tx, &mut rx, "p3", 0); // Bulk update: SET stock = 99 WHERE stock > 0 (should match p1 and p2). - let filters = vec![filter("stock", "gt", nodedb_types::Value::Integer(0))]; + let filters = vec![filter( + "stock", + FilterOp::Gt, + nodedb_types::Value::Integer(0), + )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); let updates = vec![( "stock".to_string(), @@ -153,7 +157,11 @@ fn conditional_decrement_stops_at_zero() { for i in 0..10 { let current_stock = get_stock(&mut core, &mut tx, &mut rx, "flash-deal"); - let filters = vec![filter("stock", "gte", nodedb_types::Value::Integer(1))]; + let filters = vec![filter( + "stock", + FilterOp::Gte, + nodedb_types::Value::Integer(1), + )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); let new_stock = current_stock.saturating_sub(1); @@ -209,7 +217,11 @@ fn bulk_update_zero_match_returns_zero_affected() { insert_product(&mut core, &mut tx, &mut rx, "p1", 0); - let filters = vec![filter("stock", "gte", nodedb_types::Value::Integer(100))]; + let filters = vec![filter( + "stock", + FilterOp::Gte, + nodedb_types::Value::Integer(100), + )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); let updates = vec![( "stock".to_string(), @@ -248,7 +260,11 @@ fn bulk_update_returning_returns_updated_documents() { insert_product(&mut core, &mut tx, &mut rx, "r1", 10); insert_product(&mut core, &mut tx, &mut rx, "r2", 20); - let filters = vec![filter("stock", "gt", nodedb_types::Value::Integer(0))]; + let filters = vec![filter( + "stock", + FilterOp::Gt, + nodedb_types::Value::Integer(0), + )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); let updates = vec![( "stock".to_string(), @@ -286,7 +302,11 @@ fn bulk_update_returning_zero_match_returns_affected_zero() { insert_product(&mut core, &mut tx, &mut rx, "p1", 0); - let filters = vec![filter("stock", "gte", nodedb_types::Value::Integer(100))]; + let filters = vec![filter( + "stock", + FilterOp::Gte, + nodedb_types::Value::Integer(100), + )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); let updates = vec![( "stock".to_string(), @@ -402,14 +422,14 @@ fn transaction_batch_does_not_abort_on_zero_row_update() { // Batch should NOT auto-abort on 0-row update. let filters_match = zerompk::to_msgpack_vec(&vec![filter( "stock", - "gte", + FilterOp::Gte, nodedb_types::Value::Integer(1), )]) .unwrap(); let filters_nomatch = zerompk::to_msgpack_vec(&vec![filter( "stock", - "gte", + FilterOp::Gte, nodedb_types::Value::Integer(100), )]) .unwrap(); diff --git a/nodedb/tests/inproc/cases/executor_tests/test_cross_engine_validation.rs b/nodedb/tests/inproc/cases/executor_tests/test_cross_engine_validation.rs index e6330672c..40ba6d91a 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_cross_engine_validation.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_cross_engine_validation.rs @@ -150,7 +150,7 @@ fn cross_model_query_vector_graph_relational() { // 6. Relational filter: scan papers with year >= 2023. let filter = vec![nodedb::bridge::scan_filter::ScanFilter { field: "year".into(), - op: "gte".into(), + op: nodedb::bridge::scan_filter::FilterOp::Gte, value: nodedb_types::Value::Integer(2023), clauses: Vec::new(), expr: None, diff --git a/nodedb/tests/inproc/cases/executor_tests/test_facet.rs b/nodedb/tests/inproc/cases/executor_tests/test_facet.rs index e072fca0c..17b8271ee 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_facet.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_facet.rs @@ -9,7 +9,7 @@ //! - Zero-match filter returns empty facets //! - Limit per facet (top-N truncation) -use nodedb::bridge::scan_filter::ScanFilter; +use nodedb::bridge::scan_filter::{FilterOp, ScanFilter}; use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan, QueryOp}; use super::helpers::*; @@ -133,10 +133,10 @@ fn seed_products( ); } -fn filter(field: &str, op: &str, value: nodedb_types::Value) -> ScanFilter { +fn filter(field: &str, op: FilterOp, value: nodedb_types::Value) -> ScanFilter { ScanFilter { field: field.into(), - op: op.into(), + op, value, clauses: Vec::new(), expr: None, @@ -193,7 +193,7 @@ fn filtered_facet_counts() { // Filter: brand = 'Nike' — should only count Nike products. let filters = vec![filter( "brand", - "eq", + FilterOp::Eq, nodedb_types::Value::String("Nike".into()), )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); @@ -236,7 +236,7 @@ fn zero_match_filter_returns_empty_facets() { // Filter: brand = 'NonExistent'. let filters = vec![filter( "brand", - "eq", + FilterOp::Eq, nodedb_types::Value::String("NonExistent".into()), )]; let filter_bytes = zerompk::to_msgpack_vec(&filters).unwrap(); diff --git a/nodedb/tests/inproc/cases/executor_tests/test_ollp_verification.rs b/nodedb/tests/inproc/cases/executor_tests/test_ollp_verification.rs index 9948252af..ac6fc2d29 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_ollp_verification.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_ollp_verification.rs @@ -40,7 +40,7 @@ const COLLECTION: &str = "ollp_items"; fn filter_active() -> Vec { let f = ScanFilter { field: "active".into(), - op: "eq".into(), + op: nodedb::bridge::scan_filter::FilterOp::Eq, value: nodedb_types::Value::Bool(true), clauses: Vec::new(), expr: None, diff --git a/nodedb/tests/inproc/cases/executor_tests/test_timeseries.rs b/nodedb/tests/inproc/cases/executor_tests/test_timeseries.rs index bc3e9945f..bd266b0e4 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_timeseries.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_timeseries.rs @@ -317,7 +317,7 @@ fn where_predicate_filters_count() { 0, vec![nodedb::bridge::scan_filter::ScanFilter { field: "qtype".into(), - op: "eq".into(), + op: nodedb::bridge::scan_filter::FilterOp::Eq, value: nodedb_types::Value::String("A".into()), clauses: vec![], expr: None, diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index 25c9b0ae5..7aab40103 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -261,6 +261,7 @@ mod sql_update_from; mod sql_utf8_expressions; mod sql_vector_index_ddl; mod sql_where_expressions; +mod sql_where_instant_literals; mod sql_where_vector_search; mod sql_window_frames; mod sql_window_functions; diff --git a/nodedb/tests/wire/cases/sql_where_instant_literals.rs b/nodedb/tests/wire/cases/sql_where_instant_literals.rs new file mode 100644 index 000000000..45a283a34 --- /dev/null +++ b/nodedb/tests/wire/cases/sql_where_instant_literals.rs @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A literal compared against a declared `TIMESTAMP`/`TIMESTAMPTZ` column +//! follows one rule on every engine and every path: a string literal parses +//! as a datetime, a numeric literal is epoch milliseconds, and any other +//! literal kind is a typed error. The comparison then compares instants, and +//! `SELECT` renders the column as `2020-03-05T10:00:00.000000Z` style +//! ISO-8601. + +use crate::harness::TestServer; + +/// `2020-03-05 10:00:00` as a bare datetime literal. Its epoch-millisecond +/// form is `1583402400000`, which the predicate tables below spell out +/// directly in each SQL clause. +const EARLY: &str = "2020-03-05 10:00:00"; +/// `EARLY` rendered by a `SELECT` of a `TIMESTAMP` column. +const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; + +/// A predicate against `at`, and the row ids it must select, in `ORDER BY +/// id` order. +struct Predicate { + label: &'static str, + clause: &'static str, + expected: &'static [&'static str], +} + +/// One family per operator, expressed with a numeric-ms literal. Callers +/// substitute the literal for a string form to cover the other literal kind. +const NUMERIC_PREDICATES: &[Predicate] = &[ + Predicate { + label: "= ms", + clause: "at = 1583402400000", + expected: &["r2_at"], + }, + Predicate { + label: "> ms", + clause: "at > 1583402400000", + expected: &["r3_after"], + }, + Predicate { + label: ">= ms", + clause: "at >= 1583402400000", + expected: &["r2_at", "r3_after"], + }, + Predicate { + label: "< ms", + clause: "at < 1583402400000", + expected: &["r1_before"], + }, + Predicate { + label: "<= ms", + clause: "at <= 1583402400000", + expected: &["r1_before", "r2_at"], + }, + Predicate { + label: "BETWEEN ms", + clause: "at BETWEEN 1583402400000 AND 1583406000000", + expected: &["r2_at", "r3_after"], + }, + Predicate { + label: "IN ms", + clause: "at IN (1583402400000, 1583406000000)", + expected: &["r2_at", "r3_after"], + }, +]; + +/// The same seven predicates with a space-separated datetime string literal. +const STRING_SPACE_PREDICATES: &[Predicate] = &[ + Predicate { + label: "= string", + clause: "at = '2020-03-05 10:00:00'", + expected: &["r2_at"], + }, + Predicate { + label: "> string", + clause: "at > '2020-03-05 10:00:00'", + expected: &["r3_after"], + }, + Predicate { + label: ">= string", + clause: "at >= '2020-03-05 10:00:00'", + expected: &["r2_at", "r3_after"], + }, + Predicate { + label: "< string", + clause: "at < '2020-03-05 10:00:00'", + expected: &["r1_before"], + }, + Predicate { + label: "<= string", + clause: "at <= '2020-03-05 10:00:00'", + expected: &["r1_before", "r2_at"], + }, + Predicate { + label: "BETWEEN string", + clause: "at BETWEEN '2020-03-05 10:00:00' AND '2020-03-05 11:00:00'", + expected: &["r2_at", "r3_after"], + }, + Predicate { + label: "IN string", + clause: "at IN ('2020-03-05 10:00:00', '2020-03-05 11:00:00')", + expected: &["r2_at", "r3_after"], + }, +]; + +/// The same seven predicates with an ISO-8601 `T`/`Z` string literal. +const STRING_ISO_PREDICATES: &[Predicate] = &[ + Predicate { + label: "= iso", + clause: "at = '2020-03-05T10:00:00Z'", + expected: &["r2_at"], + }, + Predicate { + label: "> iso", + clause: "at > '2020-03-05T10:00:00Z'", + expected: &["r3_after"], + }, + Predicate { + label: ">= iso", + clause: "at >= '2020-03-05T10:00:00Z'", + expected: &["r2_at", "r3_after"], + }, + Predicate { + label: "< iso", + clause: "at < '2020-03-05T10:00:00Z'", + expected: &["r1_before"], + }, + Predicate { + label: "<= iso", + clause: "at <= '2020-03-05T10:00:00Z'", + expected: &["r1_before", "r2_at"], + }, + Predicate { + label: "BETWEEN iso", + clause: "at BETWEEN '2020-03-05T10:00:00Z' AND '2020-03-05T11:00:00Z'", + expected: &["r2_at", "r3_after"], + }, + Predicate { + label: "IN iso", + clause: "at IN ('2020-03-05T10:00:00Z', '2020-03-05T11:00:00Z')", + expected: &["r2_at", "r3_after"], + }, +]; + +/// Create `name` on `engine` with `(id TEXT PRIMARY KEY, at TIMESTAMP, v +/// FLOAT)` and seed `r1_before`/`r2_at`/`r3_after` an hour apart around `EARLY`. +/// `engine` is `None` for a schemaless document collection (no `WITH` +/// clause). +async fn seed(server: &TestServer, name: &str, engine: Option<&str>) { + let with_clause = match engine { + Some(e) => format!(" WITH (engine='{e}')"), + None => String::new(), + }; + server + .exec(&format!( + "CREATE COLLECTION {name} (id TEXT PRIMARY KEY, at TIMESTAMP, v FLOAT){with_clause}" + )) + .await + .unwrap_or_else(|e| panic!("create {name}: {e}")); + server + .exec(&format!( + "INSERT INTO {name} (id, at, v) VALUES \ + ('r1_before', '2020-03-05 09:00:00', 1.0), \ + ('r2_at', '{EARLY}', 2.0), \ + ('r3_after', '2020-03-05 11:00:00', 3.0)" + )) + .await + .unwrap_or_else(|e| panic!("insert into {name}: {e}")); +} + +/// Create a timeseries collection `(at TIMESTAMP TIME_KEY, id TEXT, v FLOAT) +/// WITH (engine='timeseries')` and seed the same three rows. +async fn seed_timeseries(server: &TestServer, name: &str) { + server + .exec(&format!( + "CREATE COLLECTION {name} (at TIMESTAMP TIME_KEY, id TEXT, v FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .unwrap_or_else(|e| panic!("create {name}: {e}")); + server + .exec(&format!( + "INSERT INTO {name} (at, id, v) VALUES \ + ('2020-03-05 09:00:00', 'r1_before', 1.0), \ + ('{EARLY}', 'r2_at', 2.0), \ + ('2020-03-05 11:00:00', 'r3_after', 3.0)" + )) + .await + .unwrap_or_else(|e| panic!("insert into {name}: {e}")); +} + +/// Run every predicate in `table` against `collection` and assert the row +/// ids each one selects, each assertion message naming the predicate. +async fn assert_predicates(server: &TestServer, collection: &str, table: &[Predicate]) { + for p in table { + let rows = server + .query_text(&format!( + "SELECT id FROM {collection} WHERE {} ORDER BY id", + p.clause + )) + .await + .unwrap_or_else(|e| panic!("{collection} [{}]: {e}", p.label)); + assert_eq!( + rows, p.expected, + "{collection} [{}]: expected {:?}, got {rows:?}", + p.label, p.expected + ); + } +} + +/// Run the full literal-kind suite (numeric ms, space-separated string, +/// ISO-8601 string) against `collection`, plus the typed-error case for a +/// non-instant literal kind. +async fn assert_full_suite(server: &TestServer, collection: &str) { + assert_predicates(server, collection, NUMERIC_PREDICATES).await; + assert_predicates(server, collection, STRING_SPACE_PREDICATES).await; + assert_predicates(server, collection, STRING_ISO_PREDICATES).await; + server + .expect_error( + &format!("SELECT id FROM {collection} WHERE at = true"), + "at", + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn document_strict_instant_literal_predicates() { + let server = TestServer::start().await; + let name = "swi_strict"; + seed(&server, name, Some("document_strict")).await; + assert_full_suite(&server, name).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn schemaless_document_instant_literal_predicates() { + let server = TestServer::start().await; + let name = "swi_schemaless"; + seed(&server, name, None).await; + assert_full_suite(&server, name).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn kv_instant_literal_predicates() { + let server = TestServer::start().await; + let name = "swi_kv"; + seed(&server, name, Some("kv")).await; + assert_full_suite(&server, name).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn columnar_instant_literal_predicates() { + let server = TestServer::start().await; + let name = "swi_columnar"; + seed(&server, name, Some("columnar")).await; + assert_full_suite(&server, name).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn timeseries_instant_literal_predicates() { + let server = TestServer::start().await; + let name = "swi_timeseries"; + seed_timeseries(&server, name).await; + assert_full_suite(&server, name).await; +} + +/// Rows flushed to a timeseries partition (a low memtable budget forces the +/// flush) follow the same instant-literal rule as an unflushed memtable read. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn timeseries_instant_literal_predicates_after_flush() { + let server = TestServer::start_with_timeseries_memtable_budget(1).await; + let name = "swi_ts_flushed"; + seed_timeseries(&server, name).await; + assert_full_suite(&server, name).await; +} + +/// Rows on a flushed columnar segment follow the same instant-literal rule +/// as rows still in the write buffer. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn columnar_instant_literal_predicates_after_flush() { + let server = TestServer::start_with_columnar_flush_threshold(2).await; + let name = "swi_columnar_flushed"; + seed(&server, name, Some("columnar")).await; + assert_full_suite(&server, name).await; +} + +/// `UPDATE ... WHERE at > ` and `DELETE ... WHERE at < ` scope by the +/// same instant the read path selects by, for every DML-capable engine. +async fn assert_update_delete_scoped_by_instant(server: &TestServer, collection: &str) { + server + .exec(&format!( + "UPDATE {collection} SET v = 9 WHERE at > 1583402400000" + )) + .await + .unwrap_or_else(|e| panic!("update {collection}: {e}")); + let updated = server + .query_text(&format!("SELECT id FROM {collection} WHERE v = 9")) + .await + .unwrap_or_else(|e| panic!("select updated {collection}: {e}")); + assert_eq!( + updated, + vec!["r3_after".to_string()], + "UPDATE ... WHERE at > must touch only r3_after, got {updated:?}" + ); + + server + .exec(&format!( + "DELETE FROM {collection} WHERE at < 1583402400000" + )) + .await + .unwrap_or_else(|e| panic!("delete {collection}: {e}")); + let remaining = server + .query_text(&format!("SELECT id FROM {collection} ORDER BY id")) + .await + .unwrap_or_else(|e| panic!("select remaining {collection}: {e}")); + assert_eq!( + remaining, + vec!["r2_at".to_string(), "r3_after".to_string()], + "DELETE ... WHERE at < must remove only r1_before, got {remaining:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn document_strict_update_delete_scoped_by_instant_literal() { + let server = TestServer::start().await; + let name = "swi_strict_dml"; + seed(&server, name, Some("document_strict")).await; + assert_update_delete_scoped_by_instant(&server, name).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn columnar_update_delete_scoped_by_instant_literal() { + let server = TestServer::start().await; + let name = "swi_columnar_dml"; + seed(&server, name, Some("columnar")).await; + assert_update_delete_scoped_by_instant(&server, name).await; +} + +/// A JOIN `ON` clause comparing a timeseries time key against a numeric-ms +/// literal follows the same rule as a top-level `WHERE`: the literal is an +/// instant, so the predicate scopes the join to rows after `EARLY`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn join_on_clause_with_instant_ms_literal_scopes_the_match() { + let server = TestServer::start().await; + let ts_name = "swi_join_ts"; + let strict_name = "swi_join_strict"; + seed_timeseries(&server, ts_name).await; + seed(&server, strict_name, Some("document_strict")).await; + + let joined = server + .query_text(&format!( + "SELECT a.id FROM {ts_name} a JOIN {strict_name} b \ + ON a.id = b.id AND a.at > 1583402400000" + )) + .await + .unwrap_or_else(|e| panic!("join {ts_name}/{strict_name}: {e}")); + assert_eq!( + joined, + vec!["r3_after".to_string()], + "the ON clause's ms literal must scope the join to rows after EARLY: {joined:?}" + ); +} + +/// A `SELECT` of the declared `at` column renders `EARLY` as ISO-8601 UTC, +/// independent of which literal kind selected the row. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn selecting_the_matched_row_renders_iso8601() { + let server = TestServer::start().await; + let name = "swi_render"; + seed(&server, name, Some("document_strict")).await; + + let rows = server + .query_text(&format!("SELECT at FROM {name} WHERE at = 1583402400000")) + .await + .unwrap_or_else(|e| panic!("select at from {name}: {e}")); + assert_eq!( + rows, + vec![EARLY_ISO.to_string()], + "SELECT of a TIMESTAMP column must render ISO-8601 UTC: {rows:?}" + ); +} From 3e4b0fd7fd2e954bdd544cde943f1ad379c2c1a6 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 17 Sep 2026 11:12:54 +0800 Subject: [PATCH 18/21] feat(rls): compile policies from stored text, fail closed on typing RLS policies were stored as pre-compiled predicate JSON (compiled_predicate_json), frozen at CREATE time: an ALTER COLLECTION that added, dropped, or renamed a column left every existing policy typed against the old schema, and a literal that failed to compile was silently skipped with a warn! log, so a broken policy quietly stopped restricting rows instead of refusing to install. StoredRlsPolicy now stores predicate_text (plus the database_id its declared columns live in) and recompiles it on every read through the single entry point compile_policy_predicate: parse, validate $auth.* references, then predicate_typing::type_predicate_literals types every literal a Compare node pairs with a declared TIMESTAMP/TIMESTAMPTZ column through nodedb_sql's coerce_read_literal, the same rule a query's WHERE literal follows, so a policy and a query on the same column agree on the instant. A literal the declared type cannot read is a compile error naming the column and the literal. StoredRlsPolicy::rehydrate (catalog/rls.rs, via rls/compile.rs) is the one path that turns stored text into a runtime RlsPolicy: on a compile failure it installs a restrictive deny-all instead of skipping the policy, so an unenforceable policy fails closed rather than silently admitting every row. Boot replay, Raft post-apply, and recovery reload all call it uniformly. recompile_for_collection reruns it for every policy on a collection whenever that collection's declared columns change (ALTER ADD COLUMN, strict schema changes, and the Raft post-apply of a collection catalog entry), keeping policies current with the schema without a restart. combined_read_predicate_with_auth and combined_write_predicate_with_auth now return a Result: encoding a compiled predicate to ScanFilter bytes can fail, and every caller (read-gate checks, EXPLAIN visibility, graph index creation, the planner's RLS injection) propagates that error instead of treating it as "no policy". predicate_eval.rs splits into a directory (filters.rs for the two constant ScanFilter shapes, sets.rs, substitute.rs) alongside the new predicate_typing.rs and rls/compile.rs. --- .../catalog_entry/post_apply/collection.rs | 15 + .../control/catalog_entry/post_apply/rls.rs | 40 +- .../control/planner/catalog_adapter/mod.rs | 1 + .../planner/catalog_adapter/type_convert.rs | 2 +- .../control/planner/rls_injection/filters.rs | 12 +- nodedb/src/control/security/catalog/rls.rs | 197 +++++-- nodedb/src/control/security/explain.rs | 10 +- nodedb/src/control/security/mod.rs | 1 + nodedb/src/control/security/predicate.rs | 59 ++- nodedb/src/control/security/predicate_eval.rs | 500 ------------------ .../security/predicate_eval/filters.rs | 47 ++ .../control/security/predicate_eval/mod.rs | 19 + .../control/security/predicate_eval/sets.rs | 185 +++++++ .../security/predicate_eval/substitute.rs | 315 +++++++++++ .../src/control/security/predicate_parser.rs | 27 +- .../src/control/security/predicate_typing.rs | 271 ++++++++++ nodedb/src/control/security/rls/compile.rs | 76 +++ nodedb/src/control/security/rls/eval.rs | 186 ++++--- nodedb/src/control/security/rls/mod.rs | 3 + nodedb/src/control/security/rls/store.rs | 18 +- .../neutral/collection/alter/add_column.rs | 1 + .../neutral/collection/alter/strict_schema.rs | 25 +- .../server/shared/ddl/neutral/explain_ddl.rs | 16 +- .../server/shared/ddl/neutral/read_gate.rs | 3 + .../control/server/shared/ddl/neutral/rls.rs | 43 +- .../ddl/neutral/tree_ops/create_index.rs | 1 + .../src/control/state/init_prod/bootstrap.rs | 26 +- .../inproc/cases/catalog_recovery_check.rs | 8 +- .../cases/collection_cascade_enumeration.rs | 3 +- nodedb/tests/inproc/cases/startup_failure.rs | 3 +- .../cases/columnar_read_row_level_security.rs | 114 ++++ .../timeseries_read_row_level_security.rs | 118 +++++ 32 files changed, 1619 insertions(+), 726 deletions(-) delete mode 100644 nodedb/src/control/security/predicate_eval.rs create mode 100644 nodedb/src/control/security/predicate_eval/filters.rs create mode 100644 nodedb/src/control/security/predicate_eval/mod.rs create mode 100644 nodedb/src/control/security/predicate_eval/sets.rs create mode 100644 nodedb/src/control/security/predicate_eval/substitute.rs create mode 100644 nodedb/src/control/security/predicate_typing.rs create mode 100644 nodedb/src/control/security/rls/compile.rs diff --git a/nodedb/src/control/catalog_entry/post_apply/collection.rs b/nodedb/src/control/catalog_entry/post_apply/collection.rs index f7d28aa36..4eaabf5d4 100644 --- a/nodedb/src/control/catalog_entry/post_apply/collection.rs +++ b/nodedb/src/control/catalog_entry/post_apply/collection.rs @@ -25,6 +25,21 @@ pub fn put_owner_sync(stored: &StoredCollection, shared: Arc) { tenant_id: stored.tenant_id, owner_username: stored.owner.clone(), }); + // The collection's declared columns type its RLS policies' literals, so + // every node recompiles them against the schema this entry carries. + if let Err(e) = shared.rls.recompile_for_collection( + shared.credentials.catalog(), + stored.database_id, + stored.tenant_id, + &stored.name, + ) { + tracing::error!( + collection = %stored.name, + tenant = stored.tenant_id, + error = %e, + "post_apply: RLS policies could not be re-read for recompilation" + ); + } } /// Register-dispatch half: dispatch a `Register` request to this node's diff --git a/nodedb/src/control/catalog_entry/post_apply/rls.rs b/nodedb/src/control/catalog_entry/post_apply/rls.rs index 41ba46e74..140ef58c0 100644 --- a/nodedb/src/control/catalog_entry/post_apply/rls.rs +++ b/nodedb/src/control/catalog_entry/post_apply/rls.rs @@ -3,39 +3,27 @@ //! Post-apply side effects for RLS policy `CatalogEntry` variants. //! //! After the synchronous `apply::rls` step has written the redb row, -//! this rehydrates the runtime `RlsPolicy` (deserializing the -//! compiled predicate / deny mode JSON) and installs it into the -//! in-memory `RlsPolicyStore` on every node so the read/write -//! evaluators see the new policy on their next request. +//! this rehydrates the runtime `RlsPolicy` (compiling the stored `USING` +//! text against the collection's declared columns) and installs it into +//! the in-memory `RlsPolicyStore` on every node so the read/write +//! evaluators see the new policy on their next request. A row that cannot +//! be compiled installs as a restrictive deny-all +//! (`StoredRlsPolicy::rehydrate`). use std::sync::Arc; -use tracing::warn; - use crate::control::security::catalog::StoredRlsPolicy; use crate::control::state::SharedState; pub fn put(stored: StoredRlsPolicy, shared: Arc) { - match stored.to_runtime() { - Ok(runtime) => { - shared.rls.install_replicated_policy(runtime); - tracing::debug!( - policy = %stored.name, - collection = %stored.collection, - tenant = stored.tenant_id, - "post_apply: RLS policy replicated" - ); - } - Err(e) => { - warn!( - policy = %stored.name, - collection = %stored.collection, - tenant = stored.tenant_id, - error = %e, - "post_apply: RLS policy rehydration failed" - ); - } - } + let runtime = stored.rehydrate(shared.credentials.catalog()); + shared.rls.install_replicated_policy(runtime); + tracing::debug!( + policy = %stored.name, + collection = %stored.collection, + tenant = stored.tenant_id, + "post_apply: RLS policy replicated" + ); } pub fn delete(tenant_id: u64, collection: String, name: String, shared: Arc) { diff --git a/nodedb/src/control/planner/catalog_adapter/mod.rs b/nodedb/src/control/planner/catalog_adapter/mod.rs index fe5c8beca..668c64530 100644 --- a/nodedb/src/control/planner/catalog_adapter/mod.rs +++ b/nodedb/src/control/planner/catalog_adapter/mod.rs @@ -36,3 +36,4 @@ mod sql_catalog_impl; mod type_convert; pub use adapter::OriginCatalog; +pub(crate) use type_convert::convert_collection_type; diff --git a/nodedb/src/control/planner/catalog_adapter/type_convert.rs b/nodedb/src/control/planner/catalog_adapter/type_convert.rs index 7438b2666..e13742daf 100644 --- a/nodedb/src/control/planner/catalog_adapter/type_convert.rs +++ b/nodedb/src/control/planner/catalog_adapter/type_convert.rs @@ -6,7 +6,7 @@ use nodedb_sql::types::{ColumnInfo, EngineType, SqlDataType}; use nodedb_types::columnar::{FloatWidth, IntWidth}; /// Convert a StoredCollection to engine type, columns, and primary key. -pub(super) fn convert_collection_type( +pub(crate) fn convert_collection_type( stored: &crate::control::security::catalog::StoredCollection, ) -> (EngineType, Vec, Option) { use nodedb_types::CollectionType; diff --git a/nodedb/src/control/planner/rls_injection/filters.rs b/nodedb/src/control/planner/rls_injection/filters.rs index 2e4377948..721450a86 100644 --- a/nodedb/src/control/planner/rls_injection/filters.rs +++ b/nodedb/src/control/planner/rls_injection/filters.rs @@ -7,6 +7,9 @@ use crate::control::security::rls::RlsPolicyStore; use crate::types::TenantId; /// Fetch RLS bytes for a (tenant, collection) pair. +/// +/// An unresolvable `$auth.*` reference is the deny error; an unencodable +/// filter set is its own error. Both refuse the statement. pub(super) fn get_rls( rls_store: &RlsPolicyStore, tenant_id: u64, @@ -14,15 +17,16 @@ pub(super) fn get_rls( auth: &AuthContext, ) -> crate::Result> { rls_store - .combined_read_predicate_with_auth(tenant_id, collection, auth) + .combined_read_predicate_with_auth(tenant_id, collection, auth)? .ok_or_else(|| rls_deny_error(tenant_id, collection)) } /// Fetch the compiled write-policy bytes for a (tenant, collection) pair. /// /// Fails closed on an unresolvable `$auth.*` reference through the same deny -/// error the read fetch raises, so a write can never proceed on a predicate -/// that could not be resolved. +/// error the read fetch raises, and on an unencodable filter set through +/// its own error, so a write can never proceed on a predicate that could +/// not be resolved or encoded. pub(super) fn get_rls_write( rls_store: &RlsPolicyStore, tenant_id: u64, @@ -30,7 +34,7 @@ pub(super) fn get_rls_write( auth: &AuthContext, ) -> crate::Result> { rls_store - .combined_write_predicate_with_auth(tenant_id, collection, auth) + .combined_write_predicate_with_auth(tenant_id, collection, auth)? .ok_or_else(|| rls_deny_error(tenant_id, collection)) } diff --git a/nodedb/src/control/security/catalog/rls.rs b/nodedb/src/control/security/catalog/rls.rs index 169fc1aff..b252fc82d 100644 --- a/nodedb/src/control/security/catalog/rls.rs +++ b/nodedb/src/control/security/catalog/rls.rs @@ -2,20 +2,30 @@ //! RLS policy persistence in the system catalog. //! -//! `RlsPolicy` (the runtime shape) carries `Option` and -//! `DenyMode`, both of which are serde-only and don't fit zerompk's -//! `ToMessagePack` derive. `StoredRlsPolicy` flattens those parts into -//! JSON strings (via sonic_rs) so the whole record can be msgpack-encoded -//! by zerompk like every other catalog row. +//! `StoredRlsPolicy` is the catalog row: the policy's `USING` text, its +//! database, and the scalar settings. The compiled predicate is never +//! persisted. Every reader recompiles the text against the collection's +//! current declared columns through [`StoredRlsPolicy::to_runtime`], so a +//! literal compared against a `TIMESTAMP` column is typed the same way at +//! boot, on Raft apply, and after a schema change as it was at +//! `CREATE RLS POLICY`. `DenyMode` is serde-only and is carried as a +//! sonic_rs JSON string so the row can derive zerompk like every other +//! catalog row. //! //! Conversions: [`StoredRlsPolicy::from_runtime`] for serialization, -//! [`StoredRlsPolicy::to_runtime`] for replay on apply / boot. +//! [`StoredRlsPolicy::to_runtime`] for replay on apply / boot, and +//! [`StoredRlsPolicy::rehydrate`] for the load paths that must install a +//! policy whatever happens: a row that cannot be compiled installs as a +//! restrictive deny-all so it can never widen access. use redb::{ReadableDatabase, ReadableTable, TableDefinition}; +use tracing::error; use crate::control::security::deny::DenyMode; use crate::control::security::predicate::{PolicyMode, RlsPredicate}; +use crate::control::security::rls::compile::{compile_policy_predicate, declared_columns}; use crate::control::security::rls::{PolicyType, RlsPolicy}; +use crate::types::DatabaseId; use super::types::{SystemCatalog, catalog_err}; @@ -24,29 +34,29 @@ use super::types::{SystemCatalog, catalog_err}; pub(super) const RLS_POLICIES: TableDefinition<&str, &[u8]> = TableDefinition::new("_system.rls_policies"); -/// Catalog-shape RLS policy. JSON strings are sonic_rs-encoded -/// versions of the runtime types so zerompk can derive the encoder. +/// Catalog-shape RLS policy. /// -/// Map-encoded (`#[msgpack(map)]`) so `display_collection` could be added -/// with `#[msgpack(default)]`: records written before that field decode -/// with `display_collection = ""` instead of failing outright. +/// Map-encoded (`#[msgpack(map)]`) so a field can carry `#[msgpack(default)]`. #[derive(zerompk::ToMessagePack, zerompk::FromMessagePack, Debug, Clone)] #[msgpack(map)] pub struct StoredRlsPolicy { pub tenant_id: u64, + /// Database the policy's collection lives in. With `display_collection` + /// it names the catalog collection whose declared columns type the + /// predicate's literals. + pub database_id: u64, /// Qualified with the owning database ID — the lookup key. Never /// shown to a user; see `display_collection`. pub collection: String, /// The collection name as the user wrote it, unqualified. - /// Display-only: falls back to `collection` when empty (records - /// written before this field existed). + /// Falls back to `collection` when empty. #[msgpack(default)] pub display_collection: String, pub name: String, /// 0 = Read, 1 = Write, 2 = All. pub policy_type_tag: u8, - /// JSON-serialized `Option`. Empty string = None. - pub compiled_predicate_json: String, + /// The `USING (...)` text as written. Empty = no row filter (vacuous). + pub predicate_text: String, /// 0 = Permissive, 1 = Restrictive. pub mode_tag: u8, /// JSON-serialized `DenyMode`. @@ -57,15 +67,18 @@ pub struct StoredRlsPolicy { } impl StoredRlsPolicy { - pub fn from_runtime(p: &RlsPolicy) -> crate::Result { - let compiled_predicate_json = match &p.compiled_predicate { - Some(rp) => sonic_rs::to_string(rp).map_err(|e| catalog_err("ser compiled rls", e))?, - None => String::new(), - }; + /// The catalog row for `p`, whose predicate was compiled from + /// `predicate_text` against the collection in `database_id`. + pub fn from_runtime( + p: &RlsPolicy, + database_id: DatabaseId, + predicate_text: &str, + ) -> crate::Result { let on_deny_json = sonic_rs::to_string(&p.on_deny).map_err(|e| catalog_err("ser deny mode", e))?; Ok(Self { tenant_id: p.tenant_id, + database_id: database_id.as_u64(), collection: p.collection.clone(), display_collection: p.display_collection.clone(), name: p.name.clone(), @@ -74,7 +87,7 @@ impl StoredRlsPolicy { PolicyType::Write => 1, PolicyType::All => 2, }, - compiled_predicate_json, + predicate_text: predicate_text.to_string(), mode_tag: match p.mode { PolicyMode::Permissive => 0, PolicyMode::Restrictive => 1, @@ -86,18 +99,14 @@ impl StoredRlsPolicy { }) } - pub fn to_runtime(&self) -> crate::Result { - let policy_type = match self.policy_type_tag { - 0 => PolicyType::Read, - 1 => PolicyType::Write, - 2 => PolicyType::All, - other => { - return Err(catalog_err( - "deser rls", - format!("invalid policy_type_tag {other}"), - )); - } - }; + /// The runtime policy, with `predicate_text` compiled against the + /// collection's current declared columns. + /// + /// Errors when a tag is invalid, the deny mode does not decode, the + /// collection is absent from `catalog`, or the text does not compile + /// against its columns. + pub fn to_runtime(&self, catalog: &SystemCatalog) -> crate::Result { + let policy_type = self.policy_type()?; let mode = match self.mode_tag { 0 => PolicyMode::Permissive, 1 => PolicyMode::Restrictive, @@ -108,27 +117,23 @@ impl StoredRlsPolicy { )); } }; - let compiled_predicate: Option = if self.compiled_predicate_json.is_empty() { - None - } else { - Some( - sonic_rs::from_str(&self.compiled_predicate_json) - .map_err(|e| catalog_err("deser compiled rls", e))?, - ) - }; let on_deny: DenyMode = sonic_rs::from_str(&self.on_deny_json) .map_err(|e| catalog_err("deser deny mode", e))?; - // Records written before `display_collection` existed decode it - // empty; fall back to `collection`, which was unqualified then. - let display_collection = if self.display_collection.is_empty() { - self.collection.clone() + let compiled_predicate = if self.predicate_text.is_empty() { + None } else { - self.display_collection.clone() + let columns = declared_columns( + catalog, + DatabaseId::new(self.database_id), + self.tenant_id, + self.display_name(), + )?; + Some(compile_policy_predicate(&self.predicate_text, &columns)?) }; Ok(RlsPolicy { name: self.name.clone(), collection: self.collection.clone(), - display_collection, + display_collection: self.display_name().to_string(), tenant_id: self.tenant_id, policy_type, compiled_predicate, @@ -140,6 +145,70 @@ impl StoredRlsPolicy { }) } + /// [`Self::to_runtime`], or a restrictive deny-all policy under the same + /// key when the row cannot be compiled. + /// + /// For the load paths that install every stored row: a policy that + /// cannot be typed against its collection must still govern the + /// collection, and the only safe form is one that admits no row. The + /// error is logged at `error` level with the policy's key. + pub fn rehydrate(&self, catalog: &SystemCatalog) -> RlsPolicy { + match self.to_runtime(catalog) { + Ok(policy) => policy, + Err(e) => { + error!( + policy = %self.name, + collection = %self.collection, + tenant = self.tenant_id, + error = %e, + "RLS policy cannot be compiled; installed as restrictive deny-all" + ); + self.deny_all() + } + } + } + + /// The restrictive deny-all form of this row: same key, same enabled + /// flag, a predicate no row passes, AND-combined with every other policy. + fn deny_all(&self) -> RlsPolicy { + RlsPolicy { + name: self.name.clone(), + collection: self.collection.clone(), + display_collection: self.display_name().to_string(), + tenant_id: self.tenant_id, + // An invalid tag cannot say which path the policy governs, so + // the deny-all governs both. + policy_type: self.policy_type().unwrap_or(PolicyType::All), + compiled_predicate: Some(RlsPredicate::AlwaysFalse), + mode: PolicyMode::Restrictive, + on_deny: DenyMode::default(), + enabled: self.enabled, + created_by: self.created_by.clone(), + created_at: self.created_at, + } + } + + fn policy_type(&self) -> crate::Result { + match self.policy_type_tag { + 0 => Ok(PolicyType::Read), + 1 => Ok(PolicyType::Write), + 2 => Ok(PolicyType::All), + other => Err(catalog_err( + "deser rls", + format!("invalid policy_type_tag {other}"), + )), + } + } + + /// The unqualified collection name, falling back to `collection`. + fn display_name(&self) -> &str { + if self.display_collection.is_empty() { + &self.collection + } else { + &self.display_collection + } + } + fn redb_key(&self) -> String { rls_key(self.tenant_id, &self.collection, &self.name) } @@ -308,11 +377,11 @@ mod tests { fn put_get_delete_roundtrip() { let catalog = make_catalog(); let runtime = sample_policy(1, "users", "p1"); - let stored = StoredRlsPolicy::from_runtime(&runtime).unwrap(); + let stored = StoredRlsPolicy::from_runtime(&runtime, DatabaseId::DEFAULT, "").unwrap(); catalog.put_rls_policy(&stored).unwrap(); let loaded = catalog.get_rls_policy(1, "users", "p1").unwrap().unwrap(); - let runtime2 = loaded.to_runtime().unwrap(); + let runtime2 = loaded.to_runtime(&catalog).unwrap(); assert_eq!(runtime2.name, "p1"); assert_eq!(runtime2.collection, "users"); assert!(matches!(runtime2.policy_type, PolicyType::Read)); @@ -325,10 +394,38 @@ mod tests { fn load_all_returns_every_tenant() { let catalog = make_catalog(); for (tenant, name) in [(1, "a"), (1, "b"), (2, "c")] { - let stored = StoredRlsPolicy::from_runtime(&sample_policy(tenant, "x", name)).unwrap(); + let stored = StoredRlsPolicy::from_runtime( + &sample_policy(tenant, "x", name), + DatabaseId::DEFAULT, + "", + ) + .unwrap(); catalog.put_rls_policy(&stored).unwrap(); } let all = catalog.load_all_rls_policies().unwrap(); assert_eq!(all.len(), 3); } + + /// A row whose collection is absent from the catalog cannot be typed; + /// `rehydrate` installs it as a restrictive deny-all under the same key + /// instead of dropping it or admitting every row. + #[test] + fn rehydrate_without_the_collection_installs_a_restrictive_deny_all() { + let catalog = make_catalog(); + let mut runtime = sample_policy(1, "ghost", "p1"); + runtime.mode = PolicyMode::Permissive; + let stored = + StoredRlsPolicy::from_runtime(&runtime, DatabaseId::DEFAULT, "owner = $auth.id") + .unwrap(); + + assert!(stored.to_runtime(&catalog).is_err()); + let installed = stored.rehydrate(&catalog); + assert_eq!(installed.name, "p1"); + assert_eq!(installed.collection, "ghost"); + assert_eq!(installed.mode, PolicyMode::Restrictive); + assert!(matches!( + installed.compiled_predicate, + Some(RlsPredicate::AlwaysFalse) + )); + } } diff --git a/nodedb/src/control/security/explain.rs b/nodedb/src/control/security/explain.rs index c5b628c04..4bc204bf4 100644 --- a/nodedb/src/control/security/explain.rs +++ b/nodedb/src/control/security/explain.rs @@ -169,10 +169,12 @@ pub fn lint_predicate(predicate: &super::predicate::RlsPredicate) -> Vec super::predicate::RlsPredicate::AlwaysFalse => { warnings.push("contradiction: predicate is always false (blocks everything)".into()); } - super::predicate::RlsPredicate::Compare { value, .. } - if !value.is_auth_ref() - && matches!(value, super::predicate::PredicateValue::Literal(_)) => - { + super::predicate::RlsPredicate::Compare { + value: + super::predicate::PredicateValue::Literal(_) + | super::predicate::PredicateValue::Instant { .. }, + .. + } => { warnings .push("static predicate: no $auth reference — same result for all users".into()); } diff --git a/nodedb/src/control/security/mod.rs b/nodedb/src/control/security/mod.rs index 78275f8eb..8f73759b8 100644 --- a/nodedb/src/control/security/mod.rs +++ b/nodedb/src/control/security/mod.rs @@ -33,6 +33,7 @@ pub mod permission_tree; pub mod predicate; pub mod predicate_eval; pub mod predicate_parser; +pub mod predicate_typing; pub mod random; pub mod ratelimit; pub mod redaction; diff --git a/nodedb/src/control/security/predicate.rs b/nodedb/src/control/security/predicate.rs index 84834bc03..65395d2d8 100644 --- a/nodedb/src/control/security/predicate.rs +++ b/nodedb/src/control/security/predicate.rs @@ -31,9 +31,12 @@ //! - **Restrictive**: AND-combined. ALL restrictive policies must pass. //! - Final: `(any permissive passes) AND (all restrictive pass)` +use nodedb_types::datetime::NdbDateTime; +use nodedb_types::json_msgpack::InstantKind; use serde::{Deserialize, Serialize}; use super::auth_context::AuthContext; +use crate::bridge::scan_filter::FilterOp; /// A compiled RLS predicate expression. /// @@ -101,21 +104,21 @@ pub enum CompareOp { } impl CompareOp { - /// Convert to the ScanFilter operator string. - pub fn as_filter_op(&self) -> &'static str { + /// The `ScanFilter` operator this comparison lowers to. + pub fn as_filter_op(&self) -> FilterOp { match self { - Self::Eq => "eq", - Self::Ne => "ne", - Self::Gt => "gt", - Self::Gte => "gte", - Self::Lt => "lt", - Self::Lte => "lte", - Self::In => "in", - Self::NotIn => "not_in", - Self::Like => "like", - Self::ILike => "ilike", - Self::IsNull => "is_null", - Self::IsNotNull => "is_not_null", + Self::Eq => FilterOp::Eq, + Self::Ne => FilterOp::Ne, + Self::Gt => FilterOp::Gt, + Self::Gte => FilterOp::Gte, + Self::Lt => FilterOp::Lt, + Self::Lte => FilterOp::Lte, + Self::In => FilterOp::In, + Self::NotIn => FilterOp::NotIn, + Self::Like => FilterOp::Like, + Self::ILike => FilterOp::Ilike, + Self::IsNull => FilterOp::IsNull, + Self::IsNotNull => FilterOp::IsNotNull, } } @@ -149,6 +152,13 @@ impl CompareOp { pub enum PredicateValue { /// A literal JSON value (string, number, bool, array, null). Literal(serde_json::Value), + /// A literal typed against the declared `TIMESTAMP` / `TIMESTAMPTZ` + /// column it is compared with. Policy compilation resolves a numeric + /// (epoch milliseconds) or text (ISO-8601) literal to this at + /// `CREATE RLS POLICY` and again at every load, by the same rule the + /// planner applies to a query predicate, so the Data Plane compares an + /// instant with an instant. + Instant { at: NdbDateTime, kind: InstantKind }, /// A document field reference (resolved at Data Plane scan time). Field(String), /// A session variable reference: `$auth.id`, `$auth.roles`, etc. @@ -168,7 +178,8 @@ impl PredicateValue { /// Resolve this value using the given `AuthContext`. /// /// - `Literal`: returned as-is. - /// - `Field`: returned as-is (resolved at scan time by Data Plane). + /// - `Instant`: its ISO-8601 rendering. + /// - `Field`: `None` (resolved at scan time by the Data Plane). /// - `AuthRef`: resolved via `AuthContext::resolve_variable()`. /// - `AuthFunc`: resolved via `AuthContext` metadata (pre-computed). /// @@ -176,6 +187,7 @@ impl PredicateValue { pub fn resolve(&self, auth: &AuthContext) -> Option { match self { Self::Literal(v) => Some(v.clone()), + Self::Instant { at, .. } => Some(serde_json::Value::String(at.to_iso8601())), Self::Field(_) => None, Self::AuthRef(field) => auth.resolve_variable(field), Self::AuthFunc { func, args } => { @@ -187,6 +199,21 @@ impl PredicateValue { } } } + + /// Resolve this value to the `nodedb_types::Value` a `ScanFilter` carries. + /// + /// An `Instant` keeps its typed kind (`Value::NaiveDateTime` for a + /// `TIMESTAMP` column, `Value::DateTime` for `TIMESTAMPTZ`); every other + /// variant goes through [`Self::resolve`]. Returns `None` when a `Field` + /// is asked for a value, or an `$auth.*` reference cannot be resolved. + pub fn resolve_scan_value(&self, auth: &AuthContext) -> Option { + match self { + Self::Instant { at, kind } => Some(kind.value(*at)), + Self::Literal(_) | Self::Field(_) | Self::AuthRef(_) | Self::AuthFunc { .. } => { + self.resolve(auth).map(nodedb_types::Value::from) + } + } + } } /// Whether a policy is permissive (OR-combined) or restrictive (AND-combined). @@ -327,7 +354,7 @@ mod tests { assert_eq!(filters[0].field, "allowed_users"); assert_eq!( filters[0].op, - crate::bridge::scan_filter::FilterOp::Contains + crate::bridge::scan_filter::FilterOp::ArrayContains ); assert_eq!(filters[0].value, nodedb_types::Value::String("123".into())); } diff --git a/nodedb/src/control/security/predicate_eval.rs b/nodedb/src/control/security/predicate_eval.rs deleted file mode 100644 index 5922d66c1..000000000 --- a/nodedb/src/control/security/predicate_eval.rs +++ /dev/null @@ -1,500 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Plan-time predicate evaluation: substitute `$auth.*` references and -//! combine policies into concrete `ScanFilter` values. -//! -//! This module converts compiled [`RlsPredicate`] trees into static -//! `ScanFilter` lists that the Data Plane can evaluate without session -//! awareness. - -use super::auth_context::AuthContext; -use super::predicate::{CompareOp, PolicyMode, PredicateValue, RlsPredicate}; -use crate::bridge::scan_filter::{FilterOp, ScanFilter}; - -/// Substitute `$auth.*` references in a predicate tree and produce -/// concrete `ScanFilter` values for the Data Plane. -/// -/// This is the core plan-time substitution. After this, the resulting -/// `ScanFilter` contains only literal values and field references — no -/// session variables. The Data Plane evaluates these without any auth -/// awareness. -/// -/// Returns `None` if any required `$auth` reference cannot be resolved -/// (e.g., `$auth.org_id` when no org context). This causes the predicate -/// to evaluate as **deny** (fail-closed). -pub fn substitute_to_scan_filters( - predicate: &RlsPredicate, - auth: &AuthContext, -) -> Option> { - match predicate { - RlsPredicate::AlwaysTrue => Some(vec![ScanFilter { - field: String::new(), - op: "match_all".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]), - - RlsPredicate::AlwaysFalse => { - // Emit a filter that never matches: non-existent field must be non-null. - Some(vec![ScanFilter { - field: "__rls_deny__".into(), - op: "is_not_null".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } - - RlsPredicate::Compare { field, op, value } => { - let resolved = match value { - PredicateValue::Literal(v) => v.clone(), - PredicateValue::AuthRef(auth_field) => auth.resolve_variable(auth_field)?, - PredicateValue::AuthFunc { .. } => value.resolve(auth)?, - PredicateValue::Field(_) => { - // Field-to-field comparison not supported in ScanFilter. - return None; - } - }; - - Some(vec![ScanFilter { - field: field.clone(), - op: op.as_filter_op().into(), - value: nodedb_types::Value::from(resolved), - clauses: Vec::new(), - expr: None, - }]) - } - - RlsPredicate::Contains { set, element } => substitute_contains(set, element, auth), - - RlsPredicate::Intersects { left, right } => substitute_intersects(left, right, auth), - - RlsPredicate::And(children) => { - let mut combined = Vec::new(); - for child in children { - combined.extend(substitute_to_scan_filters(child, auth)?); - } - Some(combined) - } - - RlsPredicate::Or(children) => { - let mut clause_groups: Vec> = Vec::new(); - for child in children { - if let Some(filters) = substitute_to_scan_filters(child, auth) { - // Check for match_all (always-true) — short-circuit. - if filters.len() == 1 && filters[0].op == FilterOp::MatchAll { - return Some(filters); - } - clause_groups.push(filters); - } - } - - if clause_groups.is_empty() { - return Some(vec![ScanFilter { - field: "__rls_deny__".into(), - op: "is_not_null".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]); - } - - if clause_groups.len() == 1 { - return Some(clause_groups.into_iter().next().unwrap_or_default()); - } - - Some(vec![ScanFilter { - field: String::new(), - op: "or".into(), - value: nodedb_types::Value::Null, - clauses: clause_groups, - expr: None, - }]) - } - - RlsPredicate::Not(inner) => substitute_not(inner, auth), - } -} - -/// Combine multiple policies according to their modes. -/// -/// Final result: `(any permissive passes) AND (all restrictive pass)`. -/// -/// Returns the combined `ScanFilter` list to inject into the query. -/// Empty return = no RLS policies (allow all). -pub fn combine_policies( - policies: &[(RlsPredicate, PolicyMode)], - auth: &AuthContext, -) -> Option> { - if policies.is_empty() { - return Some(Vec::new()); // No policies → allow all - } - - let mut permissive: Vec<&RlsPredicate> = Vec::new(); - let mut restrictive: Vec<&RlsPredicate> = Vec::new(); - - for (pred, mode) in policies { - match mode { - PolicyMode::Permissive => permissive.push(pred), - PolicyMode::Restrictive => restrictive.push(pred), - } - } - - let mut combined = Vec::new(); - - // Permissive: OR-combine. If no permissive policies exist, default allow. - if permissive.len() == 1 { - combined.extend(substitute_to_scan_filters(permissive[0], auth)?); - } else if permissive.len() > 1 { - let or_children: Vec = permissive.iter().map(|p| (*p).clone()).collect(); - let or_pred = RlsPredicate::Or(or_children); - combined.extend(substitute_to_scan_filters(&or_pred, auth)?); - } - - // Restrictive: AND-combine (each becomes additional filters). - for pred in &restrictive { - combined.extend(substitute_to_scan_filters(pred, auth)?); - } - - Some(combined) -} - -// --------------------------------------------------------------------------- -// Internal helpers -// --------------------------------------------------------------------------- - -fn substitute_contains( - set: &PredicateValue, - element: &PredicateValue, - auth: &AuthContext, -) -> Option> { - match (set, element) { - // $auth.roles CONTAINS 'admin' → resolved at plan time. - (PredicateValue::AuthRef(auth_field), PredicateValue::Literal(lit)) => { - let auth_val = auth.resolve_variable(auth_field)?; - if let Some(arr) = auth_val.as_array() { - if arr.contains(lit) { - Some(vec![ScanFilter { - field: String::new(), - op: "match_all".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } else { - Some(vec![ScanFilter { - field: "__rls_deny__".into(), - op: "is_not_null".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } - } else { - None // Expected array, got scalar → deny - } - } - - // $auth.scope_status('pro:all') CONTAINS 'active' → resolved at plan time. - (PredicateValue::AuthFunc { .. }, PredicateValue::Literal(lit)) => { - let auth_val = set.resolve(auth)?; - if let Some(arr) = auth_val.as_array() { - if arr.contains(lit) { - Some(vec![ScanFilter { - field: String::new(), - op: "match_all".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } else { - Some(vec![ScanFilter { - field: "__rls_deny__".into(), - op: "is_not_null".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } - } else { - None // Expected array, got scalar → deny - } - } - - // doc_field CONTAINS $auth.id → field "contains" resolved_value. - (PredicateValue::Field(doc_field), PredicateValue::AuthRef(auth_field)) => { - let auth_val = auth.resolve_variable(auth_field)?; - Some(vec![ScanFilter { - field: doc_field.clone(), - op: "contains".into(), - value: nodedb_types::Value::from(auth_val), - clauses: Vec::new(), - expr: None, - }]) - } - - // doc_field CONTAINS $auth.scope_status('pro:all') → field "contains" resolved_value. - (PredicateValue::Field(doc_field), PredicateValue::AuthFunc { .. }) => { - let auth_val = element.resolve(auth)?; - Some(vec![ScanFilter { - field: doc_field.clone(), - op: "contains".into(), - value: nodedb_types::Value::from(auth_val), - clauses: Vec::new(), - expr: None, - }]) - } - - // doc_field CONTAINS 'literal' → field "contains" literal. - (PredicateValue::Field(doc_field), PredicateValue::Literal(lit)) => { - Some(vec![ScanFilter { - field: doc_field.clone(), - op: "contains".into(), - value: nodedb_types::Value::from(lit.clone()), - clauses: Vec::new(), - expr: None, - }]) - } - - _ => None, // Unsupported combination → deny - } -} - -fn substitute_intersects( - left: &PredicateValue, - right: &PredicateValue, - auth: &AuthContext, -) -> Option> { - match (left, right) { - // doc_field INTERSECTS $auth.groups → "any_in" operator. - (PredicateValue::Field(doc_field), PredicateValue::AuthRef(auth_field)) - | (PredicateValue::AuthRef(auth_field), PredicateValue::Field(doc_field)) => { - let auth_val = auth.resolve_variable(auth_field)?; - Some(vec![ScanFilter { - field: doc_field.clone(), - op: "any_in".into(), - value: nodedb_types::Value::from(auth_val), - clauses: Vec::new(), - expr: None, - }]) - } - - // doc_field INTERSECTS $auth.scope_status('pro:all') → "any_in" operator. - (PredicateValue::Field(doc_field), PredicateValue::AuthFunc { .. }) => { - let auth_val = right.resolve(auth)?; - Some(vec![ScanFilter { - field: doc_field.clone(), - op: "any_in".into(), - value: nodedb_types::Value::from(auth_val), - clauses: Vec::new(), - expr: None, - }]) - } - (PredicateValue::AuthFunc { .. }, PredicateValue::Field(doc_field)) => { - let auth_val = left.resolve(auth)?; - Some(vec![ScanFilter { - field: doc_field.clone(), - op: "any_in".into(), - value: nodedb_types::Value::from(auth_val), - clauses: Vec::new(), - expr: None, - }]) - } - - // $auth.groups INTERSECTS $auth.allowed → plan-time evaluation. - (PredicateValue::AuthRef(left_field), PredicateValue::AuthRef(right_field)) => { - let left_val = auth.resolve_variable(left_field)?; - let right_val = auth.resolve_variable(right_field)?; - let intersects = if let (Some(l), Some(r)) = (left_val.as_array(), right_val.as_array()) - { - l.iter().any(|v| r.contains(v)) - } else { - false - }; - - if intersects { - Some(vec![ScanFilter { - field: String::new(), - op: "match_all".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } else { - Some(vec![ScanFilter { - field: "__rls_deny__".into(), - op: "is_not_null".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } - } - - // AuthFunc on either or both sides → plan-time evaluation via resolve(). - (PredicateValue::AuthFunc { .. }, PredicateValue::AuthFunc { .. }) - | (PredicateValue::AuthRef(_), PredicateValue::AuthFunc { .. }) - | (PredicateValue::AuthFunc { .. }, PredicateValue::AuthRef(_)) => { - let left_val = left.resolve(auth)?; - let right_val = right.resolve(auth)?; - let intersects = if let (Some(l), Some(r)) = (left_val.as_array(), right_val.as_array()) - { - l.iter().any(|v| r.contains(v)) - } else { - false - }; - - if intersects { - Some(vec![ScanFilter { - field: String::new(), - op: "match_all".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } else { - Some(vec![ScanFilter { - field: "__rls_deny__".into(), - op: "is_not_null".into(), - value: nodedb_types::Value::Null, - clauses: Vec::new(), - expr: None, - }]) - } - } - - _ => None, - } -} - -fn substitute_not(inner: &RlsPredicate, auth: &AuthContext) -> Option> { - match inner { - RlsPredicate::AlwaysTrue => substitute_to_scan_filters(&RlsPredicate::AlwaysFalse, auth), - RlsPredicate::AlwaysFalse => substitute_to_scan_filters(&RlsPredicate::AlwaysTrue, auth), - RlsPredicate::Compare { field, op, value } => { - let negated_op = match op { - CompareOp::Eq => CompareOp::Ne, - CompareOp::Ne => CompareOp::Eq, - CompareOp::Gt => CompareOp::Lte, - CompareOp::Gte => CompareOp::Lt, - CompareOp::Lt => CompareOp::Gte, - CompareOp::Lte => CompareOp::Gt, - CompareOp::In => CompareOp::NotIn, - CompareOp::NotIn => CompareOp::In, - CompareOp::IsNull => CompareOp::IsNotNull, - CompareOp::IsNotNull => CompareOp::IsNull, - _ => return None, // Can't negate LIKE/ILIKE simply - }; - substitute_to_scan_filters( - &RlsPredicate::Compare { - field: field.clone(), - op: negated_op, - value: value.clone(), - }, - auth, - ) - } - _ => None, // Complex NOT not supported → deny - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::control::security::auth_context::AuthContext; - use crate::control::security::identity::{ - AuthMethod, AuthenticatedIdentity, DatabaseSet, Role, - }; - use crate::control::security::predicate::{ - CompareOp, PolicyMode, PredicateValue, RlsPredicate, - }; - use crate::types::TenantId; - use nodedb_types::id::DatabaseId; - - fn test_identity() -> AuthenticatedIdentity { - AuthenticatedIdentity::new_regular( - 42, - "alice", - TenantId::new(1), - AuthMethod::ScramSha256, - vec![Role::ReadWrite], - None, - DatabaseSet::Some(smallvec::smallvec![DatabaseId::DEFAULT]), - ) - } - - fn auth_with_database(db_id: DatabaseId) -> AuthContext { - let mut ctx = AuthContext::from_identity(&test_identity(), "s_test".into()); - ctx.database_id = Some(db_id); - ctx - } - - fn auth_without_database() -> AuthContext { - AuthContext::from_identity(&test_identity(), "s_test".into()) - } - - /// `$auth.database_id` resolves to the session's database id and the - /// predicate produces the correct ScanFilter value. - #[test] - fn database_id_auth_ref_substitutes_correctly() { - let db_id = DatabaseId::new(99); - let auth = auth_with_database(db_id); - - let predicate = RlsPredicate::Compare { - field: "owning_db".into(), - op: CompareOp::Eq, - value: PredicateValue::AuthRef("database_id".into()), - }; - - let filters = substitute_to_scan_filters(&predicate, &auth) - .expect("should resolve when database_id is set"); - assert_eq!(filters.len(), 1); - assert_eq!(filters[0].field, "owning_db"); - // The value should be the numeric database id. - match &filters[0].value { - nodedb_types::Value::Integer(n) => assert_eq!(*n as u64, db_id.as_u64()), - other => panic!("expected numeric value, got {:?}", other), - } - } - - /// When `database_id` is `None` the predicate fails closed (returns None). - #[test] - fn database_id_auth_ref_fails_closed_when_none() { - let auth = auth_without_database(); - - let predicate = RlsPredicate::Compare { - field: "owning_db".into(), - op: CompareOp::Eq, - value: PredicateValue::AuthRef("database_id".into()), - }; - - // The substitution must return None so that RLS defaults to deny. - let result = substitute_to_scan_filters(&predicate, &auth); - assert!( - result.is_none(), - "predicate must fail closed when database_id is None" - ); - } - - /// `combine_policies` with a single permissive database_id policy produces - /// the correct ScanFilter when the session has a bound database. - #[test] - fn combine_database_id_policy_passes_when_set() { - let db_id = DatabaseId::new(77); - let auth = auth_with_database(db_id); - - let predicate = RlsPredicate::Compare { - field: "db".into(), - op: CompareOp::Eq, - value: PredicateValue::AuthRef("database_id".into()), - }; - - let policies = [(predicate, PolicyMode::Permissive)]; - let filters = combine_policies(&policies, &auth); - assert!( - filters.as_ref().is_some_and(|v| !v.is_empty()), - "should produce scan filters when database_id is bound" - ); - } -} diff --git a/nodedb/src/control/security/predicate_eval/filters.rs b/nodedb/src/control/security/predicate_eval/filters.rs new file mode 100644 index 000000000..e228ba4cd --- /dev/null +++ b/nodedb/src/control/security/predicate_eval/filters.rs @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The two constant `ScanFilter` shapes policy lowering emits. + +use crate::bridge::scan_filter::{FilterOp, ScanFilter}; + +/// A filter every row passes. +pub fn match_all_filter() -> ScanFilter { + ScanFilter { + field: String::new(), + op: FilterOp::MatchAll, + value: nodedb_types::Value::Null, + clauses: Vec::new(), + expr: None, + } +} + +/// A filter no row passes: a field no document carries must be non-null. +pub fn deny_filter() -> ScanFilter { + ScanFilter { + field: "__rls_deny__".into(), + op: FilterOp::IsNotNull, + value: nodedb_types::Value::Null, + clauses: Vec::new(), + expr: None, + } +} + +/// `field value` as one filter. +pub(super) fn compare_filter(field: &str, op: FilterOp, value: nodedb_types::Value) -> ScanFilter { + ScanFilter { + field: field.to_string(), + op, + value, + clauses: Vec::new(), + expr: None, + } +} + +/// `match_all` when `passes`, the deny filter otherwise. +pub(super) fn verdict_filter(passes: bool) -> ScanFilter { + if passes { + match_all_filter() + } else { + deny_filter() + } +} diff --git a/nodedb/src/control/security/predicate_eval/mod.rs b/nodedb/src/control/security/predicate_eval/mod.rs new file mode 100644 index 000000000..242128b77 --- /dev/null +++ b/nodedb/src/control/security/predicate_eval/mod.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Plan-time predicate evaluation: substitute `$auth.*` references and +//! combine policies into concrete `ScanFilter` values. +//! +//! Converts compiled [`super::predicate::RlsPredicate`] trees into static +//! `ScanFilter` lists that the Data Plane evaluates without session +//! awareness. +//! +//! - [`substitute`] — `substitute_to_scan_filters` and `combine_policies`. +//! - [`sets`] — `CONTAINS` / `INTERSECTS` lowering. +//! - [`filters`] — the `match_all` and deny filter constructors. + +pub mod filters; +pub mod sets; +pub mod substitute; + +pub use filters::{deny_filter, match_all_filter}; +pub use substitute::{combine_policies, substitute_to_scan_filters}; diff --git a/nodedb/src/control/security/predicate_eval/sets.rs b/nodedb/src/control/security/predicate_eval/sets.rs new file mode 100644 index 000000000..dc3a99e99 --- /dev/null +++ b/nodedb/src/control/security/predicate_eval/sets.rs @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `CONTAINS` / `INTERSECTS` lowering. +//! +//! A set that lives on the session (`$auth.roles`, `$auth.scope_status(..)`) +//! is decided at plan time and lowers to `match_all` or the deny filter. A +//! set that lives on the document lowers to an `array_contains` / +//! `array_overlap` filter the Data Plane evaluates per row. A combination with no set on either +//! side, or a document set on both sides, has no `ScanFilter` form and +//! denies. + +use super::filters::{compare_filter, verdict_filter}; +use crate::bridge::scan_filter::{FilterOp, ScanFilter}; +use crate::control::security::auth_context::AuthContext; +use crate::control::security::predicate::PredicateValue; + +/// Where one operand of a set predicate resolves. +enum Operand<'a> { + /// A document field, evaluated per row by the Data Plane. + Doc(&'a str), + /// A session value, resolved at plan time. + Session(&'a PredicateValue), + /// A constant. + Constant(&'a PredicateValue), +} + +fn classify(value: &PredicateValue) -> Operand<'_> { + match value { + PredicateValue::Field(name) => Operand::Doc(name), + PredicateValue::AuthRef(_) | PredicateValue::AuthFunc { .. } => Operand::Session(value), + PredicateValue::Literal(_) | PredicateValue::Instant { .. } => Operand::Constant(value), + } +} + +/// `set CONTAINS element`. +pub(super) fn substitute_contains( + set: &PredicateValue, + element: &PredicateValue, + auth: &AuthContext, +) -> Option> { + match (classify(set), classify(element)) { + // `$auth.roles CONTAINS 'admin'`: decided at plan time. + (Operand::Session(set), Operand::Constant(element)) => { + let members = set.resolve(auth)?; + let needle = element.resolve(auth)?; + let passes = members.as_array()?.contains(&needle); + Some(vec![verdict_filter(passes)]) + } + // `doc_field CONTAINS `: per row. The + // document side is an array, so this is `array_contains` + // (membership), not `contains` (substring). + (Operand::Doc(field), Operand::Session(element) | Operand::Constant(element)) => { + let needle = element.resolve_scan_value(auth)?; + Some(vec![compare_filter(field, FilterOp::ArrayContains, needle)]) + } + (Operand::Session(_), Operand::Session(_) | Operand::Doc(_)) + | (Operand::Doc(_), Operand::Doc(_)) + | (Operand::Constant(_), _) => None, + } +} + +/// `left INTERSECTS right`. +pub(super) fn substitute_intersects( + left: &PredicateValue, + right: &PredicateValue, + auth: &AuthContext, +) -> Option> { + match (classify(left), classify(right)) { + // `doc_field INTERSECTS $auth.groups`, either orientation: per row. + (Operand::Doc(field), Operand::Session(other)) + | (Operand::Session(other), Operand::Doc(field)) => { + let members = other.resolve_scan_value(auth)?; + Some(vec![compare_filter(field, FilterOp::ArrayOverlap, members)]) + } + // `$auth.groups INTERSECTS $auth.allowed`: decided at plan time. + (Operand::Session(left), Operand::Session(right)) => { + let left = left.resolve(auth)?; + let right = right.resolve(auth)?; + let passes = match (left.as_array(), right.as_array()) { + (Some(l), Some(r)) => l.iter().any(|v| r.contains(v)), + (None, _) | (_, None) => false, + }; + Some(vec![verdict_filter(passes)]) + } + (Operand::Doc(_), Operand::Doc(_) | Operand::Constant(_)) + | (Operand::Session(_), Operand::Constant(_)) + | (Operand::Constant(_), _) => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::security::identity::{ + AuthMethod, AuthenticatedIdentity, DatabaseSet, Role, + }; + use crate::types::TenantId; + use nodedb_types::id::DatabaseId; + + fn auth() -> AuthContext { + let identity = AuthenticatedIdentity::new_regular( + 42, + "alice", + TenantId::new(1), + AuthMethod::ScramSha256, + vec![Role::ReadWrite], + None, + DatabaseSet::Some(smallvec::smallvec![DatabaseId::DEFAULT]), + ); + AuthContext::from_identity(&identity, "s_test".into()) + } + + /// A session set that holds the constant admits every row; one that + /// does not denies every row. + #[test] + fn session_set_contains_constant_is_decided_at_plan_time() { + let auth = auth(); + let held = substitute_contains( + &PredicateValue::AuthRef("roles".into()), + &PredicateValue::Literal(serde_json::json!("readwrite")), + &auth, + ) + .expect("resolves"); + assert_eq!(held[0].op, FilterOp::MatchAll); + + let missing = substitute_contains( + &PredicateValue::AuthRef("roles".into()), + &PredicateValue::Literal(serde_json::json!("admin")), + &auth, + ) + .expect("resolves"); + assert_eq!(missing[0].op, FilterOp::IsNotNull); + assert_eq!(missing[0].field, "__rls_deny__"); + } + + /// A session scalar where a set is required denies. + #[test] + fn session_scalar_as_set_denies() { + let auth = auth(); + assert!( + substitute_contains( + &PredicateValue::AuthRef("username".into()), + &PredicateValue::Literal(serde_json::json!("alice")), + &auth, + ) + .is_none() + ); + } + + /// A constant on the set side has no filter form and denies. + #[test] + fn constant_set_denies() { + let auth = auth(); + assert!( + substitute_contains( + &PredicateValue::Literal(serde_json::json!(["a"])), + &PredicateValue::Field("tags".into()), + &auth, + ) + .is_none() + ); + assert!( + substitute_intersects( + &PredicateValue::Literal(serde_json::json!(["a"])), + &PredicateValue::Field("tags".into()), + &auth, + ) + .is_none() + ); + } + + /// `doc_field INTERSECTS $auth.groups` lowers to `array_overlap` in + /// either orientation. + #[test] + fn doc_field_intersects_session_set_in_either_orientation() { + let auth = auth(); + let field = PredicateValue::Field("allowed".into()); + let session = PredicateValue::AuthRef("roles".into()); + for (l, r) in [(&field, &session), (&session, &field)] { + let filters = substitute_intersects(l, r, &auth).expect("resolves"); + assert_eq!(filters[0].field, "allowed"); + assert_eq!(filters[0].op, FilterOp::ArrayOverlap); + } + } +} diff --git a/nodedb/src/control/security/predicate_eval/substitute.rs b/nodedb/src/control/security/predicate_eval/substitute.rs new file mode 100644 index 000000000..d11f1baae --- /dev/null +++ b/nodedb/src/control/security/predicate_eval/substitute.rs @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `$auth.*` substitution over a predicate tree and policy combination. + +use super::filters::{compare_filter, deny_filter, match_all_filter}; +use super::sets::{substitute_contains, substitute_intersects}; +use crate::bridge::scan_filter::{FilterOp, ScanFilter}; +use crate::control::security::auth_context::AuthContext; +use crate::control::security::predicate::{CompareOp, PolicyMode, PredicateValue, RlsPredicate}; + +/// Substitute `$auth.*` references in a predicate tree and produce +/// concrete `ScanFilter` values for the Data Plane. +/// +/// This is the core plan-time substitution. After this, the resulting +/// `ScanFilter` contains only literal values and field references — no +/// session variables. The Data Plane evaluates these without any auth +/// awareness. +/// +/// Returns `None` if any required `$auth` reference cannot be resolved +/// (e.g., `$auth.org_id` when no org context). This causes the predicate +/// to evaluate as **deny** (fail-closed). +pub fn substitute_to_scan_filters( + predicate: &RlsPredicate, + auth: &AuthContext, +) -> Option> { + match predicate { + RlsPredicate::AlwaysTrue => Some(vec![match_all_filter()]), + RlsPredicate::AlwaysFalse => Some(vec![deny_filter()]), + + RlsPredicate::Compare { field, op, value } => { + // Field-to-field comparison is not expressible as a `ScanFilter`. + if matches!(value, PredicateValue::Field(_)) { + return None; + } + let resolved = value.resolve_scan_value(auth)?; + Some(vec![compare_filter(field, op.as_filter_op(), resolved)]) + } + + RlsPredicate::Contains { set, element } => substitute_contains(set, element, auth), + + RlsPredicate::Intersects { left, right } => substitute_intersects(left, right, auth), + + RlsPredicate::And(children) => { + let mut combined = Vec::new(); + for child in children { + combined.extend(substitute_to_scan_filters(child, auth)?); + } + Some(combined) + } + + RlsPredicate::Or(children) => substitute_or(children, auth), + + RlsPredicate::Not(inner) => substitute_not(inner, auth), + } +} + +/// Lower an `OR` node. A child that cannot be resolved contributes nothing; +/// a child that lowers to `match_all` short-circuits the whole disjunction. +fn substitute_or(children: &[RlsPredicate], auth: &AuthContext) -> Option> { + let mut clause_groups: Vec> = Vec::new(); + for child in children { + if let Some(filters) = substitute_to_scan_filters(child, auth) { + if filters.len() == 1 && filters[0].op == FilterOp::MatchAll { + return Some(filters); + } + clause_groups.push(filters); + } + } + + if clause_groups.len() == 1 + && let Some(single) = clause_groups.pop() + { + return Some(single); + } + if clause_groups.is_empty() { + return Some(vec![deny_filter()]); + } + + Some(vec![ScanFilter { + field: String::new(), + op: FilterOp::Or, + value: nodedb_types::Value::Null, + clauses: clause_groups, + expr: None, + }]) +} + +/// Combine multiple policies according to their modes. +/// +/// Final result: `(any permissive passes) AND (all restrictive pass)`. +/// +/// Returns the combined `ScanFilter` list to inject into the query. +/// Empty return = no RLS policies (allow all). +pub fn combine_policies( + policies: &[(RlsPredicate, PolicyMode)], + auth: &AuthContext, +) -> Option> { + if policies.is_empty() { + return Some(Vec::new()); // No policies → allow all + } + + let mut permissive: Vec<&RlsPredicate> = Vec::new(); + let mut restrictive: Vec<&RlsPredicate> = Vec::new(); + + for (pred, mode) in policies { + match mode { + PolicyMode::Permissive => permissive.push(pred), + PolicyMode::Restrictive => restrictive.push(pred), + } + } + + let mut combined = Vec::new(); + + // Permissive: OR-combine. If no permissive policies exist, default allow. + if permissive.len() == 1 { + combined.extend(substitute_to_scan_filters(permissive[0], auth)?); + } else if permissive.len() > 1 { + let or_children: Vec = permissive.iter().map(|p| (*p).clone()).collect(); + let or_pred = RlsPredicate::Or(or_children); + combined.extend(substitute_to_scan_filters(&or_pred, auth)?); + } + + // Restrictive: AND-combine (each becomes additional filters). + for pred in &restrictive { + combined.extend(substitute_to_scan_filters(pred, auth)?); + } + + Some(combined) +} + +/// Lower a `NOT` node by negating the operator of the one comparison it +/// wraps. `LIKE` / `ILIKE` and composite children have no single negated +/// filter, so they deny. +fn substitute_not(inner: &RlsPredicate, auth: &AuthContext) -> Option> { + match inner { + RlsPredicate::AlwaysTrue => substitute_to_scan_filters(&RlsPredicate::AlwaysFalse, auth), + RlsPredicate::AlwaysFalse => substitute_to_scan_filters(&RlsPredicate::AlwaysTrue, auth), + RlsPredicate::Compare { field, op, value } => { + let negated_op = match op { + CompareOp::Eq => CompareOp::Ne, + CompareOp::Ne => CompareOp::Eq, + CompareOp::Gt => CompareOp::Lte, + CompareOp::Gte => CompareOp::Lt, + CompareOp::Lt => CompareOp::Gte, + CompareOp::Lte => CompareOp::Gt, + CompareOp::In => CompareOp::NotIn, + CompareOp::NotIn => CompareOp::In, + CompareOp::IsNull => CompareOp::IsNotNull, + CompareOp::IsNotNull => CompareOp::IsNull, + CompareOp::Like | CompareOp::ILike => return None, + }; + substitute_to_scan_filters( + &RlsPredicate::Compare { + field: field.clone(), + op: negated_op, + value: value.clone(), + }, + auth, + ) + } + RlsPredicate::Contains { .. } + | RlsPredicate::Intersects { .. } + | RlsPredicate::And(_) + | RlsPredicate::Or(_) + | RlsPredicate::Not(_) => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::security::identity::{ + AuthMethod, AuthenticatedIdentity, DatabaseSet, Role, + }; + use crate::types::TenantId; + use nodedb_types::datetime::NdbDateTime; + use nodedb_types::id::DatabaseId; + use nodedb_types::json_msgpack::InstantKind; + + fn test_identity() -> AuthenticatedIdentity { + AuthenticatedIdentity::new_regular( + 42, + "alice", + TenantId::new(1), + AuthMethod::ScramSha256, + vec![Role::ReadWrite], + None, + DatabaseSet::Some(smallvec::smallvec![DatabaseId::DEFAULT]), + ) + } + + fn auth_with_database(db_id: DatabaseId) -> AuthContext { + let mut ctx = AuthContext::from_identity(&test_identity(), "s_test".into()); + ctx.database_id = Some(db_id); + ctx + } + + fn auth_without_database() -> AuthContext { + AuthContext::from_identity(&test_identity(), "s_test".into()) + } + + /// `$auth.database_id` resolves to the session's database id and the + /// predicate produces the correct ScanFilter value. + #[test] + fn database_id_auth_ref_substitutes_correctly() { + let db_id = DatabaseId::new(99); + let auth = auth_with_database(db_id); + + let predicate = RlsPredicate::Compare { + field: "owning_db".into(), + op: CompareOp::Eq, + value: PredicateValue::AuthRef("database_id".into()), + }; + + let filters = substitute_to_scan_filters(&predicate, &auth) + .expect("should resolve when database_id is set"); + assert_eq!(filters.len(), 1); + assert_eq!(filters[0].field, "owning_db"); + match &filters[0].value { + nodedb_types::Value::Integer(n) => assert_eq!(*n as u64, db_id.as_u64()), + other => panic!("expected numeric value, got {:?}", other), + } + } + + /// When `database_id` is `None` the predicate fails closed (returns None). + #[test] + fn database_id_auth_ref_fails_closed_when_none() { + let auth = auth_without_database(); + + let predicate = RlsPredicate::Compare { + field: "owning_db".into(), + op: CompareOp::Eq, + value: PredicateValue::AuthRef("database_id".into()), + }; + + let result = substitute_to_scan_filters(&predicate, &auth); + assert!( + result.is_none(), + "predicate must fail closed when database_id is None" + ); + } + + /// `combine_policies` with a single permissive database_id policy produces + /// the correct ScanFilter when the session has a bound database. + #[test] + fn combine_database_id_policy_passes_when_set() { + let db_id = DatabaseId::new(77); + let auth = auth_with_database(db_id); + + let predicate = RlsPredicate::Compare { + field: "db".into(), + op: CompareOp::Eq, + value: PredicateValue::AuthRef("database_id".into()), + }; + + let policies = [(predicate, PolicyMode::Permissive)]; + let filters = combine_policies(&policies, &auth); + assert!( + filters.as_ref().is_some_and(|v| !v.is_empty()), + "should produce scan filters when database_id is bound" + ); + } + + /// A typed instant literal lowers to the typed `ScanFilter.value` of its + /// declared kind, so the Data Plane compares an instant with an instant. + #[test] + fn instant_literal_lowers_to_a_typed_scan_value() { + let auth = auth_without_database(); + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + + let naive = RlsPredicate::Compare { + field: "captured_at".into(), + op: CompareOp::Gte, + value: PredicateValue::Instant { + at, + kind: InstantKind::Naive, + }, + }; + let filters = substitute_to_scan_filters(&naive, &auth).expect("an instant resolves"); + assert_eq!(filters.len(), 1); + assert_eq!(filters[0].field, "captured_at"); + assert_eq!(filters[0].op, FilterOp::Gte); + assert_eq!(filters[0].value, nodedb_types::Value::NaiveDateTime(at)); + + let zoned = RlsPredicate::Compare { + field: "captured_at".into(), + op: CompareOp::Lt, + value: PredicateValue::Instant { + at, + kind: InstantKind::Utc, + }, + }; + let filters = substitute_to_scan_filters(&zoned, &auth).expect("an instant resolves"); + assert_eq!(filters[0].value, nodedb_types::Value::DateTime(at)); + } + + /// `NOT` over an instant comparison negates the operator and keeps the + /// typed value. + #[test] + fn negated_instant_comparison_keeps_the_typed_value() { + let auth = auth_without_database(); + let at = NdbDateTime::from_micros(1_583_402_400_000_000); + let predicate = RlsPredicate::Not(Box::new(RlsPredicate::Compare { + field: "captured_at".into(), + op: CompareOp::Gte, + value: PredicateValue::Instant { + at, + kind: InstantKind::Naive, + }, + })); + let filters = substitute_to_scan_filters(&predicate, &auth).expect("resolves"); + assert_eq!(filters[0].op, FilterOp::Lt); + assert_eq!(filters[0].value, nodedb_types::Value::NaiveDateTime(at)); + } +} diff --git a/nodedb/src/control/security/predicate_parser.rs b/nodedb/src/control/security/predicate_parser.rs index f66421462..5587ec2f1 100644 --- a/nodedb/src/control/security/predicate_parser.rs +++ b/nodedb/src/control/security/predicate_parser.rs @@ -198,19 +198,18 @@ fn parse_atom(tokens: &[String], pos: &mut usize) -> Result= tokens.len() { // Standalone value — treat as boolean (truthy). - return match left_value { - PredicateValue::AuthRef(_) - | PredicateValue::Field(_) - | PredicateValue::AuthFunc { .. } => Ok(RlsPredicate::Compare { - field: match &left_value { - PredicateValue::Field(f) => f.clone(), - _ => String::new(), - }, - op: CompareOp::IsNotNull, - value: left_value, - }), - PredicateValue::Literal(_) => Ok(RlsPredicate::AlwaysTrue), + let field = match &left_value { + PredicateValue::Field(f) => f.clone(), + PredicateValue::AuthRef(_) | PredicateValue::AuthFunc { .. } => String::new(), + PredicateValue::Literal(_) | PredicateValue::Instant { .. } => { + return Ok(RlsPredicate::AlwaysTrue); + } }; + return Ok(RlsPredicate::Compare { + field, + op: CompareOp::IsNotNull, + value: left_value, + }); } let op_token = tokens[*pos].to_uppercase(); @@ -430,7 +429,9 @@ pub fn validate_auth_refs(predicate: &RlsPredicate) -> crate::Result<()> { }); } } - _ => {} + PredicateValue::Literal(_) + | PredicateValue::Instant { .. } + | PredicateValue::Field(_) => {} } Ok(()) } diff --git a/nodedb/src/control/security/predicate_typing.rs b/nodedb/src/control/security/predicate_typing.rs new file mode 100644 index 000000000..165e36ae1 --- /dev/null +++ b/nodedb/src/control/security/predicate_typing.rs @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Type the literals of a compiled RLS predicate against the collection's +//! declared columns. +//! +//! A policy literal reaches the parser as whatever the author typed: +//! `captured_at > 1583402400000` or `captured_at > '2020-03-05'`. The Data +//! Plane compares an instant column against a typed instant, and an untyped +//! integer has no order against one, so a policy left untyped matches +//! nothing on such a column. This pass resolves every literal compared +//! against a declared `TIMESTAMP` / `TIMESTAMPTZ` column through +//! [`nodedb_sql::planner::predicate_coerce::coerce_read_literal`], the same +//! rule the planner applies to a query's `WHERE` literal, so a policy and a +//! query on the same column agree on the instant. +//! +//! A literal the declared type cannot read (`captured_at >= true`) is an +//! error naming the column and the literal. The caller refuses the policy: +//! a policy that cannot be enforced as written is never stored. + +use nodedb_sql::planner::predicate_coerce::coerce_read_literal; +use nodedb_sql::types::{ColumnInfo, SqlValue}; +use nodedb_types::json_msgpack::InstantKind; + +use super::predicate::{PredicateValue, RlsPredicate}; + +/// Type every literal in `predicate` that a `Compare` node pairs with a +/// declared column of `columns`. +/// +/// Literals on `CONTAINS` / `INTERSECTS` nodes and on plan-time-only +/// comparisons (`$auth.x = 'lit'`, which name no document field) are left +/// as written: neither compares a literal against a declared scalar column. +pub fn type_predicate_literals( + predicate: RlsPredicate, + columns: &[ColumnInfo], +) -> crate::Result { + Ok(match predicate { + RlsPredicate::Compare { field, op, value } => { + let value = type_compare_literal(&field, value, columns)?; + RlsPredicate::Compare { field, op, value } + } + RlsPredicate::And(children) => RlsPredicate::And(type_children(children, columns)?), + RlsPredicate::Or(children) => RlsPredicate::Or(type_children(children, columns)?), + RlsPredicate::Not(inner) => { + RlsPredicate::Not(Box::new(type_predicate_literals(*inner, columns)?)) + } + RlsPredicate::Contains { .. } + | RlsPredicate::Intersects { .. } + | RlsPredicate::AlwaysTrue + | RlsPredicate::AlwaysFalse => predicate, + }) +} + +fn type_children( + children: Vec, + columns: &[ColumnInfo], +) -> crate::Result> { + children + .into_iter() + .map(|child| type_predicate_literals(child, columns)) + .collect() +} + +/// Type the right-hand side of `field value` when it is a literal and +/// `field` names a declared column. +fn type_compare_literal( + field: &str, + value: PredicateValue, + columns: &[ColumnInfo], +) -> crate::Result { + let PredicateValue::Literal(literal) = value else { + return Ok(value); + }; + let Some(column) = columns + .iter() + .find(|column| column.name.eq_ignore_ascii_case(field)) + else { + return Ok(PredicateValue::Literal(literal)); + }; + let Some(sql_literal) = json_to_sql_literal(&literal) else { + return Ok(PredicateValue::Literal(literal)); + }; + let coerced = + coerce_read_literal(column, sql_literal).map_err(|error| crate::Error::BadRequest { + detail: format!("RLS predicate: {error}"), + })?; + Ok(sql_literal_to_predicate_value(coerced, literal)) +} + +/// The `SqlValue` a policy literal denotes, for the literal kinds the policy +/// parser produces. An array or object literal has no scalar form and stays +/// as written. +fn json_to_sql_literal(literal: &serde_json::Value) -> Option { + match literal { + serde_json::Value::Null => Some(SqlValue::Null), + serde_json::Value::Bool(b) => Some(SqlValue::Bool(*b)), + serde_json::Value::Number(n) => n + .as_i64() + .map(SqlValue::Int) + .or_else(|| n.as_f64().map(SqlValue::Float)), + serde_json::Value::String(s) => Some(SqlValue::String(s.clone())), + serde_json::Value::Array(_) | serde_json::Value::Object(_) => None, + } +} + +/// The predicate value a coerced literal becomes: a typed instant, or the +/// original literal when the declared type imposed no representation. +fn sql_literal_to_predicate_value( + coerced: SqlValue, + original: serde_json::Value, +) -> PredicateValue { + match coerced { + SqlValue::Timestamp(at) => PredicateValue::Instant { + at, + kind: InstantKind::Naive, + }, + SqlValue::Timestamptz(at) => PredicateValue::Instant { + at, + kind: InstantKind::Utc, + }, + SqlValue::Null + | SqlValue::Bool(_) + | SqlValue::Int(_) + | SqlValue::Float(_) + | SqlValue::Decimal(_) + | SqlValue::String(_) + | SqlValue::Bytes(_) + | SqlValue::Array(_) => PredicateValue::Literal(original), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::security::predicate::CompareOp; + use crate::control::security::predicate_parser::parse_predicate; + use nodedb_sql::types::SqlDataType; + use nodedb_types::datetime::NdbDateTime; + + /// `2020-03-05T10:00:00Z` as epoch milliseconds. + const EARLY_MS: i64 = 1_583_402_400_000; + + fn early() -> NdbDateTime { + NdbDateTime::from_micros(EARLY_MS * 1_000) + } + + fn column(name: &str, data_type: SqlDataType, is_primary_key: bool) -> ColumnInfo { + ColumnInfo { + name: name.to_string(), + data_type, + nullable: true, + is_primary_key, + default: None, + raw_type: None, + int_width: None, + float_width: None, + } + } + + fn columns() -> Vec { + vec![ + column("id", SqlDataType::String, true), + column("captured_at", SqlDataType::Timestamp, false), + column("seen_at", SqlDataType::Timestamptz, false), + column("n", SqlDataType::Int64, false), + ] + } + + fn typed(text: &str) -> crate::Result { + let parsed = parse_predicate(text).expect("policy text parses"); + type_predicate_literals(parsed, &columns()) + } + + fn compare_value(predicate: &RlsPredicate) -> &PredicateValue { + match predicate { + RlsPredicate::Compare { value, .. } => value, + other => panic!("expected a comparison, got {other:?}"), + } + } + + /// A numeric literal against a `TIMESTAMP` column is epoch milliseconds + /// and becomes a naive instant. + #[test] + fn numeric_literal_on_an_instant_column_becomes_a_naive_instant() { + let predicate = typed(&format!("captured_at >= {EARLY_MS}")).expect("types"); + assert!(matches!( + compare_value(&predicate), + PredicateValue::Instant { at, kind: InstantKind::Naive } if *at == early() + )); + } + + /// A text literal against a `TIMESTAMPTZ` column is parsed as ISO-8601 + /// and becomes a zoned instant. + #[test] + fn text_literal_on_a_zoned_column_becomes_a_utc_instant() { + let predicate = typed("seen_at > '2020-03-05 10:00:00'").expect("types"); + assert!(matches!( + compare_value(&predicate), + PredicateValue::Instant { at, kind: InstantKind::Utc } if *at == early() + )); + } + + /// Typing reaches every comparison under `AND` / `OR` / `NOT`. + #[test] + fn typing_descends_through_composite_nodes() { + let predicate = typed(&format!( + "(captured_at >= {EARLY_MS} OR owner = $auth.username) AND NOT seen_at < '2020-03-05'" + )) + .expect("types"); + let RlsPredicate::And(children) = &predicate else { + panic!("expected AND, got {predicate:?}"); + }; + let RlsPredicate::Or(or_children) = &children[0] else { + panic!("expected OR, got {:?}", children[0]); + }; + assert!(matches!( + compare_value(&or_children[0]), + PredicateValue::Instant { .. } + )); + let RlsPredicate::Not(inner) = &children[1] else { + panic!("expected NOT, got {:?}", children[1]); + }; + assert!(matches!( + compare_value(inner), + PredicateValue::Instant { .. } + )); + } + + /// A literal no instant can be read from is refused, naming the column. + #[test] + fn non_instant_literal_on_an_instant_column_is_refused_naming_the_column() { + let error = typed("captured_at >= true").expect_err("a boolean is refused"); + let detail = error.to_string(); + assert!( + detail.contains("captured_at") && detail.contains("boolean"), + "error must name the column and the literal kind: {detail}" + ); + let error = typed("captured_at >= 'not a date'").expect_err("non-datetime text is refused"); + let detail = error.to_string(); + assert!( + detail.contains("captured_at") && detail.contains("not a date"), + "error must name the column and the literal: {detail}" + ); + } + + /// A column that is not an instant, an undeclared field, the primary + /// key, and an `$auth.*` reference are all left as written. + #[test] + fn non_instant_targets_are_left_as_written() { + for text in [ + "n > 5", + "undeclared > 5", + "id = '2020-03-05'", + "captured_at = $auth.id", + ] { + let predicate = typed(text).unwrap_or_else(|e| panic!("{text}: {e}")); + assert!( + !matches!(compare_value(&predicate), PredicateValue::Instant { .. }), + "{text} must not be typed as an instant: {predicate:?}" + ); + } + let predicate = typed("n > 5").expect("types"); + assert!(matches!( + predicate, + RlsPredicate::Compare { + op: CompareOp::Gt, + value: PredicateValue::Literal(_), + .. + } + )); + } +} diff --git a/nodedb/src/control/security/rls/compile.rs b/nodedb/src/control/security/rls/compile.rs new file mode 100644 index 000000000..dd0e89d9a --- /dev/null +++ b/nodedb/src/control/security/rls/compile.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Compile a policy's `USING` text against its collection's declared columns. +//! +//! One entry point, [`compile_policy_predicate`], serves `CREATE RLS POLICY` +//! and every load of a stored policy (boot replay, Raft apply, recovery +//! reload, schema change), so a policy is typed against the collection's +//! current declared columns wherever it is materialized. The compiled +//! predicate is never persisted: the catalog keeps the text, and every +//! reader recompiles. + +use nodedb_sql::types::ColumnInfo; +use nodedb_types::DatabaseId; + +use crate::control::planner::catalog_adapter::convert_collection_type; +use crate::control::security::catalog::SystemCatalog; +use crate::control::security::predicate::RlsPredicate; +use crate::control::security::predicate_parser::{parse_predicate, validate_auth_refs}; +use crate::control::security::predicate_typing::type_predicate_literals; + +/// Parse, validate, and type `text` against `columns`. +/// +/// Errors: a parse error, an unknown `$auth.*` reference, or a literal the +/// declared column type cannot read. Each names what was refused. +pub fn compile_policy_predicate(text: &str, columns: &[ColumnInfo]) -> crate::Result { + let parsed = parse_predicate(text).map_err(|error| crate::Error::BadRequest { + detail: format!("predicate parse error: {error}"), + })?; + validate_auth_refs(&parsed)?; + type_predicate_literals(parsed, columns) +} + +/// The declared columns of the active collection `collection` in +/// `database_id` / `tenant_id`, as the planner sees them. +/// +/// Errors with `CollectionNotFound` when the collection is absent or +/// dropped: a policy cannot be typed without its schema. +pub fn declared_columns( + catalog: &SystemCatalog, + database_id: DatabaseId, + tenant_id: u64, + collection: &str, +) -> crate::Result> { + let stored = catalog + .get_collection(database_id, tenant_id, collection)? + .filter(|stored| stored.is_active) + .ok_or_else(|| crate::Error::CollectionNotFound { + tenant_id: crate::types::TenantId::new(tenant_id), + collection: collection.to_string(), + })?; + let (_, columns, _) = convert_collection_type(&stored); + Ok(columns) +} + +impl super::store::RlsPolicyStore { + /// Recompile every stored policy on `collection` against its current + /// declared columns and reinstall it. + /// + /// For a schema change: a policy literal typed against a column that + /// `ALTER COLLECTION` added, dropped, or renamed must follow the new + /// schema without waiting for a restart. A row that no longer compiles + /// installs as a restrictive deny-all (`StoredRlsPolicy::rehydrate`). + pub fn recompile_for_collection( + &self, + catalog: &SystemCatalog, + database_id: DatabaseId, + tenant_id: u64, + collection: &str, + ) -> crate::Result<()> { + let qualified = nodedb_types::QualifiedCollection::new(database_id, collection); + for stored in catalog.list_rls_policies_for_collection(tenant_id, qualified.as_str())? { + self.install_replicated_policy(stored.rehydrate(catalog)); + } + Ok(()) + } +} diff --git a/nodedb/src/control/security/rls/eval.rs b/nodedb/src/control/security/rls/eval.rs index 1c0009df2..3d536f07c 100644 --- a/nodedb/src/control/security/rls/eval.rs +++ b/nodedb/src/control/security/rls/eval.rs @@ -27,83 +27,42 @@ impl RlsPolicyStore { /// Primary read-path RLS method: substitutes `$auth.*` variables /// and returns the combined `ScanFilter` bytes. /// - /// Returns `None` when a required `$auth` field is missing - /// (fail-closed semantics). + /// `Ok(Some(bytes))` is the filter set, empty when no read policy + /// restricts this identity here. `Ok(None)` means deny: a required + /// `$auth` field is missing. `Err` means the filter set could not be + /// encoded; the caller must refuse the statement, never treat it as + /// unrestricted. pub fn combined_read_predicate_with_auth( &self, tenant_id: u64, collection: &str, auth: &AuthContext, - ) -> Option> { + ) -> crate::Result>> { if auth.is_superuser() { - return Some(Vec::new()); - } - - let policies = self.read_policies(tenant_id, collection); - if policies.is_empty() { - return Some(Vec::new()); - } - - let compiled_policies: Vec<(RlsPredicate, PolicyMode)> = policies - .iter() - .filter_map(|p| p.compiled_predicate.as_ref().map(|c| (c.clone(), p.mode))) - .collect(); - - let all_filters = if !compiled_policies.is_empty() { - combine_policies(&compiled_policies, auth)? - } else { - Vec::new() - }; - - if all_filters.is_empty() { - Some(Vec::new()) - } else { - Some(zerompk::to_msgpack_vec(&all_filters).unwrap_or_default()) + return Ok(Some(Vec::new())); } + compile_policy_bytes(&self.read_policies(tenant_id, collection), auth) } /// Compile the collection's write policies into the `ScanFilter` bytes a /// write gate evaluates against a row image. /// /// The write-path twin of [`Self::combined_read_predicate_with_auth`] and - /// resolved identically — superuser bypass, vacuous policies, and the - /// fail-closed `None` on an unresolvable `$auth.*` reference all behave the - /// same — so a `FOR ALL` policy cannot compile to one predicate for reads - /// and a different one for writes. - /// - /// Empty bytes mean "no write policy restricts this identity here"; `None` - /// means deny. + /// resolved identically — superuser bypass, vacuous policies, the + /// fail-closed `None` on an unresolvable `$auth.*` reference, and the + /// `Err` on an unencodable filter set all behave the same — so a + /// `FOR ALL` policy cannot compile to one predicate for reads and a + /// different one for writes. pub fn combined_write_predicate_with_auth( &self, tenant_id: u64, collection: &str, auth: &AuthContext, - ) -> Option> { + ) -> crate::Result>> { if auth.is_superuser() { - return Some(Vec::new()); - } - - let policies = self.write_policies(tenant_id, collection); - if policies.is_empty() { - return Some(Vec::new()); - } - - let compiled_policies: Vec<(RlsPredicate, PolicyMode)> = policies - .iter() - .filter_map(|p| p.compiled_predicate.as_ref().map(|c| (c.clone(), p.mode))) - .collect(); - - let all_filters = if !compiled_policies.is_empty() { - combine_policies(&compiled_policies, auth)? - } else { - Vec::new() - }; - - if all_filters.is_empty() { - Some(Vec::new()) - } else { - Some(zerompk::to_msgpack_vec(&all_filters).unwrap_or_default()) + return Ok(Some(Vec::new())); } + compile_policy_bytes(&self.write_policies(tenant_id, collection), auth) } /// Whether any enabled, non-vacuous write policy exists in this tenant. @@ -158,6 +117,42 @@ impl RlsPolicyStore { } } +/// Combine `policies` for `auth` and encode the result. +/// +/// Empty bytes when no policy carries a predicate or the combination is +/// vacuous. `None` when an `$auth.*` reference cannot be resolved. `Err` +/// when the filter set does not encode: an unencodable policy is an error +/// the caller surfaces, never an empty (admit-all) filter set. +fn compile_policy_bytes( + policies: &[RlsPolicy], + auth: &AuthContext, +) -> crate::Result>> { + let compiled_policies: Vec<(RlsPredicate, PolicyMode)> = policies + .iter() + .filter_map(|p| p.compiled_predicate.as_ref().map(|c| (c.clone(), p.mode))) + .collect(); + if compiled_policies.is_empty() { + return Ok(Some(Vec::new())); + } + let Some(all_filters) = combine_policies(&compiled_policies, auth) else { + return Ok(None); + }; + if all_filters.is_empty() { + return Ok(Some(Vec::new())); + } + encode_scan_filters(&all_filters).map(Some) +} + +/// Encode a filter set for the bridge. +fn encode_scan_filters( + filters: &[crate::bridge::scan_filter::ScanFilter], +) -> crate::Result> { + zerompk::to_msgpack_vec(&filters).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("RLS filter serialization failed: {e}"), + }) +} + /// Decide one row image against ALREADY-COMPILED write-policy filter bytes. /// /// For Control-Plane paths that hold a post-image but not the policy store: the @@ -170,7 +165,8 @@ impl RlsPolicyStore { /// `image` is the MessagePack row body. Empty `compiled` means no write policy /// restricts this identity here. Fails closed on an undecodable payload or an /// evaluation error, so an adversarial predicate cannot become an admitted -/// write by erroring out of the check. +/// write by erroring out of the check; the evaluation error is named in the +/// denial. pub fn admit_compiled_write_image( compiled: &[u8], image: &[u8], @@ -184,13 +180,15 @@ pub fn admit_compiled_write_image( .map_err(|error| crate::Error::PlanError { detail: format!("RLS write filter deserialization failed: {error}"), })?; - if crate::bridge::scan_filter::ScanFilter::all_match_binary(&filters, image).unwrap_or(false) { - return Ok(()); - } - Err(crate::Error::RejectedAuthz { + let deny = |detail: String| crate::Error::RejectedAuthz { tenant_id: crate::types::TenantId::new(tenant_id), - resource: format!("RLS write policy on '{collection}' rejected the row"), - }) + resource: format!("RLS write policy on '{collection}' {detail}"), + }; + match crate::bridge::scan_filter::ScanFilter::all_match_binary(&filters, image) { + Ok(true) => Ok(()), + Ok(false) => Err(deny("rejected the row".into())), + Err(error) => Err(deny(format!("could not be evaluated: {error}"))), + } } fn check_compiled_write( @@ -468,7 +466,9 @@ mod tests { store.create_policy(policy).unwrap(); let auth = nonsuper_auth(); - let result = store.combined_read_predicate_with_auth(1, "orders", &auth); + let result = store + .combined_read_predicate_with_auth(1, "orders", &auth) + .expect("filters encode"); // Must return Some with non-empty filter bytes (not unrestricted). assert!( result.is_some_and(|b| !b.is_empty()), @@ -491,10 +491,64 @@ mod tests { store.create_policy(policy).unwrap(); let auth = nonsuper_auth(); - let result = store.combined_read_predicate_with_auth(1, "orders", &auth); + let result = store + .combined_read_predicate_with_auth(1, "orders", &auth) + .expect("nothing to encode"); assert!( result.is_none(), "read path must fail-closed on unresolvable auth ref; got {result:?}" ); } + + /// An unencodable filter set is an `Err`, never an empty (admit-all) + /// filter set. `encode_scan_filters` is the one seam every policy byte + /// set passes through, so the `Result` plumbing is pinned on it directly + /// with a filter set of the largest size and shape the encoder accepts. + #[test] + fn filter_encoding_result_is_propagated_not_defaulted() { + let filters = vec![crate::bridge::scan_filter::ScanFilter { + field: "owner".into(), + op: crate::bridge::scan_filter::FilterOp::Eq, + value: nodedb_types::Value::String("42".into()), + clauses: Vec::new(), + expr: None, + }]; + let bytes = encode_scan_filters(&filters).expect("a plain filter encodes"); + assert!(!bytes.is_empty()); + let decoded: Vec = + zerompk::from_msgpack(&bytes).expect("round-trips"); + assert_eq!(decoded.len(), 1); + assert_eq!(decoded[0].field, "owner"); + } + + /// A write gate that cannot evaluate its filters denies, naming the + /// evaluation error, instead of admitting the row. + #[test] + fn admit_compiled_write_image_denies_when_evaluation_errors() { + use nodedb_query::{BinaryOp, SqlExpr}; + // `1 / 0` is the one expression the evaluator refuses to decide. + let filters = vec![crate::bridge::scan_filter::ScanFilter { + field: String::new(), + op: crate::bridge::scan_filter::FilterOp::Expr, + value: nodedb_types::Value::Null, + clauses: Vec::new(), + expr: Some(SqlExpr::BinaryOp { + left: Box::new(SqlExpr::Literal(nodedb_types::Value::Integer(1))), + op: BinaryOp::Div, + right: Box::new(SqlExpr::Literal(nodedb_types::Value::Integer(0))), + }), + }]; + let compiled = encode_scan_filters(&filters).expect("encodes"); + let image = nodedb_types::json_to_msgpack_or_empty(&serde_json::json!({"qty": 1})); + let result = admit_compiled_write_image(&compiled, &image, 1, "items"); + match result { + Err(crate::Error::RejectedAuthz { resource, .. }) => { + assert!( + resource.contains("could not be evaluated"), + "denial must name the evaluation error: {resource}" + ); + } + other => panic!("expected RejectedAuthz, got {other:?}"), + } + } } diff --git a/nodedb/src/control/security/rls/mod.rs b/nodedb/src/control/security/rls/mod.rs index bfeb7ae70..7b70c8aec 100644 --- a/nodedb/src/control/security/rls/mod.rs +++ b/nodedb/src/control/security/rls/mod.rs @@ -7,6 +7,8 @@ //! //! Layout: //! - [`types`] — `RlsPolicy`, `PolicyType` data shapes. +//! - [`compile`] — `USING` text → typed `RlsPredicate` against the +//! collection's declared columns. //! - [`store`] — `RlsPolicyStore` in-memory CRUD + query methods. //! - [`eval`] — read/write predicate evaluation on `RlsPolicyStore` //! (including `$auth.*` substitution). @@ -14,6 +16,7 @@ //! to sync replicated policies into the in-memory store. //! - [`namespace`] — namespace-scoped `check_namespace_authz` helper. +pub mod compile; pub mod eval; pub mod namespace; pub mod replication; diff --git a/nodedb/src/control/security/rls/store.rs b/nodedb/src/control/security/rls/store.rs index 289a90b5e..b069cf041 100644 --- a/nodedb/src/control/security/rls/store.rs +++ b/nodedb/src/control/security/rls/store.rs @@ -148,6 +148,10 @@ impl RlsPolicyStore { /// Clear all in-memory policies and reload from the catalog. /// Used by the recovery verifier repair path. + /// + /// Every stored row is installed: a row that cannot be compiled against + /// its collection installs as a restrictive deny-all + /// (`StoredRlsPolicy::rehydrate`), never dropped. pub fn clear_and_reload( &self, catalog: &crate::control::security::catalog::SystemCatalog, @@ -160,16 +164,10 @@ impl RlsPolicyStore { } policies.clear(); for s in stored { - match s.to_runtime() { - Ok(rp) => { - affected_tenants.insert(rp.tenant_id); - let key = super::types::policy_key(rp.tenant_id, &rp.collection); - policies.entry(key).or_default().push(rp); - } - Err(e) => { - tracing::warn!(error = %e, "rls_store.clear_and_reload: skipping unparseable policy"); - } - } + let rp = s.rehydrate(catalog); + affected_tenants.insert(rp.tenant_id); + let key = super::types::policy_key(rp.tenant_id, &rp.collection); + policies.entry(key).or_default().push(rp); } drop(policies); for tenant_id in affected_tenants { diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/alter/add_column.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/alter/add_column.rs index 9a641a494..197c5bc5e 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/alter/add_column.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/alter/add_column.rs @@ -112,6 +112,7 @@ pub(super) async fn alter_table_add_column( super::super::register::dispatch_register_from_stored(state, coll) .await .map_err(|e| err("XX000", e.to_string()))?; + super::strict_schema::recompile_rls_policies(state, coll)?; } state.audit_record( diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/alter/strict_schema.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/alter/strict_schema.rs index e7cfa934d..2e48e4b01 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/alter/strict_schema.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/alter/strict_schema.rs @@ -108,7 +108,8 @@ pub(super) fn add_field(coll: &mut StoredCollection, column: &str, declared_type /// Replicate the mutated collection through the metadata raft group, /// refresh this node's Data Plane register so the in-memory shape -/// catches up with the new schema, then bump `schema_version`. +/// catches up with the new schema, recompile the collection's RLS +/// policies against it, then bump `schema_version`. pub(super) async fn persist_schema_change( state: &SharedState, updated: &StoredCollection, @@ -120,6 +121,28 @@ pub(super) async fn persist_schema_change( super::super::register::dispatch_register_from_stored(state, updated) .await .map_err(|e| err("XX000", e.to_string()))?; + recompile_rls_policies(state, updated)?; state.schema_version.bump(); Ok(()) } + +/// Recompile the RLS policies on `updated` against its new declared columns. +/// +/// A policy literal is typed against the column it compares with, so a +/// column the statement added, dropped, or renamed changes what the policy +/// compiles to. The Raft post-apply recompiles on every node in cluster mode; +/// this call covers the single-node path, where no post-apply runs. +pub(super) fn recompile_rls_policies( + state: &SharedState, + updated: &StoredCollection, +) -> Result<(), DdlError> { + state + .rls + .recompile_for_collection( + state.credentials.catalog(), + updated.database_id, + updated.tenant_id, + &updated.name, + ) + .map_err(|e| err("XX000", format!("rls recompile: {e}"))) +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/explain_ddl.rs b/nodedb/src/control/server/shared/ddl/neutral/explain_ddl.rs index e3b0ce4bd..58d646541 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/explain_ddl.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/explain_ddl.rs @@ -201,12 +201,16 @@ pub fn assert_visible( ); let scope = RequestAuthScope::builder(&target_identity, state.auth_stores()).build(); - // Check if RLS policies would filter this user. - let rls_bytes = state.rls.combined_read_predicate_with_auth( - target_identity.tenant_id.as_u64(), - collection, - scope.auth(), - ); + // Check if RLS policies would filter this user. An unencodable policy + // is an error, not an answer. + let rls_bytes = state + .rls + .combined_read_predicate_with_auth( + target_identity.tenant_id.as_u64(), + collection, + scope.auth(), + ) + .map_err(|e| DdlError::new("XX000", format!("rls compile: {e}")))?; let visible = rls_bytes.is_some_and(|b| b.is_empty()); // No filters = visible. diff --git a/nodedb/src/control/server/shared/ddl/neutral/read_gate.rs b/nodedb/src/control/server/shared/ddl/neutral/read_gate.rs index 7d37e20ab..ded823621 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/read_gate.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/read_gate.rs @@ -50,6 +50,8 @@ const INSUFFICIENT_PRIVILEGE: &str = "42501"; const FEATURE_NOT_SUPPORTED: &str = "0A000"; /// SQLSTATE for a collection the catalog does not hold. const UNDEFINED_TABLE: &str = "42P01"; +/// SQLSTATE for a policy set that could not be compiled. +const INTERNAL_ERROR: &str = "XX000"; fn gate_err(sqlstate: &str, message: impl Into) -> DdlError { DdlError::new(sqlstate, message) @@ -217,6 +219,7 @@ impl<'a> CollectionReadGate<'a> { collection, self.scope.auth(), ) + .map_err(|e| gate_err(INTERNAL_ERROR, format!("rls compile: {e}")))? .is_some_and(|filters| filters.is_empty()); if unrestricted { return Ok(()); diff --git a/nodedb/src/control/server/shared/ddl/neutral/rls.rs b/nodedb/src/control/server/shared/ddl/neutral/rls.rs index df57309cf..7cf67644d 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/rls.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/rls.rs @@ -19,8 +19,8 @@ use crate::control::security::catalog::StoredRlsPolicy; use crate::control::security::deny::{self, DenyMode}; use crate::control::security::identity::{AuthenticatedIdentity, Role}; use crate::control::security::predicate::RlsPredicate; -use crate::control::security::predicate_parser::{parse_predicate, validate_auth_refs}; use crate::control::security::rls::RlsPolicy; +use crate::control::security::rls::compile::{compile_policy_predicate, declared_columns}; use crate::control::server::response_shape::types::ShapedRows; use crate::control::state::SharedState; use crate::types::DatabaseId; @@ -52,16 +52,38 @@ struct CompiledPredicate { on_deny: DenyMode, } -/// Compile a predicate string and optional `ON DENY` raw clause into a +/// Compile a predicate string against the target collection's declared +/// columns, and the optional `ON DENY` raw clause, into a /// `CompiledPredicate`. Called by [`create_rls_policy`] after the typed AST /// fields have been validated. +/// +/// The collection must exist: a literal compared against a declared +/// `TIMESTAMP` / `TIMESTAMPTZ` column is typed here by the rule the planner +/// applies to a query predicate, and a literal that type cannot read +/// refuses the statement naming the column. A policy that cannot be +/// enforced as written is never stored. fn compile_rls_predicate( + state: &SharedState, + database_id: DatabaseId, + tenant_id: u64, + collection: &str, predicate_str: &str, on_deny_raw: Option<&str>, ) -> Result { - let compiled = parse_predicate(predicate_str) - .map_err(|e| DdlError::new("42601", format!("predicate parse error: {e}")))?; - validate_auth_refs(&compiled).map_err(|e| DdlError::new("42601", e.to_string()))?; + let columns = declared_columns( + state.credentials.catalog(), + database_id, + tenant_id, + collection, + ) + .map_err(|e| match e { + crate::Error::CollectionNotFound { .. } => { + DdlError::new("42P01", format!("collection '{collection}' does not exist")) + } + other => DdlError::new("XX000", format!("catalog read: {other}")), + })?; + let compiled = compile_policy_predicate(predicate_str, &columns) + .map_err(|e| DdlError::new("42601", e.to_string()))?; let on_deny = if let Some(deny_text) = on_deny_raw { let deny_parts: Vec<&str> = deny_text.split_whitespace().collect(); @@ -166,7 +188,14 @@ pub fn create_rls_policy( crate::control::security::predicate::PolicyMode::Permissive }; - let compiled = compile_rls_predicate(predicate_raw, on_deny_raw)?; + let compiled = compile_rls_predicate( + state, + database_id, + tenant_id, + collection, + predicate_raw, + on_deny_raw, + )?; // Pre-check duplicate so the proposing node fails fast with a // clean SQLSTATE instead of going through raft only to be a @@ -198,7 +227,7 @@ pub fn create_rls_policy( .as_secs(), }; - let stored = StoredRlsPolicy::from_runtime(&policy) + let stored = StoredRlsPolicy::from_runtime(&policy, database_id, predicate_raw) .map_err(|e| DdlError::new("XX000", format!("rls serialize: {e}")))?; let entry = CatalogEntry::PutRlsPolicy(Box::new(stored.clone())); diff --git a/nodedb/src/control/server/shared/ddl/neutral/tree_ops/create_index.rs b/nodedb/src/control/server/shared/ddl/neutral/tree_ops/create_index.rs index 6f20b8b27..9c4862551 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/tree_ops/create_index.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/tree_ops/create_index.rs @@ -121,6 +121,7 @@ pub async fn create_graph_index( if state .rls .combined_read_predicate_with_auth(tenant_id.as_u64(), &collection, scope.auth()) + .map_err(|e| ddl_err("XX000", format!("rls compile: {e}")))? .is_none_or(|filters| !filters.is_empty()) { return Err(ddl_err( diff --git a/nodedb/src/control/state/init_prod/bootstrap.rs b/nodedb/src/control/state/init_prod/bootstrap.rs index 36d26f531..8d130755d 100644 --- a/nodedb/src/control/state/init_prod/bootstrap.rs +++ b/nodedb/src/control/state/init_prod/bootstrap.rs @@ -158,27 +158,19 @@ pub(super) fn run( ep_topic_registry.load_from_catalog(catalog)?; mv_registry.load_from_catalog(catalog); sequence_registry.load_from_catalog(catalog); + // Every stored row is installed: a row that cannot be compiled + // against its collection installs as a restrictive deny-all + // (`StoredRlsPolicy::rehydrate`), never skipped. match catalog.load_all_rls_policies() { Ok(stored) => { - let mut loaded = 0usize; for s in &stored { - match s.to_runtime() { - Ok(p) => { - rls_store.install_replicated_policy(p); - loaded += 1; - } - Err(e) => { - tracing::warn!( - name = %s.name, - collection = %s.collection, - error = %e, - "boot replay: skipped invalid RLS policy" - ); - } - } + rls_store.install_replicated_policy(s.rehydrate(catalog)); } - if loaded > 0 { - tracing::info!(rls_policies = loaded, "loaded RLS policies from catalog"); + if !stored.is_empty() { + tracing::info!( + rls_policies = stored.len(), + "loaded RLS policies from catalog" + ); } } Err(e) => tracing::warn!(error = %e, "failed to load RLS policies"), diff --git a/nodedb/tests/inproc/cases/catalog_recovery_check.rs b/nodedb/tests/inproc/cases/catalog_recovery_check.rs index a8e07476e..97bfdf70d 100644 --- a/nodedb/tests/inproc/cases/catalog_recovery_check.rs +++ b/nodedb/tests/inproc/cases/catalog_recovery_check.rs @@ -207,11 +207,12 @@ async fn rls_policy_orphan_refuses_startup() { let stored = nodedb::control::security::catalog::rls::StoredRlsPolicy { tenant_id: 1, + database_id: 0, collection: "orders".to_string(), display_collection: "orders".to_string(), name: "only_own_orders".to_string(), policy_type_tag: 0, - compiled_predicate_json: String::new(), + predicate_text: String::new(), mode_tag: 0, on_deny_json: r#""Silent""#.to_string(), enabled: true, @@ -517,11 +518,12 @@ async fn rls_policy_value_mismatch_detected() { let stored = nodedb::control::security::catalog::rls::StoredRlsPolicy { tenant_id: 1, + database_id: 0, collection: "docs".to_string(), display_collection: "docs".to_string(), name: "read_own".to_string(), policy_type_tag: 0, - compiled_predicate_json: String::new(), + predicate_text: String::new(), mode_tag: 0, on_deny_json: r#""Silent""#.to_string(), enabled: true, @@ -531,7 +533,7 @@ async fn rls_policy_value_mismatch_detected() { catalog.put_rls_policy(&stored).unwrap(); // Insert into memory with enabled=false — value mismatch. - let mut policy = stored.to_runtime().unwrap(); + let mut policy = stored.to_runtime(catalog).unwrap(); policy.enabled = false; shared.rls.install_replicated_policy(policy); diff --git a/nodedb/tests/inproc/cases/collection_cascade_enumeration.rs b/nodedb/tests/inproc/cases/collection_cascade_enumeration.rs index b5df0bfc2..bb7cc4bff 100644 --- a/nodedb/tests/inproc/cases/collection_cascade_enumeration.rs +++ b/nodedb/tests/inproc/cases/collection_cascade_enumeration.rs @@ -24,11 +24,12 @@ use super::catalog_integrity_helpers::{ fn put_rls(catalog: &nodedb::control::security::catalog::SystemCatalog, name: &str, coll: &str) { let p = StoredRlsPolicy { tenant_id: TENANT, + database_id: 0, collection: coll.into(), display_collection: coll.into(), name: name.into(), policy_type_tag: 0, - compiled_predicate_json: String::new(), + predicate_text: String::new(), mode_tag: 0, on_deny_json: String::new(), enabled: true, diff --git a/nodedb/tests/inproc/cases/startup_failure.rs b/nodedb/tests/inproc/cases/startup_failure.rs index aea3e9fc5..42ddb059e 100644 --- a/nodedb/tests/inproc/cases/startup_failure.rs +++ b/nodedb/tests/inproc/cases/startup_failure.rs @@ -103,11 +103,12 @@ fn nodedb_exits_nonzero_on_catalog_integrity_violation() { catalog .put_rls_policy(&StoredRlsPolicy { tenant_id: 1, + database_id: 0, collection: "collection_that_was_never_created".to_string(), display_collection: "collection_that_was_never_created".to_string(), name: "dangling_policy".to_string(), policy_type_tag: 0, - compiled_predicate_json: String::new(), + predicate_text: String::new(), mode_tag: 0, on_deny_json: r#""Silent""#.to_string(), enabled: true, diff --git a/nodedb/tests/wire/cases/columnar_read_row_level_security.rs b/nodedb/tests/wire/cases/columnar_read_row_level_security.rs index cddeeac96..b790184c0 100644 --- a/nodedb/tests/wire/cases/columnar_read_row_level_security.rs +++ b/nodedb/tests/wire/cases/columnar_read_row_level_security.rs @@ -248,3 +248,117 @@ async fn a_spatial_select_returns_only_policy_admitted_rows() { "the read policy admits one row for this caller: {rows:?}" ); } + +/// `2020-03-05T10:00:00Z` as epoch milliseconds: the instant the policies +/// below compare against. +const CUTOFF_MS: i64 = 1_583_402_400_000; +const CUTOFF_TEXT: &str = "2020-03-05 10:00:00"; + +/// Create a columnar `collection` with a declared `TIMESTAMP` column holding +/// one row before, one at, and one after the cutoff, plus `user` with the +/// readwrite role. No policy: each test creates its own. +async fn seed_instant(server: &TestServer, collection: &str, user: &str) { + server + .exec(&format!( + "CREATE COLLECTION {collection} \ + (id TEXT PRIMARY KEY, owner TEXT, captured_at TIMESTAMP) \ + WITH (engine='columnar')" + )) + .await + .unwrap_or_else(|e| panic!("create {collection}: {e}")); + server + .exec(&format!( + "INSERT INTO {collection} (id, owner, captured_at) VALUES \ + ('before', '{user}', '2020-03-05 09:00:00'), \ + ('at', '{user}', '{CUTOFF_TEXT}'), \ + ('after', '{user}', '2020-03-05 11:00:00')" + )) + .await + .unwrap_or_else(|e| panic!("seed {collection}: {e}")); + server + .exec(&format!("CREATE USER {user} PASSWORD '{PASSWORD}'")) + .await + .unwrap_or_else(|e| panic!("create user {user}: {e}")); + server + .exec(&format!("GRANT ROLE readwrite TO {user}")) + .await + .unwrap_or_else(|e| panic!("grant readwrite to {user}: {e}")); +} + +/// A numeric policy literal compared against a declared `TIMESTAMP` column +/// is epoch milliseconds, typed at `CREATE RLS POLICY` by the rule a query +/// predicate follows: the governed user sees the rows at and after the +/// cutoff, and the row before it is excluded. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_numeric_policy_literal_on_a_timestamp_column_is_an_instant() { + let server = TestServer::start().await; + let user = "col_rls_ms_reader"; + seed_instant(&server, "col_rls_ms", user).await; + server + .exec(&format!( + "CREATE RLS POLICY col_rls_ms_recent ON col_rls_ms FOR READ \ + USING (captured_at >= {CUTOFF_MS})" + )) + .await + .expect("a numeric literal on a TIMESTAMP column is epoch milliseconds"); + + let rows = rows_as( + &server, + user, + "SELECT id FROM col_rls_ms ORDER BY captured_at", + ) + .await; + assert_eq!( + rows, + vec!["at".to_string(), "after".to_string()], + "the policy admits the rows at and after the cutoff: {rows:?}" + ); +} + +/// A text policy literal compared against a declared `TIMESTAMP` column is +/// parsed as ISO-8601 and enforced as the same instant. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_text_policy_literal_on_a_timestamp_column_is_an_instant() { + let server = TestServer::start().await; + let user = "col_rls_text_reader"; + seed_instant(&server, "col_rls_text", user).await; + server + .exec(&format!( + "CREATE RLS POLICY col_rls_text_recent ON col_rls_text FOR READ \ + USING (captured_at >= '{CUTOFF_TEXT}')" + )) + .await + .expect("a text literal on a TIMESTAMP column is parsed as an instant"); + + let rows = rows_as( + &server, + user, + "SELECT id FROM col_rls_text ORDER BY captured_at", + ) + .await; + assert_eq!( + rows, + vec!["at".to_string(), "after".to_string()], + "the policy admits the rows at and after the cutoff: {rows:?}" + ); +} + +/// A policy literal no instant can be read from is refused at +/// `CREATE RLS POLICY`, naming the column, so an unenforceable policy is +/// never stored. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_non_instant_policy_literal_on_a_timestamp_column_is_refused() { + let server = TestServer::start().await; + seed_instant(&server, "col_rls_bool", "col_rls_bool_reader").await; + let error = server + .exec( + "CREATE RLS POLICY col_rls_bool_bad ON col_rls_bool FOR READ \ + USING (captured_at >= true)", + ) + .await + .expect_err("a boolean is not an instant"); + assert!( + error.contains("captured_at"), + "the refusal must name the column: {error}" + ); +} diff --git a/nodedb/tests/wire/cases/timeseries_read_row_level_security.rs b/nodedb/tests/wire/cases/timeseries_read_row_level_security.rs index 92c41dac6..af9512198 100644 --- a/nodedb/tests/wire/cases/timeseries_read_row_level_security.rs +++ b/nodedb/tests/wire/cases/timeseries_read_row_level_security.rs @@ -245,3 +245,121 @@ async fn a_join_over_a_governed_timeseries_collection_excludes_policy_filtered_r "the join surfaced timeseries rows the read policy excludes: {rows:?}" ); } + +/// `2020-03-05T10:00:00Z` as epoch milliseconds: the instant the policies +/// below compare against. +const CUTOFF_MS: i64 = 1_583_402_400_000; +const CUTOFF_TEXT: &str = "2020-03-05 10:00:00"; + +/// Create a timeseries `collection` with a declared `TIMESTAMP` time key +/// holding one row before, one at, and one after the cutoff, plus `user` +/// with the readwrite role. No policy: each test creates its own. +async fn seed_instant(server: &TestServer, collection: &str, user: &str) { + server + .exec(&format!( + "CREATE COLLECTION {collection} \ + (captured_at TIMESTAMP TIME_KEY, label TEXT, value FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .unwrap_or_else(|e| panic!("create {collection}: {e}")); + for (at, label) in [ + ("2020-03-05 09:00:00", "before"), + (CUTOFF_TEXT, "at"), + ("2020-03-05 11:00:00", "after"), + ] { + server + .exec(&format!( + "INSERT INTO {collection} (captured_at, label, value) \ + VALUES ('{at}', '{label}', 1.0)" + )) + .await + .unwrap_or_else(|e| panic!("seed {collection} row {label}: {e}")); + } + server + .exec(&format!("CREATE USER {user} PASSWORD '{PASSWORD}'")) + .await + .unwrap_or_else(|e| panic!("create user {user}: {e}")); + server + .exec(&format!("GRANT ROLE readwrite TO {user}")) + .await + .unwrap_or_else(|e| panic!("grant readwrite to {user}: {e}")); +} + +/// A numeric policy literal compared against a declared `TIMESTAMP` time +/// key is epoch milliseconds, typed at `CREATE RLS POLICY` by the rule a +/// query predicate follows: the governed user sees the rows at and after the +/// cutoff, and the row before it is excluded. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_numeric_policy_literal_on_a_timestamp_time_key_is_an_instant() { + let server = TestServer::start().await; + let user = "ts_rls_ms_reader"; + seed_instant(&server, "ts_rls_ms", user).await; + server + .exec(&format!( + "CREATE RLS POLICY ts_rls_ms_recent ON ts_rls_ms FOR READ \ + USING (captured_at >= {CUTOFF_MS})" + )) + .await + .expect("a numeric literal on a TIMESTAMP column is epoch milliseconds"); + + let rows = rows_as( + &server, + user, + "SELECT label FROM ts_rls_ms ORDER BY captured_at", + ) + .await; + assert_eq!( + rows, + vec!["at".to_string(), "after".to_string()], + "the policy admits the rows at and after the cutoff: {rows:?}" + ); +} + +/// A text policy literal compared against a declared `TIMESTAMP` time key +/// is parsed as ISO-8601 and enforced as the same instant. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_text_policy_literal_on_a_timestamp_time_key_is_an_instant() { + let server = TestServer::start().await; + let user = "ts_rls_text_reader"; + seed_instant(&server, "ts_rls_text", user).await; + server + .exec(&format!( + "CREATE RLS POLICY ts_rls_text_recent ON ts_rls_text FOR READ \ + USING (captured_at >= '{CUTOFF_TEXT}')" + )) + .await + .expect("a text literal on a TIMESTAMP column is parsed as an instant"); + + let rows = rows_as( + &server, + user, + "SELECT label FROM ts_rls_text ORDER BY captured_at", + ) + .await; + assert_eq!( + rows, + vec!["at".to_string(), "after".to_string()], + "the policy admits the rows at and after the cutoff: {rows:?}" + ); +} + +/// A policy literal no instant can be read from is refused at +/// `CREATE RLS POLICY`, naming the column, so an unenforceable policy is +/// never stored. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_non_instant_policy_literal_on_a_timestamp_time_key_is_refused() { + let server = TestServer::start().await; + seed_instant(&server, "ts_rls_bool", "ts_rls_bool_reader").await; + let error = server + .exec( + "CREATE RLS POLICY ts_rls_bool_bad ON ts_rls_bool FOR READ \ + USING (captured_at >= true)", + ) + .await + .expect_err("a boolean is not an instant"); + assert!( + error.contains("captured_at"), + "the refusal must name the column: {error}" + ); +} From 46956dd06a6dd2f600ad7db97352809444857303 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 17 Sep 2026 12:57:54 +0800 Subject: [PATCH 19/21] feat(sql): coerce DEFAULT literals against declared column types DEFAULT materialization moves from a per-engine step in the control plane (sql_plan_convert::value::defaults::expand_row_defaults) into the planner itself: every write plan (Insert, Upsert, TimeseriesIngest) now carries rows with every declared DEFAULT already materialized and every literal already coerced to its column's declared type, plus a volatile_defaults flag replacing the old column_defaults text pairs so the plan cache still refuses a plan holding a fresh-per-row generator. Conversion no longer re-reads the catalog's DEFAULT text, and default_expr_is_volatile/defaults_are_volatile move out of volatility_scan into the planner's own defaults module. declared_type_coerce::coerce_write_literal is the new entry point a caller outside the planner uses for a literal that will reach storage under a column: it applies the same write-side coercion and range check a VALUES literal gets, exempting the primary key exactly as every VALUES/SET path does. The DDL gate (column_default.rs) now calls it for every column DEFAULT that spells a bare literal, so `DEFAULT 'not a date'` on a TIMESTAMP column and `DEFAULT 999999` on a SMALLINT column are refused at CREATE/ALTER instead of only surfacing once a row is inserted. CompiledDefault::literal() exposes the bare literal a DEFAULT spells; a generator or parsed expression has none to check yet. A refused literal DEFAULT raises the new sqlstate 42804 (datatype_mismatch) or the existing numeric-range SQLSTATE, mapped to BAD_REQUEST. catalog_adapter::declared_column_info centralizes the ColumnInfo a raw (name, type_str) DDL declaration resolves to, used both by the schemaless/columnar-family catalog arms and by the DDL gate, so a DEFAULT is judged against exactly the type its column will carry at INSERT time. planner/dml.rs and planner/dml_update_delete.rs split into a dml/ directory (insert, target, update_delete, upsert); planner/merge.rs splits into a merge/ directory (actions, plan) grouping the DML/MERGE planners with the write-side literal coercion they now perform. --- nodedb-sql/src/engine_rules/array.rs | 4 +- nodedb-sql/src/engine_rules/columnar.rs | 4 +- .../src/engine_rules/document_schemaless.rs | 4 +- .../src/engine_rules/document_strict.rs | 4 +- nodedb-sql/src/engine_rules/params.rs | 12 +- nodedb-sql/src/engine_rules/spatial.rs | 4 +- nodedb-sql/src/engine_rules/timeseries.rs | 1 + .../src/planner/declared_type_coerce.rs | 66 +++ nodedb-sql/src/planner/defaults/compiled.rs | 13 + nodedb-sql/src/planner/defaults/mod.rs | 11 +- nodedb-sql/src/planner/dml.rs | 422 ------------------ nodedb-sql/src/planner/dml/insert.rs | 281 ++++++++++++ nodedb-sql/src/planner/dml/mod.rs | 12 + nodedb-sql/src/planner/dml/target.rs | 187 ++++++++ .../update_delete.rs} | 13 +- nodedb-sql/src/planner/dml/upsert.rs | 92 ++++ .../planner/dml_helpers/declared_defaults.rs | 9 +- nodedb-sql/src/planner/dml_helpers/mod.rs | 4 +- nodedb-sql/src/planner/merge/actions.rs | 401 +++++++++++++++++ nodedb-sql/src/planner/merge/mod.rs | 8 + .../src/planner/{merge.rs => merge/plan.rs} | 139 +----- nodedb-sql/src/types/plan/cacheability.rs | 14 +- nodedb-sql/src/types/plan/mod.rs | 2 +- nodedb-sql/src/types/plan/variants.rs | 20 +- nodedb-sql/src/types/plan/volatility_scan.rs | 38 +- nodedb-sql/src/visitor/plan_visitor/args.rs | 2 - .../src/visitor/plan_visitor/dispatch.rs | 14 +- nodedb-types/src/error/sqlstate.rs | 5 + .../control/planner/catalog_adapter/mod.rs | 2 +- .../planner/catalog_adapter/type_convert.rs | 46 +- .../control/planner/context/query/planning.rs | 2 - .../array_fn_convert/aggregate.rs | 1 - .../array_fn_convert/slice.rs | 1 - .../planner/sql_plan_convert/convert.rs | 23 - .../sql_plan_convert/dml/insert/convert.rs | 16 +- .../sql_plan_convert/dml/kv_and_vector.rs | 1 - .../dml/update_delete/delete.rs | 1 - .../dml/update_delete/update.rs | 1 - .../planner/sql_plan_convert/dml/upsert.rs | 15 +- .../planner/sql_plan_convert/set_ops.rs | 2 - .../sql_plan_convert/value/defaults.rs | 45 -- .../planner/sql_plan_convert/value/mod.rs | 6 +- .../planner/sql_plan_convert/value/rows.rs | 7 +- .../sql_plan_convert/visitor/arms_dml.rs | 4 - .../neutral/collection/alter/add_column.rs | 15 + .../ddl/neutral/collection/create/build.rs | 7 +- .../shared/ddl/neutral/column_default.rs | 137 +++++- .../shared/ddl/neutral/convert/column_defs.rs | 11 +- .../ddl/neutral/convert/typeguard_columns.rs | 14 +- .../src/control/server/shared/ddl/result.rs | 2 + .../executor_tests/test_group_by_alias.rs | 1 - nodedb/tests/wire/cases/kv_column_defaults.rs | 22 +- nodedb/tests/wire/cases/mod.rs | 2 + .../wire/cases/sql_default_declared_types.rs | 174 ++++++++ .../wire/cases/sql_merge_declared_types.rs | 195 ++++++++ 55 files changed, 1736 insertions(+), 803 deletions(-) delete mode 100644 nodedb-sql/src/planner/dml.rs create mode 100644 nodedb-sql/src/planner/dml/insert.rs create mode 100644 nodedb-sql/src/planner/dml/mod.rs create mode 100644 nodedb-sql/src/planner/dml/target.rs rename nodedb-sql/src/planner/{dml_update_delete.rs => dml/update_delete.rs} (97%) create mode 100644 nodedb-sql/src/planner/dml/upsert.rs create mode 100644 nodedb-sql/src/planner/merge/actions.rs create mode 100644 nodedb-sql/src/planner/merge/mod.rs rename nodedb-sql/src/planner/{merge.rs => merge/plan.rs} (70%) delete mode 100644 nodedb/src/control/planner/sql_plan_convert/value/defaults.rs create mode 100644 nodedb/tests/wire/cases/sql_default_declared_types.rs create mode 100644 nodedb/tests/wire/cases/sql_merge_declared_types.rs diff --git a/nodedb-sql/src/engine_rules/array.rs b/nodedb-sql/src/engine_rules/array.rs index 1e9b7e8f7..523a9c3cd 100644 --- a/nodedb-sql/src/engine_rules/array.rs +++ b/nodedb-sql/src/engine_rules/array.rs @@ -96,7 +96,7 @@ mod tests { collection: "g".into(), columns: vec![], rows: vec![], - column_defaults: vec![], + volatile_defaults: false, if_absent: false, column_schema: vec![], primary_key: None, @@ -115,7 +115,7 @@ mod tests { collection: "g".into(), columns: vec![], rows: vec![], - column_defaults: vec![], + volatile_defaults: false, on_conflict_updates: vec![], column_schema: vec![], primary_key: None, diff --git a/nodedb-sql/src/engine_rules/columnar.rs b/nodedb-sql/src/engine_rules/columnar.rs index 7decfbd69..e6939dcb6 100644 --- a/nodedb-sql/src/engine_rules/columnar.rs +++ b/nodedb-sql/src/engine_rules/columnar.rs @@ -15,7 +15,7 @@ impl EngineRules for ColumnarRules { engine: EngineType::Columnar, route: WriteRoute::ColumnarFamily, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, if_absent: p.if_absent, column_schema: p.column_schema, primary_key: p.primary_key, @@ -32,7 +32,7 @@ impl EngineRules for ColumnarRules { engine: EngineType::Columnar, route: WriteRoute::ColumnarFamily, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, on_conflict_updates: p.on_conflict_updates, column_schema: p.column_schema, primary_key: p.primary_key, diff --git a/nodedb-sql/src/engine_rules/document_schemaless.rs b/nodedb-sql/src/engine_rules/document_schemaless.rs index 17937962f..bdad8b51f 100644 --- a/nodedb-sql/src/engine_rules/document_schemaless.rs +++ b/nodedb-sql/src/engine_rules/document_schemaless.rs @@ -15,7 +15,7 @@ impl EngineRules for SchemalessRules { engine: EngineType::DocumentSchemaless, route: WriteRoute::Document, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, if_absent: p.if_absent, column_schema: vec![], primary_key: p.primary_key, @@ -28,7 +28,7 @@ impl EngineRules for SchemalessRules { engine: EngineType::DocumentSchemaless, route: WriteRoute::Document, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, on_conflict_updates: p.on_conflict_updates, column_schema: vec![], primary_key: p.primary_key, diff --git a/nodedb-sql/src/engine_rules/document_strict.rs b/nodedb-sql/src/engine_rules/document_strict.rs index 0aaca86ac..3e8d25327 100644 --- a/nodedb-sql/src/engine_rules/document_strict.rs +++ b/nodedb-sql/src/engine_rules/document_strict.rs @@ -15,7 +15,7 @@ impl EngineRules for StrictRules { engine: EngineType::DocumentStrict, route: WriteRoute::Document, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, if_absent: p.if_absent, column_schema: vec![], primary_key: p.primary_key, @@ -28,7 +28,7 @@ impl EngineRules for StrictRules { engine: EngineType::DocumentStrict, route: WriteRoute::Document, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, on_conflict_updates: p.on_conflict_updates, column_schema: vec![], primary_key: p.primary_key, diff --git a/nodedb-sql/src/engine_rules/params.rs b/nodedb-sql/src/engine_rules/params.rs index 6483fe75b..e4e98a890 100644 --- a/nodedb-sql/src/engine_rules/params.rs +++ b/nodedb-sql/src/engine_rules/params.rs @@ -8,8 +8,13 @@ use crate::types::*; pub struct InsertParams { pub collection: String, pub columns: Vec, + /// Every declared DEFAULT already materialized and every literal already + /// coerced to its declared column type. An engine stores these as they + /// are; none re-reads the catalog's DEFAULT text. pub rows: Vec>, - pub column_defaults: Vec<(String, String)>, + /// Whether a DEFAULT materialized into `rows` was volatile. A plan that + /// carries one is never admitted to the plan cache. + pub volatile_defaults: bool, /// `ON CONFLICT DO NOTHING` semantics: duplicate-PK rows are skipped /// silently. `false` for plain `INSERT` (raises `unique_violation`). pub if_absent: bool, @@ -109,8 +114,11 @@ pub struct MergeParams { pub struct UpsertParams { pub collection: String, pub columns: Vec, + /// Defaults materialized and literals coerced, as in + /// `InsertParams::rows`. pub rows: Vec>, - pub column_defaults: Vec<(String, String)>, + /// Mirrors `InsertParams::volatile_defaults`. + pub volatile_defaults: bool, /// `ON CONFLICT (...) DO UPDATE SET` assignments. Empty for plain /// `UPSERT INTO ...`; populated when the caller is /// `INSERT ... ON CONFLICT ... DO UPDATE SET`. diff --git a/nodedb-sql/src/engine_rules/spatial.rs b/nodedb-sql/src/engine_rules/spatial.rs index afe708e52..6373f11e9 100644 --- a/nodedb-sql/src/engine_rules/spatial.rs +++ b/nodedb-sql/src/engine_rules/spatial.rs @@ -15,7 +15,7 @@ impl EngineRules for SpatialRules { engine: EngineType::Spatial, route: WriteRoute::ColumnarFamily, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, if_absent: p.if_absent, column_schema: p.column_schema, primary_key: p.primary_key, @@ -31,7 +31,7 @@ impl EngineRules for SpatialRules { engine: EngineType::Spatial, route: WriteRoute::ColumnarFamily, rows: p.rows, - column_defaults: p.column_defaults, + volatile_defaults: p.volatile_defaults, on_conflict_updates: p.on_conflict_updates, column_schema: p.column_schema, primary_key: p.primary_key, diff --git a/nodedb-sql/src/engine_rules/timeseries.rs b/nodedb-sql/src/engine_rules/timeseries.rs index ab6ab210d..4c4122bed 100644 --- a/nodedb-sql/src/engine_rules/timeseries.rs +++ b/nodedb-sql/src/engine_rules/timeseries.rs @@ -18,6 +18,7 @@ impl EngineRules for TimeseriesRules { Ok(vec![SqlPlan::TimeseriesIngest { collection: p.collection, rows: p.rows, + volatile_defaults: p.volatile_defaults, }]) } diff --git a/nodedb-sql/src/planner/declared_type_coerce.rs b/nodedb-sql/src/planner/declared_type_coerce.rs index dcc4955f5..d098ab358 100644 --- a/nodedb-sql/src/planner/declared_type_coerce.rs +++ b/nodedb-sql/src/planner/declared_type_coerce.rs @@ -62,9 +62,33 @@ use nodedb_types::datetime::NdbDateTime; use rust_decimal::prelude::ToPrimitive; +use super::dml_helpers::{check_declared_float_ranges, check_declared_int_ranges}; use crate::error::{Result, SqlError}; use crate::types::{ColumnInfo, SqlDataType, SqlExpr, SqlValue}; +/// Coerce one literal bound for `column` by the write-side rule, then +/// range-check it against the column's declared width. +/// +/// The one entry a caller outside the planner uses for a literal that will +/// reach storage under `column`: a DDL gate checks a column `DEFAULT` through +/// it, so a default is accepted or refused exactly as the same literal in a +/// `VALUES` clause is. The primary-key column keeps its literal as written, +/// the same exemption every `VALUES` and `SET` path carries — see +/// [`coerce_rows_to_declared_types`]. +/// +/// Errors name the column and the literal. +pub fn coerce_write_literal(column: &ColumnInfo, value: SqlValue) -> Result { + if column.is_primary_key { + return Ok(value); + } + let coerced = coerce_value(&column.name, value, &column.data_type)?; + let row = [vec![(column.name.clone(), coerced)]]; + check_declared_int_ranges(std::slice::from_ref(column), &row)?; + check_declared_float_ranges(std::slice::from_ref(column), &row)?; + let [mut row] = row; + Ok(row.swap_remove(0).1) +} + /// Coerce every value in one `(column, value)` row to its declared column /// type, in place. /// @@ -669,6 +693,48 @@ mod tests { assert_eq!(literal(&assignments[1].1), SqlValue::Float(1.5)); } + /// The public write-side entry coerces, then range-checks: a numeric + /// literal on a TIMESTAMP column becomes the instant, text that spells + /// no instant is refused naming the column, and an integer past the + /// declared SMALLINT width is refused naming the column. + #[test] + fn write_literal_coerces_then_range_checks() { + let at = column("at", SqlDataType::Timestamp); + assert_eq!( + coerce_write_literal(&at, SqlValue::Int(1_583_402_400_000)) + .expect("an integer is epoch milliseconds"), + SqlValue::Timestamp(early()) + ); + let err = coerce_write_literal(&at, SqlValue::String("not a date".into())) + .expect_err("text with no instant is refused"); + assert!(err.to_string().contains("'at'"), "{err}"); + + let mut small = column("s", SqlDataType::Int64); + small.int_width = Some(nodedb_types::columnar::IntWidth::I16); + assert_eq!( + coerce_write_literal(&small, SqlValue::Int(7)).expect("7 fits SMALLINT"), + SqlValue::Int(7) + ); + let err = coerce_write_literal(&small, SqlValue::Int(999_999)) + .expect_err("999999 does not fit SMALLINT"); + assert!( + matches!(err, SqlError::IntegerOutOfRange { ref column, .. } if column == "s"), + "{err}" + ); + } + + /// The write-side entry keeps the primary key as written, like every + /// `VALUES` and `SET` path. + #[test] + fn write_literal_exempts_the_primary_key() { + let mut id = column("id", SqlDataType::Int64); + id.is_primary_key = true; + assert_eq!( + coerce_write_literal(&id, decimal("5.0")).expect("an exempt key is never re-typed"), + decimal("5.0") + ); + } + /// A column the catalog does not declare (the KV `key`/`value` /// convention, a `ttl` pseudo-column) is left exactly as written. #[test] diff --git a/nodedb-sql/src/planner/defaults/compiled.rs b/nodedb-sql/src/planner/defaults/compiled.rs index 9905e3e23..69e25e460 100644 --- a/nodedb-sql/src/planner/defaults/compiled.rs +++ b/nodedb-sql/src/planner/defaults/compiled.rs @@ -63,6 +63,19 @@ impl CompiledDefault { &self.column } + /// The constant this DEFAULT spells, when it is a bare literal. + /// + /// A generator or a parsed expression has no value until evaluation, so + /// it yields `None`. The DDL gate checks a literal against the declared + /// column type through this, so a DEFAULT the column cannot hold is + /// refused where it is declared. + pub fn literal(&self) -> Option<&nodedb_types::Value> { + match &self.kind { + DefaultKind::Literal(value) => Some(value), + DefaultKind::Generator(_) | DefaultKind::Expr(_) => None, + } + } + /// Whether this DEFAULT produces a fresh value on every evaluation. /// /// A plan carrying one is never admitted to the plan cache, or the cache diff --git a/nodedb-sql/src/planner/defaults/mod.rs b/nodedb-sql/src/planner/defaults/mod.rs index 98b05a51b..2e19ca84c 100644 --- a/nodedb-sql/src/planner/defaults/mod.rs +++ b/nodedb-sql/src/planner/defaults/mod.rs @@ -14,13 +14,10 @@ //! The column is never omitted: an omitted column stores NULL where the //! declaration promised a value, and nothing reports it. //! -//! Lives in the SQL crate rather than beside one engine's converter because -//! every engine that materializes a DEFAULT has to produce the SAME value for -//! the same expression — a `DEFAULT now()` that means one thing on a document -//! collection and another on a key-value one would be a difference nobody -//! declared. The key-value planner also needs it BEFORE its declared-type -//! coercion and range checks run, so a materialized default is validated -//! exactly like a supplied one. +//! Lives in the planner because every engine's rows materialize their +//! DEFAULTs there, before declared-type coercion and range checks run, so a +//! materialized default is validated exactly like a supplied one and a +//! `DEFAULT now()` means the same thing on every engine. mod compiled; mod convert; diff --git a/nodedb-sql/src/planner/dml.rs b/nodedb-sql/src/planner/dml.rs deleted file mode 100644 index 350e078fd..000000000 --- a/nodedb-sql/src/planner/dml.rs +++ /dev/null @@ -1,422 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! INSERT, UPDATE, DELETE planning. - -use nodedb_types::DatabaseId; -use sqlparser::ast::{self}; - -use super::dml_helpers::{ - KvInsertParams, bind_insert_select_columns, build_kv_insert_plan, - build_vector_primary_insert_plan, check_declared_float_ranges_in_assignments, - check_declared_int_ranges_in_assignments, coerce_and_check_rows, convert_value_rows, - materialize_defaults_in_rows, resolve_insert_columns, -}; -use crate::engine_rules::{self, InsertParams}; -use crate::error::{Result, SqlError}; -use crate::parser::normalize::{normalize_insert_column, normalize_object_name_checked}; -use crate::planner::declared_type_coerce::coerce_assignments_to_declared_types; -use crate::resolver::ColumnScope; -use crate::resolver::columns::{ResolvedTable, TableScope}; -use crate::resolver::expr::convert_expr; -use crate::types::*; - -pub use dml_update_delete::{plan_delete, plan_truncate_stmt, plan_update}; - -#[path = "dml_update_delete.rs"] -mod dml_update_delete; - -/// The column namespace of an INSERT target. -fn target_scope(table_name: &str, info: &CollectionInfo) -> Result { - let mut scope = TableScope::single(ResolvedTable { - name: table_name.to_string(), - alias: None, - info: info.clone(), - })?; - // `ON CONFLICT DO UPDATE` addresses the proposed row as `excluded`. It - // carries the target's columns and is qualified-only, so a bare name in - // the SET clause names the stored row. - scope.add_qualified_only(ResolvedTable { - name: EXCLUDED_RELATION.to_string(), - alias: None, - info: info.clone(), - })?; - Ok(scope) -} - -/// The pseudo-relation `ON CONFLICT DO UPDATE` uses for the proposed row. -const EXCLUDED_RELATION: &str = "excluded"; - -/// Normalize an INSERT column list and reject a name the target does not have. -fn insert_columns(columns: &[ast::ObjectName], scope: &TableScope) -> Result> { - columns - .iter() - .map(|c| { - let col = normalize_insert_column(c)?; - scope.check_name(None, &col)?; - Ok(col) - }) - .collect() -} - -/// Classification of an `ON CONFLICT` clause attached to an INSERT. -enum OnConflict { - /// No `ON CONFLICT` clause — plain INSERT (error on duplicate PK). - None, - /// `ON CONFLICT DO NOTHING` — skip rows that would conflict, no error. - DoNothing, - /// `ON CONFLICT (...) DO UPDATE SET ...` — apply the assignments against - /// the existing row on conflict. - DoUpdate(Vec<(String, SqlExpr)>), -} - -fn classify_on_conflict(ins: &ast::Insert, scope: &TableScope) -> Result { - let Some(on) = ins.on.as_ref() else { - return Ok(OnConflict::None); - }; - let ast::OnInsert::OnConflict(oc) = on else { - return Ok(OnConflict::None); - }; - match &oc.action { - ast::OnConflictAction::DoNothing => Ok(OnConflict::DoNothing), - ast::OnConflictAction::DoUpdate(do_update) => { - let mut pairs = Vec::with_capacity(do_update.assignments.len()); - for a in &do_update.assignments { - let name = match &a.target { - ast::AssignmentTarget::ColumnName(obj) => normalize_object_name_checked(obj)?, - _ => { - return Err(SqlError::Unsupported { - detail: "ON CONFLICT DO UPDATE SET target must be a column name".into(), - }); - } - }; - scope.check_name(None, &name)?; - let expr = convert_expr(&a.value, &ColumnScope::Relations(scope))?; - pairs.push((name, expr)); - } - Ok(OnConflict::DoUpdate(pairs)) - } - } -} - -/// Plan an INSERT statement. -pub fn plan_insert(ins: &ast::Insert, catalog: &dyn SqlCatalog) -> Result> { - let table_name = match &ins.table { - ast::TableObject::TableName(name) => normalize_object_name_checked(name)?, - ast::TableObject::TableFunction(_) => { - return Err(SqlError::Unsupported { - detail: "INSERT INTO table function not supported".into(), - }); - } - // Oracle's `INSERT INTO (SELECT ...)`: the target is a subquery, so - // there is no collection to resolve or route to an engine. - ast::TableObject::TableQuery(_) => { - return Err(SqlError::Unsupported { - detail: "INSERT INTO a subquery target is not supported".into(), - }); - } - }; - let info = catalog - .get_collection(DatabaseId::DEFAULT, &table_name)? - .ok_or_else(|| SqlError::UnknownTable { - name: table_name.clone(), - })?; - let target_scope = target_scope(&table_name, &info)?; - - // `INSERT ... ON CONFLICT DO UPDATE SET` reroutes to the upsert path - // with the assignments carried through. `DO NOTHING` stays on the - // INSERT path with `if_absent=true`. - let if_absent = match classify_on_conflict(ins, &target_scope)? { - OnConflict::None => false, - OnConflict::DoNothing => true, - OnConflict::DoUpdate(updates) => { - return plan_upsert_with_on_conflict(ins, catalog, updates); - } - }; - - let columns = insert_columns(&ins.columns, &target_scope)?; - - // Check for INSERT...SELECT. - if let Some(source) = &ins.source - && let ast::SetExpr::Select(select) = &*source.body - { - let column_map = bind_insert_select_columns(catalog, &columns, select, &info)?; - let source_plan = super::select::plan_query( - source, - catalog, - &crate::functions::registry::FunctionRegistry::new(), - crate::TemporalScope::default(), - )?; - return Ok(vec![SqlPlan::InsertSelect { - target: table_name, - source: Box::new(source_plan), - limit: 0, - column_map, - }]); - } - - // VALUES clause. - let source = ins.source.as_ref().ok_or_else(|| SqlError::Parse { - detail: "INSERT requires VALUES or SELECT".into(), - })?; - - let rows_ast = match &*source.body { - ast::SetExpr::Values(values) => &values.rows, - _ => { - return Err(SqlError::Unsupported { - detail: "INSERT source must be VALUES or SELECT".into(), - }); - } - }; - - // KV engine: key and value are fundamentally separate — handle directly. - // Positional column binding (below) does not apply here: the KV path - // matches columns by name against `pk_col`/`"key"`/`"ttl"`, which is - // orthogonal to declared column order. - if info.engine == EngineType::KeyValue { - let intent = if if_absent { - KvInsertIntent::InsertIfAbsent - } else { - KvInsertIntent::Insert - }; - return build_kv_insert_plan(KvInsertParams { - collection: table_name, - columns: &columns, - rows_ast, - intent, - on_conflict_updates: Vec::new(), - pk_col: info.primary_key.as_deref(), - declared_columns: &info.columns, - catalog, - }); - } - - // Positional INSERT (no column list): bind values to the collection's - // declared column order so named projections/predicates can find them. - // No-op for named inserts and schemaless collections. - let columns = resolve_insert_columns(columns, &info, rows_ast)?; - - // Vector-primary collection: bypass document encoding. - // - // The vector path never reaches `EngineRules::plan_insert`, which is where - // every other engine hands its declared DEFAULTs on for expansion. It - // materializes them here instead, through the same helper the key-value - // path uses, and before coercion so a default is range-checked exactly like - // a supplied literal. - if info.primary == nodedb_types::PrimaryEngine::Vector - && let Some(ref vpc) = info.vector_primary - { - let mut rows_parsed = convert_value_rows(&columns, rows_ast)?; - let volatile_defaults = - materialize_defaults_in_rows(&info.columns, &mut rows_parsed, catalog)?; - coerce_and_check_rows(&info, &mut rows_parsed)?; - return build_vector_primary_insert_plan( - &table_name, - vpc, - &columns, - rows_parsed, - volatile_defaults, - ); - } - - // All other engines: delegate to engine rules. - let mut rows = convert_value_rows(&columns, rows_ast)?; - coerce_and_check_rows(&info, &mut rows)?; - let column_defaults: Vec<(String, String)> = info - .columns - .iter() - .filter_map(|c| c.default.as_ref().map(|d| (c.name.clone(), d.clone()))) - .collect(); - let column_schema: Vec<(String, String)> = info - .columns - .iter() - .filter_map(|c| c.raw_type.as_ref().map(|t| (c.name.clone(), t.clone()))) - .collect(); - let rules = engine_rules::resolve_engine_rules(info.engine); - rules.plan_insert(InsertParams { - collection: table_name, - columns, - rows, - column_defaults, - if_absent, - column_schema, - primary_key: info.primary_key.clone(), - }) -} - -/// Plan an UPSERT statement (pre-processed from `UPSERT INTO` to `INSERT INTO`). -/// -/// Same parsing as INSERT but routes through `engine_rules.plan_upsert()`. -pub fn plan_upsert(ins: &ast::Insert, catalog: &dyn SqlCatalog) -> Result> { - let table_name = match &ins.table { - ast::TableObject::TableName(name) => normalize_object_name_checked(name)?, - ast::TableObject::TableFunction(_) => { - return Err(SqlError::Unsupported { - detail: "UPSERT INTO table function not supported".into(), - }); - } - // Oracle's `INSERT INTO (SELECT ...)`: the target is a subquery, so - // there is no collection to resolve or route to an engine. - ast::TableObject::TableQuery(_) => { - return Err(SqlError::Unsupported { - detail: "UPSERT INTO a subquery target is not supported".into(), - }); - } - }; - let info = catalog - .get_collection(DatabaseId::DEFAULT, &table_name)? - .ok_or_else(|| SqlError::UnknownTable { - name: table_name.clone(), - })?; - - let columns = insert_columns(&ins.columns, &target_scope(&table_name, &info)?)?; - - let source = ins.source.as_ref().ok_or_else(|| SqlError::Parse { - detail: "UPSERT requires VALUES".into(), - })?; - - let rows_ast = match &*source.body { - ast::SetExpr::Values(values) => &values.rows, - _ => { - return Err(SqlError::Unsupported { - detail: "UPSERT source must be VALUES".into(), - }); - } - }; - - // KV: upsert is just a PUT (natural overwrite). Positional column - // binding (below) does not apply here — see `plan_insert`. - if info.engine == EngineType::KeyValue { - return build_kv_insert_plan(KvInsertParams { - collection: table_name, - columns: &columns, - rows_ast, - intent: KvInsertIntent::Put, - on_conflict_updates: Vec::new(), - pk_col: info.primary_key.as_deref(), - declared_columns: &info.columns, - catalog, - }); - } - - // Positional UPSERT (no column list): bind to the collection's declared - // column order — see `plan_insert` for the full rationale. - let columns = resolve_insert_columns(columns, &info, rows_ast)?; - - let mut rows = convert_value_rows(&columns, rows_ast)?; - coerce_and_check_rows(&info, &mut rows)?; - let column_defaults: Vec<(String, String)> = info - .columns - .iter() - .filter_map(|c| c.default.as_ref().map(|d| (c.name.clone(), d.clone()))) - .collect(); - let column_schema: Vec<(String, String)> = info - .columns - .iter() - .filter_map(|c| c.raw_type.as_ref().map(|t| (c.name.clone(), t.clone()))) - .collect(); - let rules = engine_rules::resolve_engine_rules(info.engine); - rules.plan_upsert(engine_rules::UpsertParams { - collection: table_name, - columns, - rows, - column_defaults, - on_conflict_updates: Vec::new(), - column_schema, - primary_key: info.primary_key.clone(), - }) -} - -/// Plan an `INSERT ... ON CONFLICT DO UPDATE SET` statement. -fn plan_upsert_with_on_conflict( - ins: &ast::Insert, - catalog: &dyn SqlCatalog, - mut on_conflict_updates: Vec<(String, SqlExpr)>, -) -> Result> { - let table_name = match &ins.table { - ast::TableObject::TableName(name) => normalize_object_name_checked(name)?, - ast::TableObject::TableFunction(_) => { - return Err(SqlError::Unsupported { - detail: "INSERT ... ON CONFLICT on a table function is not supported".into(), - }); - } - // Oracle's `INSERT INTO (SELECT ...)`: the target is a subquery, so - // there is no collection to resolve or route to an engine. - ast::TableObject::TableQuery(_) => { - return Err(SqlError::Unsupported { - detail: "INSERT ... ON CONFLICT on a subquery target is not supported".into(), - }); - } - }; - let info = catalog - .get_collection(DatabaseId::DEFAULT, &table_name)? - .ok_or_else(|| SqlError::UnknownTable { - name: table_name.clone(), - })?; - - let columns = insert_columns(&ins.columns, &target_scope(&table_name, &info)?)?; - - let source = ins.source.as_ref().ok_or_else(|| SqlError::Parse { - detail: "INSERT ... ON CONFLICT requires VALUES".into(), - })?; - let rows_ast = match &*source.body { - ast::SetExpr::Values(values) => &values.rows, - _ => { - return Err(SqlError::Unsupported { - detail: "INSERT ... ON CONFLICT source must be VALUES".into(), - }); - } - }; - - // KV: `INSERT ... ON CONFLICT (key) DO UPDATE SET ...` is an opt-in - // overwrite — same physical semantics as UPSERT, with the optional - // per-row assignments carried through for the Data Plane to apply - // against the existing row. - if info.engine == EngineType::KeyValue { - return build_kv_insert_plan(KvInsertParams { - collection: table_name, - columns: &columns, - rows_ast, - intent: KvInsertIntent::Put, - on_conflict_updates, - pk_col: info.primary_key.as_deref(), - declared_columns: &info.columns, - catalog, - }); - } - - // Positional UPSERT (no column list): bind to the collection's declared - // column order — see `plan_insert` for the full rationale. - let columns = resolve_insert_columns(columns, &info, rows_ast)?; - - let mut rows = convert_value_rows(&columns, rows_ast)?; - coerce_and_check_rows(&info, &mut rows)?; - // `DO UPDATE SET col = ` writes through the same path as the - // inserted row, so its literals carry the same declared-type contract. - coerce_assignments_to_declared_types( - &info.columns, - &mut on_conflict_updates, - info.primary_key.as_deref(), - )?; - check_declared_int_ranges_in_assignments(&info.columns, &on_conflict_updates)?; - check_declared_float_ranges_in_assignments(&info.columns, &on_conflict_updates)?; - let column_defaults: Vec<(String, String)> = info - .columns - .iter() - .filter_map(|c| c.default.as_ref().map(|d| (c.name.clone(), d.clone()))) - .collect(); - let column_schema: Vec<(String, String)> = info - .columns - .iter() - .filter_map(|c| c.raw_type.as_ref().map(|t| (c.name.clone(), t.clone()))) - .collect(); - let rules = engine_rules::resolve_engine_rules(info.engine); - rules.plan_upsert(engine_rules::UpsertParams { - collection: table_name, - columns, - rows, - column_defaults, - on_conflict_updates, - column_schema, - primary_key: info.primary_key.clone(), - }) -} diff --git a/nodedb-sql/src/planner/dml/insert.rs b/nodedb-sql/src/planner/dml/insert.rs new file mode 100644 index 000000000..bbbe1ce1b --- /dev/null +++ b/nodedb-sql/src/planner/dml/insert.rs @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! INSERT planning. + +use sqlparser::ast; + +use super::super::dml_helpers::{ + KvInsertParams, bind_insert_select_columns, build_kv_insert_plan, + build_vector_primary_insert_plan, resolve_insert_columns, +}; +use super::target::{ + OnConflict, classify_on_conflict, column_schema, insert_columns, resolve_target, target_scope, + typed_rows, values_rows, +}; +use super::upsert::plan_upsert_with_on_conflict; +use crate::engine_rules::{self, InsertParams}; +use crate::error::Result; +use crate::types::*; + +/// Plan an INSERT statement. +pub fn plan_insert(ins: &ast::Insert, catalog: &dyn SqlCatalog) -> Result> { + let (table_name, info) = resolve_target(ins, "INSERT", catalog)?; + let target_scope = target_scope(&table_name, &info)?; + + // `INSERT ... ON CONFLICT DO UPDATE SET` reroutes to the upsert path + // with the assignments carried through. `DO NOTHING` stays on the + // INSERT path with `if_absent=true`. + let if_absent = match classify_on_conflict(ins, &target_scope)? { + OnConflict::None => false, + OnConflict::DoNothing => true, + OnConflict::DoUpdate(updates) => { + return plan_upsert_with_on_conflict(ins, catalog, updates); + } + }; + + let columns = insert_columns(&ins.columns, &target_scope)?; + + // Check for INSERT...SELECT. + if let Some(source) = &ins.source + && let ast::SetExpr::Select(select) = &*source.body + { + let column_map = bind_insert_select_columns(catalog, &columns, select, &info)?; + let source_plan = super::super::select::plan_query( + source, + catalog, + &crate::functions::registry::FunctionRegistry::new(), + crate::TemporalScope::default(), + )?; + return Ok(vec![SqlPlan::InsertSelect { + target: table_name, + source: Box::new(source_plan), + limit: 0, + column_map, + }]); + } + + let rows_ast = values_rows(ins, "INSERT")?; + + // KV engine: key and value are fundamentally separate — handle directly. + // Positional column binding (below) does not apply here: the KV path + // matches columns by name against `pk_col`/`"key"`/`"ttl"`, which is + // orthogonal to declared column order. + if info.engine == EngineType::KeyValue { + let intent = if if_absent { + KvInsertIntent::InsertIfAbsent + } else { + KvInsertIntent::Insert + }; + return build_kv_insert_plan(KvInsertParams { + collection: table_name, + columns: &columns, + rows_ast, + intent, + on_conflict_updates: Vec::new(), + pk_col: info.primary_key.as_deref(), + declared_columns: &info.columns, + catalog, + }); + } + + // Positional INSERT (no column list): bind values to the collection's + // declared column order so named projections/predicates can find them. + // No-op for named inserts and schemaless collections. + let columns = resolve_insert_columns(columns, &info, rows_ast)?; + + // One typing pass for every remaining engine: defaults materialized, + // then every cell coerced and range-checked against its declared type. + let typed = typed_rows(&info, &columns, rows_ast, catalog)?; + + // Vector-primary collection: bypass document encoding. + if info.primary == nodedb_types::PrimaryEngine::Vector + && let Some(ref vpc) = info.vector_primary + { + return build_vector_primary_insert_plan( + &table_name, + vpc, + &columns, + typed.rows, + typed.volatile_defaults, + ); + } + + // All other engines: delegate to engine rules. + let column_schema = column_schema(&info); + let rules = engine_rules::resolve_engine_rules(info.engine); + rules.plan_insert(InsertParams { + collection: table_name, + columns, + rows: typed.rows, + volatile_defaults: typed.volatile_defaults, + if_absent, + column_schema, + primary_key: info.primary_key.clone(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::catalog::{SqlCatalog, SqlCatalogError}; + use crate::parser::statement::parse_sql; + use nodedb_types::columnar::IntWidth; + use nodedb_types::datetime::NdbDateTime; + + fn column(name: &str, data_type: SqlDataType, default: Option<&str>) -> ColumnInfo { + ColumnInfo { + name: name.to_string(), + data_type, + nullable: true, + is_primary_key: false, + default: default.map(str::to_string), + raw_type: None, + int_width: None, + float_width: None, + } + } + + fn collection(engine: EngineType, columns: Vec) -> CollectionInfo { + let primary_key = columns + .iter() + .find(|c| c.is_primary_key) + .map(|c| c.name.clone()); + CollectionInfo { + name: "t".into(), + engine, + columns, + primary_key, + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(engine), + } + } + + /// A catalog with one collection named `t`. + struct OneCollection(CollectionInfo); + + impl SqlCatalog for OneCollection { + fn get_collection( + &self, + _: nodedb_types::DatabaseId, + name: &str, + ) -> std::result::Result, SqlCatalogError> { + Ok((name == "t").then(|| self.0.clone())) + } + } + + fn plan(sql: &str, catalog: &dyn SqlCatalog) -> Result { + let statements = parse_sql(sql)?; + let sqlparser::ast::Statement::Insert(ins) = &statements[0] else { + panic!("expected an INSERT statement"); + }; + let mut plans = plan_insert(ins, catalog)?; + Ok(plans.remove(0)) + } + + /// `2020-03-05T10:00:00Z` as microseconds since the Unix epoch. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + fn cell<'a>(row: &'a [(String, SqlValue)], name: &str) -> &'a SqlValue { + &row.iter() + .find(|(n, _)| n == name) + .unwrap_or_else(|| panic!("row carries {name}")) + .1 + } + + /// A numeric DEFAULT on a TIMESTAMP column is materialized, then coerced + /// like a supplied literal: the plan carries the typed instant. + #[test] + fn a_numeric_timestamp_default_is_materialized_then_coerced() { + let mut id = column("id", SqlDataType::String, None); + id.is_primary_key = true; + let catalog = OneCollection(collection( + EngineType::DocumentSchemaless, + vec![ + id, + column("at", SqlDataType::Timestamp, Some("1583402400000")), + ], + )); + let plan = plan("INSERT INTO t (id) VALUES ('r1')", &catalog).expect("plans"); + let SqlPlan::Insert { + rows, + volatile_defaults, + .. + } = plan + else { + panic!("expected SqlPlan::Insert, got {plan:?}"); + }; + assert!(!volatile_defaults, "a literal DEFAULT is not volatile"); + assert_eq!( + cell(&rows[0], "at"), + &SqlValue::Timestamp(NdbDateTime::from_micros(EARLY_MICROS)) + ); + } + + /// A DEFAULT the column cannot hold is refused at the insert, naming the + /// column, exactly as the same literal in VALUES is. + #[test] + fn a_default_out_of_declared_range_is_refused() { + let mut id = column("id", SqlDataType::String, None); + id.is_primary_key = true; + let mut small = column("s", SqlDataType::Int64, Some("999999")); + small.int_width = Some(IntWidth::I16); + let catalog = OneCollection(collection(EngineType::DocumentStrict, vec![id, small])); + let err = plan("INSERT INTO t (id) VALUES ('r1')", &catalog) + .expect_err("DEFAULT 999999 does not fit SMALLINT"); + assert!( + matches!(err, crate::SqlError::IntegerOutOfRange { ref column, .. } if column == "s"), + "{err}" + ); + } + + /// A timeseries plan carries the declared default of an omitted column + /// in its rows, the TIME_KEY column included. + #[test] + fn a_timeseries_plan_carries_defaults_in_its_rows() { + let catalog = OneCollection(collection( + EngineType::Timeseries, + vec![ + column("at", SqlDataType::Timestamp, Some("1583402400000")), + column("host", SqlDataType::String, Some("'h0'")), + column("v", SqlDataType::Float64, None), + ], + )); + let plan = plan("INSERT INTO t (v) VALUES (1.0)", &catalog).expect("plans"); + let SqlPlan::TimeseriesIngest { + rows, + volatile_defaults, + .. + } = plan + else { + panic!("expected SqlPlan::TimeseriesIngest, got {plan:?}"); + }; + assert!(!volatile_defaults); + assert_eq!( + cell(&rows[0], "at"), + &SqlValue::Timestamp(NdbDateTime::from_micros(EARLY_MICROS)) + ); + assert_eq!(cell(&rows[0], "host"), &SqlValue::String("h0".into())); + assert_eq!(cell(&rows[0], "v"), &SqlValue::Float(1.0)); + } + + /// A volatile DEFAULT marks the plan so the plan cache never replays it. + #[test] + fn a_volatile_default_marks_the_plan() { + let mut id = column("id", SqlDataType::String, Some("UUID_V7")); + id.is_primary_key = true; + let catalog = OneCollection(collection( + EngineType::DocumentStrict, + vec![id, column("n", SqlDataType::Int64, None)], + )); + let plan = plan("INSERT INTO t (n) VALUES (1)", &catalog).expect("plans"); + assert!( + !plan.cache_eligibility().is_cacheable(), + "a plan carrying a volatile DEFAULT is never cached" + ); + } +} diff --git a/nodedb-sql/src/planner/dml/mod.rs b/nodedb-sql/src/planner/dml/mod.rs new file mode 100644 index 000000000..2e815dfc8 --- /dev/null +++ b/nodedb-sql/src/planner/dml/mod.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! INSERT, UPSERT, UPDATE, DELETE, and TRUNCATE planning. + +mod insert; +mod target; +mod update_delete; +mod upsert; + +pub use insert::plan_insert; +pub use update_delete::{plan_delete, plan_truncate_stmt, plan_update}; +pub use upsert::plan_upsert; diff --git a/nodedb-sql/src/planner/dml/target.rs b/nodedb-sql/src/planner/dml/target.rs new file mode 100644 index 000000000..0f1a46b56 --- /dev/null +++ b/nodedb-sql/src/planner/dml/target.rs @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The parts of an INSERT-shaped statement every row-writing planner shares: +//! target resolution, the column namespace, `ON CONFLICT` classification, +//! and the one row-typing pass. + +use nodedb_types::DatabaseId; +use sqlparser::ast; + +use super::super::dml_helpers::{ + coerce_and_check_rows, convert_value_rows, materialize_defaults_in_rows, +}; +use crate::error::{Result, SqlError}; +use crate::parser::normalize::{normalize_insert_column, normalize_object_name_checked}; +use crate::resolver::ColumnScope; +use crate::resolver::columns::{ResolvedTable, TableScope}; +use crate::resolver::expr::convert_expr; +use crate::types::*; + +/// The pseudo-relation `ON CONFLICT DO UPDATE` uses for the proposed row. +const EXCLUDED_RELATION: &str = "excluded"; + +/// The statement's target collection, resolved through the catalog. +/// +/// `verb` names the statement in the refusal a non-collection target gets. +pub(super) fn resolve_target( + ins: &ast::Insert, + verb: &str, + catalog: &dyn SqlCatalog, +) -> Result<(String, CollectionInfo)> { + let table_name = match &ins.table { + ast::TableObject::TableName(name) => normalize_object_name_checked(name)?, + ast::TableObject::TableFunction(_) => { + return Err(SqlError::Unsupported { + detail: format!("{verb} INTO a table function is not supported"), + }); + } + // Oracle's `INSERT INTO (SELECT ...)`: the target is a subquery, so + // there is no collection to resolve or route to an engine. + ast::TableObject::TableQuery(_) => { + return Err(SqlError::Unsupported { + detail: format!("{verb} INTO a subquery target is not supported"), + }); + } + }; + let info = catalog + .get_collection(DatabaseId::DEFAULT, &table_name)? + .ok_or_else(|| SqlError::UnknownTable { + name: table_name.clone(), + })?; + Ok((table_name, info)) +} + +/// The statement's `VALUES` rows. +/// +/// `verb` names the statement in the refusal a non-`VALUES` source gets. +pub(super) fn values_rows<'a>( + ins: &'a ast::Insert, + verb: &str, +) -> Result<&'a [ast::Parens>]> { + let source = ins.source.as_ref().ok_or_else(|| SqlError::Parse { + detail: format!("{verb} requires VALUES"), + })?; + match &*source.body { + ast::SetExpr::Values(values) => Ok(&values.rows), + _ => Err(SqlError::Unsupported { + detail: format!("{verb} source must be VALUES"), + }), + } +} + +/// The column namespace of an INSERT target. +pub(super) fn target_scope(table_name: &str, info: &CollectionInfo) -> Result { + let mut scope = TableScope::single(ResolvedTable { + name: table_name.to_string(), + alias: None, + info: info.clone(), + })?; + // `ON CONFLICT DO UPDATE` addresses the proposed row as `excluded`. It + // carries the target's columns and is qualified-only, so a bare name in + // the SET clause names the stored row. + scope.add_qualified_only(ResolvedTable { + name: EXCLUDED_RELATION.to_string(), + alias: None, + info: info.clone(), + })?; + Ok(scope) +} + +/// Normalize an INSERT column list and reject a name the target does not have. +pub(super) fn insert_columns( + columns: &[ast::ObjectName], + scope: &TableScope, +) -> Result> { + columns + .iter() + .map(|c| { + let col = normalize_insert_column(c)?; + scope.check_name(None, &col)?; + Ok(col) + }) + .collect() +} + +/// Classification of an `ON CONFLICT` clause attached to an INSERT. +pub(super) enum OnConflict { + /// No `ON CONFLICT` clause — plain INSERT (error on duplicate PK). + None, + /// `ON CONFLICT DO NOTHING` — skip rows that would conflict, no error. + DoNothing, + /// `ON CONFLICT (...) DO UPDATE SET ...` — apply the assignments against + /// the existing row on conflict. + DoUpdate(Vec<(String, SqlExpr)>), +} + +pub(super) fn classify_on_conflict(ins: &ast::Insert, scope: &TableScope) -> Result { + let Some(on) = ins.on.as_ref() else { + return Ok(OnConflict::None); + }; + let ast::OnInsert::OnConflict(oc) = on else { + return Ok(OnConflict::None); + }; + match &oc.action { + ast::OnConflictAction::DoNothing => Ok(OnConflict::DoNothing), + ast::OnConflictAction::DoUpdate(do_update) => { + let mut pairs = Vec::with_capacity(do_update.assignments.len()); + for a in &do_update.assignments { + let name = match &a.target { + ast::AssignmentTarget::ColumnName(obj) => normalize_object_name_checked(obj)?, + ast::AssignmentTarget::Tuple(_) => { + return Err(SqlError::Unsupported { + detail: "ON CONFLICT DO UPDATE SET target must be a column name".into(), + }); + } + }; + scope.check_name(None, &name)?; + let expr = convert_expr(&a.value, &ColumnScope::Relations(scope))?; + pairs.push((name, expr)); + } + Ok(OnConflict::DoUpdate(pairs)) + } + } +} + +/// A `VALUES` row set with every declared DEFAULT materialized and every +/// value coerced to its declared column type. +pub(super) struct TypedRows { + pub rows: Vec>, + /// Whether a materialized DEFAULT was volatile. A plan carrying one is + /// never admitted to the plan cache. + pub volatile_defaults: bool, +} + +/// Resolve `VALUES` literals, fill in declared DEFAULTs, then coerce and +/// range-check every cell against its declared column type. +/// +/// This is the one row-typing pass for every engine. Defaults are +/// materialized BEFORE coercion so a defaulted value is checked exactly like +/// a supplied literal: `DEFAULT 999999` on a `SMALLINT` column is refused +/// where `VALUES (999999)` is, and `DEFAULT 1583402400000` on a `TIMESTAMP` +/// column becomes the same instant `VALUES (1583402400000)` does. Nothing +/// downstream reads the catalog's DEFAULT text again. +pub(super) fn typed_rows( + info: &CollectionInfo, + columns: &[String], + rows_ast: &[ast::Parens>], + catalog: &dyn SqlCatalog, +) -> Result { + let mut rows = convert_value_rows(columns, rows_ast)?; + let volatile_defaults = materialize_defaults_in_rows(&info.columns, &mut rows, catalog)?; + coerce_and_check_rows(info, &mut rows)?; + Ok(TypedRows { + rows, + volatile_defaults, + }) +} + +/// Raw column type strings from the catalog: `(column_name, type_str)`. +/// +/// Columnar converters read these to reconstruct the exact `ColumnType` for +/// columns whose `SqlDataType` is ambiguous. +pub(super) fn column_schema(info: &CollectionInfo) -> Vec<(String, String)> { + info.columns + .iter() + .filter_map(|c| c.raw_type.as_ref().map(|t| (c.name.clone(), t.clone()))) + .collect() +} diff --git a/nodedb-sql/src/planner/dml_update_delete.rs b/nodedb-sql/src/planner/dml/update_delete.rs similarity index 97% rename from nodedb-sql/src/planner/dml_update_delete.rs rename to nodedb-sql/src/planner/dml/update_delete.rs index 5973a88b8..b49470263 100644 --- a/nodedb-sql/src/planner/dml_update_delete.rs +++ b/nodedb-sql/src/planner/dml/update_delete.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -//! UPDATE, DELETE, and TRUNCATE planning — extracted from `dml.rs`. +//! UPDATE, DELETE, and TRUNCATE planning. use nodedb_types::DatabaseId; use sqlparser::ast; @@ -38,8 +38,8 @@ pub fn plan_update(stmt: &ast::Statement, catalog: &dyn SqlCatalog) -> Result Result Result> { +fn plan_update_from( + update: &ast::Update, + from_kind: &ast::UpdateTableFromKind, + catalog: &dyn SqlCatalog, +) -> Result> { let target_name = extract_table_name_from_table_with_joins(&update.table)?; // Extract alias for the target table if present. @@ -101,7 +105,6 @@ fn plan_update_from(update: &ast::Update, catalog: &dyn SqlCatalog) -> Result = match from_kind { ast::UpdateTableFromKind::AfterSet(tables) | ast::UpdateTableFromKind::BeforeSet(tables) => tables, diff --git a/nodedb-sql/src/planner/dml/upsert.rs b/nodedb-sql/src/planner/dml/upsert.rs new file mode 100644 index 000000000..f8d75d3ce --- /dev/null +++ b/nodedb-sql/src/planner/dml/upsert.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! UPSERT and `INSERT ... ON CONFLICT DO UPDATE` planning. + +use sqlparser::ast; + +use super::super::dml_helpers::{ + KvInsertParams, build_kv_insert_plan, check_declared_float_ranges_in_assignments, + check_declared_int_ranges_in_assignments, resolve_insert_columns, +}; +use super::target::{ + column_schema, insert_columns, resolve_target, target_scope, typed_rows, values_rows, +}; +use crate::engine_rules::{self, UpsertParams}; +use crate::error::Result; +use crate::planner::declared_type_coerce::coerce_assignments_to_declared_types; +use crate::types::*; + +/// Plan an UPSERT statement (pre-processed from `UPSERT INTO` to `INSERT INTO`). +/// +/// Same parsing as INSERT but routes through `engine_rules.plan_upsert()`. +pub fn plan_upsert(ins: &ast::Insert, catalog: &dyn SqlCatalog) -> Result> { + plan_upsert_rows(ins, catalog, "UPSERT", Vec::new()) +} + +/// Plan an `INSERT ... ON CONFLICT DO UPDATE SET` statement. +pub(super) fn plan_upsert_with_on_conflict( + ins: &ast::Insert, + catalog: &dyn SqlCatalog, + on_conflict_updates: Vec<(String, SqlExpr)>, +) -> Result> { + plan_upsert_rows(ins, catalog, "INSERT ... ON CONFLICT", on_conflict_updates) +} + +/// The shared body of `UPSERT` and `INSERT ... ON CONFLICT DO UPDATE`. +/// +/// The two differ only in `verb` (for error messages) and in whether +/// `on_conflict_updates` carries per-row assignments for the Data Plane to +/// apply against the existing row. +fn plan_upsert_rows( + ins: &ast::Insert, + catalog: &dyn SqlCatalog, + verb: &str, + mut on_conflict_updates: Vec<(String, SqlExpr)>, +) -> Result> { + let (table_name, info) = resolve_target(ins, verb, catalog)?; + let columns = insert_columns(&ins.columns, &target_scope(&table_name, &info)?)?; + let rows_ast = values_rows(ins, verb)?; + + // KV: upsert is a PUT (natural overwrite), and `INSERT ... ON CONFLICT + // (key) DO UPDATE SET ...` is the same physical write with the optional + // per-row assignments carried through. Positional column binding + // (below) does not apply here — see `plan_insert`. + if info.engine == EngineType::KeyValue { + return build_kv_insert_plan(KvInsertParams { + collection: table_name, + columns: &columns, + rows_ast, + intent: KvInsertIntent::Put, + on_conflict_updates, + pk_col: info.primary_key.as_deref(), + declared_columns: &info.columns, + catalog, + }); + } + + // Positional UPSERT (no column list): bind to the collection's declared + // column order — see `plan_insert` for the full rationale. + let columns = resolve_insert_columns(columns, &info, rows_ast)?; + + let typed = typed_rows(&info, &columns, rows_ast, catalog)?; + // `DO UPDATE SET col = ` writes through the same path as the + // inserted row, so its literals carry the same declared-type contract. + coerce_assignments_to_declared_types( + &info.columns, + &mut on_conflict_updates, + info.primary_key.as_deref(), + )?; + check_declared_int_ranges_in_assignments(&info.columns, &on_conflict_updates)?; + check_declared_float_ranges_in_assignments(&info.columns, &on_conflict_updates)?; + let column_schema = column_schema(&info); + let rules = engine_rules::resolve_engine_rules(info.engine); + rules.plan_upsert(UpsertParams { + collection: table_name, + columns, + rows: typed.rows, + volatile_defaults: typed.volatile_defaults, + on_conflict_updates, + column_schema, + primary_key: info.primary_key.clone(), + }) +} diff --git a/nodedb-sql/src/planner/dml_helpers/declared_defaults.rs b/nodedb-sql/src/planner/dml_helpers/declared_defaults.rs index 5c5e6e5ab..0a6bee325 100644 --- a/nodedb-sql/src/planner/dml_helpers/declared_defaults.rs +++ b/nodedb-sql/src/planner/dml_helpers/declared_defaults.rs @@ -9,11 +9,10 @@ use crate::types::*; /// Materialize declared DEFAULTs across a whole `VALUES` row set. /// -/// The key-value and vector-primary engines store the values they are handed -/// and have no typed write path, so a DEFAULT that is not materialized HERE is -/// materialized nowhere: the catalog would keep the declaration and every read -/// return nothing for it. Documents and columnar rows expand theirs through the -/// same [`ColumnDefaults`], so one expression yields one value on every engine. +/// This is the one place a declared DEFAULT becomes a value, for every +/// engine. The plan an engine receives carries the materialized cell like a +/// supplied one, and nothing downstream reads the catalog's DEFAULT text +/// again, so one expression yields one value on every engine. /// /// Two rules the ordering encodes: /// diff --git a/nodedb-sql/src/planner/dml_helpers/mod.rs b/nodedb-sql/src/planner/dml_helpers/mod.rs index 242266de8..23a2d3446 100644 --- a/nodedb-sql/src/planner/dml_helpers/mod.rs +++ b/nodedb-sql/src/planner/dml_helpers/mod.rs @@ -29,8 +29,8 @@ pub(super) use insert_select_bind::bind_insert_select_columns; pub(super) use kv_insert::build_kv_insert_plan; pub(super) use params::KvInsertParams; pub(super) use range_check::{ - check_declared_float_ranges_in_assignments, check_declared_int_ranges_in_assignments, - coerce_and_check_rows, + check_declared_float_ranges, check_declared_float_ranges_in_assignments, + check_declared_int_ranges, check_declared_int_ranges_in_assignments, coerce_and_check_rows, }; pub(super) use value_convert::convert_value_rows; pub(super) use vector_primary_insert::build_vector_primary_insert_plan; diff --git a/nodedb-sql/src/planner/merge/actions.rs b/nodedb-sql/src/planner/merge/actions.rs new file mode 100644 index 000000000..e8fdf0730 --- /dev/null +++ b/nodedb-sql/src/planner/merge/actions.rs @@ -0,0 +1,401 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! MERGE `WHEN ... THEN` clause conversion. +//! +//! Every literal an action writes to the target passes the declared-type +//! coercion here, once, the same pass an ordinary `INSERT VALUES` or +//! `UPDATE SET` literal gets. The document-schemaless engine stores the +//! planner's value verbatim, so a literal left uncoerced would reach storage +//! as whatever the author typed: a bare integer under a `TIMESTAMP` column, +//! text under an `INT` column. +//! +//! Only literals are coerced. A source column reference or a function call +//! is evaluated on the Data Plane against the source row and carries no +//! value at plan time. + +use sqlparser::ast::{self, MergeAction, MergeClauseKind as AstMergeClauseKind, MergeInsertKind}; + +use super::super::ast_helpers::strip_and_convert_filters; +use super::super::declared_type_coerce::coerce_assignments_to_declared_types; +use super::super::dml_helpers::{ + check_declared_float_ranges_in_assignments, check_declared_int_ranges_in_assignments, + coerce_and_check_rows, +}; +use crate::error::{Result, SqlError}; +use crate::parser::normalize::normalize_object_name_checked; +use crate::resolver::ColumnScope; +use crate::resolver::columns::TableScope; +use crate::resolver::expr::convert_expr; +use crate::types::*; + +/// Convert every `WHEN` clause of a MERGE statement. +/// +/// `scope` addresses both target and source, `target_scope` the target +/// alone, and `target` carries the declared columns the action literals are +/// coerced against. +pub(super) fn convert_merge_clauses( + clauses: &[ast::MergeClause], + target_ref: &str, + scope: &TableScope, + target_scope: &TableScope, + target: &CollectionInfo, +) -> Result> { + clauses + .iter() + .map(|c| convert_one_clause(c, target_ref, scope, target_scope, target)) + .collect() +} + +fn convert_one_clause( + clause: &ast::MergeClause, + target_ref: &str, + scope: &TableScope, + target_scope: &TableScope, + target: &CollectionInfo, +) -> Result { + let kind = match clause.clause_kind { + AstMergeClauseKind::Matched => MergeClauseKind::Matched, + AstMergeClauseKind::NotMatched | AstMergeClauseKind::NotMatchedByTarget => { + MergeClauseKind::NotMatched + } + AstMergeClauseKind::NotMatchedBySource => MergeClauseKind::NotMatchedBySource, + }; + + let extra_predicate = match &clause.predicate { + Some(expr) => strip_and_convert_filters(vec![expr.clone()], target_ref, scope)?, + None => Vec::new(), + }; + + let action = convert_merge_action(&clause.action, scope, target_scope, target)?; + + Ok(MergePlanClause { + kind, + extra_predicate, + action, + }) +} + +/// Convert one `THEN` action, coercing every literal it writes to the +/// declared type of the target column it lands in. +pub(super) fn convert_merge_action( + action: &MergeAction, + scope: &TableScope, + target_scope: &TableScope, + target: &CollectionInfo, +) -> Result { + match action { + MergeAction::Update(update_expr) => { + let mut assignments = update_expr + .assignments + .iter() + .map(|a| { + let col = match &a.target { + ast::AssignmentTarget::ColumnName(name) => { + normalize_object_name_checked(name) + } + ast::AssignmentTarget::Tuple(_) => Err(SqlError::Unsupported { + detail: "tuple assignment target in MERGE UPDATE is not supported" + .into(), + }), + }?; + target_scope.check_name(None, &col)?; + let val = convert_expr(&a.value, &ColumnScope::Relations(scope))?; + Ok((col, val)) + }) + .collect::>>()?; + // `SET col = ` rewrites the stored row through the same + // path as `UPDATE ... SET`, so it carries the same declared-type + // contract, primary-key exemption included. + coerce_assignments_to_declared_types( + &target.columns, + &mut assignments, + target.primary_key.as_deref(), + )?; + check_declared_int_ranges_in_assignments(&target.columns, &assignments)?; + check_declared_float_ranges_in_assignments(&target.columns, &assignments)?; + Ok(MergePlanAction::Update { assignments }) + } + MergeAction::Delete { .. } => Ok(MergePlanAction::Delete), + MergeAction::Insert(insert_expr) => { + let columns: Vec = insert_expr + .columns + .iter() + .map(|c| { + let col = normalize_object_name_checked(c)?; + target_scope.check_name(None, &col)?; + Ok(col) + }) + .collect::>>()?; + + let mut values: Vec = match &insert_expr.kind { + MergeInsertKind::Values(vals) => { + if vals.rows.len() != 1 { + return Err(SqlError::Unsupported { + detail: format!( + "MERGE INSERT VALUES must have exactly one row; got {}", + vals.rows.len() + ), + }); + } + vals.rows[0] + .iter() + .map(|e| convert_expr(e, &ColumnScope::Relations(scope))) + .collect::>>()? + } + MergeInsertKind::Row => { + return Err(SqlError::Unsupported { + detail: "MERGE INSERT ROW is not supported; use explicit VALUES".into(), + }); + } + }; + + if !columns.is_empty() && columns.len() != values.len() { + return Err(SqlError::Parse { + detail: format!( + "MERGE INSERT column list ({}) and VALUES ({}) lengths do not match", + columns.len(), + values.len() + ), + }); + } + + let columns = bind_positional_columns(columns, values.len(), target)?; + coerce_insert_literals(&columns, &mut values, target)?; + Ok(MergePlanAction::Insert { columns, values }) + } + } +} + +/// Bind a column-less `INSERT VALUES (...)` arm to the target's declared +/// column order, the way a positional `INSERT INTO t VALUES (...)` binds. +/// +/// A target with no declared columns has no order to bind to and keeps the +/// empty list. More values than declared columns is refused: there is no +/// column name for the overflow to land under. +fn bind_positional_columns( + columns: Vec, + value_count: usize, + target: &CollectionInfo, +) -> Result> { + if !columns.is_empty() || target.columns.is_empty() { + return Ok(columns); + } + if value_count > target.columns.len() { + return Err(SqlError::InsertColumnArityMismatch { + collection: target.name.clone(), + given: value_count, + declared: target.columns.len(), + }); + } + Ok(target + .columns + .iter() + .take(value_count) + .map(|c| c.name.clone()) + .collect()) +} + +/// Coerce every literal of an INSERT arm to the declared type of the target +/// column it is bound to, and range-check it, through the one row-typing +/// pass every `VALUES` clause takes. +/// +/// Non-literal expressions are left in place: they are evaluated against the +/// source row on the Data Plane and carry no value here. +fn coerce_insert_literals( + columns: &[String], + values: &mut [SqlExpr], + target: &CollectionInfo, +) -> Result<()> { + let mut literal_row: Vec<(String, SqlValue)> = Vec::new(); + let mut literal_slots: Vec = Vec::new(); + for (slot, (column, expr)) in columns.iter().zip(values.iter()).enumerate() { + if let SqlExpr::Literal(value) = expr { + literal_row.push((column.clone(), value.clone())); + literal_slots.push(slot); + } + } + if literal_row.is_empty() { + return Ok(()); + } + let mut rows = [literal_row]; + coerce_and_check_rows(target, &mut rows)?; + let [literal_row] = rows; + for (slot, (_, value)) in literal_slots.into_iter().zip(literal_row) { + values[slot] = SqlExpr::Literal(value); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::statement::parse_sql; + use crate::resolver::columns::ResolvedTable; + use nodedb_types::columnar::IntWidth; + use nodedb_types::datetime::NdbDateTime; + + /// `2020-03-05T10:00:00Z` as microseconds since the Unix epoch. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + fn column(name: &str, data_type: SqlDataType) -> ColumnInfo { + ColumnInfo { + name: name.to_string(), + data_type, + nullable: true, + is_primary_key: false, + default: None, + raw_type: None, + int_width: None, + float_width: None, + } + } + + /// `(id INT PRIMARY KEY, n INT, at TIMESTAMP, s SMALLINT)` on the + /// schemaless engine, the engine that stores the planner's value verbatim. + fn target() -> CollectionInfo { + let mut id = column("id", SqlDataType::Int64); + id.is_primary_key = true; + let mut small = column("s", SqlDataType::Int64); + small.int_width = Some(IntWidth::I16); + CollectionInfo { + name: "t".into(), + engine: EngineType::DocumentSchemaless, + columns: vec![ + id, + column("n", SqlDataType::Int64), + column("at", SqlDataType::Timestamp), + small, + ], + primary_key: Some("id".into()), + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentSchemaless), + } + } + + /// Plan the first WHEN clause of `MERGE INTO t t USING t s ON t.id = s.id + /// `. + fn action(when: &str) -> Result { + let sql = format!("MERGE INTO t t USING t s ON t.id = s.id {when}"); + let statements = parse_sql(&sql)?; + let ast::Statement::Merge(merge) = &statements[0] else { + panic!("expected a MERGE statement"); + }; + let target = target(); + let relation = |alias: &str| ResolvedTable { + name: "t".into(), + alias: Some(alias.into()), + info: target.clone(), + }; + let target_scope = TableScope::single(relation("t"))?; + let mut scope = TableScope::new(); + scope.add(relation("t"))?; + scope.add_qualified_only(relation("s"))?; + convert_merge_action(&merge.clauses[0].action, &scope, &target_scope, &target) + } + + fn literal(expr: &SqlExpr) -> SqlValue { + match expr { + SqlExpr::Literal(value) => value.clone(), + other => panic!("expected a literal, got {other:?}"), + } + } + + /// The INSERT arm coerces each literal to its bound column's declared + /// type and leaves a source column reference untouched. + #[test] + fn insert_arm_literals_take_the_declared_column_types() { + let action = + action("WHEN NOT MATCHED THEN INSERT (id, n, at) VALUES (s.id, '1', 1583402400000)") + .expect("plans"); + let MergePlanAction::Insert { columns, values } = action else { + panic!("expected an INSERT action, got {action:?}"); + }; + assert_eq!(columns, vec!["id", "n", "at"]); + assert!( + matches!(values[0], SqlExpr::Column { .. }), + "a source reference stays an expression: {:?}", + values[0] + ); + assert_eq!(literal(&values[1]), SqlValue::Int(1)); + assert_eq!( + literal(&values[2]), + SqlValue::Timestamp(NdbDateTime::from_micros(EARLY_MICROS)) + ); + } + + /// The UPDATE arm coerces its literals the same way. + #[test] + fn update_arm_literals_take_the_declared_column_types() { + let action = + action("WHEN MATCHED THEN UPDATE SET n = '2', at = 1583402400000").expect("plans"); + let MergePlanAction::Update { assignments } = action else { + panic!("expected an UPDATE action, got {action:?}"); + }; + assert_eq!(literal(&assignments[0].1), SqlValue::Int(2)); + assert_eq!( + literal(&assignments[1].1), + SqlValue::Timestamp(NdbDateTime::from_micros(EARLY_MICROS)) + ); + } + + /// A literal the column cannot hold is refused naming the column, on + /// both arms: text into INT, a boolean into TIMESTAMP, and an integer + /// past the declared SMALLINT width. + #[test] + fn a_literal_the_column_cannot_hold_is_refused_naming_the_column() { + let err = action("WHEN NOT MATCHED THEN INSERT (id, n) VALUES (s.id, 'abc')") + .expect_err("'abc' does not fit INT"); + assert!(err.to_string().contains("'n'"), "{err}"); + + let err = + action("WHEN MATCHED THEN UPDATE SET at = true").expect_err("true is not an instant"); + assert!(err.to_string().contains("'at'"), "{err}"); + + let err = action("WHEN NOT MATCHED THEN INSERT (id, s) VALUES (s.id, 999999)") + .expect_err("999999 does not fit SMALLINT"); + assert!( + matches!(err, SqlError::IntegerOutOfRange { ref column, .. } if column == "s"), + "{err}" + ); + } + + /// The primary key keeps its literal as written on both arms, like every + /// `VALUES` and `SET` path. + #[test] + fn the_primary_key_literal_is_exempt() { + let action = action("WHEN NOT MATCHED THEN INSERT (id, n) VALUES ('7', 1)").expect("plans"); + let MergePlanAction::Insert { values, .. } = action else { + panic!("expected an INSERT action, got {action:?}"); + }; + assert_eq!(literal(&values[0]), SqlValue::String("7".into())); + } + + /// A column-less INSERT arm binds to the declared column order, so its + /// literals still find their declared types. + #[test] + fn a_column_less_insert_arm_binds_positionally() { + let planned = action("WHEN NOT MATCHED THEN INSERT VALUES (s.id, '3', 1583402400000)") + .expect("plans"); + let MergePlanAction::Insert { columns, values } = planned else { + panic!("expected an INSERT action, got {planned:?}"); + }; + assert_eq!(columns, vec!["id", "n", "at"]); + assert_eq!(literal(&values[1]), SqlValue::Int(3)); + assert_eq!( + literal(&values[2]), + SqlValue::Timestamp(NdbDateTime::from_micros(EARLY_MICROS)) + ); + + let err = action("WHEN NOT MATCHED THEN INSERT VALUES (1, 2, 3, 4, 5)") + .expect_err("more values than declared columns"); + assert!( + matches!(err, SqlError::InsertColumnArityMismatch { .. }), + "{err}" + ); + } +} diff --git a/nodedb-sql/src/planner/merge/mod.rs b/nodedb-sql/src/planner/merge/mod.rs new file mode 100644 index 000000000..25ff6b2dc --- /dev/null +++ b/nodedb-sql/src/planner/merge/mod.rs @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! MERGE statement planning. + +mod actions; +mod plan; + +pub use plan::plan_merge; diff --git a/nodedb-sql/src/planner/merge.rs b/nodedb-sql/src/planner/merge/plan.rs similarity index 70% rename from nodedb-sql/src/planner/merge.rs rename to nodedb-sql/src/planner/merge/plan.rs index 24087614f..1886ffe38 100644 --- a/nodedb-sql/src/planner/merge.rs +++ b/nodedb-sql/src/planner/merge/plan.rs @@ -1,24 +1,23 @@ // SPDX-License-Identifier: Apache-2.0 -//! MERGE statement planning. +//! MERGE statement planning: target and source resolution, the ON clause, +//! and dispatch to the engine rules. //! //! Translates `sqlparser::ast::Statement::Merge` into `SqlPlan::Merge`. //! Supported engines: `document_schemaless`, `document_strict`. //! All other engines return `SqlError::Unsupported`. use nodedb_types::DatabaseId; -use sqlparser::ast::{self, MergeAction, MergeClauseKind as AstMergeClauseKind, MergeInsertKind}; +use sqlparser::ast; -use super::ast_helpers::{qualified_ident_pair, strip_and_convert_filters}; +use super::super::ast_helpers::qualified_ident_pair; +use super::actions::convert_merge_clauses; use crate::engine_rules::{self, MergeParams, ScanParams}; use crate::error::{Result, SqlError}; use crate::parser::normalize::{normalize_ident, normalize_object_name_checked}; -use crate::resolver::ColumnScope; use crate::resolver::columns::{ResolvedTable, TableScope}; -use crate::resolver::expr::convert_expr; use crate::temporal::TemporalScope; use crate::types::*; -use crate::types::{MergeClauseKind, MergePlanAction, MergePlanClause, SqlPlan}; /// Plan a `MERGE INTO target USING source ON ... WHEN ... THEN ...` statement. pub fn plan_merge(stmt: &ast::Statement, catalog: &dyn SqlCatalog) -> Result> { @@ -73,9 +72,9 @@ pub fn plan_merge(stmt: &ast::Statement, catalog: &dyn SqlCatalog) -> Result Result> { - clauses - .iter() - .map(|c| convert_one_clause(c, target_ref, source_ref, scope, target_scope)) - .collect() -} - -fn convert_one_clause( - clause: &ast::MergeClause, - target_ref: &str, - source_ref: &str, - scope: &TableScope, - target_scope: &TableScope, -) -> Result { - let kind = match clause.clause_kind { - AstMergeClauseKind::Matched => MergeClauseKind::Matched, - AstMergeClauseKind::NotMatched | AstMergeClauseKind::NotMatchedByTarget => { - MergeClauseKind::NotMatched - } - AstMergeClauseKind::NotMatchedBySource => MergeClauseKind::NotMatchedBySource, - }; - - let extra_predicate = match &clause.predicate { - Some(expr) => strip_and_convert_filters(vec![expr.clone()], target_ref, scope)?, - None => Vec::new(), - }; - - let action = convert_merge_action(&clause.action, source_ref, scope, target_scope)?; - - Ok(MergePlanClause { - kind, - extra_predicate, - action, - }) -} - -fn convert_merge_action( - action: &MergeAction, - source_ref: &str, - scope: &TableScope, - target_scope: &TableScope, -) -> Result { - match action { - MergeAction::Update(update_expr) => { - let assignments = update_expr - .assignments - .iter() - .map(|a| { - let col = match &a.target { - ast::AssignmentTarget::ColumnName(name) => { - normalize_object_name_checked(name) - } - ast::AssignmentTarget::Tuple(_) => Err(SqlError::Unsupported { - detail: "tuple assignment target in MERGE UPDATE is not supported" - .into(), - }), - }?; - target_scope.check_name(None, &col)?; - let val = convert_expr(&a.value, &ColumnScope::Relations(scope))?; - Ok((col, val)) - }) - .collect::>>()?; - Ok(MergePlanAction::Update { assignments }) - } - MergeAction::Delete { .. } => Ok(MergePlanAction::Delete), - MergeAction::Insert(insert_expr) => { - let columns: Vec = insert_expr - .columns - .iter() - .map(|c| { - let col = normalize_object_name_checked(c)?; - target_scope.check_name(None, &col)?; - Ok(col) - }) - .collect::>>()?; - - let values: Vec = match &insert_expr.kind { - MergeInsertKind::Values(vals) => { - if vals.rows.len() != 1 { - return Err(SqlError::Unsupported { - detail: format!( - "MERGE INSERT VALUES must have exactly one row; got {}", - vals.rows.len() - ), - }); - } - vals.rows[0] - .iter() - .map(|e| convert_expr(e, &ColumnScope::Relations(scope))) - .collect::>>()? - } - MergeInsertKind::Row => { - return Err(SqlError::Unsupported { - detail: "MERGE INSERT ROW is not supported; use explicit VALUES".into(), - }); - } - }; - - if !columns.is_empty() && columns.len() != values.len() { - return Err(SqlError::Parse { - detail: format!( - "MERGE INSERT column list ({}) and VALUES ({}) lengths do not match", - columns.len(), - values.len() - ), - }); - } - - let _ = source_ref; // for future multi-row insert support - Ok(MergePlanAction::Insert { columns, values }) - } - } -} - // ── Helpers ──────────────────────────────────────────────────────────────── -pub(super) fn extract_table_factor_name_alias( - factor: &ast::TableFactor, -) -> Result<(String, Option)> { +fn extract_table_factor_name_alias(factor: &ast::TableFactor) -> Result<(String, Option)> { match factor { ast::TableFactor::Table { name, alias, .. } => { let table_name = normalize_object_name_checked(name)?; diff --git a/nodedb-sql/src/types/plan/cacheability.rs b/nodedb-sql/src/types/plan/cacheability.rs index c03d0b509..1c2b392c8 100644 --- a/nodedb-sql/src/types/plan/cacheability.rs +++ b/nodedb-sql/src/types/plan/cacheability.rs @@ -45,12 +45,18 @@ impl SqlPlan { match self { Self::ConstantResult { volatile: true, .. } => DataDependent, Self::Insert { - column_defaults, .. + volatile_defaults: true, + .. } | Self::Upsert { - column_defaults, .. - } if super::volatility_scan::defaults_are_volatile(column_defaults) => DataDependent, - Self::KvInsert { + volatile_defaults: true, + .. + } + | Self::TimeseriesIngest { + volatile_defaults: true, + .. + } + | Self::KvInsert { volatile_defaults: true, .. } diff --git a/nodedb-sql/src/types/plan/mod.rs b/nodedb-sql/src/types/plan/mod.rs index ad6020b61..371c9a13e 100644 --- a/nodedb-sql/src/types/plan/mod.rs +++ b/nodedb-sql/src/types/plan/mod.rs @@ -15,4 +15,4 @@ pub use merge_types::{MergeClauseKind, MergePlanAction, MergePlanClause}; pub use row_types::{KvInsertIntent, VectorPrimaryRow, WriteRoute}; pub use variants::{DistanceMetric, SqlPlan}; pub use vector_opts::{ArrayPrefilter, VectorAnnOptions, VectorQuantization}; -pub use volatility_scan::{default_expr_is_volatile, defaults_are_volatile, expr_is_volatile}; +pub use volatility_scan::expr_is_volatile; diff --git a/nodedb-sql/src/types/plan/variants.rs b/nodedb-sql/src/types/plan/variants.rs index f5ad2b736..7439de518 100644 --- a/nodedb-sql/src/types/plan/variants.rs +++ b/nodedb-sql/src/types/plan/variants.rs @@ -107,10 +107,14 @@ pub enum SqlPlan { /// The lowering these rows take, chosen by the engine's `EngineRules`. /// The conversion layer reads it instead of re-deciding from `engine`. route: WriteRoute, + /// Every declared DEFAULT already materialized and every literal + /// coerced to its declared column type by the planner. rows: Vec>, - /// Column defaults from schema: `(column_name, default_expr)`. - /// Used to auto-generate values for missing columns (e.g. `id` with `UUID_V7`). - column_defaults: Vec<(String, String)>, + /// Whether a DEFAULT materialized into `rows` was volatile. The + /// planner evaluates declared defaults while building this plan, so a + /// cached plan would replay one execution's value; a volatile plan is + /// never cached. + volatile_defaults: bool, /// `ON CONFLICT DO NOTHING` semantics: when true, duplicate-PK rows /// are silently skipped instead of raising `unique_violation`. Plain /// `INSERT` (no `ON CONFLICT` clause) sets this to `false`. @@ -157,8 +161,10 @@ pub enum SqlPlan { engine: EngineType, /// The lowering these rows take. Mirrors `Insert::route`. route: WriteRoute, + /// Defaults materialized and literals coerced, as in `Insert::rows`. rows: Vec>, - column_defaults: Vec<(String, String)>, + /// Mirrors `Insert::volatile_defaults`. + volatile_defaults: bool, /// `ON CONFLICT (...) DO UPDATE SET field = expr` assignments. /// When empty, upsert is a plain merge: new columns overwrite existing. /// When non-empty, the engine applies these per-row against the @@ -296,7 +302,13 @@ pub enum SqlPlan { }, TimeseriesIngest { collection: String, + /// Defaults materialized and literals coerced, as in `Insert::rows`. + /// A row that omits the `TIME_KEY` column carries its declared + /// default here when one exists; only a row with no time value at + /// all takes the ingest clock. rows: Vec>, + /// Mirrors `Insert::volatile_defaults`. + volatile_defaults: bool, }, // ── Search (first-class) ── diff --git a/nodedb-sql/src/types/plan/volatility_scan.rs b/nodedb-sql/src/types/plan/volatility_scan.rs index b72965c36..60d518d1f 100644 --- a/nodedb-sql/src/types/plan/volatility_scan.rs +++ b/nodedb-sql/src/types/plan/volatility_scan.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -//! Volatility scanning over plan expressions and column DEFAULT strings. +//! Volatility scanning over plan expressions. //! //! A plan holding a volatile call must re-plan per execution, so the plan //! cache never replays a frozen value. @@ -42,39 +42,3 @@ pub fn expr_is_volatile(expr: &SqlExpr) -> bool { } } } - -/// Column DEFAULT spellings that generate a fresh value per row but name no -/// registered function. Kept in step with the generator arms of -/// `crate::planner::defaults`. -const VOLATILE_DEFAULT_ALIASES: &[&str] = - &["uuidv7", "uuidv4", "gen_uuid_v7", "gen_uuid_v4", "gen_ulid"]; - -/// Whether a stored column DEFAULT expression re-evaluates per execution. -/// -/// The catalog stores a DEFAULT as text, in either a bare form (`UUID_V7`) or -/// a call form (`uuid_v7()`, `nextval('s')`), so the leading identifier is -/// checked before the string is parsed as an expression. -pub fn default_expr_is_volatile(expr: &str) -> bool { - let trimmed = expr.trim(); - let head: String = trimmed - .chars() - .take_while(|c| c.is_alphanumeric() || *c == '_') - .collect::() - .to_ascii_lowercase(); - if !head.is_empty() - && (default_registry().is_volatile(&head) - || VOLATILE_DEFAULT_ALIASES.contains(&head.as_str())) - { - return true; - } - crate::parse_expr_string(trimmed) - .ok() - .is_some_and(|parsed| expr_is_volatile(&parsed)) -} - -/// Whether any `(column, default_expr)` pair re-evaluates per execution. -pub fn defaults_are_volatile(defaults: &[(String, String)]) -> bool { - defaults - .iter() - .any(|(_, expr)| default_expr_is_volatile(expr)) -} diff --git a/nodedb-sql/src/visitor/plan_visitor/args.rs b/nodedb-sql/src/visitor/plan_visitor/args.rs index 81207a39a..f3678a436 100644 --- a/nodedb-sql/src/visitor/plan_visitor/args.rs +++ b/nodedb-sql/src/visitor/plan_visitor/args.rs @@ -68,7 +68,6 @@ pub struct InsertVisitArgs<'a> { /// The lowering these rows take, decided by the engine's `EngineRules`. pub route: WriteRoute, pub rows: &'a [Vec<(String, SqlValue)>], - pub column_defaults: &'a [(String, String)], pub if_absent: bool, pub column_schema: &'a [(String, String)], pub primary_key: Option<&'a str>, @@ -81,7 +80,6 @@ pub struct UpsertVisitArgs<'a> { /// The lowering these rows take, decided by the engine's `EngineRules`. pub route: WriteRoute, pub rows: &'a [Vec<(String, SqlValue)>], - pub column_defaults: &'a [(String, String)], pub on_conflict_updates: &'a [(String, SqlExpr)], pub column_schema: &'a [(String, String)], pub primary_key: Option<&'a str>, diff --git a/nodedb-sql/src/visitor/plan_visitor/dispatch.rs b/nodedb-sql/src/visitor/plan_visitor/dispatch.rs index dc8c98930..641295e96 100644 --- a/nodedb-sql/src/visitor/plan_visitor/dispatch.rs +++ b/nodedb-sql/src/visitor/plan_visitor/dispatch.rs @@ -97,7 +97,7 @@ pub fn dispatch(visitor: &mut V, plan: &SqlPlan) -> Result(visitor: &mut V, plan: &SqlPlan) -> Result(visitor: &mut V, plan: &SqlPlan) -> Result(visitor: &mut V, plan: &SqlPlan) -> Result(visitor: &mut V, plan: &SqlPlan) -> Result { - visitor.timeseries_ingest(collection, rows) - } + SqlPlan::TimeseriesIngest { + collection, + rows, + volatile_defaults: _, + } => visitor.timeseries_ingest(collection, rows), SqlPlan::VectorSearch { collection, field, diff --git a/nodedb-types/src/error/sqlstate.rs b/nodedb-types/src/error/sqlstate.rs index 076dc0758..cc8a8899c 100644 --- a/nodedb-types/src/error/sqlstate.rs +++ b/nodedb-types/src/error/sqlstate.rs @@ -148,6 +148,10 @@ pub const UNDEFINED_COLUMN: &str = "42703"; /// more than one relation in scope. pub const AMBIGUOUS_COLUMN: &str = "42702"; +/// `42804` — `datatype_mismatch` (a literal the declared column type cannot +/// represent, such as a column `DEFAULT` refused at `CREATE`) +pub const DATATYPE_MISMATCH: &str = "42804"; + /// `42846` — `cannot_coerce` pub const CANNOT_COERCE: &str = "42846"; @@ -363,6 +367,7 @@ mod tests { UNDEFINED_COLUMN, AMBIGUOUS_COLUMN, UNDEFINED_OBJECT, + DATATYPE_MISMATCH, CANNOT_COERCE, UNDEFINED_TABLE, UNDEFINED_FUNCTION, diff --git a/nodedb/src/control/planner/catalog_adapter/mod.rs b/nodedb/src/control/planner/catalog_adapter/mod.rs index 668c64530..00ed1e9dc 100644 --- a/nodedb/src/control/planner/catalog_adapter/mod.rs +++ b/nodedb/src/control/planner/catalog_adapter/mod.rs @@ -36,4 +36,4 @@ mod sql_catalog_impl; mod type_convert; pub use adapter::OriginCatalog; -pub(crate) use type_convert::convert_collection_type; +pub(crate) use type_convert::{convert_collection_type, declared_column_info}; diff --git a/nodedb/src/control/planner/catalog_adapter/type_convert.rs b/nodedb/src/control/planner/catalog_adapter/type_convert.rs index e13742daf..2a21d4f31 100644 --- a/nodedb/src/control/planner/catalog_adapter/type_convert.rs +++ b/nodedb/src/control/planner/catalog_adapter/type_convert.rs @@ -84,16 +84,7 @@ pub(crate) fn convert_collection_type( if name.eq_ignore_ascii_case(&pk_name) { continue; } - columns.push(ColumnInfo { - name: name.clone(), - data_type: parse_type_str(type_str), - nullable: true, - is_primary_key: false, - default: declared_default(type_str), - raw_type: None, - int_width: IntWidth::from_declared_type(type_str), - float_width: FloatWidth::from_declared_type(type_str), - }); + columns.push(declared_column_info(name, type_str)); } (EngineType::DocumentSchemaless, columns, Some(pk_name)) } @@ -174,16 +165,9 @@ pub(crate) fn convert_collection_type( if !profile.is_timeseries() && name.eq_ignore_ascii_case(pk_name) { continue; } - columns.push(ColumnInfo { - name: name.clone(), - data_type: parse_type_str(type_str), - nullable: true, - is_primary_key: false, - default: declared_default(type_str), - raw_type: Some(type_str.clone()), - int_width: IntWidth::from_declared_type(type_str), - float_width: FloatWidth::from_declared_type(type_str), - }); + let mut column = declared_column_info(name, type_str); + column.raw_type = Some(type_str.clone()); + columns.push(column); } let pk = if profile.is_timeseries() { None @@ -195,6 +179,28 @@ pub(crate) fn convert_collection_type( } } +/// The planner-facing column a raw DDL declaration (`name`, `type_str`) +/// resolves to: its SQL type, declared numeric width, and DEFAULT text. +/// +/// `type_str` is the text that followed the column name in the DDL, modifiers +/// included (`SMALLINT DEFAULT 5`, `TIMESTAMP TIME_KEY`). The schemaless and +/// columnar-family catalog arms read their tracked fields through this, and +/// the DDL gate checks a declared DEFAULT against the same resolution, so a +/// default is judged against exactly the type its column will carry at +/// INSERT time. +pub(crate) fn declared_column_info(name: &str, type_str: &str) -> ColumnInfo { + ColumnInfo { + name: name.to_string(), + data_type: parse_type_str(type_str), + nullable: true, + is_primary_key: false, + default: declared_default(type_str), + raw_type: None, + int_width: IntWidth::from_declared_type(type_str), + float_width: FloatWidth::from_declared_type(type_str), + } +} + /// Extract the `DEFAULT ` clause a columnar-family column declared. /// /// The columnar catalog stores each column as the raw DDL type string with its diff --git a/nodedb/src/control/planner/context/query/planning.rs b/nodedb/src/control/planner/context/query/planning.rs index 6d46558ef..3a600b923 100644 --- a/nodedb/src/control/planner/context/query/planning.rs +++ b/nodedb/src/control/planner/context/query/planning.rs @@ -141,7 +141,6 @@ impl QueryContext { .load(std::sync::atomic::Ordering::Relaxed), database_id, tenant_id, - sql_catalog: Some(Arc::clone(&catalog) as _), }; let output_schema = crate::control::planner::sql_plan_convert::output_schema::build_output_schema( @@ -423,7 +422,6 @@ impl QueryContext { .load(std::sync::atomic::Ordering::Relaxed), database_id, tenant_id, - sql_catalog: Some(Arc::clone(&catalog) as _), }; let output_schema = crate::control::planner::sql_plan_convert::output_schema::build_output_schema( diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs index c61cf62bc..d52fc535b 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs @@ -153,7 +153,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), } diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs index 845605a93..ed997b9f4 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs @@ -255,7 +255,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; diff --git a/nodedb/src/control/planner/sql_plan_convert/convert.rs b/nodedb/src/control/planner/sql_plan_convert/convert.rs index 91f6ae37b..1b38821ea 100644 --- a/nodedb/src/control/planner/sql_plan_convert/convert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/convert.rs @@ -129,12 +129,6 @@ pub struct ConvertContext { /// `DEFAULT_SHUFFLE_AGG_THRESHOLD`; overridable per-session via /// `nodedb.shuffle_agg_threshold` for operator control and test determinism. pub shuffle_agg_threshold: usize, - /// The catalog the plan was built against. INSERT/UPSERT conversion - /// evaluates each declared column DEFAULT here, and a sequence-backed - /// DEFAULT (`nextval`, `currval`) reads its value through this handle. - /// `None` for converters built by sub-planners that hold no catalog; a - /// catalog-reading DEFAULT then raises instead of dropping the column. - pub sql_catalog: Option>, } impl ConvertContext { @@ -142,22 +136,6 @@ impl ConvertContext { self.purpose == PlanningPurpose::Metadata } - /// The catalog a column DEFAULT is evaluated against. - /// - /// Raises when the converter holds none. A DEFAULT that reads the catalog - /// must fail loudly: omitting the column stores NULL where the - /// declaration promised a value. - pub fn sql_catalog( - &self, - ) -> crate::Result<&(dyn nodedb_sql::catalog::SqlCatalog + Send + Sync)> { - self.sql_catalog - .as_deref() - .ok_or_else(|| crate::Error::PlanError { - detail: "plan conversion holds no catalog, so a column DEFAULT that reads one cannot be evaluated" - .into(), - }) - } - /// Resolve an existing surrogate without creating a mapping while planning /// metadata. Execute planning retains the allocating assignment behavior. pub fn surrogate_for_pk( @@ -329,7 +307,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 0, shuffle_agg_threshold: 0, - sql_catalog: None, } } diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/insert/convert.rs b/nodedb/src/control/planner/sql_plan_convert/dml/insert/convert.rs index dc4a3b110..970986c4e 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/insert/convert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/insert/convert.rs @@ -8,7 +8,7 @@ use crate::types::{TenantId, VShardId}; use nodedb_physical::physical_plan::*; use super::super::super::convert::ConvertContext; -use super::super::super::value::{expand_row_defaults, row_to_msgpack, rows_to_msgpack_array}; +use super::super::super::value::{row_to_msgpack, rows_to_msgpack_array}; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use super::identity::{ @@ -22,8 +22,9 @@ pub(in super::super::super) struct ConvertInsertArgs<'a> { pub collection: &'a str, /// The lowering these rows take, decided by `nodedb-sql`. pub route: WriteRoute, + /// Rows with every declared DEFAULT already materialized and every + /// literal coerced to its declared column type by the planner. pub rows: &'a [Vec<(String, SqlValue)>], - pub column_defaults: &'a [(String, String)], pub column_schema: &'a [(String, String)], pub if_absent: bool, pub primary_key: &'a str, @@ -38,7 +39,6 @@ pub(in super::super::super) fn convert_insert( collection, route, rows, - column_defaults, column_schema, if_absent, primary_key, @@ -89,11 +89,6 @@ pub(in super::super::super) fn convert_insert( let mut balanced_documents: Vec<(String, Vec)> = Vec::new(); let mut balanced_surrogates: Vec = Vec::new(); - // Every engine's rows expand their DEFAULTs here, ahead of identity - // derivation. A DEFAULT materialized after the primary-key NOT NULL gate - // refuses a key the declaration supplies. - let expanded_rows = expand_row_defaults(rows, column_defaults, tenant_id, ctx)?; - // One catalog read for the whole statement. `_rowid` carries no // declaration, so it skips the read. let declared_pk = if is_auto_rowid_pk(primary_key) { @@ -102,7 +97,7 @@ pub(in super::super::super) fn convert_insert( declared_primary_key_name(ctx, collection)? }; - for row in &expanded_rows { + for row in rows { match route { WriteRoute::ColumnarFamily => { columnar_rows.push(row); @@ -283,7 +278,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; @@ -305,7 +299,6 @@ mod tests { collection: "crdt_coll", route: WriteRoute::Document, rows: &rows, - column_defaults: &[], column_schema: &[], if_absent: false, primary_key: "id", @@ -337,7 +330,6 @@ mod tests { collection: "plain", route: WriteRoute::Document, rows: &rows, - column_defaults: &[], column_schema: &[], if_absent: false, primary_key: "id", diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs b/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs index 5b3130117..b317b82f4 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs @@ -226,7 +226,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), } diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs index 6a92304b8..2161fb4d2 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs @@ -265,7 +265,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs index bfd2894fb..f9c75d93f 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs @@ -386,7 +386,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs b/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs index 70a5d679e..b93b0fed7 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs @@ -14,9 +14,7 @@ use nodedb_physical::physical_plan::ColumnarInsertIntent; use nodedb_physical::physical_plan::*; use super::super::convert::ConvertContext; -use super::super::value::{ - assignments_to_update_values, expand_row_defaults, row_to_msgpack, rows_to_msgpack_array, -}; +use super::super::value::{assignments_to_update_values, row_to_msgpack, rows_to_msgpack_array}; use super::insert::{ build_schema_bytes, columnar_row_surrogates, declared_primary_key_name, is_auto_rowid_pk, resolve_doc_identity_with_declared, @@ -28,8 +26,9 @@ pub(in super::super) struct ConvertUpsertArgs<'a> { pub collection: &'a str, /// The lowering these rows take, decided by `nodedb-sql`. pub route: WriteRoute, + /// Rows with every declared DEFAULT already materialized and every + /// literal coerced to its declared column type by the planner. pub rows: &'a [Vec<(String, SqlValue)>], - pub column_defaults: &'a [(String, String)], pub column_schema: &'a [(String, String)], pub on_conflict_updates: &'a [(String, SqlExpr)], pub primary_key: &'a str, @@ -44,7 +43,6 @@ pub(in super::super) fn convert_upsert( collection, route, rows, - column_defaults, column_schema, on_conflict_updates, primary_key, @@ -79,11 +77,6 @@ pub(in super::super) fn convert_upsert( let mut columnar_rows: Vec<&Vec<(String, SqlValue)>> = Vec::new(); - // Every engine's rows expand their DEFAULTs here, ahead of identity - // derivation, so the primary-key NOT NULL gate reads the row the - // declaration promises — see `expand_row_defaults`. - let expanded_rows = expand_row_defaults(rows, column_defaults, tenant_id, ctx)?; - // One catalog read for the whole statement, mirroring `convert_insert`. // `_rowid` carries no declaration, so it skips the read. let declared_pk = if is_auto_rowid_pk(primary_key) { @@ -92,7 +85,7 @@ pub(in super::super) fn convert_upsert( declared_primary_key_name(ctx, collection)? }; - for row in &expanded_rows { + for row in rows { match route { WriteRoute::Document => { let value_bytes = row_to_msgpack(row)?; diff --git a/nodedb/src/control/planner/sql_plan_convert/set_ops.rs b/nodedb/src/control/planner/sql_plan_convert/set_ops.rs index a9d435fe6..c24534444 100644 --- a/nodedb/src/control/planner/sql_plan_convert/set_ops.rs +++ b/nodedb/src/control/planner/sql_plan_convert/set_ops.rs @@ -412,7 +412,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }, @@ -472,7 +471,6 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }, diff --git a/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs b/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs deleted file mode 100644 index cb6e4aba2..000000000 --- a/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! The one DEFAULT materialization point for the row-shaped DML converters. - -use nodedb_sql::types::SqlValue; - -use super::super::convert::ConvertContext; -use crate::control::planner::plan_error_map::map_plan_error; -use crate::types::TenantId; - -/// Expand each row's omitted DEFAULT columns, before engine dispatch. -/// -/// INSERT and UPSERT call this once per statement, so identity derivation, -/// the primary-key NOT NULL gate, and the stored payload all read the same -/// row. A DEFAULT materialized per engine after that gate refuses a key the -/// declaration supplies. -/// -/// Each DEFAULT compiles once per statement and evaluates once per row, so a -/// `nextval` DEFAULT allocates exactly one value per row of a multi-row VALUES -/// clause and the expression is parsed once however many rows it fills. -/// -/// `column_defaults` empty means nothing to expand: the rows pass through and -/// the catalog is never read. -pub(in super::super) fn expand_row_defaults( - rows: &[Vec<(String, SqlValue)>], - column_defaults: &[(String, String)], - tenant_id: TenantId, - ctx: &ConvertContext, -) -> crate::Result>> { - if column_defaults.is_empty() { - return Ok(rows.to_vec()); - } - let catalog = ctx.sql_catalog()?; - let compiled = nodedb_sql::planner::defaults::ColumnDefaults::compile_pairs(column_defaults) - .map_err(|e| map_plan_error(e, tenant_id))?; - let mut expanded = Vec::with_capacity(rows.len()); - for row in rows { - let mut row = row.clone(); - compiled - .materialize_row(&mut row, catalog) - .map_err(|e| map_plan_error(e, tenant_id))?; - expanded.push(row); - } - Ok(expanded) -} diff --git a/nodedb/src/control/planner/sql_plan_convert/value/mod.rs b/nodedb/src/control/planner/sql_plan_convert/value/mod.rs index 851ae2e54..63293f723 100644 --- a/nodedb/src/control/planner/sql_plan_convert/value/mod.rs +++ b/nodedb/src/control/planner/sql_plan_convert/value/mod.rs @@ -1,11 +1,10 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Value conversion utilities: SqlValue ↔ nodedb_types::Value, msgpack -//! encoding, and the one DEFAULT materialization point. +//! Value conversion utilities: SqlValue ↔ nodedb_types::Value and msgpack +//! encoding. pub(super) mod assignments; pub(super) mod convert; -pub(super) mod defaults; pub(super) mod msgpack_write; pub(super) mod rows; @@ -15,7 +14,6 @@ pub(super) use assignments::{ pub(super) use convert::{ sql_value_to_bytes, sql_value_to_msgpack, sql_value_to_nodedb_value, sql_value_to_string, }; -pub(super) use defaults::expand_row_defaults; pub(super) use msgpack_write::{ row_to_msgpack, write_msgpack_array_header, write_msgpack_map_header, write_msgpack_str, write_msgpack_value, diff --git a/nodedb/src/control/planner/sql_plan_convert/value/rows.rs b/nodedb/src/control/planner/sql_plan_convert/value/rows.rs index ab35416aa..b8f29818c 100644 --- a/nodedb/src/control/planner/sql_plan_convert/value/rows.rs +++ b/nodedb/src/control/planner/sql_plan_convert/value/rows.rs @@ -6,10 +6,11 @@ use nodedb_sql::types::SqlValue; use super::convert::sql_value_to_nodedb_value; -/// Encode already-expanded rows as one msgpack array of maps. +/// Encode planner-typed rows as one msgpack array of maps. /// -/// Callers materialize DEFAULTs through `expand_row_defaults` before routing, -/// so every column the declaration promises is already present in `rows`. +/// The planner materializes every declared DEFAULT before the plan reaches +/// conversion, so every column the declaration promises is already present +/// in `rows`. pub(crate) fn rows_to_msgpack_array(rows: &[&Vec<(String, SqlValue)>]) -> crate::Result> { let mut arr: Vec = Vec::with_capacity(rows.len()); for row in rows { diff --git a/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs b/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs index 5b16ac971..607fb34f6 100644 --- a/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs +++ b/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs @@ -13,7 +13,6 @@ macro_rules! impl_dml_arms_for_convert_visitor { engine: _engine, route, rows, - column_defaults, if_absent, column_schema, primary_key, @@ -28,7 +27,6 @@ macro_rules! impl_dml_arms_for_convert_visitor { collection, route, rows, - column_defaults, column_schema, if_absent, primary_key, @@ -46,7 +44,6 @@ macro_rules! impl_dml_arms_for_convert_visitor { engine: _engine, route, rows, - column_defaults, on_conflict_updates, column_schema, primary_key, @@ -61,7 +58,6 @@ macro_rules! impl_dml_arms_for_convert_visitor { collection, route, rows, - column_defaults, column_schema, on_conflict_updates, primary_key, diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/alter/add_column.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/alter/add_column.rs index 197c5bc5e..6976f6749 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/alter/add_column.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/alter/add_column.rs @@ -15,6 +15,9 @@ use nodedb_types::DatabaseId; use crate::control::security::audit::AuditEvent; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::shared::ddl::neutral::collection::helpers::parse_origin_column_def; +use crate::control::server::shared::ddl::neutral::column_default::{ + DeclaredColumn, validate_column_default, +}; use crate::control::server::shared::ddl::result::{DdlError, DdlResult}; use crate::control::state::SharedState; @@ -52,6 +55,18 @@ pub(super) async fn alter_table_add_column( ), )); } + // A DEFAULT passes the same gate `CREATE` applies: evaluable, and a + // literal the declared type can hold. + if let Some(expr) = &column.default { + validate_column_default( + &DeclaredColumn { + name: &column.name, + declared_type: &declared_type, + primary_key: column.primary_key, + }, + expr, + )?; + } let updated = { let catalog = state.credentials.catalog(); diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs index a3ed56209..324cd03e6 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs @@ -87,9 +87,10 @@ pub async fn build_and_persist( )); } - // Refuse a DEFAULT the server cannot evaluate here, not at the first - // INSERT. It runs before any lifecycle guard or predecessor purge, so a - // rejected declaration leaves the existing state untouched. A SERIAL + // Refuse a DEFAULT the server cannot evaluate, or that the declared + // column type cannot hold, here, not at the first INSERT. It runs before + // any lifecycle guard or predecessor purge, so a rejected declaration + // leaves the existing state untouched. A SERIAL // column carries no DEFAULT text yet; the `nextval` this build generates // for it names a registered function and clears the same gate. validate_column_defaults(columns)?; diff --git a/nodedb/src/control/server/shared/ddl/neutral/column_default.rs b/nodedb/src/control/server/shared/ddl/neutral/column_default.rs index 390409a66..d7c07eb6f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/column_default.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/column_default.rs @@ -14,31 +14,86 @@ //! The check runs `nodedb_sql`'s DEFAULT classifier, which consults the same //! `FunctionRegistry` the resolver's undefined-function gate consults. No //! second registry and no second name list exist here. +//! +//! A column `DEFAULT` that spells a literal is also checked against the +//! column's declared type, through the same coercion an INSERT applies to +//! the materialized value. `DEFAULT 'not a date'` on a `TIMESTAMP` column +//! and `DEFAULT 999999` on a `SMALLINT` column are refused at the +//! declaration, so no insert can ever materialize a value the column cannot +//! hold. use nodedb_sql::SqlError; use nodedb_sql::ddl_ast::collection_type::parse_column_type_str_full; +use nodedb_sql::planner::declared_type_coerce::coerce_write_literal; +use nodedb_sql::planner::defaults::{CompiledDefault, default_value_to_sql}; use nodedb_types::error::sqlstate; use super::super::result::DdlError; +use crate::control::planner::catalog_adapter::declared_column_info; + +/// One declared column as the DDL gate sees it. +pub(super) struct DeclaredColumn<'a> { + pub name: &'a str, + /// The declared type text, modifiers included (`SMALLINT NOT NULL`, + /// `TIMESTAMP TIME_KEY`). The bare type resolves out of it the same way + /// the catalog adapter resolves a tracked field. + pub declared_type: &'a str, + /// Whether the column is the primary key. The key keeps its literal as + /// written on every write path, so its DEFAULT is not re-typed either. + pub primary_key: bool, +} -/// Refuse a declared column `DEFAULT` the server cannot evaluate. +/// Refuse a declared column `DEFAULT` the server cannot evaluate, or that the +/// column's declared type cannot hold. /// /// Each pair carries a column name and its declared type text, and the /// `DEFAULT` clause is read out of that text. pub(super) fn validate_column_defaults(columns: &[(String, String)]) -> Result<(), DdlError> { for (column, type_str) in columns { - let (_, _, _, default_expr) = parse_column_type_str_full(type_str); + let (_, primary_key, _, default_expr) = parse_column_type_str_full(type_str); let Some(expr) = default_expr else { continue; }; - validate_column_default(column, &expr)?; + validate_column_default( + &DeclaredColumn { + name: column, + declared_type: type_str, + primary_key, + }, + &expr, + )?; } Ok(()) } -/// Refuse one declared column `DEFAULT` the server cannot evaluate. -pub(super) fn validate_column_default(column: &str, expr: &str) -> Result<(), DdlError> { - validate_clause_expr("DEFAULT", column, expr) +/// Refuse one declared column `DEFAULT` the server cannot evaluate, or that +/// the column's declared type cannot hold. +/// +/// The expression is classified and parsed, never evaluated, so a +/// `DEFAULT nextval('s')` column never advances its sequence at DDL time. A +/// literal is then coerced to the declared type and range-checked exactly as +/// an INSERT coerces the materialized value; a generator or an expression has +/// no value to check until it is evaluated. +/// +/// An unregistered function name raises SQLSTATE `42883`, a literal the +/// declared type cannot represent `42804`, a literal past the declared +/// numeric width `22003`, and every other rejection `42601`. +pub(super) fn validate_column_default( + column: &DeclaredColumn<'_>, + expr: &str, +) -> Result<(), DdlError> { + let compiled = CompiledDefault::declare(column.name, expr) + .map_err(|error| clause_error("DEFAULT", column.name, &error))?; + let Some(literal) = compiled.literal() else { + return Ok(()); + }; + let mut info = declared_column_info(column.name, column.declared_type); + info.is_primary_key = column.primary_key; + let value = default_value_to_sql(column.name, literal.clone()) + .map_err(|error| clause_error("DEFAULT", column.name, &error))?; + coerce_write_literal(&info, value) + .map(|_| ()) + .map_err(|error| clause_error("DEFAULT", column.name, &error)) } /// Refuse one declared value-producing clause the server cannot evaluate. @@ -61,6 +116,10 @@ pub(super) fn validate_clause_expr(clause: &str, owner: &str, expr: &str) -> Res fn clause_error(clause: &str, owner: &str, error: &SqlError) -> DdlError { let sqlstate = match error { SqlError::UndefinedFunction { .. } => sqlstate::UNDEFINED_FUNCTION, + SqlError::TypeMismatch { .. } => sqlstate::DATATYPE_MISMATCH, + SqlError::IntegerOutOfRange { .. } | SqlError::FloatOutOfRange { .. } => { + sqlstate::NUMERIC_VALUE_OUT_OF_RANGE + } _ => sqlstate::SYNTAX_ERROR, }; DdlError::new( @@ -68,3 +127,69 @@ fn clause_error(clause: &str, owner: &str, error: &SqlError) -> DdlError { format!("{clause} for '{owner}' is invalid: {error}"), ) } + +#[cfg(test)] +mod tests { + use super::*; + + fn declared(name: &str, declared_type: &str) -> (String, String) { + (name.to_string(), declared_type.to_string()) + } + + /// Text that spells no instant is refused on a TIMESTAMP column, naming + /// the column, with the datatype-mismatch SQLSTATE. + #[test] + fn a_text_default_with_no_instant_is_refused_on_a_timestamp_column() { + let error = validate_column_defaults(&[declared("at", "TIMESTAMP DEFAULT 'not a date'")]) + .expect_err("'not a date' is not an instant"); + assert_eq!(error.sqlstate, sqlstate::DATATYPE_MISMATCH); + assert!(error.message.contains("'at'"), "{}", error.message); + } + + /// An integer past the declared SMALLINT width is refused with the + /// out-of-range SQLSTATE, naming the column. + #[test] + fn an_out_of_range_default_is_refused_on_a_smallint_column() { + let error = validate_column_defaults(&[declared("s", "SMALLINT DEFAULT 999999")]) + .expect_err("999999 does not fit SMALLINT"); + assert_eq!(error.sqlstate, sqlstate::NUMERIC_VALUE_OUT_OF_RANGE); + assert!(error.message.contains("'s'"), "{}", error.message); + } + + /// A literal the column can hold is accepted, in every spelling the + /// INSERT coercion accepts: epoch milliseconds and ISO text on a + /// TIMESTAMP column, an in-range integer on SMALLINT, numeric text on a + /// FLOAT column, and a TIME_KEY modifier before the clause. + #[test] + fn a_representable_default_is_accepted() { + validate_column_defaults(&[ + declared("at", "TIMESTAMP DEFAULT 1583402400000"), + declared("at2", "TIMESTAMP DEFAULT '2020-03-05 10:00:00'"), + declared("ts", "TIMESTAMP TIME_KEY DEFAULT 1583402400000"), + declared("s", "SMALLINT DEFAULT 7"), + declared("r", "FLOAT DEFAULT '1.5'"), + declared("name", "TEXT DEFAULT 'h0'"), + ]) + .expect("every default is representable"); + } + + /// The primary key keeps its literal as written, so its DEFAULT is not + /// re-typed; a generator has no value to check at declaration. + #[test] + fn primary_key_and_generator_defaults_pass_the_type_gate() { + validate_column_defaults(&[ + declared("id", "INT PRIMARY KEY DEFAULT 'k'"), + declared("u", "TEXT DEFAULT UUID_V7"), + declared("n", "INT DEFAULT nextval('s')"), + ]) + .expect("no literal reaches the type gate"); + } + + /// An unregistered function name keeps its own SQLSTATE. + #[test] + fn an_unregistered_function_keeps_undefined_function() { + let error = validate_column_defaults(&[declared("a", "TEXT DEFAULT no_such_fn('x')")]) + .expect_err("unknown function refused"); + assert_eq!(error.sqlstate, sqlstate::UNDEFINED_FUNCTION); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs index f72364ef9..4e571907f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs @@ -14,7 +14,7 @@ use nodedb_sql::parser::preprocess::lex::{ use crate::control::server::shared::ddl::sql_parse::{parse_ident_token, split_values}; use super::super::super::result::DdlError; -use super::super::column_default::validate_column_default; +use super::super::column_default::{DeclaredColumn, validate_column_default}; use super::support::err; use super::type_map::sql_type_to_column_type; @@ -165,7 +165,14 @@ fn parse_column_defs(s: &str) -> Result, col = col.with_primary_key(); } if let Some(expr) = default_expr { - validate_column_default(&col.name, &expr)?; + validate_column_default( + &DeclaredColumn { + name: &col.name, + declared_type: &col_type, + primary_key, + }, + &expr, + )?; col = col.with_default(expr); } columns.push(col); diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs index 0bc1749cc..85fc6e48a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs @@ -6,7 +6,7 @@ //! reads the collection's active typeguards instead. use super::super::super::result::DdlError; -use super::super::column_default::validate_column_default; +use super::super::column_default::{DeclaredColumn, validate_column_default}; use super::support::err; use super::type_map::typeguard_type_to_column_type; @@ -50,7 +50,17 @@ pub(super) fn typeguards_to_column_defs( // A guard carries either DEFAULT or VALUE, never both. Strict schema // has one materialization slot, so both land on the column `DEFAULT`. if let Some(expr) = guard.default_expr.clone().or(guard.value_expr.clone()) { - validate_column_default(&col.name, &expr)?; + // The resolved type's own spelling stands in for the declaration: + // a guard names no numeric width, so the canonical name resolves + // to the same width-less type the column will carry. + validate_column_default( + &DeclaredColumn { + name: &col.name, + declared_type: &col.column_type.to_string(), + primary_key: col.primary_key, + }, + &expr, + )?; col.default = Some(expr); } columns.push(col); diff --git a/nodedb/src/control/server/shared/ddl/result.rs b/nodedb/src/control/server/shared/ddl/result.rs index bc902d31c..3a135037c 100644 --- a/nodedb/src/control/server/shared/ddl/result.rs +++ b/nodedb/src/control/server/shared/ddl/result.rs @@ -179,6 +179,8 @@ pub fn code_for_sqlstate(sqlstate_str: &str) -> ErrorCode { // Invalid/incompatible object definition or a caller reaching a // dependent object still in use — all client-actionable, non-retriable. "42P17" | "42809" | "42P16" | "2BP01" => ErrorCode::BAD_REQUEST, + // A declared literal the column type cannot represent. + sqlstate::DATATYPE_MISMATCH => ErrorCode::BAD_REQUEST, // Default "object not in prerequisite state" meaning of `55006`; // `CLONE_DEPENDENCY` and `CLONE_WRITE_REQUIRES_MATERIALIZE` are // ambiguous-typed and cannot reach this function. diff --git a/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs b/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs index d96111c4a..3b1cab4c5 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs @@ -86,7 +86,6 @@ fn sql_to_physical(sql: &str) -> PhysicalPlan { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, - sql_catalog: None, }; let tenant_id = nodedb::types::TenantId::new(1); let tasks = convert(&plans, tenant_id, &ctx).unwrap(); diff --git a/nodedb/tests/wire/cases/kv_column_defaults.rs b/nodedb/tests/wire/cases/kv_column_defaults.rs index e0bd55a92..d0d4b056b 100644 --- a/nodedb/tests/wire/cases/kv_column_defaults.rs +++ b/nodedb/tests/wire/cases/kv_column_defaults.rs @@ -177,29 +177,17 @@ async fn upsert_materializes_an_omitted_columns_default() { ); } -/// A materialized default is validated exactly like a supplied literal: it is -/// appended to the row BEFORE the declared-type coercion and range checks run. -/// Were it filled in afterwards, a DEFAULT would be a way to store a value the -/// same literal is rejected for. +/// A `DEFAULT` beyond the declared width is refused where it is declared, +/// so no insert can reach it: the `CREATE` fails with the same range error a +/// supplied literal gets, naming the column. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_default_beyond_the_declared_width_is_rejected_like_a_supplied_literal() { let server = TestServer::start().await; - create_kv( - &server, - "kv_def_range", - "key TEXT PRIMARY KEY, n SMALLINT DEFAULT 999999", - ) - .await; - - // The same literal supplied directly is rejected... server .expect_error( - "INSERT INTO kv_def_range (key, n) VALUES ('k1', 999999)", + "CREATE COLLECTION kv_def_range (key TEXT PRIMARY KEY, n SMALLINT DEFAULT 999999) \ + WITH (engine='kv')", "range", ) .await; - // ...so arriving via the DEFAULT must not be a way around it. - server - .expect_error("INSERT INTO kv_def_range (key) VALUES ('k2')", "range") - .await; } diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index 7aab40103..da4caf948 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -166,6 +166,7 @@ mod sql_copy_from; mod sql_copy_to; mod sql_cursors; mod sql_declared_column_types; +mod sql_default_declared_types; mod sql_default_expressions; mod sql_default_vector_primary; mod sql_default_volatility; @@ -195,6 +196,7 @@ mod sql_limit_offset_bounds; mod sql_maintenance; mod sql_materialized_view_refresh; mod sql_merge; +mod sql_merge_declared_types; mod sql_multi_statement_batch; mod sql_null_predicate_parity; mod sql_object_literal_insert; diff --git a/nodedb/tests/wire/cases/sql_default_declared_types.rs b/nodedb/tests/wire/cases/sql_default_declared_types.rs new file mode 100644 index 000000000..490426e1e --- /dev/null +++ b/nodedb/tests/wire/cases/sql_default_declared_types.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Declared-type coercion for DEFAULT literals. +//! +//! A DEFAULT literal materialized at insert passes the same declared-type +//! coercion an explicit `VALUES` literal gets: a numeric literal defaulted +//! into a `TIMESTAMP` column is epoch milliseconds, a text literal is parsed, +//! and a literal the column cannot hold is refused naming the column. + +use crate::harness::TestServer; + +/// A numeric `DEFAULT` on a `TIMESTAMP` column is epoch milliseconds, the +/// same unit an explicit `VALUES` literal takes — on `document_strict` and +/// `columnar`. +async fn a_numeric_default_on_a_timestamp_column_is_epoch_milliseconds_on(engine: &str) { + let server = TestServer::start().await; + let name = format!("def_ts_epoch_{engine}"); + + server + .exec(&format!( + "CREATE COLLECTION {name} (\ + id TEXT PRIMARY KEY, \ + at TIMESTAMP DEFAULT 1583402400000) WITH (engine='{engine}')" + )) + .await + .unwrap_or_else(|e| panic!("create {name}: {e}")); + + server + .exec(&format!("INSERT INTO {name} (id) VALUES ('r1')")) + .await + .unwrap_or_else(|e| panic!("insert into {name}: {e}")); + + let rows = server + .query_text(&format!("SELECT at FROM {name} WHERE id = 'r1'")) + .await + .unwrap_or_else(|e| panic!("select from {name}: {e}")); + assert_eq!(rows.len(), 1, "row should exist"); + assert_eq!( + rows[0], "2020-03-05T10:00:00.000000Z", + "DEFAULT 1583402400000 on a TIMESTAMP column must render as epoch milliseconds: {:?}", + rows[0] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_numeric_default_on_a_timestamp_column_is_epoch_milliseconds_document_strict() { + a_numeric_default_on_a_timestamp_column_is_epoch_milliseconds_on("document_strict").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_numeric_default_on_a_timestamp_column_is_epoch_milliseconds_columnar() { + a_numeric_default_on_a_timestamp_column_is_epoch_milliseconds_on("columnar").await; +} + +/// The timeseries engine has no `id TEXT PRIMARY KEY`; its row identity is +/// the `TIME_KEY` column, so the numeric `DEFAULT` sits on it. A row that +/// omits the time key takes the declared default, not the ingest clock. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_numeric_default_on_a_timestamp_column_is_epoch_milliseconds_timeseries() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION def_ts_epoch_timeseries \ + COLUMNS (at TIMESTAMP TIME_KEY DEFAULT 1583402400000, host TEXT, v FLOAT) \ + WITH (engine='timeseries')", + ) + .await + .expect("create def_ts_epoch_timeseries"); + + server + .exec("INSERT INTO def_ts_epoch_timeseries (host, v) VALUES ('h0', 1.0)") + .await + .expect("insert into def_ts_epoch_timeseries"); + + let rows = server + .query_text("SELECT at FROM def_ts_epoch_timeseries") + .await + .expect("select from def_ts_epoch_timeseries"); + assert_eq!( + rows, + vec!["2020-03-05T10:00:00.000000Z".to_string()], + "DEFAULT 1583402400000 on TIME_KEY is the row's time key, in epoch milliseconds" + ); +} + +/// A text `DEFAULT` on a `TIMESTAMP` column is parsed the same way an +/// explicit text literal is. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_text_default_on_a_timestamp_column_is_parsed() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION def_ts_text (\ + id TEXT PRIMARY KEY, \ + at TIMESTAMP DEFAULT '2020-03-05 10:00:00') WITH (engine='document_strict')", + ) + .await + .expect("create def_ts_text"); + + server + .exec("INSERT INTO def_ts_text (id) VALUES ('r1')") + .await + .expect("insert into def_ts_text"); + + let rows = server + .query_text("SELECT at FROM def_ts_text WHERE id = 'r1'") + .await + .expect("select from def_ts_text"); + assert_eq!(rows.len(), 1, "row should exist"); + assert_eq!( + rows[0], "2020-03-05T10:00:00.000000Z", + "DEFAULT '2020-03-05 10:00:00' must parse to the same instant: {:?}", + rows[0] + ); +} + +/// A `DEFAULT` literal the declared column type cannot hold is refused at +/// `CREATE`, naming the column. A default is checked where it is declared, +/// so no insert can ever materialize a value the column cannot hold. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_default_the_column_cannot_hold_is_refused() { + let server = TestServer::start().await; + + server + .expect_error( + "CREATE COLLECTION def_bad_ts (\ + id TEXT PRIMARY KEY, \ + at TIMESTAMP DEFAULT 'not a date') WITH (engine='document_strict')", + "'at'", + ) + .await; + + server + .expect_error( + "CREATE COLLECTION def_bad_smallint (\ + id TEXT PRIMARY KEY, \ + s SMALLINT DEFAULT 999999) WITH (engine='document_strict')", + "'s'", + ) + .await; +} + +/// A `DEFAULT` on a timeseries collection's non-key column lands, the same +/// way it lands on every other engine. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_default_on_a_timeseries_collection_lands() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION def_ts_host_default \ + COLUMNS (ts BIGINT TIME_KEY, host TEXT DEFAULT 'h0', v FLOAT) \ + WITH (engine='timeseries')", + ) + .await + .expect("create def_ts_host_default"); + + server + .exec("INSERT INTO def_ts_host_default (ts, v) VALUES (1700000000000, 1.0)") + .await + .expect("insert into def_ts_host_default"); + + let rows = server + .query_text("SELECT host FROM def_ts_host_default") + .await + .expect("select from def_ts_host_default"); + assert_eq!(rows.len(), 1, "row should exist"); + assert_eq!( + rows[0], "h0", + "DEFAULT 'h0' must land on host: {:?}", + rows[0] + ); +} diff --git a/nodedb/tests/wire/cases/sql_merge_declared_types.rs b/nodedb/tests/wire/cases/sql_merge_declared_types.rs new file mode 100644 index 000000000..714a71b64 --- /dev/null +++ b/nodedb/tests/wire/cases/sql_merge_declared_types.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Declared-type coercion for MERGE literals. +//! +//! A literal reaching storage through `INSERT VALUES` or `UPDATE SET` passes +//! the declared-type coercion the planner applies once, in +//! `declared_type_coerce`. A MERGE `WHEN NOT MATCHED THEN INSERT` arm and a +//! `WHEN MATCHED THEN UPDATE SET` arm carry the same literals through the same +//! planner and must be coerced the same way, on every engine that declares +//! columns. + +use crate::harness::TestServer; + +async fn create_typed_target(server: &TestServer, name: &str, engine: &str, id_type: &str) { + server + .exec(&format!( + "CREATE COLLECTION {name} (id {id_type} PRIMARY KEY, n INT, at TIMESTAMP) \ + WITH (engine='{engine}')" + )) + .await + .unwrap_or_else(|e| panic!("create {name} ({engine}): {e}")); +} + +async fn create_typed_source(server: &TestServer, name: &str, engine: &str, id_type: &str) { + server + .exec(&format!( + "CREATE COLLECTION {name} (id {id_type} PRIMARY KEY) WITH (engine='{engine}')" + )) + .await + .unwrap_or_else(|e| panic!("create {name} ({engine}): {e}")); +} + +/// `MERGE ... WHEN NOT MATCHED THEN INSERT` runs its literals through the same +/// declared-type coercion an ordinary `INSERT VALUES` gets: a text literal +/// into `n INT` stores an integer, and an epoch-millisecond numeric literal +/// into `at TIMESTAMP` stores that instant, on every engine whose rule +/// plans MERGE (the key-value and columnar rules refuse MERGE outright). +async fn merge_insert_literals_take_the_declared_column_types_on(engine: &str, id_type: &str) { + let server = TestServer::start().await; + create_typed_target(&server, "merge_typed_target", engine, id_type).await; + create_typed_source(&server, "merge_typed_source", engine, id_type).await; + + let source_id = if id_type == "TEXT" { "'1'" } else { "1" }; + server + .exec(&format!( + "INSERT INTO merge_typed_source (id) VALUES ({source_id})" + )) + .await + .expect("seed source"); + + server + .exec( + "MERGE INTO merge_typed_target t \ + USING merge_typed_source s ON t.id = s.id \ + WHEN NOT MATCHED THEN INSERT (id, n, at) \ + VALUES (s.id, '1', 1583402400000)", + ) + .await + .expect("MERGE INSERT of a coercible literal must succeed"); + + let rows = server + .query_rows("SELECT n, at FROM merge_typed_target") + .await + .expect("select inserted row"); + assert_eq!(rows.len(), 1, "one row inserted: {rows:?}"); + assert_eq!( + rows[0], + vec!["1".to_string(), "2020-03-05T10:00:00.000000Z".to_string()], + "n and at must take the declared column types: {rows:?}" + ); + + let doubled = server + .query_rows("SELECT n * 2 FROM merge_typed_target") + .await + .expect("select n * 2"); + assert_eq!( + doubled, + vec![vec!["2".to_string()]], + "n must be numeric, not text, for arithmetic to succeed: {doubled:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn merge_insert_literals_take_the_declared_column_types_document_strict() { + merge_insert_literals_take_the_declared_column_types_on("document_strict", "INT").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn merge_insert_literals_take_the_declared_column_types_document_schemaless() { + merge_insert_literals_take_the_declared_column_types_on("document_schemaless", "INT").await; +} + +/// `MERGE ... WHEN MATCHED THEN UPDATE SET` runs its literals through the +/// same declared-type coercion, on the row it rewrites rather than the row it +/// inserts, on every engine whose rule plans MERGE. +async fn merge_update_literals_take_the_declared_column_types_on(engine: &str) { + let server = TestServer::start().await; + create_typed_target(&server, "merge_upd_target", engine, "INT").await; + create_typed_source(&server, "merge_upd_source", engine, "INT").await; + + server + .exec("INSERT INTO merge_upd_target (id, n, at) VALUES (1, 0, 1583402400000)") + .await + .expect("seed target"); + server + .exec("INSERT INTO merge_upd_source (id) VALUES (1)") + .await + .expect("seed source"); + + server + .exec( + "MERGE INTO merge_upd_target t \ + USING merge_upd_source s ON t.id = s.id \ + WHEN MATCHED THEN UPDATE SET n = '2', at = 1583402400001", + ) + .await + .expect("MERGE UPDATE of coercible literals must succeed"); + + let rows = server + .query_rows("SELECT at FROM merge_upd_target") + .await + .expect("select updated row"); + assert_eq!( + rows, + vec![vec!["2020-03-05T10:00:00.001000Z".to_string()]], + "at must take the declared TIMESTAMP type: {rows:?}" + ); + + let doubled = server + .query_rows("SELECT n * 2 FROM merge_upd_target") + .await + .expect("select n * 2"); + assert_eq!( + doubled, + vec![vec!["4".to_string()]], + "n must be numeric, not text, for arithmetic to succeed: {doubled:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn merge_update_literals_take_the_declared_column_types_document_strict() { + merge_update_literals_take_the_declared_column_types_on("document_strict").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn merge_update_literals_take_the_declared_column_types_document_schemaless() { + merge_update_literals_take_the_declared_column_types_on("document_schemaless").await; +} + +/// A literal the declared column type cannot hold is refused, naming the +/// column, on both a MERGE INSERT arm and a MERGE UPDATE arm — exactly as an +/// ordinary `INSERT VALUES` / `UPDATE SET` refuses it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn merge_refuses_a_literal_the_column_cannot_hold() { + let server = TestServer::start().await; + create_typed_target(&server, "merge_bad_target", "document_strict", "INT").await; + create_typed_source(&server, "merge_bad_source", "document_strict", "INT").await; + + server + .exec("INSERT INTO merge_bad_source (id) VALUES (1)") + .await + .expect("seed source"); + + let insert_err = server + .exec( + "MERGE INTO merge_bad_target t \ + USING merge_bad_source s ON t.id = s.id \ + WHEN NOT MATCHED THEN INSERT (id, n, at) \ + VALUES (s.id, 'abc', 1583402400000)", + ) + .await + .expect_err("MERGE INSERT of 'abc' into an INT column must be refused"); + assert!( + insert_err.contains("'n'"), + "error must name column 'n': {insert_err}" + ); + + server + .exec("INSERT INTO merge_bad_target (id, n, at) VALUES (1, 0, 1583402400000)") + .await + .expect("seed target for the UPDATE arm"); + + let update_err = server + .exec( + "MERGE INTO merge_bad_target t \ + USING merge_bad_source s ON t.id = s.id \ + WHEN MATCHED THEN UPDATE SET at = true", + ) + .await + .expect_err("MERGE UPDATE of a boolean into a TIMESTAMP column must be refused"); + assert!( + update_err.contains("'at'"), + "error must name column 'at': {update_err}" + ); +} From ffee5d0bc09c5dff045216fe04698352655ea0d6 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 17 Sep 2026 14:12:15 +0800 Subject: [PATCH 20/21] fix(timeseries): preserve instant kind across ingest and restore Register a restored collection with the Data Plane before reissuing its rows, instead of relying on the catalog row alone: without the Data Plane declaration, a restored timeseries collection had no doc_configs entry, so reissued rows were ingested with an inferred shape, the declared time key as a plain integer, and the restore-time clock instead of the original timestamp. Carry a declared TIMESTAMP/TIMESTAMPTZ time key as a typed fixext8 instant on the timeseries ingest payload, rather than the ISO 8601 text used by other engines, and decode it on the Data Plane side back into the correct instant kind. Reject a time column whose stored value the line-protocol timestamp cannot carry, rather than silently dropping it or stamping the row with the ingest clock. Read a memtable or flushed-partition time cell back typed by its declared kind during restore reissue, returning an error for a millisecond count outside the instant's representable range. --- .../backup/restore/orchestrate/restore.rs | 23 +- nodedb/src/control/backup/restore/sections.rs | 28 ++- .../backup/restore/timeseries_reissue.rs | 138 +++++++++++- .../sql_plan_convert/scan/timeseries.rs | 7 +- .../planner/sql_plan_convert/value/mod.rs | 4 +- .../sql_plan_convert/value/msgpack_write.rs | 82 ++++++- .../handlers/timeseries/ingest_formats.rs | 13 +- .../handlers/timeseries/msgpack_decode.rs | 78 ++++++- .../executor/handlers/timeseries/normalize.rs | 160 ++++++++++++-- .../handlers/timeseries/resolve_ingest.rs | 6 +- .../executor/handlers/timeseries/rls_gate.rs | 3 +- nodedb/tests/wire/cases/mod.rs | 1 + .../cases/sql_backup_restore_timeseries.rs | 209 ++++++++++++++++++ 13 files changed, 696 insertions(+), 56 deletions(-) create mode 100644 nodedb/tests/wire/cases/sql_backup_restore_timeseries.rs diff --git a/nodedb/src/control/backup/restore/orchestrate/restore.rs b/nodedb/src/control/backup/restore/orchestrate/restore.rs index 27f4b4460..0391e3f87 100644 --- a/nodedb/src/control/backup/restore/orchestrate/restore.rs +++ b/nodedb/src/control/backup/restore/orchestrate/restore.rs @@ -12,6 +12,7 @@ use nodedb_types::backup_envelope::{ use crate::Error; use crate::bridge::envelope::PhysicalPlan; +use crate::control::server::shared::ddl::neutral::collection::dispatch_register_from_stored; use crate::control::server::shared::ddl::sync_dispatch; use crate::control::state::SharedState; use crate::types::TenantId; @@ -84,7 +85,27 @@ pub async fn restore_tenant( }; if !dry_run { - apply_metadata_sections(state, tenant_id, &env)?; + let restored_collections = apply_metadata_sections(state, tenant_id, &env)?; + // Every restored collection's declaration reaches this node's Data + // Plane before any of its rows do. The catalog row alone leaves + // `doc_configs` empty for the collection, and the re-issue below + // would then ingest a timeseries collection's rows into an inferred + // shape: the declared time key becomes an integer field and the row + // is stamped with the restore-time clock. This is the same + // registration a committed DDL and the boot rehydration dispatch, + // and it replaces any registration already present, so a cluster + // applier's own register hook and a later boot seed are both + // idempotent with it. A registration failure fails the restore. + for coll in &restored_collections { + dispatch_register_from_stored(state, coll) + .await + .map_err(|e| Error::Internal { + detail: format!( + "restore: Data Plane registration of collection '{}' failed: {e}", + coll.name + ), + })?; + } } let mut merged = merge_sections(&env.sections)?; diff --git a/nodedb/src/control/backup/restore/sections.rs b/nodedb/src/control/backup/restore/sections.rs index 0153ac9ba..7614001b6 100644 --- a/nodedb/src/control/backup/restore/sections.rs +++ b/nodedb/src/control/backup/restore/sections.rs @@ -6,6 +6,7 @@ use nodedb_types::DatabaseId; use std::sync::Arc; use crate::Error; +use crate::control::security::catalog::StoredCollection; use crate::control::state::SharedState; use crate::types::{SurrogateBindEntry, TenantDataSnapshot}; @@ -81,16 +82,23 @@ pub(super) fn is_metadata_section(section: &nodedb_types::backup_envelope::Secti /// catalog-propose failure on this path is FATAL: returning the data /// restored but unqueryable on non-coordinator nodes is the /// silent-partial-success anti-pattern this codebase forbids. +/// +/// Returns every collection written to the catalog, in section order. The +/// caller registers each one with this node's Data Plane before any restored +/// row is installed or reissued: a catalog row alone leaves `doc_configs` +/// without the collection's declaration, and a reissued timeseries row would +/// then be ingested into an inferred shape. pub(super) fn apply_metadata_sections( state: &Arc, tenant_id: u64, env: &nodedb_types::backup_envelope::Envelope, -) -> Result<(), Error> { +) -> Result, Error> { use nodedb_types::backup_envelope::{ SECTION_ORIGIN_CATALOG_ROWS, SECTION_ORIGIN_SOURCE_TOMBSTONES, SourceTombstoneEntry, StoredCollectionBlob, }; let catalog = state.credentials.catalog(); + let mut restored: Vec = Vec::new(); for section in &env.sections { match section.origin_node_id { @@ -104,9 +112,7 @@ pub(super) fn apply_metadata_sections( continue; }; for blob in blobs { - let Ok(coll) = zerompk::from_msgpack::< - crate::control::security::catalog::StoredCollection, - >(&blob.bytes) else { + let Ok(coll) = zerompk::from_msgpack::(&blob.bytes) else { tracing::warn!( tenant_id, name = %blob.name, @@ -121,8 +127,9 @@ pub(super) fn apply_metadata_sections( // blocks on its local applied-index watcher, so on the // cluster path it has already applied the put via the // same applier — we must NOT also put locally (double-put). - let entry = - crate::control::catalog_entry::CatalogEntry::PutCollection(Box::new(coll)); + let entry = crate::control::catalog_entry::CatalogEntry::PutCollection( + Box::new(coll.clone()), + ); let outcome = crate::control::metadata_proposer::propose_catalog_entry(state, &entry)?; if outcome.needs_local_apply() { @@ -131,12 +138,9 @@ pub(super) fn apply_metadata_sections( // applier would have done on a clustered deployment. // A failure here is FATAL — the collection would be // unqueryable otherwise. - if let crate::control::catalog_entry::CatalogEntry::PutCollection(boxed) = - entry - { - catalog.put_collection(DatabaseId::DEFAULT, &boxed)?; - } + catalog.put_collection(DatabaseId::DEFAULT, &coll)?; } + restored.push(coll); } } SECTION_ORIGIN_SOURCE_TOMBSTONES => { @@ -184,5 +188,5 @@ pub(super) fn apply_metadata_sections( _ => {} } } - Ok(()) + Ok(restored) } diff --git a/nodedb/src/control/backup/restore/timeseries_reissue.rs b/nodedb/src/control/backup/restore/timeseries_reissue.rs index 11015bd90..3329ad0a9 100644 --- a/nodedb/src/control/backup/restore/timeseries_reissue.rs +++ b/nodedb/src/control/backup/restore/timeseries_reissue.rs @@ -14,6 +14,7 @@ use std::time::Duration; use nodedb_types::RlsWriteCheck; use nodedb_types::columnar::schema::TS_SYSTEM; +use nodedb_types::datetime::NdbDateTimeError; use nodedb_types::value::Value; use crate::Error; @@ -88,7 +89,8 @@ fn decode_memtable_rows( if name == TS_SYSTEM_COLUMN { continue; } - let cell = memtable_cell(&mt, *col_idx, *ty, idx); + let cell = memtable_cell(&mt, *col_idx, *ty, idx) + .map_err(|e| instant_cell_error(collection, name, e))?; insert_non_null(&mut map, name, cell); } rows.push(Value::Object(map)); @@ -96,10 +98,30 @@ fn decode_memtable_rows( Ok(()) } +/// A stored millisecond count that the column's instant kind cannot carry. +fn instant_cell_error(collection: &str, column: &str, e: NdbDateTimeError) -> Error { + Error::Storage { + engine: "timeseries".into(), + detail: format!("restore reissue: time column '{column}' of '{collection}': {e}"), + } +} + /// Extract one cell from a memtable column as a `Value`. -fn memtable_cell(mt: &ColumnarMemtable, col_idx: usize, ty: ColumnType, idx: usize) -> Value { - match ty { - ColumnType::Timestamp(_) => Value::Integer(mt.column(col_idx).as_timestamps()[idx]), +/// +/// A time column yields the value its kind denotes — a typed instant for a +/// declared `TIMESTAMP` / `TIMESTAMPTZ` key, the integer stored for a +/// `BIGINT` key — so the reissued row carries the cell a client INSERT +/// would have sent. +fn memtable_cell( + mt: &ColumnarMemtable, + col_idx: usize, + ty: ColumnType, + idx: usize, +) -> Result { + let value = match ty { + ColumnType::Timestamp(kind) => { + return kind.cell_value(mt.column(col_idx).as_timestamps()[idx]); + } ColumnType::Int64 => Value::Integer(mt.column(col_idx).as_i64()[idx]), ColumnType::Float64 => { let v = mt.column(col_idx).as_f64()[idx]; @@ -132,7 +154,8 @@ fn memtable_cell(mt: &ColumnarMemtable, col_idx: usize, ty: ColumnType, idx: usi } _ => Value::Null, }, - } + }; + Ok(value) } /// Decode one flushed partition directory into row objects, appending to `rows`. @@ -190,7 +213,8 @@ fn decode_partition_rows( if name == TS_SYSTEM_COLUMN { continue; } - let cell = partition_cell(&col_data[col_i], *ty, col_i, &sym_dicts, idx); + let cell = partition_cell(&col_data[col_i], *ty, col_i, &sym_dicts, idx) + .map_err(|e| instant_cell_error(collection, name, e))?; insert_non_null(&mut map, name, cell); } rows.push(Value::Object(map)); @@ -198,16 +222,20 @@ fn decode_partition_rows( Ok(()) } -/// Extract one cell from a flushed-segment column as a `Value`. +/// Extract one cell from a flushed-segment column as a `Value`. The partition +/// schema carries each time column's kind, so the cell is typed the same way +/// a memtable cell is. fn partition_cell( data: &ColumnData, ty: ColumnType, col_idx: usize, sym_dicts: &HashMap, idx: usize, -) -> Value { - match ty { - ColumnType::Timestamp(_) => Value::Integer(data.as_timestamps()[idx]), +) -> Result { + let value = match ty { + ColumnType::Timestamp(kind) => { + return kind.cell_value(data.as_timestamps()[idx]); + } ColumnType::Int64 => Value::Integer(data.as_i64()[idx]), ColumnType::Float64 => { let v = data.as_f64()[idx]; @@ -240,7 +268,8 @@ fn partition_cell( } _ => Value::Null, }, - } + }; + Ok(value) } /// Insert a cell, skipping nulls so a re-issued row carries only present fields @@ -330,3 +359,90 @@ pub async fn reissue_timeseries_durably( .await?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::engine::timeseries::columnar_memtable::{ColumnValue, ColumnarSchema, TimeKind}; + use nodedb_types::InstantKind; + use nodedb_types::datetime::NdbDateTime; + use nodedb_types::timeseries::SeriesId; + + /// `2020-03-05T10:00:00Z` in epoch milliseconds. + const EARLY_MS: i64 = 1_583_402_400_000; + + const NAIVE: ColumnType = ColumnType::Timestamp(TimeKind::Instant(InstantKind::Naive)); + const UTC: ColumnType = ColumnType::Timestamp(TimeKind::Instant(InstantKind::Utc)); + const MILLIS: ColumnType = ColumnType::Timestamp(TimeKind::Millis); + + fn early() -> NdbDateTime { + NdbDateTime::from_micros(EARLY_MS * 1_000) + } + + /// A one-row memtable whose time column has the given type. + fn memtable_with_time_column(ty: ColumnType) -> ColumnarMemtable { + let schema = ColumnarSchema { + columns: vec![ + ("captured_at".into(), ty), + ("v".into(), ColumnType::Float64), + ], + timestamp_idx: 0, + codecs: vec![], + }; + let mut mt = ColumnarMemtable::new(schema, ColumnarMemtableConfig::default()); + let series: SeriesId = 1; + mt.ingest_row( + series, + &[ColumnValue::Timestamp(EARLY_MS), ColumnValue::Float64(1.5)], + ) + .expect("ingest one row"); + mt + } + + /// A memtable time cell is reissued as the value its kind denotes. + #[test] + fn a_memtable_time_cell_is_typed_by_its_kind() { + let mt = memtable_with_time_column(NAIVE); + assert_eq!( + memtable_cell(&mt, 0, NAIVE, 0).expect("in range"), + Value::NaiveDateTime(early()) + ); + let mt = memtable_with_time_column(UTC); + assert_eq!( + memtable_cell(&mt, 0, UTC, 0).expect("in range"), + Value::DateTime(early()) + ); + let mt = memtable_with_time_column(MILLIS); + assert_eq!( + memtable_cell(&mt, 0, MILLIS, 0).expect("an integer"), + Value::Integer(EARLY_MS) + ); + } + + /// A partition time cell is reissued as the value its kind denotes. + #[test] + fn a_partition_time_cell_is_typed_by_its_kind() { + let data = ColumnData::Timestamp(vec![EARLY_MS]); + let dicts = HashMap::new(); + assert_eq!( + partition_cell(&data, NAIVE, 0, &dicts, 0).expect("in range"), + Value::NaiveDateTime(early()) + ); + assert_eq!( + partition_cell(&data, UTC, 0, &dicts, 0).expect("in range"), + Value::DateTime(early()) + ); + assert_eq!( + partition_cell(&data, MILLIS, 0, &dicts, 0).expect("an integer"), + Value::Integer(EARLY_MS) + ); + } + + /// A millisecond count outside the instant range is an error, never a + /// silently wrong cell. + #[test] + fn an_out_of_range_instant_cell_is_an_error() { + let data = ColumnData::Timestamp(vec![i64::MAX]); + assert!(partition_cell(&data, NAIVE, 0, &HashMap::new(), 0).is_err()); + } +} diff --git a/nodedb/src/control/planner/sql_plan_convert/scan/timeseries.rs b/nodedb/src/control/planner/sql_plan_convert/scan/timeseries.rs index 93ae46a3b..8333cd819 100644 --- a/nodedb/src/control/planner/sql_plan_convert/scan/timeseries.rs +++ b/nodedb/src/control/planner/sql_plan_convert/scan/timeseries.rs @@ -14,7 +14,7 @@ use super::super::aggregate::{ use super::super::expr::convert_sort_keys; use super::super::filter::serialize_filters; use super::super::scan_params::TimeseriesScanParams; -use super::super::value::{row_to_msgpack, write_msgpack_array_header}; +use super::super::value::{InstantForm, row_to_msgpack_with, write_msgpack_array_header}; use super::helpers::valid_at_from_scope; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; @@ -105,7 +105,10 @@ pub(in crate::control::planner::sql_plan_convert) fn convert_timeseries_ingest( write_msgpack_array_header(&mut payload, rows.len()); let mut surrogates: Vec = Vec::with_capacity(rows.len()); for row in rows { - let row_bytes = row_to_msgpack(row)?; + // A declared `TIMESTAMP` / `TIMESTAMPTZ` value travels as the typed + // instant ext: the ingest decoder takes it on the same arm a restore + // reissue's typed cell takes, so both spellings converge there. + let row_bytes = row_to_msgpack_with(row, InstantForm::Ext)?; payload.extend_from_slice(&row_bytes); // A timeseries row's natural identity is its (timestamp, tag-set) // tuple, which is not a cross-engine surrogate and carries no PK diff --git a/nodedb/src/control/planner/sql_plan_convert/value/mod.rs b/nodedb/src/control/planner/sql_plan_convert/value/mod.rs index 63293f723..2cf9852fd 100644 --- a/nodedb/src/control/planner/sql_plan_convert/value/mod.rs +++ b/nodedb/src/control/planner/sql_plan_convert/value/mod.rs @@ -15,7 +15,7 @@ pub(super) use convert::{ sql_value_to_bytes, sql_value_to_msgpack, sql_value_to_nodedb_value, sql_value_to_string, }; pub(super) use msgpack_write::{ - row_to_msgpack, write_msgpack_array_header, write_msgpack_map_header, write_msgpack_str, - write_msgpack_value, + InstantForm, row_to_msgpack, row_to_msgpack_with, write_msgpack_array_header, + write_msgpack_map_header, write_msgpack_str, write_msgpack_value, }; pub(super) use rows::rows_to_msgpack_array; diff --git a/nodedb/src/control/planner/sql_plan_convert/value/msgpack_write.rs b/nodedb/src/control/planner/sql_plan_convert/value/msgpack_write.rs index 41dc91abb..4a0f2f81b 100644 --- a/nodedb/src/control/planner/sql_plan_convert/value/msgpack_write.rs +++ b/nodedb/src/control/planner/sql_plan_convert/value/msgpack_write.rs @@ -5,15 +5,39 @@ //! These are the *only* msgpack producers used by the DML path — no JSON or //! zerompk intermediary. Format matches the on-wire layout read by //! `json_from_msgpack` and the Data Plane row decoders. +//! +//! A typed instant (`SqlValue::Timestamp` / `Timestamptz`) has two +//! spellings, chosen per payload by [`InstantForm`]: ISO 8601 text for the +//! document, KV, and CRDT row decoders, and the `fixext8` instant ext for +//! the timeseries ingest payload, whose decoder takes the typed form. use nodedb_sql::types::SqlValue; +use nodedb_types::{InstantKind, write_instant}; + +/// How a typed instant is written into a row payload. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum InstantForm { + /// ISO 8601 text, decoded by the Data Plane's text-coercion path. + Iso8601, + /// The ten-byte `fixext8` instant ext, decoded as a typed instant. + Ext, +} +/// A row map with instants as ISO 8601 text. pub(crate) fn row_to_msgpack(row: &[(String, SqlValue)]) -> crate::Result> { + row_to_msgpack_with(row, InstantForm::Iso8601) +} + +/// A row map with instants in the given form. +pub(crate) fn row_to_msgpack_with( + row: &[(String, SqlValue)], + instants: InstantForm, +) -> crate::Result> { let mut buf = Vec::with_capacity(row.len() * 32); write_msgpack_map_header(&mut buf, row.len()); for (key, val) in row { write_msgpack_str(&mut buf, key); - write_msgpack_value(&mut buf, val); + write_msgpack_value_with(&mut buf, val, instants); } Ok(buf) } @@ -60,7 +84,13 @@ pub(crate) fn write_msgpack_str(buf: &mut Vec, s: &str) { buf.extend_from_slice(bytes); } +/// Write one value with instants as ISO 8601 text. pub(crate) fn write_msgpack_value(buf: &mut Vec, val: &SqlValue) { + write_msgpack_value_with(buf, val, InstantForm::Iso8601) +} + +/// Write one value with instants in the given form. +pub(crate) fn write_msgpack_value_with(buf: &mut Vec, val: &SqlValue, instants: InstantForm) { match val { SqlValue::Null => buf.push(0xC0), SqlValue::Bool(true) => buf.push(0xC3), @@ -82,15 +112,18 @@ pub(crate) fn write_msgpack_value(buf: &mut Vec, val: &SqlValue) { SqlValue::Array(arr) => { write_msgpack_array_header(buf, arr.len()); for item in arr { - write_msgpack_value(buf, item); + write_msgpack_value_with(buf, item, instants); } } SqlValue::Bytes(b) => write_msgpack_bin(buf, b), - // Timestamp/Timestamptz: write as ISO 8601 string for the Data Plane - // to decode via its standard text-coercion path. - SqlValue::Timestamp(dt) | SqlValue::Timestamptz(dt) => { - write_msgpack_str(buf, &dt.to_iso8601()) - } + SqlValue::Timestamp(dt) => match instants { + InstantForm::Iso8601 => write_msgpack_str(buf, &dt.to_iso8601()), + InstantForm::Ext => write_instant(buf, InstantKind::Naive, dt.micros), + }, + SqlValue::Timestamptz(dt) => match instants { + InstantForm::Iso8601 => write_msgpack_str(buf, &dt.to_iso8601()), + InstantForm::Ext => write_instant(buf, InstantKind::Utc, dt.micros), + }, } } @@ -164,4 +197,39 @@ mod tests { write_msgpack_value(&mut buf, &SqlValue::String("hi".into())); assert_eq!(buf, vec![0xA2, b'h', b'i']); } + + /// `2020-03-05T10:00:00Z` in epoch microseconds. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + /// The default form spells an instant as ISO 8601 text. + #[test] + fn an_instant_is_iso8601_text_by_default() { + let dt = nodedb_types::NdbDateTime::from_micros(EARLY_MICROS); + for val in [SqlValue::Timestamp(dt), SqlValue::Timestamptz(dt)] { + let mut buf = Vec::new(); + write_msgpack_value(&mut buf, &val); + let mut expected = Vec::new(); + write_msgpack_str(&mut expected, "2020-03-05T10:00:00.000000Z"); + assert_eq!(buf, expected); + } + } + + /// The ext form writes the `fixext8` instant, tagged by the SQL type. + #[test] + fn an_instant_in_ext_form_is_a_typed_fixext8() { + let dt = nodedb_types::NdbDateTime::from_micros(EARLY_MICROS); + let cases = [ + (SqlValue::Timestamp(dt), InstantKind::Naive), + (SqlValue::Timestamptz(dt), InstantKind::Utc), + ]; + for (val, kind) in cases { + let mut buf = Vec::new(); + write_msgpack_value_with(&mut buf, &val, InstantForm::Ext); + assert_eq!( + nodedb_types::read_instant(&buf, 0), + Some((kind, EARLY_MICROS)), + "{val:?}" + ); + } + } } diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest_formats.rs b/nodedb/src/data/executor/handlers/timeseries/ingest_formats.rs index 410cd6740..813a64957 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest_formats.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest_formats.rs @@ -125,7 +125,18 @@ impl CoreLoop { .declared_ts_time_key(task.request.database_id, tid, collection) .map(str::to_string); - let ilp_buf = normalize::msgpack_rows_to_ilp(&rows, measurement, time_key.as_deref()); + let ilp_buf = match normalize::msgpack_rows_to_ilp(&rows, measurement, time_key.as_deref()) + { + Ok(buf) => buf, + Err(error) => { + return self.response_error( + task, + ErrorCode::RejectedPrevalidation { + reason: format!("timeseries ingest: {error}"), + }, + ); + } + }; if ilp_buf.is_empty() { return self.response_error( diff --git a/nodedb/src/data/executor/handlers/timeseries/msgpack_decode.rs b/nodedb/src/data/executor/handlers/timeseries/msgpack_decode.rs index 2b5437a6e..83976ff9f 100644 --- a/nodedb/src/data/executor/handlers/timeseries/msgpack_decode.rs +++ b/nodedb/src/data/executor/handlers/timeseries/msgpack_decode.rs @@ -3,13 +3,22 @@ //! Lightweight msgpack decoder for timeseries ingest rows. //! //! Decodes a msgpack array of maps into `Vec>`. -//! Only supports the value types produced by `row_to_msgpack` in the planner. +//! Supports the value types the planner's row writer and the native `Value` +//! encoder produce for a timeseries row: scalars, strings, and the `fixext8` +//! instant ext (`nodedb_types::read_instant`). A `bin` value decodes as +//! `Null`. Any other ext, array, or map value is an error. + +use nodedb_types::json_msgpack::INSTANT_EXT_LEN; +use nodedb_types::read_instant; pub(super) enum MsgpackValue { Int(i64), Float(f64), Str(String), Bool(bool), + /// A typed instant, as epoch microseconds. The column's declared kind, + /// not the cell's tag, decides how the engine stores and reads it. + Instant(i64), Null, } @@ -165,7 +174,19 @@ fn read_value(buf: &[u8], pos: &mut usize) -> Result let bytes = read_bytes::<8>(buf, pos)?; Ok(MsgpackValue::Int(u64::from_be_bytes(bytes) as i64)) } - // Skip bin/ext/array/map values we don't use for timeseries fields + // fixext8: an instant of either kind. Any other ext type is + // unsupported, like every other ext marker. + 0xD7 => { + let start = *pos - 1; + if buf.len() < start + INSTANT_EXT_LEN { + return Err("unexpected EOF in fixext8"); + } + let (_, micros) = + read_instant(buf, start).ok_or("unsupported msgpack ext type in timeseries row")?; + *pos = start + INSTANT_EXT_LEN; + Ok(MsgpackValue::Instant(micros)) + } + // Skip bin values; ext/array/map values are unsupported. _ => { skip_msgpack_value(b, buf, pos)?; Ok(MsgpackValue::Null) @@ -233,6 +254,59 @@ fn read_be_f64(buf: &[u8], pos: &mut usize) -> Result { #[cfg(test)] mod tests { use super::*; + use nodedb_types::InstantKind; + + /// `2020-03-05T10:00:00Z` in epoch microseconds. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + /// A row map `{"captured_at": }` for the given kind. + fn row_with_instant(kind: InstantKind) -> Vec { + let mut payload = vec![0x91, 0x81]; + payload.extend_from_slice(&[0xAB]); + payload.extend_from_slice(b"captured_at"); + nodedb_types::write_instant(&mut payload, kind, EARLY_MICROS); + payload + } + + /// A `fixext8` instant of either kind decodes as the typed instant. + #[test] + fn a_fixext8_instant_decodes_as_a_typed_instant() { + for kind in [InstantKind::Naive, InstantKind::Utc] { + let rows = decode_msgpack_rows(&row_with_instant(kind)).expect("decode"); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].len(), 1); + assert_eq!(rows[0][0].0, "captured_at"); + assert!( + matches!( + rows[0][0].1, + MsgpackValue::Instant(micros) if micros == EARLY_MICROS + ), + "instant of kind {kind:?} must decode typed" + ); + } + } + + /// A `fixext8` of a non-instant type is unsupported, like every other ext. + #[test] + fn a_fixext8_of_another_type_is_rejected() { + let mut payload = vec![0x91, 0x81, 0xA1, b'k']; + payload.extend_from_slice(&[0xD7, 0x09, 0, 0, 0, 0, 0, 0, 0, 1]); + assert!(matches!( + decode_msgpack_rows(&payload), + Err("unsupported msgpack ext type in timeseries row") + )); + } + + /// A truncated instant is an EOF error, never a partial value. + #[test] + fn a_truncated_instant_is_rejected() { + let mut payload = row_with_instant(InstantKind::Utc); + payload.truncate(payload.len() - 1); + assert!(matches!( + decode_msgpack_rows(&payload), + Err("unexpected EOF in fixext8") + )); + } #[test] fn huge_array_count_with_tiny_payload_is_rejected_without_reservation() { diff --git a/nodedb/src/data/executor/handlers/timeseries/normalize.rs b/nodedb/src/data/executor/handlers/timeseries/normalize.rs index fb1e6ee7f..a16ce4d6c 100644 --- a/nodedb/src/data/executor/handlers/timeseries/normalize.rs +++ b/nodedb/src/data/executor/handlers/timeseries/normalize.rs @@ -6,6 +6,7 @@ //! Anything reasoning about the persisted row (RLS gate, resolve pass) must //! go through here, not the submitted values. +use nodedb_types::datetime::NdbDateTime; use sonic_rs::{JsonContainerTrait, JsonValueTrait}; use super::msgpack_decode::MsgpackValue; @@ -15,6 +16,16 @@ use crate::engine::timeseries::ilp::{self, IlpError}; /// stored time column is milliseconds. const NANOS_PER_MILLI: i64 = 1_000_000; +/// Nanoseconds per microsecond — a typed instant carries epoch microseconds. +const NANOS_PER_MICRO: i64 = 1_000; + +/// The ISO 8601 text of a typed instant, for a non-time column that stores +/// it: the ILP line carries it as a string field, the same way a client +/// that spells the instant as text sends it. +fn instant_to_iso8601(micros: i64) -> String { + NdbDateTime::from_micros(micros).to_iso8601() +} + /// Is `column` the time column of the row being ingested? Matches only the /// declared `TIME_KEY` when DDL exists; falls back to conventional names /// (`ts`/`timestamp`/`time`) only for a measurement with no DDL behind it. @@ -68,30 +79,25 @@ fn push_line(buf: &mut String, measurement: &str, fields: &[String], timestamp_n } /// Normalize decoded MessagePack rows into line protocol. +/// +/// A row whose time column is absent or NULL carries no timestamp, and the +/// ingest clock stamps it. A time column that holds a value the line cannot +/// carry — text that is not a datetime, a boolean, or an instant past the +/// nanosecond range — is an error: stamping such a row with the clock would +/// store a point at a time the client never wrote. pub(in crate::data::executor) fn msgpack_rows_to_ilp( rows: &[Vec<(String, MsgpackValue)>], measurement: &str, time_key: Option<&str>, -) -> String { +) -> Result { let mut ilp_buf = String::new(); - for row in rows { + for (line_number, row) in rows.iter().enumerate() { let mut fields = Vec::new(); let mut timestamp_ns: Option = None; for (key, val) in row { if is_time_column(key, time_key) { - match val { - MsgpackValue::Str(s) => { - timestamp_ns = parse_ts_string_to_nanos(s); - } - MsgpackValue::Int(n) => { - timestamp_ns = Some(*n * NANOS_PER_MILLI); - } - MsgpackValue::Float(f) => { - timestamp_ns = Some(*f as i64 * NANOS_PER_MILLI); - } - _ => {} - } + timestamp_ns = time_column_nanos(val, key, line_number + 1)?; continue; } @@ -111,14 +117,58 @@ pub(in crate::data::executor) fn msgpack_rows_to_ilp( fields.push(format!("{key}=\"{}\"", s.replace('\"', "\\\""))); } } + // An instant in a non-time column is carried as ISO 8601 text, + // the form a text-spelled instant field takes above. + MsgpackValue::Instant(micros) => { + fields.push(format!("{key}=\"{}\"", instant_to_iso8601(*micros))); + } MsgpackValue::Bool(b) => fields.push(format!("{key}={b}")), - _ => {} + MsgpackValue::Null => {} } } push_line(&mut ilp_buf, measurement, &fields, timestamp_ns); } - ilp_buf + Ok(ilp_buf) +} + +/// The nanosecond timestamp a time-column cell denotes: `None` when the cell +/// is NULL (the clock stamps the row), an error when the cell holds a value +/// no timestamp can carry. +fn time_column_nanos( + val: &MsgpackValue, + column: &str, + line_number: usize, +) -> Result, IlpError> { + let invalid = |detail: &str| { + IlpError::new( + line_number, + &format!("{column}={detail}"), + 0..0, + ilp::IlpErrorKind::InvalidTimestamp, + ) + }; + match val { + MsgpackValue::Null => Ok(None), + MsgpackValue::Str(s) => parse_ts_string_to_nanos(s) + .map(Some) + .ok_or_else(|| invalid(&format!("\"{s}\""))), + MsgpackValue::Int(n) => n + .checked_mul(NANOS_PER_MILLI) + .map(Some) + .ok_or_else(|| invalid(&n.to_string())), + MsgpackValue::Float(f) => (*f as i64) + .checked_mul(NANOS_PER_MILLI) + .map(Some) + .ok_or_else(|| invalid(&f.to_string())), + // A typed instant of either kind: the stored time column is the + // instant's millisecond count whatever the kind. + MsgpackValue::Instant(micros) => micros + .checked_mul(NANOS_PER_MICRO) + .map(Some) + .ok_or_else(|| invalid(&instant_to_iso8601(*micros))), + MsgpackValue::Bool(b) => Err(invalid(&b.to_string())), + } } /// Normalize decoded JSON rows into line protocol. The JSON value model @@ -195,6 +245,84 @@ pub(in crate::data::executor) fn stamp_timestamps( mod tests { use super::*; + /// `2020-03-05T10:00:00Z` in epoch microseconds. + const EARLY_MICROS: i64 = 1_583_402_400_000_000; + + fn instant() -> MsgpackValue { + MsgpackValue::Instant(EARLY_MICROS) + } + + /// A typed instant under the declared time key is the line's timestamp — + /// the same nanosecond count its text spelling gives. + #[test] + fn a_typed_instant_time_key_is_the_line_timestamp() { + let rows = vec![vec![ + ("captured_at".to_string(), instant()), + ("v".to_string(), MsgpackValue::Float(1.5)), + ]]; + let ilp = msgpack_rows_to_ilp(&rows, "m", Some("captured_at")).expect("ilp"); + assert_eq!(ilp, "m v=1.5 1583402400000000000\n"); + let text = vec![vec![ + ( + "captured_at".to_string(), + MsgpackValue::Str("2020-03-05 10:00:00".into()), + ), + ("v".to_string(), MsgpackValue::Float(1.5)), + ]]; + assert_eq!( + msgpack_rows_to_ilp(&text, "m", Some("captured_at")).expect("ilp"), + "m v=1.5 1583402400000000000\n" + ); + } + + /// A typed instant in a non-time column is a string field carrying its + /// ISO 8601 text. + #[test] + fn a_typed_instant_field_is_iso8601_text() { + let rows = vec![vec![ + ("ts".to_string(), MsgpackValue::Int(7)), + ("seen".to_string(), instant()), + ]]; + assert_eq!( + msgpack_rows_to_ilp(&rows, "m", Some("ts")).expect("ilp"), + "m seen=\"2020-03-05T10:00:00.000000Z\" 7000000\n" + ); + } + + /// A time-column value the nanosecond timestamp cannot carry is an + /// error naming the column: the row is neither wrapped nor stamped with + /// the clock. + #[test] + fn a_time_key_the_line_cannot_carry_is_an_error() { + for bad in [ + MsgpackValue::Int(i64::MAX), + MsgpackValue::Str("not a date".into()), + MsgpackValue::Bool(true), + ] { + let rows = vec![vec![ + ("ts".to_string(), bad), + ("v".to_string(), MsgpackValue::Int(1)), + ]]; + let err = msgpack_rows_to_ilp(&rows, "m", Some("ts")).expect_err("refused"); + assert_eq!(err.kind, ilp::IlpErrorKind::InvalidTimestamp); + assert!(err.raw.starts_with("ts="), "{err:?}"); + } + } + + /// A NULL time column carries no timestamp, so the ingest clock stamps + /// the row. + #[test] + fn a_null_time_key_carries_no_timestamp() { + let rows = vec![vec![ + ("ts".to_string(), MsgpackValue::Null), + ("v".to_string(), MsgpackValue::Int(1)), + ]]; + assert_eq!( + msgpack_rows_to_ilp(&rows, "m", Some("ts")).expect("ilp"), + "m v=1i\n" + ); + } + /// A line without a timestamp takes the batch default; one that carries its /// own keeps it byte for byte. #[test] diff --git a/nodedb/src/data/executor/handlers/timeseries/resolve_ingest.rs b/nodedb/src/data/executor/handlers/timeseries/resolve_ingest.rs index 201bbc371..60dac3c85 100644 --- a/nodedb/src/data/executor/handlers/timeseries/resolve_ingest.rs +++ b/nodedb/src/data/executor/handlers/timeseries/resolve_ingest.rs @@ -150,7 +150,11 @@ impl CoreLoop { reason: format!("timeseries resolve: msgpack decode error: {error}"), } })?; - Ok(normalize::msgpack_rows_to_ilp(&rows, measurement, time_key)) + normalize::msgpack_rows_to_ilp(&rows, measurement, time_key).map_err(|error| { + ErrorCode::RejectedPrevalidation { + reason: format!("timeseries resolve: {error}"), + } + }) } "json" => { let rows: sonic_rs::Array = sonic_rs::from_slice(payload).map_err(|error| { diff --git a/nodedb/src/data/executor/handlers/timeseries/rls_gate.rs b/nodedb/src/data/executor/handlers/timeseries/rls_gate.rs index f28c8424e..f249fdc82 100644 --- a/nodedb/src/data/executor/handlers/timeseries/rls_gate.rs +++ b/nodedb/src/data/executor/handlers/timeseries/rls_gate.rs @@ -63,7 +63,8 @@ pub(in crate::data::executor) fn admit_msgpack_rows( ), }; let rows = msgpack_decode::decode_msgpack_rows(payload).map_err(|_| undecodable())?; - let batch = normalize::msgpack_rows_to_ilp(&rows, measurement, time_key); + let batch = + normalize::msgpack_rows_to_ilp(&rows, measurement, time_key).map_err(|_| undecodable())?; let parsed = ilp::parse_batch(&batch).map_err(|_| undecodable())?; admit_ilp_lines( rls_write_check, diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index da4caf948..257088f8d 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -154,6 +154,7 @@ mod sql_alter_after_drop; mod sql_arithmetic_overflow; mod sql_backup_restore_columnar; mod sql_backup_restore_columnar_restart; +mod sql_backup_restore_timeseries; mod sql_backup_restore_vector_params; mod sql_backup_restore_vector_restart; mod sql_backup_restore_wire; diff --git a/nodedb/tests/wire/cases/sql_backup_restore_timeseries.rs b/nodedb/tests/wire/cases/sql_backup_restore_timeseries.rs new file mode 100644 index 000000000..a113e11e9 --- /dev/null +++ b/nodedb/tests/wire/cases/sql_backup_restore_timeseries.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! BACKUP / RESTORE of a timeseries collection with a declared time key. +//! +//! A row restored into a timeseries collection reads back exactly as the row +//! that was inserted: a declared `TIMESTAMP` / `TIMESTAMPTZ` time key is the +//! same instant, a `BIGINT` time key the same integer, and the other columns +//! are intact. This holds whether the rows sat in the memtable or had been +//! flushed to a partition at backup time, and it survives a restart of the +//! restored server. +//! +//! The restore target is a genuinely clean one: the collection is hard-purged +//! before the restore (Data Plane registration, memtable and partitions all +//! gone), or the restore lands on a fresh server. A restore that inferred the +//! collection shape from the reissued rows, instead of registering the +//! declared one first, would stamp the rows with the restore-time clock and +//! leave the time key as an integer column. + +use crate::harness::TestServer; + +use bytes::Bytes; +use futures::SinkExt; +use futures::StreamExt; + +const TENANT: u64 = 1; + +/// Event time, years in the past, so a restore-time "now" stamp is trivially +/// separable from the inserted value. +const EARLY: &str = "2020-03-05 10:00:00"; +/// `EARLY` as a declared instant column renders it over pgwire. +const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; +/// A `BIGINT` time key value in epoch milliseconds. +const EARLY_MS: &str = "1700000000000"; + +async fn drain_backup(server: &TestServer, tenant: u64) -> Vec { + let stream = server + .client + .copy_out(&format!("COPY (BACKUP TENANT {tenant}) TO STDOUT")) + .await + .expect("copy_out: BACKUP TENANT"); + let mut bytes = Vec::new(); + let mut s = Box::pin(stream); + while let Some(chunk) = s.next().await { + bytes.extend_from_slice(&chunk.expect("copy_out chunk")); + } + bytes +} + +async fn push_restore(server: &TestServer, tenant: u64, bytes: Vec) { + let sink = server + .client + .copy_in::<_, Bytes>(&format!("COPY tenant_restore({tenant}) FROM STDIN")) + .await + .expect("copy_in: RESTORE TENANT"); + let mut sink = Box::pin(sink); + sink.as_mut() + .send(Bytes::from(bytes)) + .await + .expect("send backup bytes"); + sink.as_mut() + .finish() + .await + .expect("finish copy_in: RESTORE TENANT"); +} + +/// Create the collection, insert one row, and return what the time key reads +/// back as before the backup. +async fn create_and_insert( + srv: &TestServer, + name: &str, + time_key_type: &str, + time: &str, +) -> String { + srv.exec(&format!( + "CREATE COLLECTION {name} \ + (captured_at {time_key_type} TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .unwrap_or_else(|e| panic!("CREATE COLLECTION {name}: {e}")); + srv.exec(&format!( + "INSERT INTO {name} (captured_at, host, v) VALUES ('{time}', 'h1', 1.5)" + )) + .await + .unwrap_or_else(|e| panic!("INSERT INTO {name}: {e}")); + let rows = srv + .query_text(&format!("SELECT captured_at FROM {name}")) + .await + .unwrap_or_else(|e| panic!("SELECT captured_at FROM {name}: {e}")); + assert_eq!(rows.len(), 1, "one inserted row must read back: {rows:?}"); + rows[0].clone() +} + +/// The restored row carries the same time key and the same other columns. +async fn assert_restored_row(srv: &TestServer, name: &str, expected_time: &str) { + let times = srv + .query_text(&format!("SELECT captured_at FROM {name}")) + .await + .unwrap_or_else(|e| panic!("post-restore SELECT captured_at FROM {name}: {e}")); + assert_eq!( + times, + vec![expected_time.to_string()], + "the restored time key must read back as the inserted value" + ); + let hosts = srv + .query_text(&format!("SELECT host FROM {name}")) + .await + .unwrap_or_else(|e| panic!("post-restore SELECT host FROM {name}: {e}")); + assert_eq!( + hosts, + vec!["h1".to_string()], + "the restored row must carry its other columns intact" + ); +} + +/// Backup, hard-purge the collection on the same server, restore, and check +/// the row. The purge removes the catalog row, the Data Plane registration, +/// the memtable and every partition, so the restore lands on a clean target. +async fn backup_purge_restore(srv: &TestServer, name: &str, expected_time: &str) { + let backup_bytes = drain_backup(srv, TENANT).await; + assert!( + !backup_bytes.is_empty(), + "backup envelope must not be empty" + ); + + srv.exec(&format!("DROP COLLECTION {name} PURGE")) + .await + .unwrap_or_else(|e| panic!("DROP COLLECTION {name} PURGE: {e}")); + + push_restore(srv, TENANT, backup_bytes).await; + assert_restored_row(srv, name, expected_time).await; +} + +#[tokio::test] +async fn a_declared_time_key_survives_backup_and_restore() { + let srv = TestServer::start().await; + let before = create_and_insert(&srv, "ts_bk_naive", "TIMESTAMP", EARLY).await; + assert_eq!( + before, EARLY_ISO, + "the inserted instant reads back before the backup" + ); + backup_purge_restore(&srv, "ts_bk_naive", &before).await; +} + +/// The same round trip once every row has been flushed to a partition: the +/// partition schema carries the time kind, and the restore reads it from +/// there rather than from the memtable snapshot. +#[tokio::test] +async fn a_declared_time_key_survives_backup_and_restore_from_a_flushed_partition() { + let srv = TestServer::start_with_timeseries_memtable_budget(1).await; + let before = create_and_insert(&srv, "ts_bk_flushed", "TIMESTAMP", EARLY).await; + assert_eq!( + before, EARLY_ISO, + "the inserted instant reads back before the backup" + ); + backup_purge_restore(&srv, "ts_bk_flushed", &before).await; +} + +#[tokio::test] +async fn a_declared_timestamptz_time_key_survives_backup_and_restore() { + let srv = TestServer::start().await; + let before = create_and_insert(&srv, "ts_bk_utc", "TIMESTAMPTZ", EARLY).await; + assert_eq!( + before, EARLY_ISO, + "the inserted instant reads back before the backup" + ); + backup_purge_restore(&srv, "ts_bk_utc", &before).await; +} + +/// A `BIGINT` time key is not an instant: it restores as the integer that +/// was inserted. +#[tokio::test] +async fn a_bigint_time_key_stays_an_integer_after_restore() { + let srv = TestServer::start().await; + let before = create_and_insert(&srv, "ts_bk_bigint", "BIGINT", EARLY_MS).await; + assert_eq!( + before, EARLY_MS, + "the inserted integer reads back before the backup" + ); + backup_purge_restore(&srv, "ts_bk_bigint", &before).await; +} + +/// Restore into a fresh server, then restart it on the same data directory: +/// the reissued row is WAL-durable and the boot-time registration of the +/// restored collection types it the same way the restore-time one did. +#[tokio::test] +async fn a_declared_time_key_survives_restore_and_restart() { + let srv_a = TestServer::start().await; + let before = create_and_insert(&srv_a, "ts_bk_restart", "TIMESTAMP", EARLY).await; + assert_eq!( + before, EARLY_ISO, + "the inserted instant reads back before the backup" + ); + let backup_bytes = drain_backup(&srv_a, TENANT).await; + assert!( + !backup_bytes.is_empty(), + "backup envelope must not be empty" + ); + drop(srv_a); + + let srv_b = TestServer::start().await; + push_restore(&srv_b, TENANT, backup_bytes).await; + assert_restored_row(&srv_b, "ts_bk_restart", &before).await; + + let (srv_b, dir) = srv_b.take_dir(); + srv_b.graceful_shutdown().await; + let (srv_c, _dir) = TestServer::open_on_path(dir).await; + assert_restored_row(&srv_c, "ts_bk_restart", &before).await; +} From 4a61bfffc47b1d2586eaae6e3a886844f86b83bf Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 17 Sep 2026 16:23:45 +0800 Subject: [PATCH 21/21] test(response-shape): improve failure diagnostics in cell text test Index into scalars by reference instead of consuming the vector, and include the scalar's index in the assertion message so a failure identifies which case broke. --- nodedb/src/control/server/response_shape/cell.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nodedb/src/control/server/response_shape/cell.rs b/nodedb/src/control/server/response_shape/cell.rs index 061c4f7b1..c3c977d21 100644 --- a/nodedb/src/control/server/response_shape/cell.rs +++ b/nodedb/src/control/server/response_shape/cell.rs @@ -172,11 +172,11 @@ mod tests { inclusive: false, }, ]; - for v in scalars { + for (index, v) in scalars.iter().enumerate() { assert_eq!( - cell_text(&v), - wire_json_text(&value_to_wire_json(&v)), - "{v:?} must render the same text either way" + cell_text(v), + wire_json_text(&value_to_wire_json(v)), + "scalar {index} must render the same text either way" ); } }