From d1e6145dcc26de7d3507b1aef88b087f0ed4d2f1 Mon Sep 17 00:00:00 2001 From: Serhii Savchuk Date: Sat, 19 Sep 2026 02:10:31 +0300 Subject: [PATCH 01/23] fix(if): translate only the FILE-domain cells an if branch reads An `if` whose branch scans a SYM column with a FILE-domain vocabulary translated its cells into runtime ids through the domain's whole- vocabulary LUT. The first request interns every vocabulary entry into the global symbol table (with the dotted split), permanently: on a 100M-row table with a 19.7M-entry column that is about 45 GB for a branch that reads 1.7M distinct values. The branch now carries its own translation table: slots start unresolved and a position is interned the first time the branch reads it, from the raw vocabulary snapshot. The serial scatter path resolves on the fly; the parallel eager fill pre-resolves the positions it will read before dispatching, so workers only read (sym.c frozen-table rule). The ids produced are the ones the LUT would have given. Co-Authored-By: Claude Fable 5.1 --- src/ops/pivot.c | 144 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 127 insertions(+), 17 deletions(-) diff --git a/src/ops/pivot.c b/src/ops/pivot.c index c02254f2..3bfdbb0c 100644 --- a/src/ops/pivot.c +++ b/src/ops/pivot.c @@ -25,6 +25,7 @@ #include "ops/hash.h" #include "ops/idxop.h" #include "ops/rowsel.h" +#include "table/domain.h" /* raw vocabulary snapshot for if_sym_side_t */ #include "core/pool.h" /* Resolved string atom (borrowed) of a SYM-scalar broadcast input @@ -499,7 +500,91 @@ static ray_t* if_scatter_str(ray_t* result, ray_t* value, int64_t* ids, return result; } -static int64_t if_sym_cell_value(ray_t* value, int64_t src) { +/* Runtime-id translation for one SYM branch of an `if`. + * + * A branch that scans a FILE-domain column used to go through the + * domain's whole-vocabulary LUT (ray_sym_domain_runtime_lut): the first + * request interns EVERY entry of the vocabulary into the global table — + * tens of GB, permanently, for a 20M-entry column — even when the branch + * touches a few of them. This translator interns only the positions the + * branch actually reads: `lut` starts at -1 and a slot is resolved (raw + * file bytes when available, the domain atom otherwise) the first time + * that position is met. Resolution interns, so it must run on the + * calling thread: the serial scatter path resolves on the fly, the + * parallel fill pre-resolves every used position (if_sym_side_prepare) + * before dispatching, after which workers only read the table. + * Runtime-domain, STR and scalar branches need no table. */ +typedef struct { + ray_t* v; + struct ray_sym_domain_s* dom; /* non-runtime domain, else NULL */ + int64_t dn; + int64_t* lut; /* dn slots, -1 = unresolved */ + ray_t* lut_hdr; + ray_sym_domain_raw_t raw; + bool raw_ok; +} if_sym_side_t; + +static bool if_sym_side_init(if_sym_side_t* side, ray_t* v) { + memset(side, 0, sizeof(*side)); + side->v = v; + if (!v || ray_is_atom(v) || !RAY_IS_SYM(v->type)) return true; + struct ray_sym_domain_s* dom = ray_sym_vec_domain(v); + if (!dom || dom == ray_sym_runtime_domain()) return true; + int64_t dn = ray_sym_domain_count(dom); + if (dn <= 0) return true; + side->lut = (int64_t*)scratch_alloc(&side->lut_hdr, (size_t)dn * sizeof(int64_t)); + if (!side->lut) return false; + memset(side->lut, 0xff, (size_t)dn * sizeof(int64_t)); + side->dom = dom; + side->dn = dn; + side->raw_ok = ray_sym_domain_raw_pin(dom, &side->raw); + return true; +} + +static void if_sym_side_free(if_sym_side_t* side) { + if (side->raw_ok) ray_sym_domain_raw_unpin(side->dom); + scratch_free(side->lut_hdr); + side->lut = NULL; + side->lut_hdr = NULL; +} + +/* Runtime id of the vocabulary position `pos`; interns on first use. + * Calling thread only. */ +static int64_t if_sym_side_resolve(if_sym_side_t* side, int64_t pos) { + if (pos < 0 || pos >= side->dn) return -1; + int64_t id = side->lut[pos]; + if (id >= 0) return id; + const char* sp = NULL; + size_t sl = 0; + if (side->raw_ok && pos < side->raw.count) { + sp = ray_sym_domain_raw_str(&side->raw, pos, &sl); + } else { + ray_t* s = ray_sym_domain_str(side->dom, pos); + if (s) { sp = ray_str_ptr(s); sl = ray_str_len(s); } + } + id = ray_sym_intern(sp ? sp : "", sp ? sl : 0); + if (id >= 0) side->lut[pos] = id; + return id; +} + +/* Pre-resolve the positions rows [0, len) read on this side under the + * mask (`want` selects which mask value picks this side). Serial. */ +static void if_sym_side_prepare(if_sym_side_t* side, const uint8_t* cond, + int64_t len, uint8_t want) { + if (!side->dom) return; + const void* base = ray_data(side->v); + for (int64_t i = 0; i < len; i++) { + if ((cond[i] != 0) != (want != 0)) continue; + int64_t pos = ray_read_sym(base, i, side->v->type, side->v->attrs); + if (pos >= 0 && pos < side->dn && side->lut[pos] < 0) + (void)if_sym_side_resolve(side, pos); + } +} + +/* Cell value as a runtime id. A FILE-domain cell reads the side's table + * (resolving on the calling thread when not prepared). */ +static int64_t if_sym_cell_value(if_sym_side_t* side, int64_t src) { + ray_t* value = side->v; if (value->type == -RAY_STR) return ray_sym_intern(ray_str_ptr(value), ray_str_len(value)); if (value->type == RAY_STR) { size_t sl = 0; @@ -507,19 +592,26 @@ static int64_t if_sym_cell_value(ray_t* value, int64_t src) { return ray_sym_intern(sp ? sp : "", sp ? sl : 0); } if (ray_is_atom(value)) return sym_scalar_runtime_id(value); - return sym_cell_runtime_id(value, src); + if (!side->dom) return sym_cell_runtime_id(value, src); + int64_t pos = ray_read_sym(ray_data(value), src, value->type, value->attrs); + if (pos >= 0 && pos < side->dn && side->lut[pos] >= 0) return side->lut[pos]; + return if_sym_side_resolve(side, pos); } static bool if_scatter_sym(ray_t* result, ray_t* value, int64_t* ids, int64_t count, int64_t nrows) { if (!value) return true; + if_sym_side_t side; + if (!if_sym_side_init(&side, value)) return false; int64_t* dst = (int64_t*)ray_data(result); - for (int64_t j = 0; j < count; j++) { + bool ok = true; + for (int64_t j = 0; j < count && ok; j++) { int64_t src = if_value_index(value, ids, j, count, nrows); - if (src < 0) return false; - dst[ids[j]] = if_sym_cell_value(value, src); + if (src < 0) { ok = false; break; } + dst[ids[j]] = if_sym_cell_value(&side, src); } - return true; + if_sym_side_free(&side); + return ok; } static bool if_lazy_supported_type(int8_t out_type) { @@ -709,6 +801,8 @@ typedef struct { int64_t t_i64, e_i64; void* dst; int8_t out_type; + if_sym_side_t* t_side; /* SYM output: per-side id translation */ + if_sym_side_t* e_side; } if_fill_ctx_t; static void if_fill_range(const if_fill_ctx_t* c, int64_t i0, int64_t i1) { @@ -733,8 +827,8 @@ static void if_fill_range(const if_fill_ctx_t* c, int64_t i0, int64_t i1) { case RAY_SYM: { int64_t* dst = (int64_t*)c->dst; for (int64_t i = i0; i < i1; i++) { - int64_t tv = c->then_scalar ? c->t_i64 : if_sym_cell_value(c->then_v, i); - int64_t ev = c->else_scalar ? c->e_i64 : if_sym_cell_value(c->else_v, i); + int64_t tv = c->then_scalar ? c->t_i64 : if_sym_cell_value(c->t_side, i); + int64_t ev = c->else_scalar ? c->e_i64 : if_sym_cell_value(c->e_side, i); dst[i] = cond_p[i] ? tv : ev; } break; } @@ -946,27 +1040,43 @@ static ray_t* exec_if_eager(ray_graph_t* g, ray_op_t* op) { ray_pool_t* pool = ray_pool_get(); bool par = ray_pool_par_dispatch_ok(pool, len, RAY_PARALLEL_THRESHOLD); + if_sym_side_t t_side, e_side; + bool sides_ok = true; + if (out_type == RAY_SYM) { + sides_ok = if_sym_side_init(&t_side, then_scalar ? NULL : then_v) && + if_sym_side_init(&e_side, else_scalar ? NULL : else_v); + fc.t_side = &t_side; + fc.e_side = &e_side; + } + if (!sides_ok) { + if (out_type == RAY_SYM) { if_sym_side_free(&t_side); if_sym_side_free(&e_side); } + ray_release(cond_v); ray_release(then_v); ray_release(else_v); + ray_release(result); + return ray_error("oom", NULL); + } if (par && out_type == RAY_SYM) { - /* Vector STR sides intern per element — serial only. Non-STR - * vector sides must be SYM columns; warm each non-runtime - * domain's runtime-id LUT HERE (sym.c frozen-table rule — - * the first LUT request interns the vocabulary, never allowed - * inside a worker; mirrors window.c's sequential warm-up). */ + /* Vector STR sides intern per element — serial only. A SYM + * column side is dispatch-safe once every position the fill + * will read has a runtime id (sym.c frozen-table rule: no + * interning inside a worker) — resolve those here, serially, + * and only those; the column's whole vocabulary is never + * interned. */ ray_t* sides[2] = { then_scalar ? NULL : then_v, else_scalar ? NULL : else_v }; for (int s = 0; s < 2 && par; s++) { if (!sides[s]) continue; if (sides[s]->type != RAY_SYM) { par = false; break; } - struct ray_sym_domain_s* dom = ray_sym_vec_domain(sides[s]); - if (dom != ray_sym_runtime_domain() && - !ray_sym_domain_runtime_lut(dom)) - par = false; /* LUT OOM → safe serial fallback */ + } + if (par) { + if_sym_side_prepare(&t_side, cond_p, len, 1); + if_sym_side_prepare(&e_side, cond_p, len, 0); } } if (par) ray_pool_dispatch(pool, if_fill_par_fn, &fc, len); else if_fill_range(&fc, 0, len); + if (out_type == RAY_SYM) { if_sym_side_free(&t_side); if_sym_side_free(&e_side); } } ray_release(cond_v); ray_release(then_v); ray_release(else_v); From 44864731b9fda6a313de5e95b10432eea601205b Mon Sep 17 00:00:00 2001 From: Serhii Savchuk Date: Sat, 19 Sep 2026 02:19:06 +0300 Subject: [PATCH 02/23] perf(group): evaluate a derived symbol key in chunks over the vocabulary bytes The distinct-symbol evaluation of a computed group key fed the DAG a SYM column whose ids live in a FILE domain, so every string step of the expression became a permanent cost: substr and str-find intern each distinct result into the global symbol table (with the dotted split, about 1 KB per URL-shaped value), and an `if` that keeps the column itself translates its cells into runtime ids as well. On a 100M-row table with a 19.7M-entry column the key expression left about 65 GB in the symbol table that never came back. When the expression's result over the SYM column is SYM and the column's vocabulary is readable through the raw snapshot, the column is now presented to the same DAG as a STR column built from the vocabulary bytes, two million distinct values at a time. Every intermediate is a chunk-sized STR vector that dies with its chunk; only the final key strings are interned, once per distinct value, giving the ids the SYM evaluation would have produced. Other shapes keep the one-shot SYM evaluation. Co-Authored-By: Claude Fable 5.1 --- src/ops/query.c | 119 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 2 deletions(-) diff --git a/src/ops/query.c b/src/ops/query.c index 0a07c70d..dab02c77 100644 --- a/src/ops/query.c +++ b/src/ops/query.c @@ -2884,6 +2884,117 @@ static int64_t derived_key_name(ray_t* by_expr) { return ray_sym_intern("key", 3); } #define DERIVED_KEY_MAX_DOMAIN (64LL * 1024 * 1024) +/* The distinct-symbol evaluation, in chunks and over STR. + * + * Feeding the DAG a SYM column whose ids live in a FILE domain makes every + * string step of the expression a permanent cost: substr/str-find intern + * each distinct result into the global runtime table (with the dotted + * split, about 1 KB per URL) and an `if` that keeps the column itself + * translates its cells into runtime ids too. Over a 20M-entry vocabulary + * that is tens of GB that never come back. + * + * Here the referenced column is presented to the same DAG as a STR column + * built from the raw vocabulary bytes, CHUNK rows at a time: every + * intermediate is then a chunk-sized STR vector that dies with the chunk, + * and only the FINAL key strings are interned — once per distinct value, + * exactly the ids the SYM evaluation would have produced for them. + * Applies only when the expression's result over the SYM column is SYM + * (so the interned result has the type the caller expects) and the + * column's vocabulary is readable through the raw snapshot. Any other + * shape returns NULL and the caller takes the one-shot SYM evaluation. */ +#define DERIVED_KEY_CHUNK (1LL << 21) +static ray_t* derived_key_str_chunks(ray_t* by_expr, int64_t col_sym, ray_t* dom_vec, + struct ray_sym_domain_s* dom, int64_t du) { + if (!dom || dom == ray_sym_runtime_domain() || du <= 0) return NULL; + ray_sym_domain_raw_t raw; + if (!ray_sym_domain_raw_pin(dom, &raw)) return NULL; + + /* The result type the SYM evaluation would give: compile (only) the + * expression against the SYM column. */ + int8_t sym_out = 0; + { + ray_t* probe = ray_table_new(0); + if (probe && !RAY_IS_ERR(probe)) { + ray_retain(dom_vec); + probe = ray_table_add_col(probe, col_sym, dom_vec); + } + if (!probe || RAY_IS_ERR(probe)) { if (probe) ray_error_free(probe); goto unpin_null; } + ray_graph_t* gp = ray_graph_new(probe); + if (gp) { + ray_op_t* kop = compile_expr_dag(gp, by_expr); + if (kop) sym_out = kop->out_type; + ray_graph_free(gp); + } + ray_release(probe); + } + if (sym_out != RAY_SYM) goto unpin_null; + + ray_t* key_dom = ray_vec_new(RAY_SYM, du); + if (!key_dom || RAY_IS_ERR(key_dom)) { if (key_dom) ray_error_free(key_dom); goto unpin_null; } + key_dom->len = du; + int64_t* kd = (int64_t*)ray_data(key_dom); + const void* dv = ray_data(dom_vec); + + for (int64_t lo = 0; lo < du; lo += DERIVED_KEY_CHUNK) { + int64_t n = du - lo < DERIVED_KEY_CHUNK ? du - lo : DERIVED_KEY_CHUNK; + ray_t* sv = ray_vec_new(RAY_STR, n); + if (!sv || RAY_IS_ERR(sv)) { if (sv) ray_error_free(sv); goto fail; } + for (int64_t i = 0; i < n; i++) { + int64_t pos = ray_read_sym(dv, lo + i, dom_vec->type, dom_vec->attrs); + const char* sp = NULL; + size_t sl = 0; + if (pos >= 0 && pos < raw.count) { + sp = ray_sym_domain_raw_str(&raw, pos, &sl); + } else { + ray_t* a = ray_sym_domain_str(dom, pos); + if (a) { sp = ray_str_ptr(a); sl = ray_str_len(a); } + } + sv = ray_str_vec_append(sv, sp ? sp : "", sp ? sl : 0); + if (!sv || RAY_IS_ERR(sv)) { if (sv) ray_error_free(sv); goto fail; } + } + ray_t* mini = ray_table_new(0); + if (mini && !RAY_IS_ERR(mini)) mini = ray_table_add_col(mini, col_sym, sv); + ray_release(sv); + if (!mini || RAY_IS_ERR(mini)) { if (mini) ray_error_free(mini); goto fail; } + ray_t* kc = NULL; + ray_graph_t* g2 = ray_graph_new(mini); + if (g2) { + ray_op_t* kop = compile_expr_dag(g2, by_expr); + if (kop) kop = ray_optimize(g2, kop); + if (kop) kc = ray_execute(g2, kop); + ray_graph_free(g2); + } + ray_release(mini); + if (kc && !RAY_IS_ERR(kc) && ray_is_lazy(kc)) kc = ray_lazy_materialize(kc); + if (!kc || RAY_IS_ERR(kc)) { if (kc) ray_error_free(kc); goto fail; } + if (!ray_is_vec(kc) || kc->len != n) { ray_release(kc); goto fail; } + if (kc->type == RAY_STR) { + for (int64_t i = 0; i < n; i++) { + size_t sl = 0; + const char* sp = ray_str_vec_get(kc, i, &sl); + int64_t id = ray_sym_intern(sp ? sp : "", sp ? sl : 0); + if (id < 0) { ray_release(kc); goto fail; } + kd[lo + i] = id; + } + } else if (RAY_IS_SYM(kc->type)) { + /* The STR evaluation still produced symbols (e.g. a literal + * symbol branch): take them cell by cell as runtime ids. */ + for (int64_t i = 0; i < n; i++) + kd[lo + i] = sym_cell_runtime_id(kc, i); + } else { + ray_release(kc); + goto fail; + } + ray_release(kc); + } + ray_sym_domain_raw_unpin(dom); + return key_dom; +fail: + ray_release(key_dom); +unpin_null: + ray_sym_domain_raw_unpin(dom); + return NULL; +} static ray_t* derived_key_over_sym_domain(ray_t* by_expr, ray_t* tbl) { if (!by_expr || by_expr->type != RAY_LIST || !tbl) return NULL; int64_t ref_syms[2]; @@ -2956,8 +3067,11 @@ static ray_t* derived_key_over_sym_domain(ray_t* by_expr, ray_t* tbl) { /* Evaluate the expression over the du distinct symbols through the * same DAG compiler the row-wise key would take, against a one-column * table holding the distinct vector under the referenced name. */ - ray_t* key_dom = NULL; - ray_t* mini = ray_table_new(0); + ray_t* key_dom = derived_key_str_chunks(by_expr, ref_syms[0], dom_vec, dom, du); + if (key_dom && RAY_IS_ERR(key_dom)) { ray_error_free(key_dom); key_dom = NULL; } + ray_t* mini = key_dom ? NULL : ray_table_new(0); + if (key_dom) ray_release(dom_vec); + else { if (mini && !RAY_IS_ERR(mini)) mini = ray_table_add_col(mini, ref_syms[0], dom_vec); ray_release(dom_vec); if (!mini || RAY_IS_ERR(mini)) { if (mini) ray_error_free(mini); scratch_free(pos_hdr); return NULL; } @@ -2972,6 +3086,7 @@ static ray_t* derived_key_over_sym_domain(ray_t* by_expr, ray_t* tbl) { if (key_dom && !RAY_IS_ERR(key_dom) && ray_is_lazy(key_dom)) key_dom = ray_lazy_materialize(key_dom); if (!key_dom || RAY_IS_ERR(key_dom)) { if (key_dom) ray_error_free(key_dom); scratch_free(pos_hdr); return NULL; } if (!ray_is_vec(key_dom) || key_dom->len != du) { ray_release(key_dom); scratch_free(pos_hdr); return NULL; } + } /* Pass 2: spread by slot. */ ray_t* ids = ray_vec_new(RAY_I64, nrows); From 77f41e5809e21ce37e3a80f9f835ce9165ec96cb Mon Sep 17 00:00:00 2001 From: Serhii Savchuk Date: Sat, 19 Sep 2026 12:10:27 +0300 Subject: [PATCH 03/23] test(symbol): if over a FILE-domain column equals the in-memory result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trivial branches through the pooled fill (past the parallel threshold), a computed branch through the selected path, the column on both sides under a row selection, the expression as a group key, and the empty symbol on either side — each against the same expression over the in-memory table. Co-Authored-By: Claude Fable 5.1 --- test/rfl/symbol/file_domain_if.rfl | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 test/rfl/symbol/file_domain_if.rfl diff --git a/test/rfl/symbol/file_domain_if.rfl b/test/rfl/symbol/file_domain_if.rfl new file mode 100644 index 00000000..56fb5d01 --- /dev/null +++ b/test/rfl/symbol/file_domain_if.rfl @@ -0,0 +1,49 @@ +;; `if` over a SYM column whose ids live in a FILE domain (a splayed table +;; read back from disk). The branch that keeps the column translates its +;; cells into runtime ids; it must do so for the cells it reads, not by +;; interning the column's whole vocabulary. Every answer must equal the +;; same expression over the in-memory table. Sizes past the parallel +;; threshold (65536) so the pooled fill runs for trivial branches, plus a +;; computed branch for the selected path. +(.sys.exec "rm -rf /tmp/rfl_fdom_if/") -- 0 +(set N 70000) +(set i (til N)) +(set hosts (map (fn [k] (format "h%.example.com" k)) (til 300))) +(set ref (as 'SYMBOL (map (fn [k] (if (== 0 (% k 11)) "" (if (== 0 (% k 7)) (format "https://www.%/p/%" (at hosts (% (* 31 k) 300)) (% k 13)) (format "http://%/a/%" (at hosts (% (* 31 k) 300)) (% k 17))))) i))) +(set v (as 'I64 (% (* i 7) 101))) +(set T (table [ref v] (list ref v))) +(.db.splayed.set "/tmp/rfl_fdom_if/" T) +(set F (.db.splayed.get "/tmp/rfl_fdom_if/")) +(count F) -- 70000 +(== (type (at F 'ref)) 'SYM) -- true +(set S (fn [r] (as 'STR r))) + +;; trivial branches (column vs literal, column vs column): the pooled fill +(set A1 (select {from: F k: (if (> v 50) ref 'other)})) +(set B1 (select {from: T k: (if (> v 50) ref 'other)})) +(all (== (S (at A1 'k)) (S (at B1 'k)))) -- true +(set A2 (select {from: F k: (if (== 0 (% v 3)) 'left ref)})) +(set B2 (select {from: T k: (if (== 0 (% v 3)) 'left ref)})) +(all (== (S (at A2 'k)) (S (at B2 'k)))) -- true + +;; computed branch against the column: the selected path +(set A3 (select {from: F k: (if (== (str-find ref "www.") 8) (substr ref 13 -1) ref)})) +(set B3 (select {from: T k: (if (== (str-find ref "www.") 8) (substr ref 13 -1) ref)})) +(all (== (S (at A3 'k)) (S (at B3 'k)))) -- true +(count (distinct (at A3 (quote k)))) -- 9001 + +;; the column on both sides, under a row selection +(set A4 (select {from: F k: (if (> v 50) ref ref) where: (!= ref "")})) +(set B4 (select {from: T k: (if (> v 50) ref ref) where: (!= ref "")})) +(all (== (S (at A4 'k)) (S (at B4 'k)))) -- true + +;; as a group key: same groups and counts as in memory +(set FP (fn [r] (ser (xasc (xasc (table [k c] (list (S (at (value r) 0)) (at r 'c))) 'k) 'c)))) +(set A5 (select {from: F by: (if (== (str-find ref "www.") 8) (substr ref 13 -1) ref) c: (count v) where: (!= ref "")})) +(set B5 (select {from: T by: (if (== (str-find ref "www.") 8) (substr ref 13 -1) ref) c: (count v) where: (!= ref "")})) +(all (== (FP A5) (FP B5))) -- true + +;; the empty symbol (a null cell) survives the translation on either side +(set A6 (select {from: F k: (if (> v 50) ref 'x)})) +(== (count (select {from: A6 where: (== k "")})) (count (select {from: T where: (and (> v 50) (== ref ""))}))) -- true +(.sys.exec "rm -rf /tmp/rfl_fdom_if/") -- 0 From 0aeac9deaced3cec1b8b6082e5a29b08e6f23e1f Mon Sep 17 00:00:00 2001 From: Serhii Savchuk Date: Sat, 19 Sep 2026 12:10:44 +0300 Subject: [PATCH 04/23] test(group): derived key over a FILE-domain column, chunked The existing key expressions over a splayed copy of the table against the in-memory oracle, with the default chunk and with a 100-value chunk so the vocabulary crosses several chunks (RAY_DERIVED_KEY_CHUNK overrides the chunk length for this), a non-symbol key result that keeps the one-shot evaluation, and the null-bearing column. Co-Authored-By: Claude Fable 5.1 --- src/ops/query.c | 15 ++++++++++-- test/rfl/group/derived_key_sym_domain.rfl | 30 +++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/ops/query.c b/src/ops/query.c index dab02c77..0d1b90f6 100644 --- a/src/ops/query.c +++ b/src/ops/query.c @@ -2903,6 +2903,16 @@ static int64_t derived_key_name(ray_t* by_expr) { * column's vocabulary is readable through the raw snapshot. Any other * shape returns NULL and the caller takes the one-shot SYM evaluation. */ #define DERIVED_KEY_CHUNK (1LL << 21) +/* Chunk length; RAY_DERIVED_KEY_CHUNK overrides it so a test can drive + * several chunks through a small vocabulary. */ +static int64_t derived_key_chunk_rows(void) { + const char* env = getenv("RAY_DERIVED_KEY_CHUNK"); + if (env && *env) { + long v = strtol(env, NULL, 10); + if (v > 0) return (int64_t)v; + } + return DERIVED_KEY_CHUNK; +} static ray_t* derived_key_str_chunks(ray_t* by_expr, int64_t col_sym, ray_t* dom_vec, struct ray_sym_domain_s* dom, int64_t du) { if (!dom || dom == ray_sym_runtime_domain() || du <= 0) return NULL; @@ -2935,8 +2945,9 @@ static ray_t* derived_key_str_chunks(ray_t* by_expr, int64_t col_sym, ray_t* dom int64_t* kd = (int64_t*)ray_data(key_dom); const void* dv = ray_data(dom_vec); - for (int64_t lo = 0; lo < du; lo += DERIVED_KEY_CHUNK) { - int64_t n = du - lo < DERIVED_KEY_CHUNK ? du - lo : DERIVED_KEY_CHUNK; + const int64_t chunk = derived_key_chunk_rows(); + for (int64_t lo = 0; lo < du; lo += chunk) { + int64_t n = du - lo < chunk ? du - lo : chunk; ray_t* sv = ray_vec_new(RAY_STR, n); if (!sv || RAY_IS_ERR(sv)) { if (sv) ray_error_free(sv); goto fail; } for (int64_t i = 0; i < n; i++) { diff --git a/test/rfl/group/derived_key_sym_domain.rfl b/test/rfl/group/derived_key_sym_domain.rfl index d89b4c4b..6521305b 100644 --- a/test/rfl/group/derived_key_sym_domain.rfl +++ b/test/rfl/group/derived_key_sym_domain.rfl @@ -84,3 +84,33 @@ (count RQ) -- 1 (== (at (value RQ) 0) (at (value OQ) 0)) -- [true] (== (at RQ 'c) (at OQ 'c)) -- [true] + +;; ── the column's ids in a FILE domain (a splayed table read back) ────── +;; The per-symbol evaluation then runs over the vocabulary's bytes, a +;; chunk of distinct values at a time, and interns only the final key. +;; Same groups and aggregates as the in-memory table; a chunk far smaller +;; than the vocabulary drives the expression through several chunks. +(.sys.exec "rm -rf /tmp/rfl_dkey_file/") -- 0 +(.db.splayed.set "/tmp/rfl_dkey_file/" T) +(set F (.db.splayed.get "/tmp/rfl_dkey_file/")) +(count F) -- 20000 +(== (type (at F 'ref)) 'SYM) -- true +(set RF (KEYQ F)) +(count RF) -- 37 +(all (== (FP RF) (FP O))) -- true +(== (at (cols RF) 0) 'p) -- true +(== (.os.setenv "RAY_DERIVED_KEY_CHUNK" "100") "100") -- true +(set RC (KEYQ F)) +(all (== (FP RC) (FP O))) -- true +(all (== (FP2 (select {from: F by: (if (== (str-find ref "www.") 0) (substr ref 4 -1) ref) c: (count ref)})) (FP2 O2))) -- true +;; a key whose result is not a symbol keeps the one-shot evaluation +(set RL (select {from: F by: (strlen ref) c: (count ref)})) +(set OL (select {from: T by: (strlen ref) c: (count ref)})) +(all (== (FP2 RL) (FP2 OL))) -- true +(== (.os.setenv "RAY_DERIVED_KEY_CHUNK" "") "") -- true +;; nulls in the file-backed column +(.sys.exec "rm -rf /tmp/rfl_dkey_filen/") -- 0 +(.db.splayed.set "/tmp/rfl_dkey_filen/" TN) +(set FN (.db.splayed.get "/tmp/rfl_dkey_filen/")) +(all (== (FP (KEYQ FN)) (FP ON))) -- true +(.sys.exec "rm -rf /tmp/rfl_dkey_file/ /tmp/rfl_dkey_filen/") -- 0 From d6d60801ede9f0af91fdbe0f6aa69d2d2417fbc9 Mon Sep 17 00:00:00 2001 From: Serhii Savchuk Date: Sat, 19 Sep 2026 12:24:00 +0300 Subject: [PATCH 05/23] fix(if): the parallel fill reads one side per row and never interns The pre-resolution pass covered the cells the mask selects, but the SYM fill read both sides of every row before choosing, so workers met unresolved positions and interned them inside the pool. The fill now reads only the chosen side, through a read-only lookup once the sides are prepared. Both side tables are initialised before either result is checked, so the OOM path frees no uninitialised side. The table holds 32-bit ids (the symbol table counts them in 32 bits), half the scratch per vocabulary position. Test: a FILE-domain column on both sides of the `if` without a row selection, through the pooled fill. Co-Authored-By: Claude Fable 5.1 --- src/ops/pivot.c | 58 ++++++++++++++++++++++-------- test/rfl/symbol/file_domain_if.rfl | 11 +++++- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/src/ops/pivot.c b/src/ops/pivot.c index 3bfdbb0c..f3b6eb52 100644 --- a/src/ops/pivot.c +++ b/src/ops/pivot.c @@ -509,16 +509,18 @@ static ray_t* if_scatter_str(ray_t* result, ray_t* value, int64_t* ids, * touches a few of them. This translator interns only the positions the * branch actually reads: `lut` starts at -1 and a slot is resolved (raw * file bytes when available, the domain atom otherwise) the first time - * that position is met. Resolution interns, so it must run on the - * calling thread: the serial scatter path resolves on the fly, the - * parallel fill pre-resolves every used position (if_sym_side_prepare) - * before dispatching, after which workers only read the table. - * Runtime-domain, STR and scalar branches need no table. */ + * that position is met. Resolution interns, so it runs on the calling + * thread only: the serial scatter path resolves on the fly; the parallel + * fill first walks the mask serially and resolves exactly the positions + * the fill will read — a row's then-cell where the mask is set, its + * else-cell where it is clear (if_sym_side_prepare) — and the fill then + * reads only that side of each row through if_sym_side_lookup, which + * never interns. Runtime-domain, STR and scalar branches need no table. */ typedef struct { ray_t* v; struct ray_sym_domain_s* dom; /* non-runtime domain, else NULL */ int64_t dn; - int64_t* lut; /* dn slots, -1 = unresolved */ + int32_t* lut; /* dn slots, -1 = unresolved (ids fit: the table counts them in 32 bits) */ ray_t* lut_hdr; ray_sym_domain_raw_t raw; bool raw_ok; @@ -532,9 +534,9 @@ static bool if_sym_side_init(if_sym_side_t* side, ray_t* v) { if (!dom || dom == ray_sym_runtime_domain()) return true; int64_t dn = ray_sym_domain_count(dom); if (dn <= 0) return true; - side->lut = (int64_t*)scratch_alloc(&side->lut_hdr, (size_t)dn * sizeof(int64_t)); + side->lut = (int32_t*)scratch_alloc(&side->lut_hdr, (size_t)dn * sizeof(int32_t)); if (!side->lut) return false; - memset(side->lut, 0xff, (size_t)dn * sizeof(int64_t)); + memset(side->lut, 0xff, (size_t)dn * sizeof(int32_t)); side->dom = dom; side->dn = dn; side->raw_ok = ray_sym_domain_raw_pin(dom, &side->raw); @@ -563,10 +565,20 @@ static int64_t if_sym_side_resolve(if_sym_side_t* side, int64_t pos) { if (s) { sp = ray_str_ptr(s); sl = ray_str_len(s); } } id = ray_sym_intern(sp ? sp : "", sp ? sl : 0); - if (id >= 0) side->lut[pos] = id; + if (id >= 0 && id <= INT32_MAX) side->lut[pos] = (int32_t)id; return id; } +/* Read-only lookup for pool workers: the position must have been + * prepared; nothing is interned here. An unprepared position (a bug in + * the caller's prepare pass) maps to the empty symbol rather than + * touching the table from a worker. */ +static inline int64_t if_sym_side_lookup(const if_sym_side_t* side, int64_t pos) { + if (pos < 0 || pos >= side->dn) return 0; + int32_t id = side->lut[pos]; + return id >= 0 ? (int64_t)id : 0; +} + /* Pre-resolve the positions rows [0, len) read on this side under the * mask (`want` selects which mask value picks this side). Serial. */ static void if_sym_side_prepare(if_sym_side_t* side, const uint8_t* cond, @@ -803,8 +815,20 @@ typedef struct { int8_t out_type; if_sym_side_t* t_side; /* SYM output: per-side id translation */ if_sym_side_t* e_side; + bool sides_prepared; /* positions resolved up front: read-only fill (workers) */ } if_fill_ctx_t; +/* One SYM cell of a fill side as a runtime id: the selected side only. */ +static inline int64_t if_fill_sym_cell(const if_fill_ctx_t* c, if_sym_side_t* side, + int64_t i) { + ray_t* v = side->v; + if (c->sides_prepared && side->dom) { + int64_t pos = ray_read_sym(ray_data(v), i, v->type, v->attrs); + return if_sym_side_lookup(side, pos); + } + return if_sym_cell_value(side, i); +} + static void if_fill_range(const if_fill_ctx_t* c, int64_t i0, int64_t i1) { const uint8_t* cond_p = c->cond; switch (c->out_type) { @@ -826,10 +850,13 @@ static void if_fill_range(const if_fill_ctx_t* c, int64_t i0, int64_t i1) { break; } case RAY_SYM: { int64_t* dst = (int64_t*)c->dst; + /* Only the chosen side is read: the prepare pass resolved exactly + * these cells, and the other side's cell may be unresolved. */ for (int64_t i = i0; i < i1; i++) { - int64_t tv = c->then_scalar ? c->t_i64 : if_sym_cell_value(c->t_side, i); - int64_t ev = c->else_scalar ? c->e_i64 : if_sym_cell_value(c->e_side, i); - dst[i] = cond_p[i] ? tv : ev; + if (cond_p[i]) + dst[i] = c->then_scalar ? c->t_i64 : if_fill_sym_cell(c, c->t_side, i); + else + dst[i] = c->else_scalar ? c->e_i64 : if_fill_sym_cell(c, c->e_side, i); } break; } case RAY_I32: case RAY_TIME: case RAY_DATE: { @@ -1043,8 +1070,10 @@ static ray_t* exec_if_eager(ray_graph_t* g, ray_op_t* op) { if_sym_side_t t_side, e_side; bool sides_ok = true; if (out_type == RAY_SYM) { - sides_ok = if_sym_side_init(&t_side, then_scalar ? NULL : then_v) && - if_sym_side_init(&e_side, else_scalar ? NULL : else_v); + /* Both inits run (each zeroes its side) so both frees are safe. */ + bool t_ok = if_sym_side_init(&t_side, then_scalar ? NULL : then_v); + bool e_ok = if_sym_side_init(&e_side, else_scalar ? NULL : else_v); + sides_ok = t_ok && e_ok; fc.t_side = &t_side; fc.e_side = &e_side; } @@ -1070,6 +1099,7 @@ static ray_t* exec_if_eager(ray_graph_t* g, ray_op_t* op) { if (par) { if_sym_side_prepare(&t_side, cond_p, len, 1); if_sym_side_prepare(&e_side, cond_p, len, 0); + fc.sides_prepared = true; } } if (par) diff --git a/test/rfl/symbol/file_domain_if.rfl b/test/rfl/symbol/file_domain_if.rfl index 56fb5d01..0a2bfd6c 100644 --- a/test/rfl/symbol/file_domain_if.rfl +++ b/test/rfl/symbol/file_domain_if.rfl @@ -11,7 +11,8 @@ (set hosts (map (fn [k] (format "h%.example.com" k)) (til 300))) (set ref (as 'SYMBOL (map (fn [k] (if (== 0 (% k 11)) "" (if (== 0 (% k 7)) (format "https://www.%/p/%" (at hosts (% (* 31 k) 300)) (% k 13)) (format "http://%/a/%" (at hosts (% (* 31 k) 300)) (% k 17))))) i))) (set v (as 'I64 (% (* i 7) 101))) -(set T (table [ref v] (list ref v))) +(set ref2 (at ref (% (+ i 1) N))) +(set T (table [ref ref2 v] (list ref ref2 v))) (.db.splayed.set "/tmp/rfl_fdom_if/" T) (set F (.db.splayed.get "/tmp/rfl_fdom_if/")) (count F) -- 70000 @@ -25,6 +26,14 @@ (set A2 (select {from: F k: (if (== 0 (% v 3)) 'left ref)})) (set B2 (select {from: T k: (if (== 0 (% v 3)) 'left ref)})) (all (== (S (at A2 'k)) (S (at B2 'k)))) -- true +;; a FILE-domain column on BOTH sides, no row selection: the pooled fill +;; reads one side per row and must have every such cell resolved up front +(set A7 (select {from: F k: (if (> v 50) ref ref2)})) +(set B7 (select {from: T k: (if (> v 50) ref ref2)})) +(all (== (S (at A7 'k)) (S (at B7 'k)))) -- true +(set A8 (select {from: F k: (if (== 0 (% v 3)) ref2 ref)})) +(set B8 (select {from: T k: (if (== 0 (% v 3)) ref2 ref)})) +(all (== (S (at A8 'k)) (S (at B8 'k)))) -- true ;; computed branch against the column: the selected path (set A3 (select {from: F k: (if (== (str-find ref "www.") 8) (substr ref 13 -1) ref)})) From 73265b503d3ddbe8b36d267bea1927b167bb41e4 Mon Sep 17 00:00:00 2001 From: Serhii Savchuk Date: Sat, 19 Sep 2026 12:24:03 +0300 Subject: [PATCH 06/23] fix(group): drop the extra retain on the distinct vector in the type probe ray_table_add_col retains the column itself, so the retain before it left one reference on the distinct-symbol vector after every derived-key evaluation over a file-backed column, and with it the column's domain. The fallback block is re-indented as the else branch it is. Co-Authored-By: Claude Fable 5.1 --- src/ops/query.c | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/src/ops/query.c b/src/ops/query.c index 0d1b90f6..3f32ca45 100644 --- a/src/ops/query.c +++ b/src/ops/query.c @@ -2924,10 +2924,8 @@ static ray_t* derived_key_str_chunks(ray_t* by_expr, int64_t col_sym, ray_t* dom int8_t sym_out = 0; { ray_t* probe = ray_table_new(0); - if (probe && !RAY_IS_ERR(probe)) { - ray_retain(dom_vec); - probe = ray_table_add_col(probe, col_sym, dom_vec); - } + /* add_col retains the column itself; the caller keeps its own ref. */ + if (probe && !RAY_IS_ERR(probe)) probe = ray_table_add_col(probe, col_sym, dom_vec); if (!probe || RAY_IS_ERR(probe)) { if (probe) ray_error_free(probe); goto unpin_null; } ray_graph_t* gp = ray_graph_new(probe); if (gp) { @@ -3080,23 +3078,24 @@ static ray_t* derived_key_over_sym_domain(ray_t* by_expr, ray_t* tbl) { * table holding the distinct vector under the referenced name. */ ray_t* key_dom = derived_key_str_chunks(by_expr, ref_syms[0], dom_vec, dom, du); if (key_dom && RAY_IS_ERR(key_dom)) { ray_error_free(key_dom); key_dom = NULL; } - ray_t* mini = key_dom ? NULL : ray_table_new(0); - if (key_dom) ray_release(dom_vec); - else { - if (mini && !RAY_IS_ERR(mini)) mini = ray_table_add_col(mini, ref_syms[0], dom_vec); - ray_release(dom_vec); - if (!mini || RAY_IS_ERR(mini)) { if (mini) ray_error_free(mini); scratch_free(pos_hdr); return NULL; } - ray_graph_t* g2 = ray_graph_new(mini); - if (g2) { - ray_op_t* kop = compile_expr_dag(g2, by_expr); - if (kop) kop = ray_optimize(g2, kop); - if (kop) key_dom = ray_execute(g2, kop); - ray_graph_free(g2); - } - ray_release(mini); - if (key_dom && !RAY_IS_ERR(key_dom) && ray_is_lazy(key_dom)) key_dom = ray_lazy_materialize(key_dom); - if (!key_dom || RAY_IS_ERR(key_dom)) { if (key_dom) ray_error_free(key_dom); scratch_free(pos_hdr); return NULL; } - if (!ray_is_vec(key_dom) || key_dom->len != du) { ray_release(key_dom); scratch_free(pos_hdr); return NULL; } + if (key_dom) { + ray_release(dom_vec); + } else { + ray_t* mini = ray_table_new(0); + if (mini && !RAY_IS_ERR(mini)) mini = ray_table_add_col(mini, ref_syms[0], dom_vec); + ray_release(dom_vec); + if (!mini || RAY_IS_ERR(mini)) { if (mini) ray_error_free(mini); scratch_free(pos_hdr); return NULL; } + ray_graph_t* g2 = ray_graph_new(mini); + if (g2) { + ray_op_t* kop = compile_expr_dag(g2, by_expr); + if (kop) kop = ray_optimize(g2, kop); + if (kop) key_dom = ray_execute(g2, kop); + ray_graph_free(g2); + } + ray_release(mini); + if (key_dom && !RAY_IS_ERR(key_dom) && ray_is_lazy(key_dom)) key_dom = ray_lazy_materialize(key_dom); + if (!key_dom || RAY_IS_ERR(key_dom)) { if (key_dom) ray_error_free(key_dom); scratch_free(pos_hdr); return NULL; } + if (!ray_is_vec(key_dom) || key_dom->len != du) { ray_release(key_dom); scratch_free(pos_hdr); return NULL; } } /* Pass 2: spread by slot. */ From 6f3b75f4dfba0041c4761768b035215b0944518c Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 11:43:39 +0200 Subject: [PATCH 07/23] perf(group): scale grouping with cores: cache-bounded slabs, v2 top-N routing, composite key compaction - bound replicated dense slabs by the last-level cache (ray_cache_llc_bytes) - keep multi-key arithmetic-over-aggregates on the DAG engine; admit binary aggs into hidden slots - run desc/asc take: N group queries on v2 when a dense plan exists, trim afterwards - parallelize the filtered dense-plan prescan - compact interleaved symbol codes so composite symbol keys pack densely --- docs/docs/architecture/pipeline.md | 12 + docs/docs/queries/select.md | 7 + docs/grouping-engine-scaling-plan.md | 45 ++ src/core/platform.c | 152 ++++++ src/core/platform.h | 4 + src/ops/agg_engine.c | 458 +++++++++++++++--- src/ops/agg_engine.h | 14 + src/ops/group.c | 78 +-- src/ops/query.c | 61 ++- test/rfl/group/agg_arith_multikey.rfl | 64 +++ .../group/dense_composite_sym_compaction.rfl | 57 +++ test/rfl/group/emit_filter_v2_route.rfl | 62 +++ test/test_agg_contract.c | 72 +++ test/test_group_extra.c | 21 +- 14 files changed, 997 insertions(+), 110 deletions(-) create mode 100644 test/rfl/group/agg_arith_multikey.rfl create mode 100644 test/rfl/group/dense_composite_sym_compaction.rfl create mode 100644 test/rfl/group/emit_filter_v2_route.rfl diff --git a/docs/docs/architecture/pipeline.md b/docs/docs/architecture/pipeline.md index f90488dd..ca35ce74 100644 --- a/docs/docs/architecture/pipeline.md +++ b/docs/docs/architecture/pipeline.md @@ -211,6 +211,18 @@ The join pipeline: 2. **Build** — Build per-partition hash tables (each fits in L2). For inner joins, the executor selects the build side at runtime using actual materialized row counts: the smaller input becomes the build side, keeping hash tables as compact as possible. LEFT, FULL, and ANTI joins always build on the right to preserve left-row semantics. The small-input (chained) path also always builds on the right. During the per-partition open-addressing build, the executor tracks per-key duplicate counts; when a single key exceeds the duplication threshold (`RADIX_DUP_RUN_MAX = 512`), it abandons the radix attempt and re-runs the whole join through the chained hash table, which is O(n) regardless of duplication. No join (INNER, LEFT, or FULL) can degrade to quadratic build cost on a skewed key. 3. **Probe** — Probe partitions in parallel across worker threads. Inner-join output order is partition- and thread-dependent; it is not guaranteed to be stable. +### Group-By Aggregation + +Grouped aggregates run on the parallel aggregation engine, which picks a strategy from the key columns and the aggregates: + +- **Dense direct index** — integer, temporal, and symbol keys whose value ranges pack into a bounded slot space (a group id is the mixed-radix offset of the key tuple, no hashing). Composite keys whose raw ranges multiply out too far are first *compacted*: one parallel pass records the codes each key actually uses, and the plan maps each key through a code-to-component table. This matters for symbol keys, which share one domain and therefore interleave their codes with every other symbol column's. +- **Task-local slabs** — each task accumulates into its own dense slab, merged once at the end. The number of replicated slabs is bounded by the last-level cache: every task updates random slots of its slab, so once the slabs together outgrow the cache each update misses to memory and more tasks make the query slower. The engine reads the cache size at startup (`ray_cache_llc_bytes`) and keeps replicated slabs within three quarters of it; when fewer than three slabs fit, it switches to partition ownership. +- **Partition ownership** — rows are scattered by slot into partitions that each own a cache-sized slab; every group is reduced exactly once, with no merge. Used for large dense domains and for shared extrema that concurrent updates handle directly. +- **Radix** — unbounded integer or symbol keys (many-million-group inputs) are hash-partitioned and reduced per partition. +- **Shared directory** — float, string, GUID, and list keys use a shared parallel key directory. + +Arithmetic over aggregates (`(- (max v1) (min v2))`, `(pow (pearson_corr a b) 2)`) is decomposed at compile time: the aggregates run as hidden slots inside the same group pass, and the outer expression is evaluated once over the grouped result. This applies to any number of keys and to binary aggregates. Ordered top-N clauses (`desc: c take: 10`) run the full parallel grouping and trim the result to the top-N superset before the final sort; only unbounded key domains still use the older ordered-emit path. + ### Per-Thread Heaps Each worker thread has its own heap (`heap_id` in the pool header). Vectors allocated during parallel execution are tagged with their owning heap via the pool they reside in. Cross-heap frees are deferred to a lock-free LIFO and reclaimed when the owning heap flushes. See [Memory Model](memory.md) for details. diff --git a/docs/docs/queries/select.md b/docs/docs/queries/select.md index b4048e89..aa2f4d36 100644 --- a/docs/docs/queries/select.md +++ b/docs/docs/queries/select.md @@ -121,6 +121,13 @@ Group by multiple keys: ; GOOG Buy 125 ``` +Expressions over aggregates work with any number of keys and with binary aggregates such as `pearson_corr`, `cov`, or `wsum`; the aggregates are computed in the grouping pass and the outer expression once per group: + +```lisp +(select {from: trades spread: (- (max price) (min price)) by: {sym: sym venue: venue}}) +(select {from: trades r2: (pow (pearson_corr price size) 2) by: {sym: sym}}) +``` + ### Filter + Group-by Filter first, then group by. Chain two `select` calls for correct results: diff --git a/docs/grouping-engine-scaling-plan.md b/docs/grouping-engine-scaling-plan.md index 65285f1d..af940ceb 100644 --- a/docs/grouping-engine-scaling-plan.md +++ b/docs/grouping-engine-scaling-plan.md @@ -253,3 +253,48 @@ buffer for binary and mixed streaming reductions. The existing native-width readers, source selection and partition ownership remain shared. Chunk sizing bounds the active payload footprint; scratch consists of partition cursors, without another row-sized buffer. + +## Follow-up: core scaling on cache-bounded and legacy-routed shapes + +A scaling sweep over 1/8/16/28 threads on a 10M-row grouping input found five +reasons queries stopped speeding up, or slowed down, as cores were added. None +was in the pool; all were routing or footprint decisions. + +- **Replicated slab footprint.** The task-local dense strategy gave every worker a + full slab, and the strategy choice was deliberately independent of cache size. + Once the slabs together outgrew the last-level cache, every update missed to + memory: a 100k-group sum measured 6 ms with 8 slabs and 21 ms with 28 slabs on + the same pool. The runtime now reads the last-level cache size (summed over + every cache instance, so multi-die parts count each die) and bounds replicated + slabs to three quarters of it; below three slabs it prefers partition + ownership, whose per-partition slabs are cache-sized by construction. The + memory budgets stay data-derived; the cache bound only limits replication. + Task assignment stays static so floating-point sums remain deterministic for a + given pool size. +- **Multi-key arithmetic over aggregates.** Any non-aggregate output on a + two-or-more-key `by:` forced eval-level grouping (no parallelism, ~850 ms), + because that decision ran before the arithmetic-over-aggregates decomposition. + Routing now probes decomposability first, and the hidden-slot decomposition + also admits binary aggregates and literal-probability quantiles. +- **Top-N emit filter.** `desc: AGG take: N` armed an emit filter the parallel + engine does not implement, so every such query ran on the legacy ladder + (0.1x parallel at 28 threads for a three-aggregate query). Shapes the + parallel engine admits with a bounded dense plan now run there and are + trimmed to the top-N superset; unbounded key domains stay on the ladder + because the radix route's first-seen ordering and emission tail is still + serial-heavy there. +- **Filtered prescan.** The selected-row dense plan walked the selection + serially (~13 ms per 10M rows at any core count); it is now split across the + pool. +- **Composite symbol keys.** Symbol columns share one domain, so their codes + interleave and a two-key raw range product overflowed the dense limit, + sending 10k-group queries to radix (505 ms vs 52 ms on one core). Plans now + compact keys to the codes they use when the raw product overflows or exceeds + 65,536 slots. The hot loops select the compacted or raw slot computation once + per run: a per-row check made the two-key loop 12x slower. + +Still open: the radix route's post-reduce stages (an input-sized order map to +restore first-seen order, then emission) scale poorly on many-million-group +keys; the legacy ladder remains faster for those top-N shapes. An unordered +`take:` on a grouped select still uses radix's bounded emit rather than the +dense plan. diff --git a/src/core/platform.c b/src/core/platform.c index e1e835af..3879fea6 100644 --- a/src/core/platform.c +++ b/src/core/platform.c @@ -40,6 +40,8 @@ #include #include #include +#include +#include #if defined(RAY_OS_MACOS) #include /* sysctlbyname — hw.physicalcpu */ #endif @@ -294,6 +296,125 @@ uint32_t ray_physical_core_count(void) { #endif } +/* Total last-level cache capacity across every LLC instance, in bytes. + * Replicated per-task state (dense group slabs) stops scaling the moment + * its total footprint leaves the LLC: a 100k-group sum measured 6 ms with + * 8 slabs (26 MB, inside a 33 MB L3) and 21 ms with 28 slabs (92 MB) on the + * same 28-thread pool. Callers bound such replication by this figure. + * + * Linux reads sysfs: the highest-level unified cache of cpu0 gives the + * per-instance size, and its shared_cpu_list gives the instance width, so + * multi-die parts (one LLC per die) report the sum of every die's LLC. + * Falls back to sysconf's L3 (then L2) size; 0 when nothing is known. */ +#if defined(RAY_OS_LINUX) +static uint32_t cache_cpu_list_count(const char* list) { + /* "0-27" / "0-3,8-11" / "5" → number of CPUs named. */ + uint32_t n = 0; + const char* p = list; + while (*p) { + char* end; + long lo = strtol(p, &end, 10); + if (end == p) break; + long hi = lo; + p = end; + if (*p == '-') { hi = strtol(p + 1, &end, 10); if (end == p + 1) break; p = end; } + if (hi >= lo) n += (uint32_t)(hi - lo + 1); + while (*p == ',' || *p == ' ' || *p == '\n') p++; + } + return n; +} +static uint64_t cache_sysfs_llc_bytes(void) { + uint64_t best = 0; long best_level = 0; + uint32_t logical = ray_thread_count(); + for (int index = 0; index < 8; index++) { + char path[128], buf[256]; + FILE* f; + long level = 0; + snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu0/cache/index%d/level", index); + f = fopen(path, "r"); + if (!f) break; + if (fscanf(f, "%ld", &level) != 1) level = 0; + fclose(f); + snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu0/cache/index%d/type", index); + f = fopen(path, "r"); + if (!f) continue; + buf[0] = 0; + if (!fgets(buf, sizeof(buf), f)) buf[0] = 0; + fclose(f); + if (strncmp(buf, "Unified", 7) != 0 || level <= best_level) continue; + snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu0/cache/index%d/size", index); + f = fopen(path, "r"); + if (!f) continue; + buf[0] = 0; + if (!fgets(buf, sizeof(buf), f)) buf[0] = 0; + fclose(f); + char* unit = NULL; + unsigned long long size = strtoull(buf, &unit, 10); + if (unit && (*unit == 'K' || *unit == 'k')) size <<= 10; + else if (unit && (*unit == 'M' || *unit == 'm')) size <<= 20; + if (size == 0) continue; + snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu0/cache/index%d/shared_cpu_list", index); + f = fopen(path, "r"); + uint32_t width = 0; + if (f) { + buf[0] = 0; + if (fgets(buf, sizeof(buf), f)) width = cache_cpu_list_count(buf); + fclose(f); + } + uint32_t instances = width > 0 && logical > width ? (logical + width - 1) / width : 1; + best = (uint64_t)size * instances; + best_level = level; + } + return best; +} +#endif + +uint64_t ray_cache_llc_bytes(void) { + static uint64_t cached = UINT64_MAX; + if (cached != UINT64_MAX) return cached; + uint64_t bytes = 0; +#if defined(RAY_OS_MACOS) + uint64_t v = 0; size_t len = sizeof(v); + if (sysctlbyname("hw.l3cachesize", &v, &len, NULL, 0) == 0 && v > 0) bytes = v; + else { + /* No L3 (recent ARM desktop parts): the per-cluster L2 is the last level. + * Sum every cluster's L2 from the perflevel topology. */ + for (int level = 0; level < 2 && bytes < UINT64_MAX / 2; level++) { + char name[64]; + uint64_t l2 = 0; int cpus = 0, per_l2 = 0; + size_t l2_len = sizeof(l2), cpus_len = sizeof(cpus), per_len = sizeof(per_l2); + snprintf(name, sizeof(name), "hw.perflevel%d.l2cachesize", level); + if (sysctlbyname(name, &l2, &l2_len, NULL, 0) != 0 || l2 == 0) break; + snprintf(name, sizeof(name), "hw.perflevel%d.physicalcpu", level); + if (sysctlbyname(name, &cpus, &cpus_len, NULL, 0) != 0 || cpus <= 0) cpus = 1; + snprintf(name, sizeof(name), "hw.perflevel%d.cpusperl2", level); + if (sysctlbyname(name, &per_l2, &per_len, NULL, 0) != 0 || per_l2 <= 0) per_l2 = cpus; + bytes += l2 * (uint64_t)((cpus + per_l2 - 1) / per_l2); + } + if (bytes == 0) { + len = sizeof(v); + if (sysctlbyname("hw.l2cachesize", &v, &len, NULL, 0) == 0) bytes = v; + } + } +#elif defined(RAY_OS_LINUX) + bytes = cache_sysfs_llc_bytes(); + if (bytes == 0) { +#if defined(_SC_LEVEL3_CACHE_SIZE) + long l3 = sysconf(_SC_LEVEL3_CACHE_SIZE); + if (l3 > 0) bytes = (uint64_t)l3; +#endif + } + if (bytes == 0) { +#if defined(_SC_LEVEL2_CACHE_SIZE) + long l2 = sysconf(_SC_LEVEL2_CACHE_SIZE); + if (l2 > 0) bytes = (uint64_t)l2 * ray_physical_core_count(); +#endif + } +#endif + cached = bytes; + return bytes; +} + /* -------------------------------------------------------------------------- * Semaphore * -------------------------------------------------------------------------- */ @@ -486,6 +607,36 @@ uint32_t ray_physical_core_count(void) { return ray_thread_count(); } +/* Sum of every level-3 cache instance reported by the processor topology + * (each SYSTEM_LOGICAL_PROCESSOR_INFORMATION cache record is one instance). + * 0 when the query fails. */ +uint64_t ray_cache_llc_bytes(void) { + static uint64_t cached = UINT64_MAX; + if (cached != UINT64_MAX) return cached; + uint64_t bytes = 0; + DWORD len = 0; + GetLogicalProcessorInformation(NULL, &len); + if (len > 0) { + SYSTEM_LOGICAL_PROCESSOR_INFORMATION* info = ray_sys_alloc(len); + if (info) { + if (GetLogicalProcessorInformation(info, &len)) { + DWORD n = len / sizeof(*info); + BYTE best_level = 0; + for (DWORD i = 0; i < n; i++) { + if (info[i].Relationship != RelationCache) continue; + BYTE level = info[i].Cache.Level; + if (info[i].Cache.Type != CacheUnified && info[i].Cache.Type != CacheData) continue; + if (level > best_level) { best_level = level; bytes = 0; } + if (level == best_level) bytes += info[i].Cache.Size; + } + } + ray_sys_free(info); + } + } + cached = bytes; + return bytes; +} + /* -------------------------------------------------------------------------- * Semaphore * -------------------------------------------------------------------------- */ @@ -617,6 +768,7 @@ ray_err_t ray_thread_join(ray_thread_t t) { } uint32_t ray_thread_count(void) { return 1; } +uint64_t ray_cache_llc_bytes(void) { return 0; } /* Semaphore — counter-only. Single-threaded so wait never blocks (the * counter must already be positive when wait fires). */ diff --git a/src/core/platform.h b/src/core/platform.h index 6f2a9882..ea3633c5 100644 --- a/src/core/platform.h +++ b/src/core/platform.h @@ -176,6 +176,10 @@ uint32_t ray_thread_count(void); /* Physical cores (SMT siblings collapsed); falls back to the logical * count when topology is unavailable. */ uint32_t ray_physical_core_count(void); +/* Total last-level cache capacity summed over every LLC instance (bytes); + * 0 when the platform cannot report it. Bounds replicated per-task state + * whose random-access working set must stay cache-resident to scale. */ +uint64_t ray_cache_llc_bytes(void); void ray_parallel_begin(void); void ray_parallel_end(void); diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index b0810f9b..f8800b95 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -8,6 +8,7 @@ #include "lang/internal.h" /* sym_domain_rep */ #include "table/domain.h" #include "table/sym.h" /* ray_read_sym */ +#include "core/platform.h" /* ray_cache_llc_bytes — replicated slab bound */ #include #include @@ -190,6 +191,56 @@ bool agg_v2_can_handle(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { return agg_v2_admission(g, op, tbl) == AGG_V2_ADMITTED; } +/* True when the v2 engine would run this group through a bounded dense + * (direct-index) plan over the whole table. Callers use it to predict the + * strategy class before committing: dense plans scale with the pool, while + * the unbounded radix route still pays a large serial ordering/emission + * tail on many-million-group inputs. Costs one parallel min/max prescan + * of the non-SYM keys; SYM keys resolve from their domain bounds. */ +/* The probe's plan is kept for the run that follows on this thread so the + * key prescan is not paid twice (4 ms serial on a 10M-row I64 key). The + * run consumes it only when the same key columns and row count are seen. */ +typedef struct { + bool valid; + uint32_t n_keys; + int64_t nrows; + ray_t* key_cols[16]; + dense_plan_t dp; +} agg_dense_plan_cache_t; +static _Thread_local agg_dense_plan_cache_t g_dense_plan_cache; + +static bool agg_dense_plan_cached(ray_t** key_cols, uint32_t n_keys, int64_t nrows, + dense_plan_t* out) { + agg_dense_plan_cache_t* c = &g_dense_plan_cache; + bool hit = c->valid && c->n_keys == n_keys && c->nrows == nrows; + for (uint32_t k = 0; hit && k < n_keys; k++) hit = c->key_cols[k] == key_cols[k]; + if (c->valid && !hit) agg_dense_plan_free(&c->dp); + c->valid = false; /* one run per probe */ + if (!hit) return false; + *out = c->dp; /* compaction tables transfer to the run */ + return true; +} + +bool agg_v2_dense_plan_available(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { + if (!g || !op || !tbl) return false; + ray_op_ext_t* ext = find_ext(g, op->id); + if (!ext || ext->n_keys < 1 || ext->n_keys > 16) return false; + agg_dense_plan_cache_t* c = &g_dense_plan_cache; + if (c->valid) agg_dense_plan_free(&c->dp); + c->valid = false; + for (uint32_t k = 0; k < ext->n_keys; k++) { + ray_op_t* key = op_node(g, ext->keys[k]); + ray_op_ext_t* kext = key ? find_ext(g, key->id) : NULL; + c->key_cols[k] = kext ? ray_table_get_col(tbl, kext->sym) : NULL; + if (!c->key_cols[k]) return false; + } + c->n_keys = ext->n_keys; + c->nrows = ray_table_nrows(tbl); + bool ok = agg_dense_plan(c->key_cols, c->n_keys, NULL, 0, c->nrows, &c->dp); + c->valid = ok; + return ok; +} + /* ── Dense grouping eligibility selector (mirrors group.c DA path) ──────── * Decides whether the key tuple packs into a bounded direct-index slot space * (gid = sum_k (key_k - min_k)*strides[k]) so grouping can skip hashing. @@ -206,10 +257,34 @@ static int64_t agg_key_null(int8_t type) { default: return NULL_I64; } } -static inline int64_t agg_dense_component(const dense_plan_t* dp, uint32_t k, int64_t v) { +/* Raw-range component: the hot loops use this whenever the plan has no + * compacted key. A per-row check of the remap pointer inside the row loop + * defeats the compiler's pipelining of the multi-key packing (measured 12x + * slower on a two-integer-key sum), so callers hoist `dp->compacted` out of + * their loops and pick one of the two forms. */ +static inline int64_t agg_dense_component_raw(const dense_plan_t* dp, uint32_t k, int64_t v) { return dp->nullable[k] && v == dp->nulls[k] ? dp->ranges[k] - 1 : v - dp->mins[k]; } +static inline int64_t agg_dense_component(const dense_plan_t* dp, uint32_t k, int64_t v) { + if (dp->nullable[k] && v == dp->nulls[k]) return dp->ranges[k] - 1; + int64_t c = v - dp->mins[k]; + return dp->remap[k] ? dp->remap[k][c] : c; +} +/* Original key code of a dense component (inverse of agg_dense_component). */ +static inline int64_t agg_dense_code(const dense_plan_t* dp, uint32_t k, int64_t component) { + if (dp->nullable[k] && component == dp->ranges[k] - 1) return dp->nulls[k]; + return dp->inverse[k] ? dp->inverse[k][component] : dp->mins[k] + component; +} + +void agg_dense_plan_free(dense_plan_t* dp) { + if (!dp) return; + for (uint32_t k = 0; k < 16; k++) { + ray_free_raw(dp->remap[k]); dp->remap[k] = NULL; + ray_free_raw(dp->inverse[k]); dp->inverse[k] = NULL; + } + dp->compacted = false; +} static bool agg_dense_range(dense_plan_t* dp, uint32_t k, int64_t mn, int64_t mx) { if (mx < mn) { dp->mins[k] = 0; dp->ranges[k] = 1; return dp->nullable[k]; } uint64_t span = (uint64_t)mx - (uint64_t)mn; @@ -300,12 +375,149 @@ static void agg_key_bounds_parallel(ray_t* key, int64_t rows, bool nullable, agg_key_bounds(key, 0, rows, nullable, null, lo, hi); } +/* ── Key compaction for composite plans ────────────────────────────────── + * A composite plan multiplies per-key ranges. SYM columns of a shared + * domain (one runtime domain across every column, or a splayed store's + * shared symfile) interleave their codes with every other column's, so two + * 100-value SYM keys can each span a 100k-code range and their raw product + * (10^10 slots) rejects the plan although only 10^4 groups exist. The + * ranges of sparse integer keys multiply out the same way. When the raw + * product overflows, one parallel pass marks the codes each candidate key + * actually uses; if the product of the used-code counts fits, the plan maps + * each key through a code→component table (agg_dense_component) and emits + * keys through its inverse (agg_dense_code). Only keys whose remap table is + * bounded (AGG_COMPACT_MAX_RANGE codes) are compacted. */ +enum { AGG_COMPACT_MAX_RANGE = 1 << 22, /* largest per-key remap table (codes) */ + AGG_COMPACT_TRY_SLOTS = 1 << 16 }; /* raw products above this try compaction */ + +typedef struct { + ray_t** key_cols; + const dense_plan_t* dp; + const bool* candidate; + uint32_t n_keys; + int64_t rows; + uint32_t tasks; + uint64_t** task_bits; /* [tasks][n_keys] private bitmaps, NULL when not a candidate */ +} agg_compact_ctx_t; + +static void agg_compact_mark_fn(void* raw, uint32_t wid, int64_t start, int64_t end) { + (void)wid; + agg_compact_ctx_t* c = raw; + for (int64_t task = start; task < end; task++) { + int64_t begin = c->rows / c->tasks * task; + int64_t limit = task + 1 == c->tasks ? c->rows : c->rows / c->tasks * (task + 1); + for (uint32_t k = 0; k < c->n_keys; k++) { + if (!c->candidate[k]) continue; + uint64_t* bits = c->task_bits[(size_t)task * c->n_keys + k]; + ray_t* kc = c->key_cols[k]; + const void* d = ray_data(kc); + int64_t mn = c->dp->mins[k]; + bool nullable = c->dp->nullable[k]; + int64_t null = c->dp->nulls[k]; + for (int64_t r = begin; r < limit; r++) { + int64_t v = agg_read_key_i64(kc, d, r); + if (nullable && v == null) continue; + uint64_t code = (uint64_t)(v - mn); + bits[code >> 6] |= UINT64_C(1) << (code & 63); + } + } + } +} + +static bool agg_dense_plan_compact(ray_t** key_cols, uint32_t n_keys, int64_t nrows, + int64_t dense_limit, dense_plan_t* out) { + bool candidate[16]; + bool any = false; + for (uint32_t k = 0; k < n_keys; k++) { + int64_t raw = out->ranges[k] - out->nullable[k]; + candidate[k] = raw > 1 && raw <= AGG_COMPACT_MAX_RANGE; + any |= candidate[k]; + } + if (!any) return false; + ray_pool_t* pool = ray_pool_get(); + uint32_t tasks = pool && nrows >= RAY_PARALLEL_THRESHOLD ? ray_pool_total_workers(pool) : 1; + if ((int64_t)tasks > nrows) tasks = (uint32_t)nrows; + size_t words_total = 0; + for (uint32_t k = 0; k < n_keys; k++) + if (candidate[k]) words_total += (size_t)((out->ranges[k] - out->nullable[k] + 63) / 64); + uint64_t* bits = ray_calloc_raw(words_total * tasks * sizeof(uint64_t)); + uint64_t** task_bits = ray_calloc_raw((size_t)tasks * n_keys * sizeof(uint64_t*)); + if (!bits || !task_bits) { ray_free_raw(bits); ray_free_raw(task_bits); return false; } + { + uint64_t* cursor = bits; + for (uint32_t t = 0; t < tasks; t++) + for (uint32_t k = 0; k < n_keys; k++) { + if (!candidate[k]) continue; + task_bits[(size_t)t * n_keys + k] = cursor; + cursor += (out->ranges[k] - out->nullable[k] + 63) / 64; + } + } + agg_compact_ctx_t c = { .key_cols = key_cols, .dp = out, .candidate = candidate, + .n_keys = n_keys, .rows = nrows, .tasks = tasks, .task_bits = task_bits }; + if (tasks > 1) ray_pool_dispatch_n(pool, agg_compact_mark_fn, &c, tasks); + else agg_compact_mark_fn(&c, 0, 0, 1); + /* Reduce task bitmaps into task 0, count used codes, re-check the product. */ + int64_t used[16]; + int64_t total = 1; + bool fits = true; + for (uint32_t k = 0; k < n_keys; k++) { + int64_t rng = out->ranges[k]; + if (candidate[k]) { + size_t words = (size_t)((rng - out->nullable[k] + 63) / 64); + uint64_t* acc = task_bits[k]; + for (uint32_t t = 1; t < tasks; t++) { + const uint64_t* tb = task_bits[(size_t)t * n_keys + k]; + for (size_t w = 0; w < words; w++) acc[w] |= tb[w]; + } + int64_t n = 0; + for (size_t w = 0; w < words; w++) n += __builtin_popcountll(acc[w]); + used[k] = n; + rng = n + out->nullable[k]; + if (rng <= 0) rng = 1; + } else used[k] = rng; + if (fits && total > dense_limit / rng) fits = false; + if (fits) total *= rng; + } + if (!fits) { ray_free_raw(bits); ray_free_raw(task_bits); return false; } + /* Build remap/inverse for every candidate key that actually shrank. */ + for (uint32_t k = 0; k < n_keys; k++) { + if (!candidate[k]) continue; + int64_t raw = out->ranges[k] - out->nullable[k]; + if (used[k] >= raw) continue; /* every code used: raw range is already dense */ + int32_t* remap = ray_alloc_raw((size_t)raw * sizeof(int32_t)); + int64_t* inverse = ray_alloc_raw((size_t)(used[k] > 0 ? used[k] : 1) * sizeof(int64_t)); + if (!remap || !inverse) { + ray_free_raw(remap); ray_free_raw(inverse); + ray_free_raw(bits); ray_free_raw(task_bits); + agg_dense_plan_free(out); + return false; + } + const uint64_t* acc = task_bits[k]; + int64_t next = 0; + for (int64_t code = 0; code < raw; code++) { + bool on = (acc[code >> 6] >> (code & 63)) & 1; + remap[code] = on ? (int32_t)next : -1; + if (on) inverse[next++] = out->mins[k] + code; + } + out->remap[k] = remap; + out->inverse[k] = inverse; + out->compacted = true; + out->ranges[k] = used[k] + out->nullable[k]; + if (out->ranges[k] <= 0) out->ranges[k] = 1; + } + ray_free_raw(bits); ray_free_raw(task_bits); + return true; +} + bool agg_dense_plan(ray_t** key_cols, uint32_t n_keys, const agg_vtable_t** vts, uint32_t n_aggs, int64_t nrows, dense_plan_t* out) { (void)vts; (void)n_aggs; /* agg kind no longer gates dense eligibility */ out->ok = false; out->n_keys = n_keys; + out->compacted = false; + memset(out->remap, 0, sizeof(out->remap)); + memset(out->inverse, 0, sizeof(out->inverse)); /* Dense direct-index routing self-limits to <=16 keys: dense_plan_t's * mins/ranges/strides are fixed [16] (the direct-index packing bound). * n_keys is uint32_t (untruncated ext->n_keys), so a wider shape is @@ -364,12 +576,33 @@ bool agg_dense_plan(ray_t** key_cols, uint32_t n_keys, int64_t total = 1; int64_t dense_limit = nrows < (int64_t)UINT32_MAX ? nrows : (int64_t)UINT32_MAX; - for (uint32_t k = 0; k < n_keys; k++) { + bool overflow = false; + for (uint32_t k = 0; k < n_keys && !overflow; k++) { int64_t rng = out->ranges[k]; if (rng <= 0) return false; - if (total > dense_limit / rng) return false; + if (total > dense_limit / rng) overflow = true; + else total *= rng; + } + if (overflow || (n_keys >= 2 && total > AGG_COMPACT_TRY_SLOTS)) { + /* Raw ranges multiply out too far — or far enough past a + * cache-resident slab that sparse keys would waste it (a W16 SYM key + * reports its whole 65,536-code width): compact the keys to the codes + * they use and retry the packing over the compacted ranges. Keys + * whose every code is used keep their raw range. */ + bool compacted_ok = n_keys >= 2 && + agg_dense_plan_compact(key_cols, n_keys, nrows, dense_limit, out); + if (overflow && !compacted_ok) return false; + total = 1; + for (uint32_t k = 0; k < n_keys; k++) { + int64_t rng = out->ranges[k]; + if (rng <= 0 || total > dense_limit / rng) { agg_dense_plan_free(out); return false; } + total *= rng; + } + } + total = 1; + for (uint32_t k = 0; k < n_keys; k++) { out->strides[k] = total; - total *= rng; + total *= out->ranges[k]; } out->total_slots = total; @@ -859,11 +1092,57 @@ static int agg_sel_accum_chunk(const agg_valdesc_t* vd, agg_sel_scratch_t* sc, /* n_keys is uint32_t (untruncated ext->n_keys): the >16 shapes are rejected by * the same dense direct-index bound as agg_dense_plan and take v2's unbounded * hash/radix path; the fixed [16] mins/ranges/strides are only read at <=16. */ +typedef struct { + ray_t** key_cols; + const void** key_data; + const bool* nullable; + const int64_t* nulls; + uint32_t n_keys; + ray_t* sel; + const int64_t* sel_prefix; + int64_t n_sel; + uint32_t tasks; + int64_t* task_bounds; /* [tasks][2][n_keys]: min row, max row per key */ +} agg_sel_bounds_ctx_t; + +/* One task's min/max over its slice of selected-row space. */ +static void agg_sel_bounds_fn(void* raw, uint32_t wid, int64_t start, int64_t end) { + (void)wid; + agg_sel_bounds_ctx_t* c = raw; + int64_t rows[AGG_SEL_CHUNK]; + for (int64_t task = start; task < end; task++) { + int64_t* mins = c->task_bounds + (2 * (size_t)task) * c->n_keys; + int64_t* maxs = mins + c->n_keys; + for (uint32_t k = 0; k < c->n_keys; k++) { mins[k] = INT64_MAX; maxs[k] = INT64_MIN; } + int64_t begin = c->n_sel / c->tasks * task; + int64_t limit = task + 1 == c->tasks ? c->n_sel : c->n_sel / c->tasks * (task + 1); + agg_sel_cursor_t cur; + agg_sel_cursor_init(&cur, c->sel, c->sel_prefix, begin, limit); + int64_t cn; + while ((cn = agg_sel_cursor_next(&cur, rows)) > 0) { + for (uint32_t k = 0; k < c->n_keys; k++) { + ray_t* kc = c->key_cols[k]; const void* d = c->key_data[k]; + int64_t mn = mins[k], mx = maxs[k]; + for (int64_t i = 0; i < cn; i++) { + int64_t v = agg_read_key_i64(kc, d, rows[i]); + if (c->nullable[k] && v == c->nulls[k]) continue; + if (v < mn) mn = v; + if (v > mx) mx = v; + } + mins[k] = mn; maxs[k] = mx; + } + } + } +} + static bool agg_dense_plan_sel(ray_t** key_cols, uint32_t n_keys, int64_t n_sel, ray_t* sel, const int64_t* sel_prefix, dense_plan_t* out) { out->ok = false; out->n_keys = n_keys; + out->compacted = false; + memset(out->remap, 0, sizeof(out->remap)); + memset(out->inverse, 0, sizeof(out->inverse)); if (n_keys < 1 || n_keys > 16) return false; if (n_sel <= 0) return false; @@ -880,7 +1159,9 @@ static bool agg_dense_plan_sel(ray_t** key_cols, uint32_t n_keys, int64_t n_sel, out->mins[k] = INT64_MAX; out->ranges[k] = 0; /* sentinels; filled below */ } - /* One pass over the selected rows updating every key's min/max together. */ + /* One pass over the selected rows updating every key's min/max together. + * The pass is split across the pool: a serial walk of a 10M-row selection + * cost ~13 ms on every filtered group-by regardless of core count. */ ray_t* pre_hdr; void* pre = scratch_alloc(&pre_hdr, 3u * (size_t)n_keys * 8); /* mins,maxs,key_data */ if (!pre) return false; @@ -890,23 +1171,26 @@ static bool agg_dense_plan_sel(ray_t** key_cols, uint32_t n_keys, int64_t n_sel, for (uint32_t k = 0; k < n_keys; k++) { mins[k] = INT64_MAX; maxs[k] = INT64_MIN; } for (uint32_t k = 0; k < n_keys; k++) key_data[k] = ray_data(key_cols[k]); - int64_t rows[AGG_SEL_CHUNK]; - agg_sel_cursor_t cur; - agg_sel_cursor_init(&cur, sel, sel_prefix, 0, n_sel); - int64_t cn; - while ((cn = agg_sel_cursor_next(&cur, rows)) > 0) { + ray_pool_t* pool = ray_pool_get(); + uint32_t tasks = pool && n_sel >= RAY_PARALLEL_THRESHOLD ? ray_pool_total_workers(pool) * 4 : 1; + if (tasks > RAY_POOL_INIT_TASKS) tasks = RAY_POOL_INIT_TASKS; + if ((int64_t)tasks > n_sel) tasks = (uint32_t)n_sel; + agg_sel_bounds_ctx_t bc = { .key_cols = key_cols, .key_data = key_data, + .nullable = out->nullable, .nulls = out->nulls, .n_keys = n_keys, + .sel = sel, .sel_prefix = sel_prefix, .n_sel = n_sel, .tasks = tasks }; + ray_t* tb_hdr; + bc.task_bounds = (int64_t*)scratch_alloc(&tb_hdr, 2u * (size_t)tasks * n_keys * sizeof(int64_t)); + if (!bc.task_bounds) { scratch_free(pre_hdr); return false; } + if (tasks > 1) ray_pool_dispatch_n(pool, agg_sel_bounds_fn, &bc, tasks); + else agg_sel_bounds_fn(&bc, 0, 0, 1); + for (uint32_t t = 0; t < tasks; t++) for (uint32_t k = 0; k < n_keys; k++) { - ray_t* kc = key_cols[k]; const void* d = key_data[k]; - int64_t mn = mins[k], mx = maxs[k]; - for (int64_t i = 0; i < cn; i++) { - int64_t v = agg_read_key_i64(kc, d, rows[i]); - if (out->nullable[k] && v == out->nulls[k]) continue; - if (v < mn) mn = v; - if (v > mx) mx = v; - } - mins[k] = mn; maxs[k] = mx; + int64_t mn = bc.task_bounds[(2 * (size_t)t) * n_keys + k]; + int64_t mx = bc.task_bounds[(2 * (size_t)t + 1) * n_keys + k]; + if (mn < mins[k]) mins[k] = mn; + if (mx > maxs[k]) maxs[k] = mx; } - } + scratch_free(tb_hdr); for (uint32_t k = 0; k < n_keys; k++) { if (!agg_dense_range(out, k, mins[k], maxs[k])) { scratch_free(pre_hdr); return false; } } @@ -1127,11 +1411,18 @@ static void agg_dense_emit_fn(void* raw, uint32_t wid, int64_t start, int64_t en if (any) atomic_store_explicit(&c->any_null, true, memory_order_relaxed); } -/* Compute the dense slot for row r via mixed-radix packing. */ -static inline int64_t agg_dense_slot(const agg_dense_ctx_t* c, int64_t r) { +/* Compute the dense slot for row r via mixed-radix packing. `compacted` + * is loop-invariant (dp->compacted); callers pass it from a local so the + * raw form stays branch-free. */ +static inline int64_t agg_dense_slot(const agg_dense_ctx_t* c, int64_t r, bool compacted) { int64_t slot = 0; - for (uint32_t k = 0; k < c->n_keys; k++) - slot += agg_dense_component(c->dp, k, agg_read_key_i64(c->key_cols[k], c->key_data[k], r)) * c->dp->strides[k]; + if (compacted) { + for (uint32_t k = 0; k < c->n_keys; k++) + slot += agg_dense_component(c->dp, k, agg_read_key_i64(c->key_cols[k], c->key_data[k], r)) * c->dp->strides[k]; + } else { + for (uint32_t k = 0; k < c->n_keys; k++) + slot += agg_dense_component_raw(c->dp, k, agg_read_key_i64(c->key_cols[k], c->key_data[k], r)) * c->dp->strides[k]; + } return slot; } @@ -1163,10 +1454,11 @@ static void agg_dense_phaseA_fn(void* vctx, uint32_t wid, int64_t start, int64_t agg_sel_cursor_t cur; agg_sel_cursor_init(&cur, c->sel, c->sel_prefix, start, end); int64_t cn; + const bool compacted = c->dp->compacted; while ((cn = agg_sel_cursor_next(&cur, rows)) > 0) { for (int64_t i = 0; i < cn; i++) { int64_t r = rows[i]; - int64_t slot = agg_dense_slot(c, r); /* provably in [0,total_slots) */ + int64_t slot = agg_dense_slot(c, r, compacted); /* provably in [0,total_slots) */ gid[i] = (uint32_t)slot; if (agg_dense_first(loc, slot)) { for (uint32_t a = 0; a < c->n_aggs; a++) @@ -1242,7 +1534,7 @@ static void agg_dense_phaseA_fn(void* vctx, uint32_t wid, int64_t start, int64_t break; default: for (int64_t r = start; r < end; r++) { - int64_t slot = agg_dense_slot(c, r); + int64_t slot = agg_dense_slot(c, r, false); /* single key: never compacted */ cg[r - start] = (uint32_t)slot; if (r < fr[slot]) fr[slot] = r; } @@ -1251,15 +1543,19 @@ static void agg_dense_phaseA_fn(void* vctx, uint32_t wid, int64_t start, int64_t #undef DENSE_SLOT1_LAZY #undef DENSE_SLOT1_EAGER } else { - for (int64_t r = start; r < end; r++) { - int64_t slot = agg_dense_slot(c, r); /* provably in [0,total_slots) */ - cgid[r - start] = (uint32_t)slot; - if (agg_dense_first(loc, slot)) { - for (uint32_t a = 0; a < c->n_aggs; a++) - c->vts[a]->init(loc->states + (size_t)slot * c->block + c->off[a]); - loc->first_row[slot] = r; + #define DENSE_SLOT_MULTI(COMPACTED) \ + for (int64_t r = start; r < end; r++) { \ + int64_t slot = agg_dense_slot(c, r, COMPACTED); /* provably in [0,total_slots) */ \ + cgid[r - start] = (uint32_t)slot; \ + if (agg_dense_first(loc, slot)) { \ + for (uint32_t a = 0; a < c->n_aggs; a++) \ + c->vts[a]->init(loc->states + (size_t)slot * c->block + c->off[a]); \ + loc->first_row[slot] = r; \ + } \ } - } + if (c->dp->compacted) { DENSE_SLOT_MULTI(true); } + else { DENSE_SLOT_MULTI(false); } + #undef DENSE_SLOT_MULTI } for (uint32_t a = 0; a < c->n_aggs; a++) { @@ -1328,9 +1624,7 @@ static void agg_dense_key_emit(void* raw, uint32_t wid, int64_t start, int64_t e int64_t slot = c->bits ? ((index % c->part_slots) << c->bits) + index / c->part_slots : index; uint32_t k = c->component; int64_t component = (slot / c->plan->strides[k]) % c->plan->ranges[k]; - int64_t value = c->plan->nullable[k] && component == c->plan->ranges[k] - 1 - ? c->plan->nulls[k] : c->plan->mins[k] + component; - write_col_i64(data, r, value, c->out->type, c->out->attrs); + write_col_i64(data, r, agg_dense_code(c->plan, k, component), c->out->type, c->out->attrs); } } @@ -1474,20 +1768,24 @@ static void agg_dense_partition_count(void* raw, uint32_t wid, int64_t start, in int64_t limit = task + 1 == c->sources ? c->rows : c->rows / c->sources * (task + 1); uint64_t* count = c->counts + task * c->parts; if (c->plan->n_keys > 1) { - for (int64_t r = begin; r < limit; r++) { - int64_t source = c->selected_rows ? c->selected_rows[r] : r; - int64_t slot = 0; - for (uint32_t k = 0; k < c->plan->n_keys; k++) - slot += agg_dense_component(c->plan, k, agg_read_key_i64(c->keys[k], c->key_data[k], source)) * c->plan->strides[k]; - c->gids[r] = (uint32_t)slot; - count[slot & (c->parts - 1)]++; - } + #define PART_SLOT_MULTI(COMPONENT) \ + for (int64_t r = begin; r < limit; r++) { \ + int64_t source = c->selected_rows ? c->selected_rows[r] : r; \ + int64_t slot = 0; \ + for (uint32_t k = 0; k < c->plan->n_keys; k++) \ + slot += COMPONENT(c->plan, k, agg_read_key_i64(c->keys[k], c->key_data[k], source)) * c->plan->strides[k]; \ + c->gids[r] = (uint32_t)slot; \ + count[slot & (c->parts - 1)]++; \ + } + if (c->plan->compacted) { PART_SLOT_MULTI(agg_dense_component); } + else { PART_SLOT_MULTI(agg_dense_component_raw); } + #undef PART_SLOT_MULTI continue; } #define PART_COUNT(T) do { \ const T* p = data; \ for (int64_t r = begin; r < limit; r++) { \ - uint32_t slot = (uint32_t)agg_dense_component(c->plan, 0, (int64_t)p[c->selected_rows ? c->selected_rows[r] : r]); \ + uint32_t slot = (uint32_t)agg_dense_component_raw(c->plan, 0, (int64_t)p[c->selected_rows ? c->selected_rows[r] : r]); \ c->gids[r] = slot; \ count[slot & (c->parts - 1)]++; \ } \ @@ -1839,7 +2137,7 @@ static void agg_dense_shared_rows(void* raw, uint32_t wid, int64_t start, int64_ #define SHARED_GIDS(T) do { \ const T* p = data; \ for (int64_t i = 0; i < n; i++) { \ - uint32_t slot = (uint32_t)agg_dense_component(c->plan, 0, (int64_t)p[begin + i]); \ + uint32_t slot = (uint32_t)agg_dense_component_raw(c->plan, 0, (int64_t)p[begin + i]); \ gids[i] = slot; \ uint64_t bit = UINT64_C(1) << (slot % 64); \ _Atomic(uint64_t)* word = &c->occupied[slot / 64]; \ @@ -4133,8 +4431,16 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, * not O(nrows)) — both correct (grouped keys ARE the selected rows) and cheap * for high-selectivity filters, and it can pull a sparse-but-low-card selected * key set into the dense path that the full-table range would have rejected. */ + /* Every run consumes the probe's cached plan (agg_dense_plan_cached + * invalidates it), so a plan can never outlive the query it was probed + * for and match a recycled column pointer later. Selected runs plan + * over the selected rows instead and discard the cached full-table plan. */ dense_plan_t dp; + dense_plan_t probed; + bool probed_hit = agg_dense_plan_cached(key_cols, ext->n_keys, nrows, &probed); + if (probed_hit && sel) { agg_dense_plan_free(&probed); probed_hit = false; } bool dense = sel ? agg_dense_plan_sel(key_cols, ext->n_keys, n_sel, sel, sel_prefix, &dp) + : probed_hit ? (dp = probed, true) : agg_dense_plan(key_cols, ext->n_keys, vts, ext->n_aggs, nrows, &dp); route_stats.dense_plan_available = dense; @@ -4144,7 +4450,7 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, #define AGG_RUN_COMPACT_FALLBACK() \ do { \ agg_vo_free(&vo); /* not needed by the compact recursion */ \ - scratch_free(kc_hdr); /* key_cols dead: the recursion rebuilds them */ \ + agg_dense_plan_free(&dp); scratch_free(kc_hdr); /* key_cols dead: the recursion rebuilds them */ \ ray_t* idxb = ray_rowsel_to_indices(sel); \ if (!idxb) return ray_error("oom", NULL); \ ray_t* compact = agg_build_compact(g, op, tbl, (int64_t*)ray_data(idxb), n_sel); \ @@ -4183,6 +4489,24 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (dense_budget > (double)SIZE_MAX) dense_budget = (double)SIZE_MAX; if (watermark > 0 && dense_budget > (double)watermark / 4) dense_budget = (double)watermark / 4; double slab_bytes = dp.ok ? (double)dp.total_slots * (block + sizeof(int64_t) + 1) : 0; + /* Replicated slabs must stay resident in the last-level cache. Each + * task updates random slots of its own slab, so once the slabs + * together outgrow the LLC every update misses to DRAM and adding + * tasks makes the query slower: 100k groups over 10M rows measured + * 6 ms with 8 slabs (26 MB inside a 33 MB L3) and 21 ms with 28 + * slabs (92 MB) on the same 28-thread pool. Bound the number of + * replicated task slabs by three quarters of the LLC (the rest + * streams the input); an unreported cache assumes 32 MB. The + * memory budgets below remain data-derived; this bound only limits + * replication, and larger domains use partition ownership, which + * scales with cores because every partition slab is L1-sized. */ + uint64_t llc_bytes = ray_cache_llc_bytes(); + double cache_budget = (llc_bytes ? (double)llc_bytes : 32.0 * 1048576.0) * 0.75; + uint32_t cache_tasks = dense_workers; + if (slab_bytes > 0 && slab_bytes * dense_workers > cache_budget) { + double fit = cache_budget / slab_bytes; + cache_tasks = fit >= 1 ? (uint32_t)fit : 0; + } /* A bounded group emit selects first-seen groups. Dense slot order * cannot satisfy that contract; retain radix's bounded selection. */ bool shared_plan = dp.ok && group_limit <= 0 && !sel && ext->n_keys == 1 && dp.total_slots >= 4096; @@ -4193,7 +4517,7 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, agg_route_record(AGG_ROUTE_V2_DENSE); ray_t* result = exec_group_v2_parallel_dense(g, op, tbl, key_cols, key_syms, ext, nrows, pool, &dp, dense_workers, AGG_DENSE_SHARED, sel, sel_prefix, n_sel); - agg_vo_free(&vo); scratch_free(kc_hdr); + agg_vo_free(&vo); agg_dense_plan_free(&dp); scratch_free(kc_hdr); return result; } /* Partition ownership amortizes scatter through concurrent reducers. @@ -4220,7 +4544,8 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, * pass. Prefer partition ownership only once replicated state * traffic exceeds its row traffic; large pools/ranges still use * bounded shared storage. */ - double local_traffic = dense_workers * slab_bytes; + uint32_t local_tasks = cache_tasks < dense_workers ? cache_tasks : dense_workers; + double local_traffic = local_tasks * slab_bytes; double partition_traffic = (double)eff_n * (sizeof(uint32_t) + record_size); /* Compare the complete partition allocation against radix's * payload plus its worst-case per-row group state. A payload-only @@ -4230,18 +4555,22 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (partition_budget > (double)SIZE_MAX) partition_budget = (double)SIZE_MAX; if (watermark > 0 && partition_budget > (double)watermark / 4) partition_budget = (double)watermark / 4; - if (bytes <= partition_budget && local_traffic > partition_traffic) { + /* Fewer than three cache-resident slabs cannot use the pool; + * partition ownership keeps every core busy instead. */ + bool cache_starved = local_tasks < 3 && local_tasks < dense_workers; + if (bytes <= partition_budget && (local_traffic > partition_traffic || cache_starved)) { route_stats.dense_worker_budget = false; agg_route_record(AGG_ROUTE_V2_DENSE); ray_t* result = exec_group_v2_parallel_dense(g, op, tbl, key_cols, key_syms, ext, nrows, pool, &dp, dense_workers, AGG_DENSE_PARTITIONED, sel, sel_prefix, n_sel); - agg_vo_free(&vo); scratch_free(kc_hdr); + agg_vo_free(&vo); agg_dense_plan_free(&dp); scratch_free(kc_hdr); return result; } } if (dp.ok && slab_bytes > 0) { double fit = (dense_budget - (double)eff_n * sizeof(uint32_t)) / slab_bytes - 1; if (fit < dense_workers) dense_workers = fit >= 2 ? (uint32_t)fit : 0; + if (cache_tasks < dense_workers) dense_workers = cache_tasks >= 2 ? cache_tasks : 0; } bool dense_par_ok = dp.ok && group_limit <= 0 && dense_workers > 0; /* Allocation size alone misses repeated wide-range worker updates. @@ -4296,6 +4625,7 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, uint32_t extra_tasks = dense_workers * 4; if (extra_tasks > RAY_POOL_INIT_TASKS) extra_tasks = RAY_POOL_INIT_TASKS; if (dense_workers > 1 && extra_tasks * slab_bytes <= scatter_budget / 8 && + extra_tasks * slab_bytes <= cache_budget && (extra_tasks + 1.0) * slab_bytes + (double)eff_n * sizeof(uint32_t) <= dense_budget) dense_workers = extra_tasks; route_stats.dense_tasks = dense_workers; @@ -4305,7 +4635,7 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, agg_route_record(AGG_ROUTE_V2_DENSE); ray_t* r = exec_group_v2_parallel_dense(g, op, tbl, key_cols, key_syms, ext, nrows, pool, &dp, dense_workers, AGG_DENSE_TASK_LOCAL, sel, sel_prefix, n_sel); - agg_vo_free(&vo); scratch_free(kc_hdr); return r; + agg_vo_free(&vo); agg_dense_plan_free(&dp); scratch_free(kc_hdr); return r; } if (keys_intsym) { agg_route_record(AGG_ROUTE_V2_RADIX); @@ -4313,13 +4643,13 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, ray_t* r = exec_group_v2_parallel_radix(g, op, tbl, nrows, key_cols, key_syms, vts, off, block, sel, sel_prefix, n_sel, group_limit); - agg_vo_free(&vo); scratch_free(kc_hdr); return r; + agg_vo_free(&vo); agg_dense_plan_free(&dp); scratch_free(kc_hdr); return r; } /* Hash fallback (F64 / STR keys): not a chunked strategy — compact. */ if (sel) AGG_RUN_COMPACT_FALLBACK(); agg_route_record(AGG_ROUTE_V2_INDEXED); { ray_t* r = agg_indexed_run(g, op, tbl, key_cols, key_syms, nrows); - agg_vo_free(&vo); scratch_free(kc_hdr); return r; } + agg_vo_free(&vo); agg_dense_plan_free(&dp); scratch_free(kc_hdr); return r; } } /* Serial path does not consult the vts/off tables (per-agg vt is re-resolved @@ -4334,18 +4664,18 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, agg_groups_t groups = {0}; int grp_rc = dense ? agg_group_keys_dense(key_cols, nrows, &dp, &groups) : agg_group_keys(key_cols, ext->n_keys, nrows, &groups); - if (grp_rc != 0) { scratch_free(kc_hdr); return ray_error("oom", NULL); } + if (grp_rc != 0) { agg_dense_plan_free(&dp); scratch_free(kc_hdr); return ray_error("oom", NULL); } ray_t* result = ray_table_new(ext->n_keys + ext->n_aggs); - if (!result || RAY_IS_ERR(result)) { scratch_free(kc_hdr); agg_groups_free(&groups); return ray_error("oom", NULL); } + if (!result || RAY_IS_ERR(result)) { agg_dense_plan_free(&dp); scratch_free(kc_hdr); agg_groups_free(&groups); return ray_error("oom", NULL); } for (uint32_t k = 0; k < ext->n_keys; k++) { ray_t* kc = ray_group_gather(key_cols[k], groups.first_row, groups.ngroups); - if (!kc || RAY_IS_ERR(kc)) { scratch_free(kc_hdr); agg_groups_free(&groups); ray_release(result); return kc ? kc : ray_error("oom", NULL); } + if (!kc || RAY_IS_ERR(kc)) { agg_dense_plan_free(&dp); scratch_free(kc_hdr); agg_groups_free(&groups); ray_release(result); return kc ? kc : ray_error("oom", NULL); } result = ray_table_add_col(result, key_syms[k], kc); ray_release(kc); } - scratch_free(kc_hdr); /* key_cols/key_syms done — agg loop below reads neither */ + agg_dense_plan_free(&dp); scratch_free(kc_hdr); /* key_cols/key_syms done — agg loop below reads neither */ for (uint32_t a = 0; a < ext->n_aggs; a++) { ray_op_ext_t* ie = find_ext(g, ext->agg_ins[a]); @@ -4598,10 +4928,16 @@ static int agg_group_keys_dense(ray_t** key_cols, int64_t nrows, } for (int64_t s = 0; s < dp->total_slots; s++) slot2gid[s] = -1; int64_t ngroups = 0; + const bool compacted = dp->compacted; for (int64_t r = 0; r < nrows; r++) { int64_t slot = 0; - for (uint32_t k = 0; k < dp->n_keys; k++) - slot += agg_dense_component(dp, k, agg_read_key_i64(key_cols[k], data[k], r)) * dp->strides[k]; + if (compacted) { + for (uint32_t k = 0; k < dp->n_keys; k++) + slot += agg_dense_component(dp, k, agg_read_key_i64(key_cols[k], data[k], r)) * dp->strides[k]; + } else { + for (uint32_t k = 0; k < dp->n_keys; k++) + slot += agg_dense_component_raw(dp, k, agg_read_key_i64(key_cols[k], data[k], r)) * dp->strides[k]; + } /* slot is provably in [0,total_slots): each key in [min_k,max_k] so * (key-min) in [0,range_k), and the composite is a mixed-radix index * < total_slots (dp->ok from the same prescan). */ diff --git a/src/ops/agg_engine.h b/src/ops/agg_engine.h index 7cd84d75..bee911fa 100644 --- a/src/ops/agg_engine.h +++ b/src/ops/agg_engine.h @@ -75,6 +75,10 @@ void agg_route_reason(agg_v2_reason_t reason); * Conservative: any uncertainty → false → caller uses the existing engine. */ bool agg_v2_can_handle(ray_graph_t* g, ray_op_t* op, ray_t* tbl); +/* True when v2 would group this node through a bounded dense plan (see the + * definition for the strategy-prediction contract). */ +bool agg_v2_dense_plan_available(ray_graph_t* g, ray_op_t* op, ray_t* tbl); + /* Precondition: agg_v2_can_handle(g, op, tbl) returned true. * `group_limit` is the HEAD(GROUP) row-limit HINT (0 = no limit): when * positive, an engine strategy may emit only the first `group_limit` groups in @@ -149,8 +153,18 @@ typedef struct { int64_t ranges[16]; /* [16]: dense direct-index routing self-limits to <=16 keys (agg_dense_plan) */ int64_t strides[16]; /* [16]: dense self-limit <=16; composite packing: slot = sum_k (key_k - min_k)*strides[k] */ int64_t total_slots; /* product of ranges */ + /* Compacted keys (composite plans whose raw range product overflowed): + * remap[k][code - mins[k]] is the dense component of a code that occurs + * in the input, inverse[k][component] the original code. NULL for keys + * that use their raw range. Owned by the plan: agg_dense_plan_free. */ + int32_t* remap[16]; + int64_t* inverse[16]; + bool compacted; /* any remap set: hot loops select the remap form once */ } dense_plan_t; +/* Release a plan's compaction tables (no-op for raw-range plans). */ +void agg_dense_plan_free(dense_plan_t* dp); + /* Decide if dense grouping applies to (key_cols, aggs). Eligible iff: * - every key type in {I64,I32,I16,U8,BOOL,DATE,TIME,TIMESTAMP,SYM} with a dedicated slot for nullable keys * - product of per-key ranges is no larger than the contributing row count diff --git a/src/ops/group.c b/src/ops/group.c index b79276e2..c63c3926 100644 --- a/src/ops/group.c +++ b/src/ops/group.c @@ -11409,50 +11409,54 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (admission == AGG_V2_ADMITTED) return exec_group_v2(g, op, tbl, group_limit); - /* Emit-filter shape on a wide-domain SYM key: the sp dense/sparse - * ladder below is single-threaded and its dense array scales with the - * store's SHARED sym domain (splayed stores keep one domain across all - * SYM columns — often 10M+ ids), so the scatter becomes the query's - * serial wall (ClickBench q13: 88ms of a 110ms query, flat multi-core - * scaling). Run the PARALLEL v2 engine instead and trim its full - * result to the filter's top-N superset — the emit filter is purely an - * optimization; the DAG's sort+take downstream produces the final - * order/limit either way. */ + /* Emit-filter shapes (`desc: c take: N`, `where (> c N)` on the group + * result) used to run on the legacy ladder because v2 does not + * implement the top-count emit filter. That ladder is single-threaded + * in its scatter and its dense array scales with the store's SHARED sym + * domain, so every top-N group query lost the parallel engine: a + * three-aggregate 100k-group query measured 800 ms on one core and + * 777 ms on a 28-thread pool (0.1x parallel), while the same grouping + * without the filter ran in 47 ms. The emit filter is purely + * an optimization — the DAG's sort+take downstream produces the final + * order/limit either way — so run the PARALLEL v2 engine on every shape + * it admits with a bounded dense plan and trim its full result to the + * filter's top-N superset. Unbounded key domains (many-million-group + * inputs) stay on the legacy ladder for now: v2's radix route still + * pays a serial first-seen ordering and emission tail there (a 10M-group + * three-key count measured 154 ms against the ladder's 54 ms on 28 + * threads). One unbounded shape still prefers v2: a single wide-domain + * SYM key on a bounded input, where the ladder's dense scatter scales + * with the store's SHARED sym domain (splayed stores keep one domain + * across all SYM columns — often 10M+ ids) and became the serial wall + * (88 ms of a 110 ms benchmark query, flat multi-core scaling). */ { ray_group_emit_filter_t ef = ray_group_emit_filter_get(); - if (ray_agg_engine_v2 && group_limit == 0 && ef.enabled - && ext->n_keys == 1 + bool wide_sym_key = false; + if (ef.enabled && ext->n_keys == 1 && ray_table_nrows(tbl) <= (int64_t)(4u << 20) && (ef.agg_op == 0 || ef.agg_op == OP_COUNT || ef.agg_op == OP_SUM - || ef.agg_op == OP_MIN || ef.agg_op == OP_MAX) - && agg_v2_can_handle(g, op, tbl)) { + || ef.agg_op == OP_MIN || ef.agg_op == OP_MAX)) { ray_op_t* k0 = op_node(g, ext->keys[0]); ray_op_ext_t* k0e = k0 ? find_ext(g, k0->id) : NULL; ray_t* k0c = (k0e && k0e->base.opcode == OP_SCAN) ? ray_table_get_col(tbl, k0e->sym) : NULL; - /* Input-size gate: on RAW-table inputs (10M+ rows) consecutive - * rows repeat keys, so the serial dense scatter mostly hits - * cache and beats the radix pipeline (ClickBench q33/q34: - * 56ms serial vs 148ms via v2). The pathological case is the - * count-distinct SECOND phase, whose distinct-pairs - * intermediate (~1M rows) has no locality — every increment - * misses (q13: 88ms serial). Intermediates are bounded by - * their distinct count; raw fact tables are not. */ - if (k0c && k0c->type == RAY_SYM && - ray_table_nrows(tbl) <= (int64_t)(4u << 20) && - ray_sym_domain_count(ray_sym_vec_domain(k0c)) > (1 << 21)) { - /* Suppress the filter for the v2 run (v2 ignores it anyway; - * clearing keeps recursion/asserts honest), restore after. */ - ray_group_emit_filter_t saved = ray_group_emit_filter_get(); - ray_group_emit_filter_t off = {0}; - ray_group_emit_filter_set(off); - ray_t* r = exec_group_v2(g, op, tbl, 0); - ray_group_emit_filter_set(saved); - if (r && !RAY_IS_ERR(r)) - return group_emit_filter_trim(r, ext->n_keys, - ext->n_aggs, ef); - if (r) return r; - /* v2 declined at runtime — continue on the legacy ladder. */ - } + wide_sym_key = k0c && k0c->type == RAY_SYM && + ray_sym_domain_count(ray_sym_vec_domain(k0c)) > (1 << 21); + } + if (ray_agg_engine_v2 && group_limit == 0 && ef.enabled + && agg_v2_can_handle(g, op, tbl) + && (wide_sym_key || agg_v2_dense_plan_available(g, op, tbl))) { + /* Suppress the filter for the v2 run (v2 ignores it anyway; + * clearing keeps recursion/asserts honest), restore after. */ + ray_group_emit_filter_t saved = ray_group_emit_filter_get(); + ray_group_emit_filter_t off = {0}; + ray_group_emit_filter_set(off); + ray_t* r = exec_group_v2(g, op, tbl, 0); + ray_group_emit_filter_set(saved); + if (r && !RAY_IS_ERR(r)) + return group_emit_filter_trim(r, ext->n_keys, + ext->n_aggs, ef); + if (r) return r; + /* v2 declined at runtime — continue on the legacy ladder. */ } } diff --git a/src/ops/query.c b/src/ops/query.c index 0a07c70d..5f23f2d2 100644 --- a/src/ops/query.c +++ b/src/ops/query.c @@ -2009,15 +2009,33 @@ static int sg_agg_expr_ok(ray_t* expr) { return 0; } +/* Aggregate shapes the hidden-slot compiler accepts: unary aggregates, + * binary aggregates over two bare columns (pearson, covariance, wsum, ...), + * and quantile with its literal probability. List-valued top/bottom K and + * anything wider keep the per-group scatter path, whose element-wise + * semantics differ from a post-group column expression. */ +static int hidden_agg_shape_ok(ray_t* expr) { + int64_t n = ray_len(expr); + if (n == 2) return 1; + if (n != 3) return 0; + ray_t** e = (ray_t**)ray_data(expr); + uint16_t op = resolve_agg_opcode(e[0]->i64); + if (agg_is_binary_agg(op)) + return e[1] && e[1]->type == -RAY_SYM && !(e[1]->attrs & ATTR_QUOTED) && + e[2] && e[2]->type == -RAY_SYM && !(e[2]->attrs & ATTR_QUOTED); + return op == OP_QUANTILE; +} + static ray_t* agg_arith_rewrite(ray_t* expr, ray_t* tbl, ray_t** hexprs, int64_t* hnames, int* n_hidden, int cap, int* ok) { if (!*ok || !expr) { *ok = 0; return NULL; } if (expr->type == RAY_LIST && is_group_dag_agg_expr(expr) && - ray_len(expr) == 2) { + hidden_agg_shape_ok(expr)) { ray_t** el = (ray_t**)ray_data(expr); /* nested agg inside the agg argument: not a DAG shape — bail */ if (expr_contains_agg(el[1])) { *ok = 0; return NULL; } + if (ray_len(expr) > 2 && expr_contains_agg(el[2])) { *ok = 0; return NULL; } if (*n_hidden >= cap) { *ok = 0; return NULL; } char buf[24]; int bn = snprintf(buf, sizeof buf, "__ha%d", *n_hidden); @@ -2085,6 +2103,23 @@ static ray_t* try_decompose_agg_arith(ray_t* val_expr, ray_t* tbl, return rw; } +/* Dry run of the arith-of-aggs decomposition: true when `expr` is an + * arithmetic combination of DAG-shaped aggregates that the hidden-slot path + * evaluates after grouping. Routing consults this before deciding that a + * non-aggregate output needs the per-group scatter or eval-level grouping; + * the real rewrite runs later in the classification pass. */ +static int is_decomposable_agg_compound(ray_t* expr, ray_t* tbl) { + enum { PROBE_CAP = 64 }; + ray_t* hexprs[PROBE_CAP]; + int64_t hnames[PROBE_CAP]; + int n_hidden = 0; + ray_t* rw = try_decompose_agg_arith(expr, tbl, hexprs, hnames, + &n_hidden, PROBE_CAP); + if (!rw) return 0; + ray_release(rw); + return 1; +} + static int expr_contains_call_named(ray_t* expr, const char* name, size_t name_len) { if (!expr) return 0; if (expr->type != RAY_LIST) return 0; @@ -7465,6 +7500,10 @@ ray_t* ray_select(ray_t** args, int64_t n) { if (is_single_group_key_projection(by_expr, dict_elems[i + 1])) continue; if (is_group_dag_agg_expr(dict_elems[i + 1])) continue; + /* Arithmetic over aggregates is served by hidden agg slots + * plus one post-group evaluation on any key shape; it must + * not push multi-key queries onto eval-level grouping. */ + if (is_decomposable_agg_compound(dict_elems[i + 1], tbl)) continue; any_nonagg = 1; if (can_atom_broadcast(dict_elems[i + 1])) continue; if (!match_count_distinct(dict_elems[i + 1])) { any_true_nonagg = 1; break; } @@ -9368,7 +9407,25 @@ ray_t* ray_select(ray_t** args, int64_t n) { } agg_ins2[n_aggs] = NULL; agg_k[n_aggs] = 0; - if (hop == OP_TOP_N || hop == OP_BOT_N) { + if (agg_is_binary_agg(hop)) { + /* hidden_agg_shape_ok admitted two bare column arguments */ + agg_ins2[n_aggs] = compile_expr_dag(g, he[2]); + if (!agg_ins2[n_aggs]) { + for (int ci = 0; ci < n_compound; ci++) + ray_release(compound_rw[ci]); + ray_graph_free(g); ray_release(tbl); + scratch_free(sel_slots_hdr); DICT_VIEW_CLOSE(dv); return ray_error("domain", "select by: failed to compile binary aggregation second argument"); + } + if (agg_ins2[n_aggs]->out_type > 0 && + !agg_type_admitted(hop, agg_ins2[n_aggs]->out_type)) { + int8_t in_t = agg_ins2[n_aggs]->out_type; + for (int ci = 0; ci < n_compound; ci++) + ray_release(compound_rw[ci]); + ray_graph_free(g); ray_release(tbl); + scratch_free(sel_slots_hdr); DICT_VIEW_CLOSE(dv); return ray_error("type", "select by: binary aggregation does not admit second input type %s", ray_type_name(in_t)); + } + has_binary_agg = 1; + } else if (hop == OP_TOP_N || hop == OP_BOT_N) { has_list_agg = 1; if (ray_len(hidden_agg_exprs[hi]) < 3) { for (int ci = 0; ci < n_compound; ci++) diff --git a/test/rfl/group/agg_arith_multikey.rfl b/test/rfl/group/agg_arith_multikey.rfl new file mode 100644 index 00000000..0718238f --- /dev/null +++ b/test/rfl/group/agg_arith_multikey.rfl @@ -0,0 +1,64 @@ +;; Arithmetic over aggregates on multi-key group-bys. +;; +;; Before: any non-aggregate output (even one composed only of aggregates) +;; with a two-or-more-key by: clause forced the eval-level per-group path +;; (~850 ms for 10M rows, no parallelism). Only single-key queries used the +;; hidden-slot decomposition (aggregates computed by the DAG engine, the +;; outer expression evaluated once over the grouped result). +;; Now a decomposable compound never forces eval-level grouping, and binary +;; aggregates (pearson_corr, cov, wsum, ...) and quantile with a literal +;; probability are extracted into hidden slots too. +;; +;; Every case compares the compound result against the same arithmetic +;; applied to separately grouped plain aggregates. + +(set n 20000) +(set i (til n)) +(set t (table [a b c x y] (list (% i 7) (% i 11) (% i 3) (as 'F64 (% (* i 13) 101)) (as 'F64 (% (* i 29) 97))))) + +;; two keys, arithmetic over two unary aggregates +(set r1 (xasc (select {from: t d: (- (max x) (min y)) by: {a: a b: b}}) ['a 'b])) +(set p1 (xasc (select {from: t mx: (max x) mn: (min y) by: {a: a b: b}}) ['a 'b])) +(count r1) -- 77 +(all (== (at r1 'd) (- (at p1 'mx) (at p1 'mn)))) -- true +(all (== (at r1 'a) (at p1 'a))) -- true +(all (== (at r1 'b) (at p1 'b))) -- true + +;; three keys, scalar function over an aggregate, plus a plain aggregate beside it +(set r2 (xasc (select {from: t n: (count x) s: (pow (sum x) 2) by: {a: a b: b c: c}}) ['a 'b 'c])) +(set p2 (xasc (select {from: t n: (count x) s: (sum x) by: {a: a b: b c: c}}) ['a 'b 'c])) +(count r2) -- 231 +(all (== (at r2 'n) (at p2 'n))) -- true +(all (< (abs (- (at r2 's) (pow (at p2 's) 2))) 1e-6)) -- true + +;; binary aggregate inside arithmetic: single key and two keys +(set r3 (xasc (select {from: t r2: (pow (pearson_corr x y) 2) by: {a: a}}) ['a])) +(set p3 (xasc (select {from: t p: (pearson_corr x y) by: {a: a}}) ['a])) +(all (< (abs (- (at r3 'r2) (pow (at p3 'p) 2))) 1e-12)) -- true +(set r4 (xasc (select {from: t r2: (pow (pearson_corr x y) 2) by: {a: a b: b}}) ['a 'b])) +(set p4 (xasc (select {from: t p: (pearson_corr x y) by: {a: a b: b}}) ['a 'b])) +(count r4) -- 77 +(all (< (abs (- (at r4 'r2) (pow (at p4 'p) 2))) 1e-12)) -- true + +;; covariance and weighted sum inside arithmetic +(set r5 (xasc (select {from: t v: (* 2 (cov x y)) w: (+ 1 (wsum x y)) by: {a: a b: b}}) ['a 'b])) +(set p5 (xasc (select {from: t v: (cov x y) w: (wsum x y) by: {a: a b: b}}) ['a 'b])) +(all (< (abs (- (at r5 'v) (* 2 (at p5 'v)))) 1e-9)) -- true +(all (< (abs (- (at r5 'w) (+ 1 (at p5 'w)))) 1e-9)) -- true + +;; quantile with a literal probability inside arithmetic +(set r6 (xasc (select {from: t q: (+ 1 (quantile x 0.25)) by: {a: a b: b}}) ['a 'b])) +(set p6 (xasc (select {from: t q: (quantile x 0.25) by: {a: a b: b}}) ['a 'b])) +(all (== (at r6 'q) (+ 1 (at p6 'q)))) -- true + +;; a genuinely row-dependent projection still takes the per-group path and +;; keeps its list-per-group shape, unchanged by the routing fix +(set r7 (select {from: t d: (+ x 1) by: {a: a}})) +(count r7) -- 7 +(type (at (at r7 'd) 0)) -- 'F64 + +;; a where: filter composes with the compound on a multi-key by +(set r8 (xasc (select {from: t d: (- (max x) (min y)) by: {a: a b: b} where: (> x 50)}) ['a 'b])) +(set p8 (xasc (select {from: t mx: (max x) mn: (min y) by: {a: a b: b} where: (> x 50)}) ['a 'b])) +(count r8) -- 77 +(all (== (at r8 'd) (- (at p8 'mx) (at p8 'mn)))) -- true diff --git a/test/rfl/group/dense_composite_sym_compaction.rfl b/test/rfl/group/dense_composite_sym_compaction.rfl new file mode 100644 index 00000000..ded37c36 --- /dev/null +++ b/test/rfl/group/dense_composite_sym_compaction.rfl @@ -0,0 +1,57 @@ +;; Composite dense plans over symbol keys with interleaved codes. +;; +;; Symbol columns share one domain, so their codes interleave with every +;; other symbol column's: two 100-value keys each span a 100k-code range and +;; their raw product (10^10) rejected the dense plan although only 10^4 +;; groups exist, sending the query to radix (505 ms vs 52 ms on one core for +;; 10M rows). The plan now compacts each key to the codes it uses when the +;; raw product overflows. The reference below groups by the integer indices +;; the symbols were generated from; names are zero-padded so symbol order +;; equals index order. + +(set n 40000) +(set i (til n)) +(set ai (% i 100)) +(set bi (% (div i 100) 100)) +(set ci (% (* i 7) 3000)) +;; interleave: intern one c symbol between every a and b symbol +(set names-a (map (fn [x] (format "a%" (+ 1000 x))) (til 100))) +(set names-b (map (fn [x] (format "b%" (+ 1000 x))) (til 100))) +(set names-c (map (fn [x] (format "c%" (+ 10000 x))) (til 3000))) +(set mix (raze (map (fn [x] (list (at names-c (% (* x 31) 3000)) (at names-a (% x 100)) (at names-c (% (* x 17) 3000)) (at names-b (% x 100)))) (til 3000)))) +(set mixsym (as 'SYM mix)) +(set syms-a (as 'SYM names-a)) +(set syms-b (as 'SYM names-b)) +(set syms-c (as 'SYM names-c)) +(set a (at syms-a ai)) +(set b (at syms-b bi)) +(set c (at syms-c ci)) +(set v (% (* i 11) 101)) +(set t (table [a b c ai bi ci v] (list a b c ai bi ci v))) + +;; two symbol keys: 10,000 groups +(set r (xasc (select {from: t s: (sum v) n: (count v) by: {a: a b: b}}) ['a 'b])) +(set p (xasc (select {from: t s: (sum v) n: (count v) by: {ai: ai bi: bi}}) ['ai 'bi])) +(count r) -- 10000 +(count p) -- 10000 +(all (== (at r 's) (at p 's))) -- true +(all (== (at r 'n) (at p 'n))) -- true +(all (== (at r 'a) (at syms-a (at p 'ai)))) -- true +(all (== (at r 'b) (at syms-b (at p 'bi)))) -- true + +;; three keys, one wide: 40,000 rows, key c has 3,000 codes +(set r3 (xasc (select {from: t s: (sum v) by: {a: a c: c}}) ['a 'c])) +(set p3 (xasc (select {from: t s: (sum v) by: {ai: ai ci: ci}}) ['ai 'ci])) +(== (count r3) (count p3)) -- true +(all (== (at r3 's) (at p3 's))) -- true + +;; mixed symbol + integer keys, with a where filter and top-N +(set r4 (select {from: t s: (sum v) by: {a: a bi: bi} where: (> v 50) desc: s take: 5})) +(set p4 (take (xdesc (select {from: t s: (sum v) by: {ai: ai bi: bi} where: (> v 50)}) 's) 5)) +(all (== (at r4 's) (at p4 's))) -- true +(all (== (at r4 'bi) (at p4 'bi))) -- true + +;; arithmetic over aggregates on the compacted composite key +(set r5 (xasc (select {from: t d: (- (max v) (min v)) by: {a: a b: b}}) ['a 'b])) +(set p5 (xasc (select {from: t mx: (max v) mn: (min v) by: {ai: ai bi: bi}}) ['ai 'bi])) +(all (== (at r5 'd) (- (at p5 'mx) (at p5 'mn)))) -- true diff --git a/test/rfl/group/emit_filter_v2_route.rfl b/test/rfl/group/emit_filter_v2_route.rfl new file mode 100644 index 00000000..566f5a0a --- /dev/null +++ b/test/rfl/group/emit_filter_v2_route.rfl @@ -0,0 +1,62 @@ +;; `by: ... desc:|asc: AGG take: N` arms the top-N emit filter. These shapes +;; used to bypass the parallel grouping engine entirely and run on the legacy +;; ladder (single-threaded scatter, state sized by the shared sym domain): +;; a three-aggregate top-N over 100k groups took 800 ms on one core and did +;; not speed up on a 28-thread pool. Now every shape the parallel engine +;; admits with a bounded dense plan runs there and its full result is +;; trimmed to the filter's top-N superset; the DAG's sort+take finalizes the +;; exact order and limit exactly as before. +;; +;; Each case compares the filtered query with the same query grouped +;; without take:, then sorted and cut down the same way. + +(set n 30000) +(set i (til n)) +(set t (table [k j s v w] (list (% (* i 7) 1000) (% i 5) (at ['a 'b 'c 'd 'e 'f 'g] (% (* i 3) 7)) (% (* i 13) 97) (as 'F64 (% (* i 31) 89))))) + +;; single int key, count desc +(set a1 (select {from: t by: k c: (count v) desc: c take: 7})) +(set b1 (take (xdesc (select {from: t by: k c: (count v)}) 'c) 7)) +(count a1) -- 7 +(all (== (at a1 'c) (at b1 'c))) -- true +(all (== (at a1 'k) (at b1 'k))) -- true + +;; three aggregates, ordered by sum desc (the multi-aggregate shape that was 60x slower) +(set a2 (select {from: t by: k s: (sum v) c: (count v) a: (avg w) desc: s take: 10})) +(set b2 (take (xdesc (select {from: t by: k s: (sum v) c: (count v) a: (avg w)}) 's) 10)) +(count a2) -- 10 +(all (== (at a2 's) (at b2 's))) -- true +(all (== (at a2 'k) (at b2 'k))) -- true +(all (< (abs (- (at a2 'a) (at b2 'a))) 1e-9)) -- true + +;; asc on min, multi-key by +(set a3 (select {from: t by: [k j] m: (min v) asc: m take: 5})) +(set b3 (take (xasc (select {from: t by: [k j] m: (min v)}) 'm) 5)) +(count a3) -- 5 +(all (== (at a3 'm) (at b3 'm))) -- true + +;; max desc with a where filter (filtered dense plan, parallel prescan) +(set a4 (select {from: t by: k m: (max v) where: (> w 40) desc: m take: 8})) +(set b4 (take (xdesc (select {from: t by: k m: (max v) where: (> w 40)}) 'm) 8)) +(count a4) -- 8 +(all (== (at a4 'm) (at b4 'm))) -- true + +;; symbol key, count desc: the top-N SET is fully determined here because +;; every symbol's count is distinct +(set a5 (select {from: t by: s c: (count v) desc: c take: 3})) +(set b5 (take (xdesc (select {from: t by: s c: (count v)}) 'c) 3)) +(count a5) -- 3 +(all (== (at a5 'c) (at b5 'c))) -- true + +;; take larger than the group count returns every group +(set a6 (select {from: t by: j c: (count v) desc: c take: 100})) +(count a6) -- 5 +(sum (at a6 'c)) -- 30000 + +;; ties: the emitted rows are a valid top-N (every emitted count is >= the +;; N-th largest count of the full grouping) +(set a7 (select {from: t by: k c: (count v) desc: c take: 20})) +(set full7 (xdesc (select {from: t by: k c: (count v)}) 'c)) +(count a7) -- 20 +(all (>= (at a7 'c) (at (at full7 'c) 19))) -- true +(== (sum (at a7 'c)) (sum (take (at full7 'c) 20))) -- true diff --git a/test/test_agg_contract.c b/test/test_agg_contract.c index b5f1531d..0c92875f 100644 --- a/test/test_agg_contract.c +++ b/test/test_agg_contract.c @@ -6,6 +6,7 @@ #include "ops/agg_engine.h" #include "ops/agg_registry.h" #include "core/pool.h" +#include "core/platform.h" #include "mem/heap.h" #include "ops/fused_pred.h" #include "ops/cdfuse.h" @@ -1729,6 +1730,75 @@ static test_result_t test_cancelled_group(void) { PASS(); } +/* Replicated task-local slabs are bounded by the last-level cache: with a + * 20-worker pool and a 100k-slot slab the raw replication (20 slabs) leaves + * most caches, and the run must use at most floor(0.75 * LLC / slab) task + * slabs (never fewer than the pool when everything fits). The result is + * identical either way. */ +static test_result_t test_dense_cache_bound(void) { + ray_pool_destroy(); + TEST_ASSERT_EQ_I(ray_pool_init_total(20), RAY_OK); + ray_t* setup = ray_eval_str( + "(set cb_i (til 4000000)) " + "(set cb_t (table [k v] (list (as 'I32 (% (* cb_i 7919) 100000)) (% cb_i 13))))"); + TEST_ASSERT_NOT_NULL(setup); TEST_ASSERT_FALSE(RAY_IS_ERR(setup)); ray_release(setup); + agg_route_reset(); + ray_t* r = ray_eval_str("(select {from:cb_t by:k s:(sum v)})"); + TEST_ASSERT_NOT_NULL(r); TEST_ASSERT_FALSE(RAY_IS_ERR(r)); + agg_route_stats_t stats = agg_route_stats(); + TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_V2_DENSE], 1); + TEST_ASSERT_EQ_I(stats.dense_strategy, AGG_DENSE_TASK_LOCAL); + TEST_ASSERT_TRUE(stats.dense_tasks >= 2 && stats.dense_tasks <= 20); + TEST_ASSERT_EQ_I(ray_table_nrows(r), 100000); + uint64_t llc = ray_cache_llc_bytes(); + if (llc > 0) { + size_t block = agg_resolve(OP_SUM, RAY_I64)->state_size; + double slots = (double)stats.dense_local_slots / stats.dense_tasks; + double slab = slots * (block + sizeof(int64_t) + 1); + double budget = (double)llc * 0.75; + uint32_t cap = slab * 20 > budget ? (uint32_t)(budget / slab) : 20; + if (cap < 2) cap = 2; + TEST_ASSERT_EQ_I(stats.dense_tasks, cap); + } + /* The bounded run computes the same sums as the serial engine. */ + ray_t* check = ray_eval_str( + "(all (== (at (xasc (select {from:cb_t by:k s:(sum v)}) 'k) 's) " + "(at (xasc (select {from:(select {from:cb_t k:k v:v}) by:k s:(sum v)}) 'k) 's)))"); + TEST_ASSERT_NOT_NULL(check); TEST_ASSERT_FALSE(RAY_IS_ERR(check)); + TEST_ASSERT_EQ_I(check->i64, 1); + ray_release(check); ray_release(r); + ray_release(ray_eval_str("(set cb_t 0) (set cb_i 0)")); + PASS(); +} + +/* Composite symbol keys whose codes interleave in the shared domain get a + * compacted dense plan: two 40-value keys over a 4,000-code domain pack into + * 1,600 slots instead of falling to radix. */ +static test_result_t test_dense_composite_compaction(void) { + ray_t* setup = ray_eval_str( + "(set cc_i (til 400000)) " + "(set cc_syms (as 'SYM (map (fn [x] (format \"s%\" (+ 10000 x))) (til 4000)))) " + "(set cc_a (at cc_syms (* (% cc_i 40) 100))) " + "(set cc_b (at cc_syms (+ 1 (* (% (div cc_i 40) 40) 100)))) " + "(set cc_t (table [a b v] (list cc_a cc_b (% cc_i 11))))"); + TEST_ASSERT_NOT_NULL(setup); TEST_ASSERT_FALSE(RAY_IS_ERR(setup)); ray_release(setup); + agg_route_reset(); + ray_t* r = ray_eval_str("(select {from:cc_t by:[a b] s:(sum v) n:(count v)})"); + TEST_ASSERT_NOT_NULL(r); TEST_ASSERT_FALSE(RAY_IS_ERR(r)); + agg_route_stats_t stats = agg_route_stats(); + TEST_ASSERT_TRUE(stats.dense_plan_available); + TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_V2_DENSE], 1); + TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_V2_RADIX], 0); + TEST_ASSERT_EQ_I(ray_table_nrows(r), 1600); + ray_t* check = ray_eval_str( + "(== (sum (at (select {from:cc_t by:[a b] s:(sum v)}) 's)) (sum (at cc_t 'v)))"); + TEST_ASSERT_NOT_NULL(check); TEST_ASSERT_FALSE(RAY_IS_ERR(check)); + TEST_ASSERT_EQ_I(check->i64, 1); + ray_release(check); ray_release(r); + ray_release(ray_eval_str("(set cc_t 0) (set cc_a 0) (set cc_b 0) (set cc_syms 0) (set cc_i 0)")); + PASS(); +} + const test_entry_t agg_contract_entries[] = { { "agg_contract/empty_inference_errors", test_empty_inference_errors, contract_setup, contract_teardown }, { "agg_contract/nth_bounds", test_nth_bounds, contract_setup, contract_teardown }, @@ -1743,6 +1813,8 @@ const test_entry_t agg_contract_entries[] = { { "agg_contract/dense_strategies", test_dense_strategies, contract_setup, contract_teardown }, { "agg_contract/dense_symbol_output", test_dense_symbol_output, contract_setup, contract_teardown }, { "agg_contract/dense_task_local", test_dense_task_local, contract_setup, contract_teardown }, + { "agg_contract/dense_cache_bound", test_dense_cache_bound, contract_setup, contract_teardown }, + { "agg_contract/dense_composite_compaction", test_dense_composite_compaction, contract_setup, contract_teardown }, { "agg_contract/rank_widths_nulls_slices", test_rank_widths_nulls_and_slices, contract_setup, contract_teardown }, { "agg_contract/nullable_differential", test_nullable_differential, contract_setup, contract_teardown }, { "agg_contract/wide_key_routes", test_wide_key_routes, contract_setup, contract_teardown }, diff --git a/test/test_group_extra.c b/test/test_group_extra.c index bfccd310..6160c460 100644 --- a/test/test_group_extra.c +++ b/test/test_group_extra.c @@ -1108,10 +1108,11 @@ static test_result_t test_i16_group_top_count_emit_filter(void) { ray_release(res); /* Same filter with desc = 0 (`asc: c take: 2`): the emit filter must NOT - * keep the two LARGEST groups. Trimming here is desc-only machinery, so - * the asc request falls through to the full group set and the caller's - * sort+take picks the smallest — what must never happen is the result - * losing the small groups (issue #408: asc returned the desc answer). */ + * keep the two LARGEST groups (issue #408: asc returned the desc answer). + * This shape now runs on the parallel engine and is trimmed to the + * smallest-N superset, so the result is exactly the two smallest groups; + * the caller's sort+take finalizes the order. What must never happen is + * the result losing the small groups. */ filter.desc = 0; ray_group_emit_filter_set(filter); res = ray_execute(g, grp); @@ -1121,17 +1122,17 @@ static test_result_t test_i16_group_top_count_emit_filter(void) { out_cnt = ray_table_get_col_idx(res, 1); TEST_ASSERT_NOT_NULL(out_key); TEST_ASSERT_NOT_NULL(out_cnt); - /* The asc request falls through to the FULL group set (all 5 groups) — - * pin the row count too, so this can tell "full fall-through" apart - * from a hypothetical asc top-2, and cannot pass by accident. */ - TEST_ASSERT_EQ_I(ray_table_nrows(res), 5); - int got_smallest = 0; + /* Counts are 5,4,3,2,1 with no ties: the asc top-2 superset is exactly + * {k=4 (2), k=5 (1)}. */ + TEST_ASSERT_EQ_I(ray_table_nrows(res), 2); + int got_smallest = 0, got_second = 0; for (int64_t i = 0; i < ray_table_nrows(res); i++) { int16_t k = ((int16_t*)ray_data(out_key))[i]; int64_t c = ((int64_t*)ray_data(out_cnt))[i]; if (k == 5 && c == 1) got_smallest = 1; + if (k == 4 && c == 2) got_second = 1; } - TEST_ASSERT_TRUE(got_smallest); + TEST_ASSERT_TRUE(got_smallest && got_second); ray_release(res); ray_graph_free(g); From bb88f0d2616b7dd35db456d604be18660702c89b Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 11:47:29 +0200 Subject: [PATCH 08/23] feat(group): shared top-N keep decision for the emit filter --- src/ops/agg_engine.c | 43 ++++++++++++++++++++++++++++++++++++++++++ src/ops/agg_engine.h | 8 ++++++++ test/test_agg_engine.c | 35 ++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index f8800b95..dfb39812 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -1392,6 +1392,49 @@ static inline bool agg_finalize_value(const agg_vtable_t* vt, const void* state, return is_null; } +/* Threshold = N-th value in the keep direction, found by quickselect on a + * copy: O(n) and exact, so ties at the threshold are kept. Callers + * parallelize the value fill; the selection itself runs on the caller. */ +int64_t agg_topn_keep(const double* vals, int64_t n, + const ray_group_emit_filter_t* ef, uint8_t* keep) { + if (n <= 0) return 0; + double thr = 0.0; + bool have_thr = false; + if (ef->top_count_take > 0 && n > ef->top_count_take) { + ray_t* hdr = NULL; + double* sv = (double*)scratch_alloc(&hdr, (size_t)n * sizeof(double)); + if (sv) { + memcpy(sv, vals, (size_t)n * sizeof(double)); + int64_t k = ef->desc ? (n - ef->top_count_take) : (ef->top_count_take - 1); + int64_t lo = 0, hi = n - 1; + while (lo < hi) { + double pivot = sv[k]; + int64_t i = lo, j = hi; + while (i <= j) { + while (sv[i] < pivot) i++; + while (sv[j] > pivot) j--; + if (i <= j) { double t = sv[i]; sv[i] = sv[j]; sv[j] = t; i++; j--; } + } + if (k <= j) hi = j; + else if (k >= i) lo = i; + else break; + } + thr = sv[k]; + have_thr = true; + scratch_free(hdr); + } + } + int64_t kept = 0; + for (int64_t i = 0; i < n; i++) { + double v = vals[i]; + bool ok = !(ef->min_count_exclusive > 0 && !(v > (double)ef->min_count_exclusive)); + if (ok && have_thr && (ef->desc ? (v < thr) : (v > thr))) ok = false; + keep[i] = (uint8_t)ok; + kept += ok; + } + return kept; +} + typedef struct { const agg_vtable_t* vt; ray_t* out; diff --git a/src/ops/agg_engine.h b/src/ops/agg_engine.h index bee911fa..2090285a 100644 --- a/src/ops/agg_engine.h +++ b/src/ops/agg_engine.h @@ -79,6 +79,14 @@ bool agg_v2_can_handle(ray_graph_t* g, ray_op_t* op, ray_t* tbl); * definition for the strategy-prediction contract). */ bool agg_v2_dense_plan_available(ray_graph_t* g, ray_op_t* op, ray_t* tbl); +/* Top-N keep decision shared by every strategy: keep[i] = 1 when group i + * passes min_count_exclusive and (when top_count_take > 0) lies within the + * top-N by value in the filter's direction, ties included (a superset of N; + * the DAG's sort+take downstream finalizes order and limit). Returns the + * number kept. vals may be NULL when n == 0. */ +int64_t agg_topn_keep(const double* vals, int64_t n, + const ray_group_emit_filter_t* ef, uint8_t* keep); + /* Precondition: agg_v2_can_handle(g, op, tbl) returned true. * `group_limit` is the HEAD(GROUP) row-limit HINT (0 = no limit): when * positive, an engine strategy may emit only the first `group_limit` groups in diff --git a/test/test_agg_engine.c b/test/test_agg_engine.c index 63147467..a98707f8 100644 --- a/test/test_agg_engine.c +++ b/test/test_agg_engine.c @@ -2029,6 +2029,40 @@ DIFF_SHAPE(test_diff_group_pearson_2k, diff_make_pearson_2k, gb_pearson_2k, 2) /* Shape 3: heterogeneous sum(x)+pearson(x,y)+count over single I64 key. */ DIFF_SHAPE(test_diff_group_pearson_mixed, diff_make_pearson_1k, gb_sum_pearson_count, 1) +/* Shared top-N keep decision for the emit filter: ties included, direction + * from .desc, min_count_exclusive as a pre-filter, take beyond n keeps all. */ +static test_result_t test_topn_keep(void) { + ray_heap_init(); + (void)ray_sym_init(); + double vals[] = { 5, 1, 4, 4, 2, 9, 4 }; + uint8_t keep[7]; + ray_group_emit_filter_t ef = {0}; + ef.enabled = 1; ef.top_count_take = 3; ef.desc = 1; + int64_t n = agg_topn_keep(vals, 7, &ef, keep); + /* top-3 largest are 9,5,4 — every 4 ties in: {9,5,4,4,4} */ + TEST_ASSERT_EQ_I(n, 5); + TEST_ASSERT_TRUE(keep[0] && !keep[1] && keep[2] && keep[3] && !keep[4] && keep[5] && keep[6]); + ef.desc = 0; + n = agg_topn_keep(vals, 7, &ef, keep); /* smallest 3: 1,2,4 (+ ties) */ + TEST_ASSERT_EQ_I(n, 5); + TEST_ASSERT_TRUE(keep[1] && keep[4] && keep[2] && keep[3] && keep[6] && !keep[0] && !keep[5]); + ef.desc = 1; ef.top_count_take = 0; ef.min_count_exclusive = 4; + n = agg_topn_keep(vals, 7, &ef, keep); /* > 4: {5, 9} */ + TEST_ASSERT_EQ_I(n, 2); + TEST_ASSERT_TRUE(keep[0] && keep[5] && !keep[2]); + ef.top_count_take = 100; + n = agg_topn_keep(vals, 7, &ef, keep); /* take beyond n keeps all passing */ + TEST_ASSERT_EQ_I(n, 2); + ef.min_count_exclusive = 0; ef.top_count_take = 2; + n = agg_topn_keep(vals, 7, &ef, keep); /* top-2: 9 and 5, no ties */ + TEST_ASSERT_EQ_I(n, 2); + TEST_ASSERT_TRUE(keep[0] && keep[5]); + TEST_ASSERT_EQ_I(agg_topn_keep(NULL, 0, &ef, keep), 0); + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + const test_entry_t agg_engine_entries[] = { { "pearson_old_engine_r_vs_r2", test_pearson_old_engine_r_vs_r2, NULL, NULL }, { "diff_group_pearson_1k", test_diff_group_pearson_1k, NULL, NULL }, @@ -2096,6 +2130,7 @@ const test_entry_t agg_engine_entries[] = { { "dense_plan_single_i64", test_dense_plan_single_i64, NULL, NULL }, { "dense_plan_two_keys", test_dense_plan_two_keys, NULL, NULL }, { "dense_plan_huge_range", test_dense_plan_huge_range, NULL, NULL }, + { "topn_keep", test_topn_keep, NULL, NULL }, { "dense_plan_f64_key", test_dense_plan_f64_key, NULL, NULL }, { "dense_plan_buffered_agg", test_dense_plan_buffered_agg, NULL, NULL }, { "dense_plan_nullable_key", test_dense_plan_nullable_key, NULL, NULL }, From efcd2c146ab0cb6dc50fb2bb5d1a866a89060e20 Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 11:54:54 +0200 Subject: [PATCH 09/23] feat(group): double view of a finalized aggregate per group --- src/ops/agg_engine.c | 34 +++++++++++++++++++++++++++++++++ src/ops/agg_engine.h | 8 ++++++++ test/test_agg_engine.c | 43 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index dfb39812..e7ff3ac0 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -10,6 +10,7 @@ #include "table/sym.h" /* ray_read_sym */ #include "core/platform.h" /* ray_cache_llc_bytes — replicated slab bound */ #include +#include #include /* Radix output address: high 32 bits partition, low 32 bits local group. */ @@ -1435,6 +1436,39 @@ int64_t agg_topn_keep(const double* vals, int64_t n, return kept; } +bool agg_group_values_f64(const agg_vtable_t* vt, const char* states, + size_t stride, size_t off, const int64_t* slots, + int64_t n, int64_t param, uint8_t desc, double* out) { + int8_t t = vt->out_type; + switch (t) { + case RAY_F64: case RAY_F32: case RAY_I64: case RAY_TIMESTAMP: + case RAY_I32: case RAY_DATE: case RAY_TIME: case RAY_I16: + case RAY_U8: case RAY_BOOL: break; + default: return false; + } + double null_sink = desc ? -INFINITY : INFINITY; + ray_t* cell = ray_vec_new(t, 1); + if (!cell || RAY_IS_ERR(cell)) { if (cell) ray_error_free(cell); return false; } + cell->len = 1; + for (int64_t i = 0; i < n; i++) { + int64_t g = slots ? slots[i] : i; + const void* state = states + (size_t)g * stride + off; + bool is_null = agg_finalize_value(vt, state, cell, 0, param); + double v; + switch (t) { + case RAY_F64: v = ((const double*)ray_data(cell))[0]; break; + case RAY_F32: v = ((const float*)ray_data(cell))[0]; break; + case RAY_I64: case RAY_TIMESTAMP: v = (double)((const int64_t*)ray_data(cell))[0]; break; + case RAY_I32: case RAY_DATE: case RAY_TIME: v = (double)((const int32_t*)ray_data(cell))[0]; break; + case RAY_I16: v = (double)((const int16_t*)ray_data(cell))[0]; break; + default: v = (double)((const uint8_t*)ray_data(cell))[0]; break; + } + out[i] = is_null || v != v ? null_sink : v; + } + ray_release(cell); + return true; +} + typedef struct { const agg_vtable_t* vt; ray_t* out; diff --git a/src/ops/agg_engine.h b/src/ops/agg_engine.h index 2090285a..b46a2f4e 100644 --- a/src/ops/agg_engine.h +++ b/src/ops/agg_engine.h @@ -87,6 +87,14 @@ bool agg_v2_dense_plan_available(ray_graph_t* g, ray_op_t* op, ray_t* tbl); int64_t agg_topn_keep(const double* vals, int64_t n, const ray_group_emit_filter_t* ef, uint8_t* keep); +/* Double view of aggregate `vt` for n groups: group i's state is at + * states + (slots ? slots[i] : i) * stride + off. Nulls (and NaN) sink to + * the far end of the keep direction (`desc`) so they never enter a top-N. + * Returns false for an out_type without a scalar order (LIST, STR, ...). */ +bool agg_group_values_f64(const agg_vtable_t* vt, const char* states, + size_t stride, size_t off, const int64_t* slots, + int64_t n, int64_t param, uint8_t desc, double* out); + /* Precondition: agg_v2_can_handle(g, op, tbl) returned true. * `group_limit` is the HEAD(GROUP) row-limit HINT (0 = no limit): when * positive, an engine strategy may emit only the first `group_limit` groups in diff --git a/test/test_agg_engine.c b/test/test_agg_engine.c index a98707f8..804002a5 100644 --- a/test/test_agg_engine.c +++ b/test/test_agg_engine.c @@ -2063,6 +2063,48 @@ static test_result_t test_topn_keep(void) { PASS(); } +/* Double view of a finalized aggregate per group, contiguous and via a slot + * list; nulls sink to the far end of the keep direction. */ +static test_result_t test_group_values_f64(void) { + ray_heap_init(); + (void)ray_sym_init(); + const agg_vtable_t* vt = agg_resolve(OP_SUM, RAY_I64); + TEST_ASSERT_NOT_NULL(vt); + enum { STRIDE = 64 }; + char states[4 * STRIDE]; + for (int i = 0; i < 4; i++) vt->init(states + i * STRIDE); + uint32_t gids[] = {0, 1, 1, 2, 2, 2}; + int64_t vals[] = {10, 1, 2, 5, 5, 5}; + ray_valid_t valid = { vals, RAY_I64, false }; + vt->update_batch(states, STRIDE, gids, vals, &valid, 6, NULL); + double out[4]; + TEST_ASSERT_TRUE(agg_group_values_f64(vt, states, STRIDE, 0, NULL, 3, 0, 1, out)); + TEST_ASSERT_TRUE(out[0] == 10 && out[1] == 3 && out[2] == 15); + /* slot list picks groups 2 and 0 */ + int64_t slots[] = {2, 0}; + TEST_ASSERT_TRUE(agg_group_values_f64(vt, states, STRIDE, 0, slots, 2, 0, 1, out)); + TEST_ASSERT_TRUE(out[0] == 15 && out[1] == 10); + /* a group whose only input is null finalizes to a null minimum: it sinks + * below every value for desc and above every value for asc */ + const agg_vtable_t* mn = agg_resolve(OP_MIN, RAY_I64); + TEST_ASSERT_NOT_NULL(mn); + for (int i = 0; i < 4; i++) mn->init(states + i * STRIDE); + uint32_t mgids[] = {0, 1, 3}; + int64_t mvals[] = {7, 2, NULL_I64}; + ray_valid_t mvalid = { mvals, RAY_I64, true }; + mn->update_batch(states, STRIDE, mgids, mvals, &mvalid, 3, NULL); + TEST_ASSERT_TRUE(agg_group_values_f64(mn, states, STRIDE, 0, NULL, 4, 0, 1, out)); + TEST_ASSERT_TRUE(out[0] == 7 && out[1] == 2 && out[3] < out[1]); + TEST_ASSERT_TRUE(agg_group_values_f64(mn, states, STRIDE, 0, NULL, 4, 0, 0, out)); + TEST_ASSERT_TRUE(out[3] > out[0]); + /* a list-valued aggregate has no scalar view */ + const agg_vtable_t* top = agg_resolve(OP_TOP_N, RAY_I64); + if (top) TEST_ASSERT_FALSE(agg_group_values_f64(top, states, STRIDE, 0, NULL, 1, 3, 1, out)); + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + const test_entry_t agg_engine_entries[] = { { "pearson_old_engine_r_vs_r2", test_pearson_old_engine_r_vs_r2, NULL, NULL }, { "diff_group_pearson_1k", test_diff_group_pearson_1k, NULL, NULL }, @@ -2131,6 +2173,7 @@ const test_entry_t agg_engine_entries[] = { { "dense_plan_two_keys", test_dense_plan_two_keys, NULL, NULL }, { "dense_plan_huge_range", test_dense_plan_huge_range, NULL, NULL }, { "topn_keep", test_topn_keep, NULL, NULL }, + { "group_values_f64", test_group_values_f64, NULL, NULL }, { "dense_plan_f64_key", test_dense_plan_f64_key, NULL, NULL }, { "dense_plan_buffered_agg", test_dense_plan_buffered_agg, NULL, NULL }, { "dense_plan_nullable_key", test_dense_plan_nullable_key, NULL, NULL }, From 5915cde9ac2c987d88b44225232a9ffe56c8bc04 Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 12:08:30 +0200 Subject: [PATCH 10/23] perf(group): native top-N selection in the radix path Radix partitions finalize the emit filter's aggregate into one double per group in parallel, keep a bounded candidate heap each, take the threshold from the union, and emit only the kept superset in first-row order. v2 now reads the emit filter itself; every shape it admits runs there (the ladder keeps only shapes v2 declines). 10M rows, three-key count, desc take 10: 1 core 988 -> 287 ms; 28 cores 154 -> 43 ms (legacy ladder: 47 ms). --- src/ops/agg_engine.c | 310 ++++++++++++++++++++++++++++++++------- src/ops/agg_engine.h | 8 + src/ops/group.c | 54 ++----- test/test_agg_contract.c | 34 +++++ 4 files changed, 312 insertions(+), 94 deletions(-) diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index e7ff3ac0..15edb6c0 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -1396,35 +1396,34 @@ static inline bool agg_finalize_value(const agg_vtable_t* vt, const void* state, /* Threshold = N-th value in the keep direction, found by quickselect on a * copy: O(n) and exact, so ties at the threshold are kept. Callers * parallelize the value fill; the selection itself runs on the caller. */ -int64_t agg_topn_keep(const double* vals, int64_t n, - const ray_group_emit_filter_t* ef, uint8_t* keep) { - if (n <= 0) return 0; - double thr = 0.0; - bool have_thr = false; - if (ef->top_count_take > 0 && n > ef->top_count_take) { - ray_t* hdr = NULL; - double* sv = (double*)scratch_alloc(&hdr, (size_t)n * sizeof(double)); - if (sv) { - memcpy(sv, vals, (size_t)n * sizeof(double)); - int64_t k = ef->desc ? (n - ef->top_count_take) : (ef->top_count_take - 1); - int64_t lo = 0, hi = n - 1; - while (lo < hi) { - double pivot = sv[k]; - int64_t i = lo, j = hi; - while (i <= j) { - while (sv[i] < pivot) i++; - while (sv[j] > pivot) j--; - if (i <= j) { double t = sv[i]; sv[i] = sv[j]; sv[j] = t; i++; j--; } - } - if (k <= j) hi = j; - else if (k >= i) lo = i; - else break; - } - thr = sv[k]; - have_thr = true; - scratch_free(hdr); +bool agg_topn_threshold(const double* vals, int64_t n, + const ray_group_emit_filter_t* ef, double* thr) { + if (ef->top_count_take <= 0 || n <= ef->top_count_take) return false; + ray_t* hdr = NULL; + double* sv = (double*)scratch_alloc(&hdr, (size_t)n * sizeof(double)); + if (!sv) return false; + memcpy(sv, vals, (size_t)n * sizeof(double)); + int64_t k = ef->desc ? (n - ef->top_count_take) : (ef->top_count_take - 1); + int64_t lo = 0, hi = n - 1; + while (lo < hi) { + double pivot = sv[k]; + int64_t i = lo, j = hi; + while (i <= j) { + while (sv[i] < pivot) i++; + while (sv[j] > pivot) j--; + if (i <= j) { double t = sv[i]; sv[i] = sv[j]; sv[j] = t; i++; j--; } } + if (k <= j) hi = j; + else if (k >= i) lo = i; + else break; } + *thr = sv[k]; + scratch_free(hdr); + return true; +} + +int64_t agg_topn_mark(const double* vals, int64_t n, const ray_group_emit_filter_t* ef, + bool have_thr, double thr, uint8_t* keep) { int64_t kept = 0; for (int64_t i = 0; i < n; i++) { double v = vals[i]; @@ -1436,6 +1435,48 @@ int64_t agg_topn_keep(const double* vals, int64_t n, return kept; } +int64_t agg_topn_keep(const double* vals, int64_t n, + const ray_group_emit_filter_t* ef, uint8_t* keep) { + if (n <= 0) return 0; + double thr = 0.0; + bool have_thr = agg_topn_threshold(vals, n, ef, &thr); + return agg_topn_mark(vals, n, ef, have_thr, thr, keep); +} + +/* Bounded candidate heap: the N best values of a range in the keep + * direction. For desc a min-heap of the N largest; for asc a max-heap of + * the N smallest. Every partition contributes one such set, and the + * threshold of their union equals the threshold of the whole. */ +static void agg_topn_candidates(const double* vals, int64_t n, int64_t cap, uint8_t desc, + double* heap, int64_t* hn) { + int64_t h = 0; + #define CAND_WORSE(a, b) (desc ? ((a) < (b)) : ((a) > (b))) /* a is worse than b */ + for (int64_t i = 0; i < n; i++) { + double v = vals[i]; + if (h < cap) { + int64_t j = h++; + heap[j] = v; + while (j > 0) { /* sift up: root is the worst kept */ + int64_t par = (j - 1) / 2; + if (!CAND_WORSE(heap[j], heap[par])) break; + double t = heap[par]; heap[par] = heap[j]; heap[j] = t; j = par; + } + } else if (CAND_WORSE(heap[0], v)) { + heap[0] = v; + int64_t j = 0; + for (;;) { /* sift down */ + int64_t l = 2 * j + 1, r = l + 1, m = j; + if (l < cap && CAND_WORSE(heap[l], heap[m])) m = l; + if (r < cap && CAND_WORSE(heap[r], heap[m])) m = r; + if (m == j) break; + double t = heap[m]; heap[m] = heap[j]; heap[j] = t; j = m; + } + } + } + #undef CAND_WORSE + *hn = h; +} + bool agg_group_values_f64(const agg_vtable_t* vt, const char* states, size_t stride, size_t off, const int64_t* slots, int64_t n, int64_t param, uint8_t desc, double* out) { @@ -2514,7 +2555,7 @@ static ray_t* exec_group_v2_parallel_radix( ray_t** key_cols, int64_t* key_syms, const agg_vtable_t** vts, const size_t* off, size_t block, ray_t* sel, const int64_t* sel_prefix, int64_t n_sel, - int64_t group_limit); + int64_t group_limit, const ray_group_emit_filter_t* efp); typedef struct { ray_t** key_cols; @@ -3390,6 +3431,149 @@ static void agg_radix_finalize_fn(void* vctx, uint32_t wid, int64_t start, int64 } } +/* Heap sort of (keys, pairs) ascending by key, moving both arrays together. + * Used for the small selected sets of the bounded and top-N emits. */ +static void agg_sort_pairs_by_key(agg_radix_order_t* pairs, int64_t* keys, int64_t n) { + #define PAIRS_SIFT_DOWN(START, END) do { \ + int64_t i_ = (START); \ + for (;;) { \ + int64_t l = 2 * i_ + 1, r = l + 1, m = i_; \ + if (l < (END) && keys[l] > keys[m]) m = l; \ + if (r < (END) && keys[r] > keys[m]) m = r; \ + if (m == i_) break; \ + int64_t tk = keys[m]; keys[m] = keys[i_]; keys[i_] = tk; \ + int64_t tp = pairs[m].idx; pairs[m].idx = pairs[i_].idx; pairs[i_].idx = tp; \ + i_ = m; \ + } \ + } while (0) + for (int64_t start = n / 2 - 1; start >= 0; start--) PAIRS_SIFT_DOWN(start, n); + for (int64_t end = n - 1; end > 0; end--) { + int64_t tk = keys[0]; keys[0] = keys[end]; keys[end] = tk; + int64_t tp = pairs[0].idx; pairs[0].idx = pairs[end].idx; pairs[end].idx = tp; + PAIRS_SIFT_DOWN(0, end); + } + #undef PAIRS_SIFT_DOWN +} + +/* Top-N selection for the emit filter: every partition finalizes the filter + * aggregate into one double per group in parallel, the shared keep decision + * picks the kept superset, and only those groups are emitted — in ascending + * first-row order, so the emitted prefix is deterministic. Replaces the + * full path's input-sized order map, full emission and post-trim for + * `desc: AGG take: N` shapes (a 10M-group three-key count spent 110 of its + * 154 ms there). `*rc`: 0 ok, 1 oom, 2 the aggregate has no scalar order + * (caller keeps the full path and trims). */ +typedef struct { + const agg_radix_part_t* parts; + const agg_vtable_t* vt; + size_t off, block; + int64_t param; + const ray_group_emit_filter_t* ef; + double* vals; + uint8_t* keep; + const int64_t* base; + double* cand; /* [n_parts * cap] per-partition candidate heaps */ + int64_t* cand_n; /* [n_parts] */ + int64_t cap; + bool have_thr; + double thr; + int64_t* kept; /* [n_parts] */ + _Atomic(int) fail; +} agg_radix_vals_ctx_t; + +/* Pass 1 per partition: finalize values, then the bounded candidate heap. */ +static void agg_radix_vals_fn(void* raw, uint32_t wid, int64_t start, int64_t end) { + (void)wid; + agg_radix_vals_ctx_t* c = raw; + for (int64_t p = start; p < end; p++) { + double* v = c->vals + c->base[p]; + if (!agg_group_values_f64(c->vt, c->parts[p].states, c->block, c->off, NULL, + c->parts[p].ng, c->param, c->ef->desc, v)) { + atomic_store_explicit(&c->fail, 1, memory_order_relaxed); + continue; + } + if (c->cap > 0) + agg_topn_candidates(v, c->parts[p].ng, c->cap, c->ef->desc, + c->cand + (size_t)p * c->cap, &c->cand_n[p]); + } +} + +/* Pass 2 per partition: mark the kept groups against the global threshold. */ +static void agg_radix_mark_fn(void* raw, uint32_t wid, int64_t start, int64_t end) { + (void)wid; + agg_radix_vals_ctx_t* c = raw; + for (int64_t p = start; p < end; p++) + c->kept[p] = agg_topn_mark(c->vals + c->base[p], c->parts[p].ng, c->ef, + c->have_thr, c->thr, c->keep + c->base[p]); +} + +static agg_radix_order_t* __attribute__((noinline)) +agg_radix_select_topn(ray_pool_t* pool, const agg_radix_part_t* parts, uint32_t n_parts, + const agg_vtable_t* vt, size_t off, size_t block, int64_t param, + const ray_group_emit_filter_t* ef, int64_t* n_emit, int* rc) { + *rc = 0; + int64_t ng = 0; + int64_t* base = ray_alloc_raw(((size_t)n_parts + 1) * sizeof(int64_t)); + if (!base) { *rc = 1; return NULL; } + for (uint32_t p = 0; p < n_parts; p++) { base[p] = ng; ng += parts[p].ng; } + base[n_parts] = ng; + double* vals = ray_alloc_raw((size_t)(ng > 0 ? ng : 1) * sizeof(double)); + uint8_t* keep = ray_alloc_raw((size_t)(ng > 0 ? ng : 1)); + if (!vals || !keep) { + ray_free_raw(base); ray_free_raw(vals); ray_free_raw(keep); *rc = 1; return NULL; + } + int64_t cap = ef->top_count_take > 0 && ng > ef->top_count_take ? ef->top_count_take : 0; + double* cand = ray_alloc_raw((size_t)(cap > 0 ? cap * n_parts : 1) * sizeof(double)); + int64_t* cand_n = ray_calloc_raw((size_t)n_parts * 2 * sizeof(int64_t)); + if (!cand || !cand_n) { + ray_free_raw(base); ray_free_raw(vals); ray_free_raw(keep); + ray_free_raw(cand); ray_free_raw(cand_n); *rc = 1; return NULL; + } + agg_radix_vals_ctx_t c = { .parts = parts, .vt = vt, .off = off, .block = block, + .param = param, .ef = ef, .vals = vals, .keep = keep, .base = base, + .cand = cand, .cand_n = cand_n, .cap = cap, .kept = cand_n + n_parts }; + atomic_init(&c.fail, 0); + if (pool) ray_pool_dispatch_n(pool, agg_radix_vals_fn, &c, n_parts); + else agg_radix_vals_fn(&c, 0, 0, n_parts); + if (atomic_load_explicit(&c.fail, memory_order_relaxed)) { + ray_free_raw(base); ray_free_raw(vals); ray_free_raw(keep); + ray_free_raw(cand); ray_free_raw(cand_n); *rc = 2; return NULL; + } + /* The union of every partition's N best contains the global N best, so + * its N-th value is the global threshold: O(N * parts) serial work. */ + if (cap > 0) { + int64_t nc = 0; + for (uint32_t p = 0; p < n_parts; p++) { + memmove(cand + nc, cand + (size_t)p * cap, (size_t)cand_n[p] * sizeof(double)); + nc += cand_n[p]; + } + c.have_thr = agg_topn_threshold(cand, nc, ef, &c.thr); + } + if (pool) ray_pool_dispatch_n(pool, agg_radix_mark_fn, &c, n_parts); + else agg_radix_mark_fn(&c, 0, 0, n_parts); + int64_t kept = 0; + for (uint32_t p = 0; p < n_parts; p++) kept += c.kept[p]; + ray_free_raw(cand); ray_free_raw(cand_n); + agg_radix_order_t* sel = ray_alloc_raw((size_t)(kept > 0 ? kept : 1) * sizeof(agg_radix_order_t)); + int64_t* fr = ray_alloc_raw((size_t)(kept > 0 ? kept : 1) * sizeof(int64_t)); + if (!sel || !fr) { + ray_free_raw(sel); ray_free_raw(fr); + ray_free_raw(base); ray_free_raw(vals); ray_free_raw(keep); *rc = 1; return NULL; + } + int64_t k = 0; + for (uint32_t p = 0; p < n_parts; p++) + for (int64_t gg = 0; gg < parts[p].ng; gg++) + if (keep[base[p] + gg]) { + sel[k].idx = ((int64_t)p << 32) | (uint32_t)gg; + fr[k] = parts[p].first_row[gg]; + k++; + } + agg_sort_pairs_by_key(sel, fr, k); + ray_free_raw(fr); ray_free_raw(base); ray_free_raw(vals); ray_free_raw(keep); + *n_emit = k; + return sel; +} + /* Bounded first-seen selection for the HEAD(GROUP) limit hint: pick the * `n_emit` groups with the SMALLEST first_row across all partitions — which * ARE the first `n_emit` groups in first-seen order — and return them as the @@ -3449,24 +3633,10 @@ agg_radix_select_first_n(const agg_radix_part_t* parts, uint32_t n_parts, } } } - /* Heap-sort the selected entries into ascending first_row order. */ + /* Sort the selected entries into ascending first_row order (the heap + * above is already a max-heap, so the sort's heapify is a no-op). */ if (ok) { - for (int64_t end = hn - 1; end > 0; end--) { - int64_t tk = hkey[0]; hkey[0] = hkey[end]; hkey[end] = tk; - int64_t tp = sel_pairs[0].idx; - sel_pairs[0].idx = sel_pairs[end].idx; sel_pairs[end].idx = tp; - int64_t i = 0; - for (;;) { - int64_t l = 2 * i + 1, r = l + 1, m = i; - if (l < end && hkey[l] > hkey[m]) m = l; - if (r < end && hkey[r] > hkey[m]) m = r; - if (m == i) break; - tk = hkey[m]; hkey[m] = hkey[i]; hkey[i] = tk; - tp = sel_pairs[m].idx; - sel_pairs[m].idx = sel_pairs[i].idx; sel_pairs[i].idx = tp; - i = m; - } - } + agg_sort_pairs_by_key(sel_pairs, hkey, hn); ok = hn == n_emit; } scratch_free(hk_hdr); @@ -3479,7 +3649,7 @@ static ray_t* exec_group_v2_parallel_radix( ray_t** key_cols, int64_t* key_syms, const agg_vtable_t** vts, const size_t* off, size_t block, ray_t* sel, const int64_t* sel_prefix, int64_t n_sel, - int64_t group_limit) { + int64_t group_limit, const ray_group_emit_filter_t* efp) { ray_op_ext_t* ext = find_ext(g, op->id); uint32_t n_keys = ext->n_keys, n_aggs = ext->n_aggs; ray_pool_t* pool = ray_pool_get(); @@ -3598,7 +3768,28 @@ static ray_t* exec_group_v2_parallel_radix( * emitted prefix is byte-identical to trimming the full result to N. */ int64_t n_emit = ng; agg_radix_order_t* pairs = NULL; - if (group_limit > 0 && ng > group_limit) { + if (efp && group_limit == 0) { + int rc = 0; + int64_t kept = 0; + agg_radix_order_t* sel_pairs = agg_radix_select_topn(pool, parts, n_parts, + vts[efp->agg_index], off[efp->agg_index], block, + ext->agg_k ? ext->agg_k[efp->agg_index] : 0, efp, &kept, &rc); + if (sel_pairs) { + pairs = sel_pairs; + n_emit = kept; + route_stats.topn_native = true; + } else if (rc == 1) { + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); + for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); + ray_free_raw(bufs); ray_free_raw(parts); + agg_desc_free(&d); + return ray_error("oom", NULL); + } + /* rc == 2: no scalar order for this aggregate — full path below. */ + } + if (pairs) { + /* native top-N selected above */ + } else if (group_limit > 0 && ng > group_limit) { int rc = 0; n_emit = group_limit; agg_radix_order_t* sel_pairs = agg_radix_select_first_n( @@ -4447,13 +4638,19 @@ static bool agg_shared_sample(ray_graph_t* g, ray_op_ext_t* ext, ray_t* tbl, static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t nrows, ray_t* sel, const int64_t* sel_prefix, int64_t n_sel, - int64_t group_limit) { + int64_t group_limit, + const ray_group_emit_filter_t* efp) { agg_route_reason(AGG_V2_ADMITTED); + route_stats.topn_native = false; route_stats.nullable_key = false; route_stats.dense_plan_available = false; route_stats.dense_worker_budget = false; route_stats.dense_tasks = 0; ray_op_ext_t* ext = find_ext(g, op->id); + /* The top-N emit filter names an aggregate slot; an out-of-range slot or a + * filter armed for a different node means "no filter" here (the caller's + * sort+take still produces the final answer from the full result). */ + if (efp && (!efp->enabled || efp->agg_index >= ext->n_aggs)) efp = NULL; /* Exact-size carve for the per-key column pointers + syms (one block, both * 8-byte): unbounded key count, freed at every exit of this function @@ -4488,7 +4685,7 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (!idx || RAY_IS_ERR(idx)) { scratch_free(kc_hdr); return idx ? idx : ray_error("oom", NULL); } ray_t* compact = agg_build_compact(g, op, tbl, ray_data(idx), n_sel); ray_t* result = compact && !RAY_IS_ERR(compact) - ? exec_group_v2_run(g, op, compact, n_sel, NULL, NULL, 0, group_limit) : compact; + ? exec_group_v2_run(g, op, compact, n_sel, NULL, NULL, 0, group_limit, efp) : compact; if (compact && !RAY_IS_ERR(compact)) ray_release(compact); ray_release(idx); scratch_free(kc_hdr); return result; } @@ -4536,7 +4733,7 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, return compact ? compact : ray_error("oom", NULL); \ } \ ray_t* r = exec_group_v2_run(g, op, compact, n_sel, NULL, NULL, 0, \ - group_limit); \ + group_limit, efp); \ ray_release(compact); ray_release(idxb); \ return r; \ } while (0) @@ -4719,7 +4916,7 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, /* Sparse ranges and excessive dense worker traffic use radix. */ ray_t* r = exec_group_v2_parallel_radix(g, op, tbl, nrows, key_cols, key_syms, vts, off, block, sel, sel_prefix, n_sel, - group_limit); + group_limit, efp); agg_vo_free(&vo); agg_dense_plan_free(&dp); scratch_free(kc_hdr); return r; } /* Hash fallback (F64 / STR keys): not a chunked strategy — compact. */ @@ -4867,9 +5064,14 @@ static ray_t* agg_build_compact(ray_graph_t* g, ray_op_t* op, ray_t* tbl, ray_t* exec_group_v2(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t group_limit) { if (agg_cancelled()) return ray_error("cancel", NULL); + /* The top-N emit filter (`desc: AGG take: N`) is read once here and + * handed to every strategy: radix and the dense finishes select the kept + * groups themselves; the other routes trim their full result. */ + ray_group_emit_filter_t ef = ray_group_emit_filter_get(); + const ray_group_emit_filter_t* efp = ef.enabled ? &ef : NULL; if (!g || !g->selection) return exec_group_v2_run(g, op, tbl, ray_table_nrows(tbl), NULL, NULL, 0, - group_limit); + group_limit, efp); int64_t src_nrows = ray_table_nrows(tbl); ray_rowsel_t* sm = ray_rowsel_meta(g->selection); @@ -4877,7 +5079,7 @@ ray_t* exec_group_v2(ray_graph_t* g, ray_op_t* op, ray_t* tbl, * applied here — fall back to the unfiltered run (matches the scalar-agg * guard in group.c, which also only honors a selection when nrows match). */ if (sm->nrows != src_nrows) - return exec_group_v2_run(g, op, tbl, src_nrows, NULL, NULL, 0, group_limit); + return exec_group_v2_run(g, op, tbl, src_nrows, NULL, NULL, 0, group_limit, efp); int64_t n_sel = sm->total_pass; ray_t* prefix_block = agg_sel_build_prefix(g->selection); @@ -4889,7 +5091,7 @@ ray_t* exec_group_v2(ray_graph_t* g, ray_op_t* op, ray_t* tbl, ray_t* saved_sel = g->selection; g->selection = NULL; ray_t* result = exec_group_v2_run(g, op, tbl, src_nrows, saved_sel, sel_prefix, - n_sel, group_limit); + n_sel, group_limit, efp); g->selection = saved_sel; ray_release(prefix_block); diff --git a/src/ops/agg_engine.h b/src/ops/agg_engine.h index b46a2f4e..bc1c23e2 100644 --- a/src/ops/agg_engine.h +++ b/src/ops/agg_engine.h @@ -64,6 +64,7 @@ typedef struct { uint64_t dense_local_slots; /* allocated group-state slots, including partials */ uint32_t dense_tasks; /* local/partition tasks; worker count for shared updates */ uint64_t key_domain_evals; /* computed keys evaluated once per distinct symbol */ + bool topn_native; /* last v2 run selected the emit filter's top-N itself */ } agg_route_stats_t; void agg_route_reset(void); void agg_route_note_key_domain(void); @@ -86,6 +87,13 @@ bool agg_v2_dense_plan_available(ray_graph_t* g, ray_op_t* op, ray_t* tbl); * number kept. vals may be NULL when n == 0. */ int64_t agg_topn_keep(const double* vals, int64_t n, const ray_group_emit_filter_t* ef, uint8_t* keep); +/* The two halves of agg_topn_keep, for callers that reduce candidates in + * parallel: the N-th value in the keep direction (false when every value is + * kept), and the keep marking against a known threshold. */ +bool agg_topn_threshold(const double* vals, int64_t n, + const ray_group_emit_filter_t* ef, double* thr); +int64_t agg_topn_mark(const double* vals, int64_t n, const ray_group_emit_filter_t* ef, + bool have_thr, double thr, uint8_t* keep); /* Double view of aggregate `vt` for n groups: group i's state is at * states + (slots ? slots[i] : i) * stride + off. Nulls (and NaN) sink to diff --git a/src/ops/group.c b/src/ops/group.c index c63c3926..2e2e20fd 100644 --- a/src/ops/group.c +++ b/src/ops/group.c @@ -11410,51 +11410,25 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, return exec_group_v2(g, op, tbl, group_limit); /* Emit-filter shapes (`desc: c take: N`, `where (> c N)` on the group - * result) used to run on the legacy ladder because v2 does not - * implement the top-count emit filter. That ladder is single-threaded - * in its scatter and its dense array scales with the store's SHARED sym - * domain, so every top-N group query lost the parallel engine: a - * three-aggregate 100k-group query measured 800 ms on one core and - * 777 ms on a 28-thread pool (0.1x parallel), while the same grouping - * without the filter ran in 47 ms. The emit filter is purely - * an optimization — the DAG's sort+take downstream produces the final - * order/limit either way — so run the PARALLEL v2 engine on every shape - * it admits with a bounded dense plan and trim its full result to the - * filter's top-N superset. Unbounded key domains (many-million-group - * inputs) stay on the legacy ladder for now: v2's radix route still - * pays a serial first-seen ordering and emission tail there (a 10M-group - * three-key count measured 154 ms against the ladder's 54 ms on 28 - * threads). One unbounded shape still prefers v2: a single wide-domain - * SYM key on a bounded input, where the ladder's dense scatter scales - * with the store's SHARED sym domain (splayed stores keep one domain - * across all SYM columns — often 10M+ ids) and became the serial wall - * (88 ms of a 110 ms benchmark query, flat multi-core scaling). */ + * result) used to run on the legacy ladder because v2 did not implement + * the top-count emit filter. That ladder is single-threaded in its + * scatter and its dense array scales with the store's SHARED sym domain, + * so every top-N group query lost the parallel engine: a three-aggregate + * 100k-group query measured 800 ms on one core and 777 ms on a 28-thread + * pool (0.1x parallel), while the same grouping without the filter ran + * in 47 ms. v2 now reads the filter itself: radix and the dense finishes + * select the kept groups natively, the remaining routes return the full + * result and are trimmed here to the filter's top-N superset. The DAG's + * sort+take downstream produces the final order/limit either way. The + * legacy ladder remains only for shapes v2 declines. */ { ray_group_emit_filter_t ef = ray_group_emit_filter_get(); - bool wide_sym_key = false; - if (ef.enabled && ext->n_keys == 1 && ray_table_nrows(tbl) <= (int64_t)(4u << 20) - && (ef.agg_op == 0 || ef.agg_op == OP_COUNT || ef.agg_op == OP_SUM - || ef.agg_op == OP_MIN || ef.agg_op == OP_MAX)) { - ray_op_t* k0 = op_node(g, ext->keys[0]); - ray_op_ext_t* k0e = k0 ? find_ext(g, k0->id) : NULL; - ray_t* k0c = (k0e && k0e->base.opcode == OP_SCAN) - ? ray_table_get_col(tbl, k0e->sym) : NULL; - wide_sym_key = k0c && k0c->type == RAY_SYM && - ray_sym_domain_count(ray_sym_vec_domain(k0c)) > (1 << 21); - } if (ray_agg_engine_v2 && group_limit == 0 && ef.enabled - && agg_v2_can_handle(g, op, tbl) - && (wide_sym_key || agg_v2_dense_plan_available(g, op, tbl))) { - /* Suppress the filter for the v2 run (v2 ignores it anyway; - * clearing keeps recursion/asserts honest), restore after. */ - ray_group_emit_filter_t saved = ray_group_emit_filter_get(); - ray_group_emit_filter_t off = {0}; - ray_group_emit_filter_set(off); + && agg_v2_can_handle(g, op, tbl)) { ray_t* r = exec_group_v2(g, op, tbl, 0); - ray_group_emit_filter_set(saved); if (r && !RAY_IS_ERR(r)) - return group_emit_filter_trim(r, ext->n_keys, - ext->n_aggs, ef); + return agg_route_stats().topn_native + ? r : group_emit_filter_trim(r, ext->n_keys, ext->n_aggs, ef); if (r) return r; /* v2 declined at runtime — continue on the legacy ladder. */ } diff --git a/test/test_agg_contract.c b/test/test_agg_contract.c index 0c92875f..3d299b6d 100644 --- a/test/test_agg_contract.c +++ b/test/test_agg_contract.c @@ -1799,6 +1799,39 @@ static test_result_t test_dense_composite_compaction(void) { PASS(); } +/* A many-million-group top-N must stay on the parallel engine and emit only + * the kept superset: radix selects by aggregate value per partition. */ +static test_result_t test_radix_native_topn(void) { + ray_pool_destroy(); + TEST_ASSERT_EQ_I(ray_pool_init_total(8), RAY_OK); + ray_t* setup = ray_eval_str( + "(set rt_i (til 2000000)) " + "(set rt_t (table [k j v] (list (% (* rt_i 7919) 1500000) (% rt_i 3) (% rt_i 5))))"); + TEST_ASSERT_NOT_NULL(setup); TEST_ASSERT_FALSE(RAY_IS_ERR(setup)); ray_release(setup); + agg_route_reset(); + ray_t* r = ray_eval_str("(select {from:rt_t by:[k j] c:(count v) desc:c take:10})"); + TEST_ASSERT_NOT_NULL(r); TEST_ASSERT_FALSE(RAY_IS_ERR(r)); + agg_route_stats_t stats = agg_route_stats(); + TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_LEGACY], 0); + TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_V2_RADIX], 1); + TEST_ASSERT_TRUE(stats.topn_native); + TEST_ASSERT_EQ_I(ray_table_nrows(r), 10); + ray_t* check = ray_eval_str( + "(== (at (at (select {from:rt_t by:[k j] c:(count v) desc:c take:10}) 'c) 0) " + "(max (at (select {from:rt_t by:[k j] c:(count v)}) 'c)))"); + TEST_ASSERT_NOT_NULL(check); TEST_ASSERT_FALSE(RAY_IS_ERR(check)); + TEST_ASSERT_EQ_I(check->i64, 1); + ray_release(check); ray_release(r); + /* asc keeps the smallest counts */ + r = ray_eval_str("(select {from:rt_t by:[k j] c:(count v) asc:c take:3})"); + TEST_ASSERT_NOT_NULL(r); TEST_ASSERT_FALSE(RAY_IS_ERR(r)); + TEST_ASSERT_EQ_I(ray_table_nrows(r), 3); + TEST_ASSERT_EQ_I(((int64_t*)ray_data(ray_table_get_col_idx(r, 2)))[0], 1); + ray_release(r); + ray_release(ray_eval_str("(set rt_t 0) (set rt_i 0)")); + PASS(); +} + const test_entry_t agg_contract_entries[] = { { "agg_contract/empty_inference_errors", test_empty_inference_errors, contract_setup, contract_teardown }, { "agg_contract/nth_bounds", test_nth_bounds, contract_setup, contract_teardown }, @@ -1815,6 +1848,7 @@ const test_entry_t agg_contract_entries[] = { { "agg_contract/dense_task_local", test_dense_task_local, contract_setup, contract_teardown }, { "agg_contract/dense_cache_bound", test_dense_cache_bound, contract_setup, contract_teardown }, { "agg_contract/dense_composite_compaction", test_dense_composite_compaction, contract_setup, contract_teardown }, + { "agg_contract/radix_native_topn", test_radix_native_topn, contract_setup, contract_teardown }, { "agg_contract/rank_widths_nulls_slices", test_rank_widths_nulls_and_slices, contract_setup, contract_teardown }, { "agg_contract/nullable_differential", test_nullable_differential, contract_setup, contract_teardown }, { "agg_contract/wide_key_routes", test_wide_key_routes, contract_setup, contract_teardown }, From 83fe1612a0f4cfc358c8186a383c522610d42ea4 Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 12:16:49 +0200 Subject: [PATCH 11/23] perf(group): apply the top-N emit filter inside dense finishes Dense task-local, partitioned and shared finishes select the kept groups over the occupied slot list (parallel value fill, bounded candidate heaps, one threshold) before any output column is built. Single-key count with desc take at 28 threads: 6.0 -> 5.4 ms; at one thread 28 -> 17 ms. --- src/ops/agg_engine.c | 141 +++++++++++++++++++++++++++++++++++---- test/test_agg_contract.c | 37 ++++++++++ 2 files changed, 166 insertions(+), 12 deletions(-) diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index 15edb6c0..97c26ab5 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -1746,12 +1746,107 @@ static void agg_dense_key_emit(void* raw, uint32_t wid, int64_t start, int64_t e } } +/* Top-N selection over a slot list: tasks finalize the filter aggregate for + * disjoint ranges of `slots`, keep a bounded candidate heap each, the union's + * N-th value is the threshold, and a second pass marks the kept groups. + * Returns the kept count, or -1 when the aggregate has no scalar order or + * memory ran out (the caller then emits every group and the query trims). */ +typedef struct { + const agg_vtable_t* vt; + const char* states; + size_t block, off; + const int64_t* slots; + int64_t n, param; + const ray_group_emit_filter_t* ef; + double* vals; + uint8_t* keep; + uint32_t tasks; + double* cand; + int64_t* cand_n; + int64_t cap; + bool have_thr; + double thr; + int64_t* kept; + _Atomic(int) fail; +} agg_slots_topn_ctx_t; + +static void agg_slots_topn_range(const agg_slots_topn_ctx_t* c, int64_t task, int64_t* begin, int64_t* end) { + *begin = c->n / c->tasks * task; + *end = task + 1 == (int64_t)c->tasks ? c->n : c->n / c->tasks * (task + 1); +} + +static void agg_slots_vals_fn(void* raw, uint32_t wid, int64_t start, int64_t end) { + (void)wid; + agg_slots_topn_ctx_t* c = raw; + for (int64_t task = start; task < end; task++) { + int64_t b, e; + agg_slots_topn_range(c, task, &b, &e); + if (!agg_group_values_f64(c->vt, c->states, c->block, c->off, c->slots + b, + e - b, c->param, c->ef->desc, c->vals + b)) { + atomic_store_explicit(&c->fail, 1, memory_order_relaxed); + continue; + } + if (c->cap > 0) + agg_topn_candidates(c->vals + b, e - b, c->cap, c->ef->desc, + c->cand + (size_t)task * c->cap, &c->cand_n[task]); + } +} + +static void agg_slots_mark_fn(void* raw, uint32_t wid, int64_t start, int64_t end) { + (void)wid; + agg_slots_topn_ctx_t* c = raw; + for (int64_t task = start; task < end; task++) { + int64_t b, e; + agg_slots_topn_range(c, task, &b, &e); + c->kept[task] = agg_topn_mark(c->vals + b, e - b, c->ef, c->have_thr, c->thr, c->keep + b); + } +} + +static int64_t agg_slots_topn_select(ray_pool_t* pool, const agg_vtable_t* vt, + const char* states, size_t block, size_t off, const int64_t* slots, int64_t n, + int64_t param, const ray_group_emit_filter_t* ef, uint8_t* keep) { + if (n <= 0) return 0; + uint32_t tasks = pool && n >= RAY_PARALLEL_THRESHOLD ? ray_pool_total_workers(pool) * 4 : 1; + if (tasks > RAY_POOL_INIT_TASKS) tasks = RAY_POOL_INIT_TASKS; + if ((int64_t)tasks > n) tasks = (uint32_t)n; + int64_t cap = ef->top_count_take > 0 && n > ef->top_count_take ? ef->top_count_take : 0; + double* vals = ray_alloc_raw((size_t)n * sizeof(double)); + double* cand = ray_alloc_raw((size_t)(cap > 0 ? cap * tasks : 1) * sizeof(double)); + int64_t* cand_n = ray_calloc_raw((size_t)tasks * 2 * sizeof(int64_t)); + int64_t kept = -1; + if (vals && cand && cand_n) { + agg_slots_topn_ctx_t c = { .vt = vt, .states = states, .block = block, .off = off, + .slots = slots, .n = n, .param = param, .ef = ef, .vals = vals, .keep = keep, + .tasks = tasks, .cand = cand, .cand_n = cand_n, .cap = cap, .kept = cand_n + tasks }; + atomic_init(&c.fail, 0); + if (tasks > 1) ray_pool_dispatch_n(pool, agg_slots_vals_fn, &c, tasks); + else agg_slots_vals_fn(&c, 0, 0, 1); + if (!atomic_load_explicit(&c.fail, memory_order_relaxed)) { + if (cap > 0) { + int64_t nc = 0; + for (uint32_t t = 0; t < tasks; t++) { + memmove(cand + nc, cand + (size_t)t * cap, (size_t)cand_n[t] * sizeof(double)); + nc += cand_n[t]; + } + c.have_thr = agg_topn_threshold(cand, nc, ef, &c.thr); + } + if (tasks > 1) ray_pool_dispatch_n(pool, agg_slots_mark_fn, &c, tasks); + else agg_slots_mark_fn(&c, 0, 0, 1); + kept = 0; + for (uint32_t t = 0; t < tasks; t++) kept += c.kept[t]; + } + } + ray_free_raw(vals); ray_free_raw(cand); ray_free_raw(cand_n); + return kept; +} + /* Shared output stage. Takes ownership of states/first, borrows the * prepared aggregate layout and descriptors. */ static ray_t* agg_dense_finish(ray_t** key_cols, int64_t* key_syms, ray_op_ext_t* ext, ray_pool_t* pool, const agg_vo_t* vo, const agg_desc_t* d, int64_t total_slots, char* gstates, int64_t* gfirst, - const dense_plan_t* key_plan, int64_t key_part_slots, uint32_t key_bits) { + const dense_plan_t* key_plan, int64_t key_part_slots, uint32_t key_bits, + const ray_group_emit_filter_t* ef) { uint32_t n_keys = ext->n_keys, n_aggs = ext->n_aggs; const agg_vtable_t** vts = vo->vts; const size_t* off = vo->off; @@ -1780,6 +1875,26 @@ static ray_t* agg_dense_finish(ray_t** key_cols, int64_t* key_syms, ray_op_ext_t occupied_slot[i] = s; i++; } } + /* Top-N emit filter: keep only the groups the filter would keep, in the + * same slot order. Nothing below allocates for the dropped groups. */ + if (ef && ng > 0) { + uint8_t* keep = ray_alloc_raw((size_t)ng); + int64_t kept = keep ? agg_slots_topn_select(pool, vts[ef->agg_index], gstates, block, + off[ef->agg_index], occupied_slot, ng, + ext->agg_k ? ext->agg_k[ef->agg_index] : 0, ef, keep) : -1; + if (kept >= 0) { + int64_t w = 0; + for (int64_t i = 0; i < ng; i++) if (keep[i]) { + occupied_slot[w] = occupied_slot[i]; + if (first_row_ordered) first_row_ordered[w] = first_row_ordered[i]; + w++; + } + ng = kept; + route_stats.topn_native = true; + } + ray_free_raw(keep); + ray_profile_tick("dense: selected top-N groups"); + } ray_t* result = ray_table_new(n_keys + n_aggs); if (!result || RAY_IS_ERR(result)) { @@ -2142,7 +2257,8 @@ static uint32_t agg_dense_partition_parts(uint32_t sources, int64_t slots) { static ray_t* agg_dense_partitioned(ray_t** key_cols, int64_t* key_syms, ray_op_ext_t* ext, ray_pool_t* pool, const dense_plan_t* plan, int64_t rows, - const agg_vo_t* vo, const agg_desc_t* d, ray_t* selection) { + const agg_vo_t* vo, const agg_desc_t* d, ray_t* selection, + const ray_group_emit_filter_t* ef) { uint32_t workers = ray_pool_total_workers(pool); uint32_t sources = workers; if (sources > RAY_POOL_INIT_TASKS / 2) sources = RAY_POOL_INIT_TASKS / 2; @@ -2216,7 +2332,7 @@ static ray_t* agg_dense_partitioned(ray_t** key_cols, int64_t* key_syms, ray_op_ route_stats.dense_strategy = AGG_DENSE_PARTITIONED; route_stats.dense_tasks = c.n_tasks; route_stats.dense_local_slots = slots + extra_slots; - return agg_dense_finish(key_cols, key_syms, ext, pool, vo, d, slots, c.states, c.first, plan, part_slots, bits); + return agg_dense_finish(key_cols, key_syms, ext, pool, vo, d, slots, c.states, c.first, plan, part_slots, bits, ef); failed: ray_free_raw(c.counts); ray_free_raw(c.gids); ray_free_raw(c.tasks); ray_free_raw(c.payload); ray_free_raw(c.value_offsets); ray_release(c.selection_indices); ray_free_raw(c.states); ray_free_raw(c.first); @@ -2296,7 +2412,7 @@ static void agg_dense_shared_occupied(void* raw, uint32_t wid, int64_t start, in static ray_t* agg_dense_shared(ray_t** key_cols, int64_t* key_syms, ray_op_ext_t* ext, ray_pool_t* pool, const dense_plan_t* plan, int64_t rows, - const agg_vo_t* vo, const agg_desc_t* d) { + const agg_vo_t* vo, const agg_desc_t* d, const ray_group_emit_filter_t* ef) { int64_t slots = plan->total_slots; agg_dense_shared_ctx_t c = {.key = key_cols[0], .plan = plan, .layout = vo, .desc = d, .n_aggs = ext->n_aggs}; c.states = ray_alloc_raw((size_t)slots * vo->block); @@ -2316,7 +2432,7 @@ static ray_t* agg_dense_shared(ray_t** key_cols, int64_t* key_syms, ray_op_ext_t route_stats.dense_strategy = AGG_DENSE_SHARED; route_stats.dense_tasks = ray_pool_total_workers(pool); route_stats.dense_local_slots = slots; - return agg_dense_finish(key_cols, key_syms, ext, pool, vo, d, slots, c.states, c.first, plan, slots, 0); + return agg_dense_finish(key_cols, key_syms, ext, pool, vo, d, slots, c.states, c.first, plan, slots, 0, ef); failed: ray_free_raw(c.states); ray_free_raw(c.first); ray_free_raw(c.occupied); return ray_error(agg_cancelled() ? "cancel" : "oom", NULL); @@ -2326,7 +2442,8 @@ static ray_t* exec_group_v2_parallel_dense( ray_graph_t* g, ray_op_t* op, ray_t* tbl, ray_t** key_cols, int64_t* key_syms, ray_op_ext_t* ext, int64_t nrows, ray_pool_t* pool, const dense_plan_t* dp, uint32_t nw, agg_dense_strategy_t strategy, - ray_t* sel, const int64_t* sel_prefix, int64_t n_sel) { + ray_t* sel, const int64_t* sel_prefix, int64_t n_sel, + const ray_group_emit_filter_t* efp) { uint32_t n_keys = ext->n_keys, n_aggs = ext->n_aggs; int64_t total_slots = dp->total_slots; @@ -2344,12 +2461,12 @@ static ray_t* exec_group_v2_parallel_dense( const bool* val2_hasnull = d.val2_hasnull; const uint8_t* val2_esz = d.val2_esz; if (strategy == AGG_DENSE_SHARED) { - ray_t* result = agg_dense_shared(key_cols, key_syms, ext, pool, dp, nrows, &vo, &d); + ray_t* result = agg_dense_shared(key_cols, key_syms, ext, pool, dp, nrows, &vo, &d, efp); agg_vo_free(&vo); agg_desc_free(&d); return result; } if (strategy == AGG_DENSE_PARTITIONED) { - ray_t* result = agg_dense_partitioned(key_cols, key_syms, ext, pool, dp, sel ? n_sel : nrows, &vo, &d, sel); + ray_t* result = agg_dense_partitioned(key_cols, key_syms, ext, pool, dp, sel ? n_sel : nrows, &vo, &d, sel, efp); agg_vo_free(&vo); agg_desc_free(&d); return result; } @@ -2458,7 +2575,7 @@ static ray_t* exec_group_v2_parallel_dense( ray_free_raw(locals); ray_t* result = agg_dense_finish(key_cols, key_syms, ext, pool, &vo, &d, - total_slots, gstates, gfirst, NULL, 0, 0); + total_slots, gstates, gfirst, NULL, 0, 0, efp); agg_vo_free(&vo); agg_desc_free(&d); return result; } @@ -4790,7 +4907,7 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, route_stats.dense_worker_budget = false; agg_route_record(AGG_ROUTE_V2_DENSE); ray_t* result = exec_group_v2_parallel_dense(g, op, tbl, key_cols, key_syms, ext, - nrows, pool, &dp, dense_workers, AGG_DENSE_SHARED, sel, sel_prefix, n_sel); + nrows, pool, &dp, dense_workers, AGG_DENSE_SHARED, sel, sel_prefix, n_sel, efp); agg_vo_free(&vo); agg_dense_plan_free(&dp); scratch_free(kc_hdr); return result; } @@ -4836,7 +4953,7 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, route_stats.dense_worker_budget = false; agg_route_record(AGG_ROUTE_V2_DENSE); ray_t* result = exec_group_v2_parallel_dense(g, op, tbl, key_cols, key_syms, ext, - nrows, pool, &dp, dense_workers, AGG_DENSE_PARTITIONED, sel, sel_prefix, n_sel); + nrows, pool, &dp, dense_workers, AGG_DENSE_PARTITIONED, sel, sel_prefix, n_sel, efp); agg_vo_free(&vo); agg_dense_plan_free(&dp); scratch_free(kc_hdr); return result; } @@ -4908,7 +5025,7 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, route_stats.dense_local_slots += dp.total_slots; agg_route_record(AGG_ROUTE_V2_DENSE); ray_t* r = exec_group_v2_parallel_dense(g, op, tbl, key_cols, key_syms, ext, nrows, pool, &dp, dense_workers, AGG_DENSE_TASK_LOCAL, - sel, sel_prefix, n_sel); + sel, sel_prefix, n_sel, efp); agg_vo_free(&vo); agg_dense_plan_free(&dp); scratch_free(kc_hdr); return r; } if (keys_intsym) { diff --git a/test/test_agg_contract.c b/test/test_agg_contract.c index 3d299b6d..adc42cd0 100644 --- a/test/test_agg_contract.c +++ b/test/test_agg_contract.c @@ -1832,6 +1832,42 @@ static test_result_t test_radix_native_topn(void) { PASS(); } +/* Dense finishes select the emit filter's top-N themselves: no full emission + * and no post-trim for a bounded-domain key. */ +static test_result_t test_dense_native_topn(void) { + ray_t* setup = ray_eval_str( + "(set dt_i (til 1000000)) " + "(set dt_t (table [k v] (list (as 'I32 (% (* dt_i 7919) 50000)) (% dt_i 13))))"); + TEST_ASSERT_NOT_NULL(setup); TEST_ASSERT_FALSE(RAY_IS_ERR(setup)); ray_release(setup); + agg_route_reset(); + ray_t* r = ray_eval_str("(select {from:dt_t by:k s:(sum v) c:(count v) desc:s take:5})"); + TEST_ASSERT_NOT_NULL(r); TEST_ASSERT_FALSE(RAY_IS_ERR(r)); + agg_route_stats_t stats = agg_route_stats(); + TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_V2_DENSE], 1); + TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_LEGACY], 0); + TEST_ASSERT_TRUE(stats.topn_native); + TEST_ASSERT_EQ_I(ray_table_nrows(r), 5); + ray_t* check = ray_eval_str( + "(all (== (at (select {from:dt_t by:k s:(sum v) c:(count v) desc:s take:5}) 's) " + "(take (at (xdesc (select {from:dt_t by:k s:(sum v)}) 's) 's) 5)))"); + TEST_ASSERT_NOT_NULL(check); TEST_ASSERT_FALSE(RAY_IS_ERR(check)); + TEST_ASSERT_EQ_I(check->i64, 1); + ray_release(check); ray_release(r); + /* the asc direction on the count keeps the smallest groups */ + agg_route_reset(); + r = ray_eval_str("(select {from:dt_t by:k c:(count v) asc:c take:3})"); + TEST_ASSERT_NOT_NULL(r); TEST_ASSERT_FALSE(RAY_IS_ERR(r)); + TEST_ASSERT_TRUE(agg_route_stats().topn_native); + check = ray_eval_str( + "(== (at (at (select {from:dt_t by:k c:(count v) asc:c take:3}) 'c) 0) " + "(min (at (select {from:dt_t by:k c:(count v)}) 'c)))"); + TEST_ASSERT_NOT_NULL(check); TEST_ASSERT_FALSE(RAY_IS_ERR(check)); + TEST_ASSERT_EQ_I(check->i64, 1); + ray_release(check); ray_release(r); + ray_release(ray_eval_str("(set dt_t 0) (set dt_i 0)")); + PASS(); +} + const test_entry_t agg_contract_entries[] = { { "agg_contract/empty_inference_errors", test_empty_inference_errors, contract_setup, contract_teardown }, { "agg_contract/nth_bounds", test_nth_bounds, contract_setup, contract_teardown }, @@ -1849,6 +1885,7 @@ const test_entry_t agg_contract_entries[] = { { "agg_contract/dense_cache_bound", test_dense_cache_bound, contract_setup, contract_teardown }, { "agg_contract/dense_composite_compaction", test_dense_composite_compaction, contract_setup, contract_teardown }, { "agg_contract/radix_native_topn", test_radix_native_topn, contract_setup, contract_teardown }, + { "agg_contract/dense_native_topn", test_dense_native_topn, contract_setup, contract_teardown }, { "agg_contract/rank_widths_nulls_slices", test_rank_widths_nulls_and_slices, contract_setup, contract_teardown }, { "agg_contract/nullable_differential", test_nullable_differential, contract_setup, contract_teardown }, { "agg_contract/wide_key_routes", test_wide_key_routes, contract_setup, contract_teardown }, From 330a1dcf3e595d74062368decab182cf75980d8f Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 12:24:17 +0200 Subject: [PATCH 12/23] refactor(group): v2 owns the top-N emit filter; drop the ladder carve-outs The emit filter no longer rejects a shape from the parallel engine. Routes that cannot select natively return through one wrapper that trims the full result with the shared keep decision; the legacy trim helper and the wide-domain carve-out in exec_group_run are gone. --- src/ops/agg_engine.c | 64 ++++++++++++ src/ops/agg_engine.h | 7 +- src/ops/group.c | 124 +----------------------- test/rfl/group/emit_filter_v2_route.rfl | 13 +++ 4 files changed, 88 insertions(+), 120 deletions(-) diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index 97c26ab5..80b3df2e 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -4752,11 +4752,75 @@ static bool agg_shared_sample(ray_graph_t* g, ray_op_ext_t* ext, ray_t* tbl, * compact table of the selected rows once and recurse with sel=NULL — the * unmodified strategy then runs over the compact table. This is the documented * compact fallback the design permits for the non-chunked shapes. */ +static ray_t* exec_group_v2_run_inner(ray_graph_t* g, ray_op_t* op, ray_t* tbl, + int64_t nrows, ray_t* sel, + const int64_t* sel_prefix, int64_t n_sel, + int64_t group_limit, + const ray_group_emit_filter_t* efp); + +/* Trim a full group result to the emit filter's keep set (the same decision + * the native selections make, over the finished aggregate column). Row + * order is preserved. Consumes `result`; a failure keeps the full result, + * which is still a valid answer for the sort+take downstream. */ +ray_t* agg_emit_filter_trim(ray_t* result, uint32_t n_keys, uint32_t n_aggs, + const ray_group_emit_filter_t* ef) { + if (!result || RAY_IS_ERR(result) || result->type != RAY_TABLE) return result; + if (ef->agg_index >= n_aggs) return result; + int64_t nrows = ray_table_nrows(result); + if (nrows <= 0) return result; + ray_t* vcol = ray_table_get_col_idx(result, (int64_t)n_keys + ef->agg_index); + if (!vcol || (vcol->type != RAY_I64 && vcol->type != RAY_F64)) return result; + double* vals = ray_alloc_raw((size_t)nrows * sizeof(double)); + uint8_t* keep = ray_alloc_raw((size_t)nrows); + ray_t* idx = ray_vec_new(RAY_I64, nrows); + if (!vals || !keep || !idx || RAY_IS_ERR(idx)) { + ray_free_raw(vals); ray_free_raw(keep); + if (idx && RAY_IS_ERR(idx)) ray_error_free(idx); else ray_release(idx); + return result; + } + if (vcol->type == RAY_F64) { + const double* vf = (const double*)ray_data(vcol); + for (int64_t r = 0; r < nrows; r++) vals[r] = vf[r]; + } else { + const int64_t* vi = (const int64_t*)ray_data(vcol); + for (int64_t r = 0; r < nrows; r++) vals[r] = (double)vi[r]; + } + int64_t kept = agg_topn_keep(vals, nrows, ef, keep); + int64_t* ix = (int64_t*)ray_data(idx); + int64_t w = 0; + for (int64_t r = 0; r < nrows; r++) if (keep[r]) ix[w++] = r; + idx->len = w; + ray_free_raw(vals); ray_free_raw(keep); + if (kept == nrows) { ray_release(idx); return result; } + ray_t* out = ray_at_fn(result, idx); + ray_release(idx); + if (!out || RAY_IS_ERR(out)) { if (out) ray_error_free(out); return result; } + ray_release(result); + return out; +} + +/* Every v2 strategy returns through here: a route that could not select + * the emit filter's top-N itself hands back its full result and is trimmed + * to the kept superset, so callers see one contract regardless of route. */ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t nrows, ray_t* sel, const int64_t* sel_prefix, int64_t n_sel, int64_t group_limit, const ray_group_emit_filter_t* efp) { + ray_t* r = exec_group_v2_run_inner(g, op, tbl, nrows, sel, sel_prefix, n_sel, + group_limit, efp); + if (efp && r && !RAY_IS_ERR(r) && !route_stats.topn_native) { + ray_op_ext_t* ext = find_ext(g, op->id); + if (ext) r = agg_emit_filter_trim(r, ext->n_keys, ext->n_aggs, efp); + } + return r; +} + +static ray_t* exec_group_v2_run_inner(ray_graph_t* g, ray_op_t* op, ray_t* tbl, + int64_t nrows, ray_t* sel, + const int64_t* sel_prefix, int64_t n_sel, + int64_t group_limit, + const ray_group_emit_filter_t* efp) { agg_route_reason(AGG_V2_ADMITTED); route_stats.topn_native = false; route_stats.nullable_key = false; diff --git a/src/ops/agg_engine.h b/src/ops/agg_engine.h index bc1c23e2..662b948d 100644 --- a/src/ops/agg_engine.h +++ b/src/ops/agg_engine.h @@ -21,7 +21,7 @@ typedef enum { AGG_V2_BUFFERED, AGG_V2_PARAMETER, AGG_V2_DISABLED, - AGG_V2_EMIT_FILTER, + AGG_V2_EMIT_FILTER, /* no longer produced: v2 owns the emit filter */ AGG_V2_PARALLEL_WIDE, } agg_v2_reason_t; @@ -103,6 +103,11 @@ bool agg_group_values_f64(const agg_vtable_t* vt, const char* states, size_t stride, size_t off, const int64_t* slots, int64_t n, int64_t param, uint8_t desc, double* out); +/* Trim a finished group result to the emit filter's kept superset (row order + * preserved; consumes `result`). Used by routes that emit every group. */ +ray_t* agg_emit_filter_trim(ray_t* result, uint32_t n_keys, uint32_t n_aggs, + const ray_group_emit_filter_t* ef); + /* Precondition: agg_v2_can_handle(g, op, tbl) returned true. * `group_limit` is the HEAD(GROUP) row-limit HINT (0 = no limit): when * positive, an engine strategy may emit only the first `group_limit` groups in diff --git a/src/ops/group.c b/src/ops/group.c index 2e2e20fd..ad5d3fe3 100644 --- a/src/ops/group.c +++ b/src/ops/group.c @@ -9829,96 +9829,6 @@ static ray_t* exec_group_parted(ray_graph_t* g, ray_op_t* op, ray_t* parted_tbl, static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t group_limit); -/* Trim a full group result to the emit filter's keep set: rows whose - * filtered-agg value passes min_count_exclusive and, when top_count_take - * is set, lies within the top-N by that value (ties INCLUDED — the result - * is a superset of N rows; the DAG's sort+take downstream finalizes the - * exact order/limit, exactly as it would on an untrimmed result). - * Row order is preserved. Consumes `result`, returns an owned table. */ -static ray_t* group_emit_filter_trim(ray_t* result, uint32_t n_keys, - uint32_t n_aggs, - ray_group_emit_filter_t ef) { - if (!result || RAY_IS_ERR(result) || result->type != RAY_TABLE) - return result; - if (ef.agg_index >= n_aggs) return result; - int64_t nrows = ray_table_nrows(result); - if (nrows <= 0) return result; - ray_t* vcol = ray_table_get_col_idx(result, - (int64_t)n_keys + ef.agg_index); - if (!vcol || (vcol->type != RAY_I64 && vcol->type != RAY_F64)) - return result; - bool is_f64 = (vcol->type == RAY_F64); - /* Direction comes straight from .desc (mirrors the v2_emit topn path): - * every arming site sets it, COUNT included. Coercing COUNT to - * largest-first here made `asc: take: N` keep the largest N - * (issue #408). */ - const int64_t* vi = (const int64_t*)ray_data(vcol); - const double* vf = (const double*)ray_data(vcol); - #define EF_VAL_D(r) (is_f64 ? vf[(r)] : (double)vi[(r)]) - - double thr = 0.0; - bool have_thr = false; - if (ef.top_count_take > 0 && nrows > ef.top_count_take) { - /* Quickselect (on a copy) for the N-th value in the keep - * direction: desc keeps the N largest -> threshold is the - * (nrows-N)-th ascending element; asc keeps the N smallest. */ - ray_t* sel_hdr = NULL; - double* sv = (double*)scratch_alloc(&sel_hdr, - (size_t)nrows * sizeof(double)); - if (sv) { - for (int64_t r = 0; r < nrows; r++) sv[r] = EF_VAL_D(r); - int64_t k = ef.desc ? (nrows - ef.top_count_take) - : (ef.top_count_take - 1); - int64_t lo = 0, hi = nrows - 1; - while (lo < hi) { - double pivot = sv[k]; - int64_t i = lo, j = hi; - while (i <= j) { - while (sv[i] < pivot) i++; - while (sv[j] > pivot) j--; - if (i <= j) { - double t = sv[i]; sv[i] = sv[j]; sv[j] = t; - i++; j--; - } - } - if (k <= j) hi = j; - else if (k >= i) lo = i; - else break; - } - thr = sv[k]; - have_thr = true; - scratch_free(sel_hdr); - } - } - - ray_t* idx = ray_vec_new(RAY_I64, nrows); - if (!idx || RAY_IS_ERR(idx)) { - if (idx) ray_error_free(idx); - return result; /* trim is an optimization — full result is valid */ - } - int64_t* ix = (int64_t*)ray_data(idx); - int64_t kept = 0; - for (int64_t r = 0; r < nrows; r++) { - double v = EF_VAL_D(r); - if (ef.min_count_exclusive > 0 && !(v > (double)ef.min_count_exclusive)) - continue; - if (have_thr && (ef.desc ? (v < thr) : (v > thr))) - continue; - ix[kept++] = r; - } - #undef EF_VAL_D - idx->len = kept; - if (kept == nrows) { ray_release(idx); return result; } - ray_t* out = ray_at_fn(result, idx); - ray_release(idx); - if (!out || RAY_IS_ERR(out)) { - if (out) ray_error_free(out); - return result; - } - ray_release(result); - return out; -} - /* Map an I32 dictionary-code result column back to strings via the source * column: code -> first_occ[code] -> the string at that row. */ static ray_t* dict_codes_to_str(const ray_t* codes_col, ray_t* src_col, @@ -11395,45 +11305,21 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, ray_op_ext_t* ext = find_ext(g, op->id); if (!ext) return ray_error("nyi", NULL); - /* v2 doesn't implement the top-count emit filter (old-engine feature); - * when one is active, stay on the legacy path that honors it. */ /* group_limit is a HINT (HEAD(GROUP) fusion), not a semantic: v2 threads * it down to the radix strategy's bounded emit and the caller trims the * result either way, so a positive limit stays on v2 rather than falling - * back to the (slower, full-materialization) legacy ladder. */ + * back to the (slower, full-materialization) legacy ladder. The top-N + * emit filter (`desc: AGG take: N`) is likewise owned by v2: radix and + * the dense finishes select the kept groups natively and every other + * route trims its full result (exec_group_v2), so an armed filter never + * routes a shape v2 admits onto the ladder. */ agg_v2_reason_t admission = !ray_agg_engine_v2 ? AGG_V2_DISABLED : group_limit < 0 ? AGG_V2_SHAPE - : ray_group_emit_filter_get().enabled ? AGG_V2_EMIT_FILTER : agg_v2_admission(g, op, tbl); agg_route_reason(admission); if (admission == AGG_V2_ADMITTED) return exec_group_v2(g, op, tbl, group_limit); - /* Emit-filter shapes (`desc: c take: N`, `where (> c N)` on the group - * result) used to run on the legacy ladder because v2 did not implement - * the top-count emit filter. That ladder is single-threaded in its - * scatter and its dense array scales with the store's SHARED sym domain, - * so every top-N group query lost the parallel engine: a three-aggregate - * 100k-group query measured 800 ms on one core and 777 ms on a 28-thread - * pool (0.1x parallel), while the same grouping without the filter ran - * in 47 ms. v2 now reads the filter itself: radix and the dense finishes - * select the kept groups natively, the remaining routes return the full - * result and are trimmed here to the filter's top-N superset. The DAG's - * sort+take downstream produces the final order/limit either way. The - * legacy ladder remains only for shapes v2 declines. */ - { - ray_group_emit_filter_t ef = ray_group_emit_filter_get(); - if (ray_agg_engine_v2 && group_limit == 0 && ef.enabled - && agg_v2_can_handle(g, op, tbl)) { - ray_t* r = exec_group_v2(g, op, tbl, 0); - if (r && !RAY_IS_ERR(r)) - return agg_route_stats().topn_native - ? r : group_emit_filter_trim(r, ext->n_keys, ext->n_aggs, ef); - if (r) return r; - /* v2 declined at runtime — continue on the legacy ladder. */ - } - } - /* v2 with EXPRESSION agg inputs: v2 admission requires plain-column * scans, so a group like {sum(a*b), stddev(c), cor(x,y)} — where ONE * input is a MUL — used to drop the whole 6-agg pass onto the legacy diff --git a/test/rfl/group/emit_filter_v2_route.rfl b/test/rfl/group/emit_filter_v2_route.rfl index 566f5a0a..57996ec4 100644 --- a/test/rfl/group/emit_filter_v2_route.rfl +++ b/test/rfl/group/emit_filter_v2_route.rfl @@ -60,3 +60,16 @@ (count a7) -- 20 (all (>= (at a7 'c) (at (at full7 'c) 19))) -- true (== (sum (at a7 'c)) (sum (take (at full7 'c) 20))) -- true + +;; routes that emit every group and trim afterwards: a string key (shared +;; directory) and a buffered aggregate beside the ordering count +(set ts (table [s v] (list (at ["alpha" "beta" "gamma" "delta" "eps"] (% i 5)) (% (* i 13) 97)))) +(set a8 (select {from: ts by: s c: (count v) desc: c take: 2})) +(set b8 (take (xdesc (select {from: ts by: s c: (count v)}) 'c) 2)) +(count a8) -- 2 +(all (== (at a8 'c) (at b8 'c))) -- true +(set a9 (select {from: t by: k m: (med v) c: (count v) desc: c take: 4})) +(set b9 (take (xdesc (select {from: t by: k m: (med v) c: (count v)}) 'c) 4)) +(count a9) -- 4 +(all (== (at a9 'c) (at b9 'c))) -- true +(all (== (at a9 'm) (at b9 'm))) -- true From eab2659817c6925a308927976d17f0145cea0667 Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 12:29:33 +0200 Subject: [PATCH 13/23] perf(group): run the radix first-seen ordering across the pool The scatter, chunk count and compaction were dispatched by element grain over 128 partitions / 77 chunks, which yields one task, so the ordering ran serially. Dispatch by task count and compact out of place into an ng-sized buffer (the input-sized map is freed right after). 10M groups on 28 threads: ordering 63 -> 13.5 ms, whole query 194 -> 143 ms. Also lifts the rfl driver's file cap to 1024. --- src/ops/agg_engine.c | 63 ++++++++++++-------- test/main.c | 70 ++++++++++++++++++++++- test/rfl/group/radix_first_seen_order.rfl | 10 ++++ 3 files changed, 117 insertions(+), 26 deletions(-) create mode 100644 test/rfl/group/radix_first_seen_order.rfl diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index 80b3df2e..8ff23149 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -3463,6 +3463,7 @@ static void agg_ord_scatter_fn(void* vctx, uint32_t wid, int64_t start, typedef struct { agg_radix_order_t* pairs; + agg_radix_order_t* dst; /* [ng] compacted output (out of place) */ int64_t input_count; int64_t chunk; /* elements per chunk */ int64_t* counts; /* [n_chunks]: pass A out / pass B prefix in */ @@ -3493,7 +3494,7 @@ static void agg_ord_compact_fn(void* vctx, uint32_t wid, int64_t start, if (hi > c->input_count) hi = c->input_count; int64_t w = c->counts[ch]; /* prefix offset for this chunk */ for (int64_t i = lo; i < hi; i++) - if (c->pairs[i].idx != -1) c->pairs[w++].idx = c->pairs[i].idx; + if (c->pairs[i].idx != -1) c->dst[w++].idx = c->pairs[i].idx; } } @@ -3761,6 +3762,13 @@ agg_radix_select_first_n(const agg_radix_part_t* parts, uint32_t n_parts, return sel_pairs; } +typedef struct { agg_radix_order_t* pairs; } agg_ord_fill_ctx_t; +static void agg_ord_fill_fn(void* raw, uint32_t wid, int64_t start, int64_t end) { + (void)wid; + agg_ord_fill_ctx_t* c = raw; + memset(c->pairs + start, 0xFF, (size_t)(end - start) * sizeof(agg_radix_order_t)); +} + static ray_t* exec_group_v2_parallel_radix( ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t nrows, ray_t** key_cols, int64_t* key_syms, @@ -3930,45 +3938,46 @@ static ray_t* exec_group_v2_parallel_radix( agg_desc_free(&d); return ray_error("oom", NULL); } - /* -1 fill as bytes: 0xFF.. == -1 for int64, and memset vectorizes — - * this is an 80MB serial touch on a 10M-row input, worth the idiom. */ - memset(pairs, 0xFF, - (size_t)(input_count > 0 ? input_count : 1) * sizeof(agg_radix_order_t)); + /* -1 fill: on a 10M-row input this is an 80MB first touch of fresh + * pages, so it runs across the pool (each task faults and fills its own + * range); the serial memset it replaces cost ~20 ms of a 63 ms phase. */ + { + agg_ord_fill_ctx_t fctx = { .pairs = pairs }; + if (pool && input_count >= RAY_PARALLEL_THRESHOLD) + ray_pool_dispatch(pool, agg_ord_fill_fn, &fctx, input_count); + else + memset(pairs, 0xFF, (size_t)(input_count > 0 ? input_count : 1) * sizeof(agg_radix_order_t)); + } bool order_ok = true; int64_t ordered = 0; bool ord_parallel_done = false; - /* The compaction below writes into `pairs` in place. Its safety - * argument (see agg_ord_compact_fn) only shows that an EARLIER chunk - * cannot clobber a LATER chunk's unread input; the reverse is not - * true, because a later chunk writes at its prefix offset, which sits - * far below its own input range whenever groups are sparse relative - * to rows — i.e. any ordinary group-by with duplicates. Running the - * chunks in ascending order on one task is what makes that safe, and - * ray_pool_dispatch only ever produced one task here because its - * grain is 8192 ELEMENTS and the extent is n_chunks. Say so instead - * of depending on it: above the grain the compaction runs serially - * rather than racing (#556). */ + /* Every stage below is dispatched by TASK COUNT (ray_pool_dispatch_n): + * the element-grain dispatch used before produced a single task for a + * 128-partition scatter and a 77-chunk compaction, so this "parallel" + * ordering ran serially (63 ms of a 10M-group query on 28 threads). The + * compaction writes out of place into an ng-sized buffer, so chunks are + * independent and the input-sized map is released right after. */ const int64_t ORD_CHUNK = 1 << 17; - const int64_t ORD_MAX_CHUNKS = RAY_DISPATCH_MORSELS * RAY_MORSEL_ELEMS; + const int64_t ORD_MAX_CHUNKS = RAY_POOL_MAX_TASKS / 4; if (pool && nw > 1 && input_count >= (1 << 20) && (input_count + ORD_CHUNK - 1) / ORD_CHUNK <= ORD_MAX_CHUNKS) { int64_t n_chunks = (input_count + ORD_CHUNK - 1) / ORD_CHUNK; ray_t* ordcnt_hdr = NULL; int64_t* ord_counts = (int64_t*)scratch_alloc(&ordcnt_hdr, (size_t)n_chunks * sizeof(int64_t)); - if (ord_counts) { + agg_radix_order_t* compacted = ray_alloc_raw((size_t)(ng > 0 ? ng : 1) * sizeof(agg_radix_order_t)); + if (ord_counts && compacted) { agg_ord_scatter_ctx_t sctx = { .parts = parts, .pairs = pairs, .input_count = input_count, .fail = 0, }; - ray_pool_dispatch(pool, agg_ord_scatter_fn, &sctx, - (int64_t)n_parts); + ray_pool_dispatch_n(pool, agg_ord_scatter_fn, &sctx, (uint32_t)n_parts); if (!atomic_load_explicit(&sctx.fail, memory_order_relaxed)) { agg_ord_compact_ctx_t cctx = { - .pairs = pairs, .input_count = input_count, + .pairs = pairs, .dst = compacted, .input_count = input_count, .chunk = ORD_CHUNK, .counts = ord_counts, }; - ray_pool_dispatch(pool, agg_ord_count_fn, &cctx, n_chunks); + ray_pool_dispatch_n(pool, agg_ord_count_fn, &cctx, (uint32_t)n_chunks); /* Exclusive prefix (serial over a few hundred chunks). */ int64_t run = 0; for (int64_t ch = 0; ch < n_chunks; ch++) { @@ -3976,16 +3985,22 @@ static ray_t* exec_group_v2_parallel_radix( ord_counts[ch] = run; run += n; } - ray_pool_dispatch(pool, agg_ord_compact_fn, &cctx, n_chunks); ordered = run; order_ok = ordered == ng; + if (order_ok) { + ray_pool_dispatch_n(pool, agg_ord_compact_fn, &cctx, (uint32_t)n_chunks); + ray_free_raw(pairs); + pairs = compacted; + compacted = NULL; + } ord_parallel_done = true; } else { order_ok = false; ord_parallel_done = true; /* bounds violation → error path */ } - scratch_free(ordcnt_hdr); } + ray_free_raw(compacted); + scratch_free(ordcnt_hdr); } if (!ord_parallel_done) { for (uint32_t p = 0; p < n_parts && order_ok; p++) { diff --git a/test/main.c b/test/main.c index 21d94af9..54904c56 100644 --- a/test/main.c +++ b/test/main.c @@ -218,7 +218,7 @@ static const test_entry_t* const compiled_groups[] = { * evaluates it under a fresh runtime (via rfl_setup/rfl_teardown). */ -#define RFL_THUNK_CAPACITY 512 +#define RFL_THUNK_CAPACITY 1024 static char g_rfl_paths[RFL_THUNK_CAPACITY][512]; static char g_rfl_names[RFL_THUNK_CAPACITY][256]; @@ -572,7 +572,71 @@ static void rfl_teardown(void) { X(480) X(481) X(482) X(483) X(484) X(485) X(486) X(487) \ X(488) X(489) X(490) X(491) X(492) X(493) X(494) X(495) \ X(496) X(497) X(498) X(499) X(500) X(501) X(502) X(503) \ - X(504) X(505) X(506) X(507) X(508) X(509) X(510) X(511) + X(504) X(505) X(506) X(507) X(508) X(509) X(510) X(511) \ + X(512) X(513) X(514) X(515) X(516) X(517) X(518) X(519) \ + X(520) X(521) X(522) X(523) X(524) X(525) X(526) X(527) \ + X(528) X(529) X(530) X(531) X(532) X(533) X(534) X(535) \ + X(536) X(537) X(538) X(539) X(540) X(541) X(542) X(543) \ + X(544) X(545) X(546) X(547) X(548) X(549) X(550) X(551) \ + X(552) X(553) X(554) X(555) X(556) X(557) X(558) X(559) \ + X(560) X(561) X(562) X(563) X(564) X(565) X(566) X(567) \ + X(568) X(569) X(570) X(571) X(572) X(573) X(574) X(575) \ + X(576) X(577) X(578) X(579) X(580) X(581) X(582) X(583) \ + X(584) X(585) X(586) X(587) X(588) X(589) X(590) X(591) \ + X(592) X(593) X(594) X(595) X(596) X(597) X(598) X(599) \ + X(600) X(601) X(602) X(603) X(604) X(605) X(606) X(607) \ + X(608) X(609) X(610) X(611) X(612) X(613) X(614) X(615) \ + X(616) X(617) X(618) X(619) X(620) X(621) X(622) X(623) \ + X(624) X(625) X(626) X(627) X(628) X(629) X(630) X(631) \ + X(632) X(633) X(634) X(635) X(636) X(637) X(638) X(639) \ + X(640) X(641) X(642) X(643) X(644) X(645) X(646) X(647) \ + X(648) X(649) X(650) X(651) X(652) X(653) X(654) X(655) \ + X(656) X(657) X(658) X(659) X(660) X(661) X(662) X(663) \ + X(664) X(665) X(666) X(667) X(668) X(669) X(670) X(671) \ + X(672) X(673) X(674) X(675) X(676) X(677) X(678) X(679) \ + X(680) X(681) X(682) X(683) X(684) X(685) X(686) X(687) \ + X(688) X(689) X(690) X(691) X(692) X(693) X(694) X(695) \ + X(696) X(697) X(698) X(699) X(700) X(701) X(702) X(703) \ + X(704) X(705) X(706) X(707) X(708) X(709) X(710) X(711) \ + X(712) X(713) X(714) X(715) X(716) X(717) X(718) X(719) \ + X(720) X(721) X(722) X(723) X(724) X(725) X(726) X(727) \ + X(728) X(729) X(730) X(731) X(732) X(733) X(734) X(735) \ + X(736) X(737) X(738) X(739) X(740) X(741) X(742) X(743) \ + X(744) X(745) X(746) X(747) X(748) X(749) X(750) X(751) \ + X(752) X(753) X(754) X(755) X(756) X(757) X(758) X(759) \ + X(760) X(761) X(762) X(763) X(764) X(765) X(766) X(767) \ + X(768) X(769) X(770) X(771) X(772) X(773) X(774) X(775) \ + X(776) X(777) X(778) X(779) X(780) X(781) X(782) X(783) \ + X(784) X(785) X(786) X(787) X(788) X(789) X(790) X(791) \ + X(792) X(793) X(794) X(795) X(796) X(797) X(798) X(799) \ + X(800) X(801) X(802) X(803) X(804) X(805) X(806) X(807) \ + X(808) X(809) X(810) X(811) X(812) X(813) X(814) X(815) \ + X(816) X(817) X(818) X(819) X(820) X(821) X(822) X(823) \ + X(824) X(825) X(826) X(827) X(828) X(829) X(830) X(831) \ + X(832) X(833) X(834) X(835) X(836) X(837) X(838) X(839) \ + X(840) X(841) X(842) X(843) X(844) X(845) X(846) X(847) \ + X(848) X(849) X(850) X(851) X(852) X(853) X(854) X(855) \ + X(856) X(857) X(858) X(859) X(860) X(861) X(862) X(863) \ + X(864) X(865) X(866) X(867) X(868) X(869) X(870) X(871) \ + X(872) X(873) X(874) X(875) X(876) X(877) X(878) X(879) \ + X(880) X(881) X(882) X(883) X(884) X(885) X(886) X(887) \ + X(888) X(889) X(890) X(891) X(892) X(893) X(894) X(895) \ + X(896) X(897) X(898) X(899) X(900) X(901) X(902) X(903) \ + X(904) X(905) X(906) X(907) X(908) X(909) X(910) X(911) \ + X(912) X(913) X(914) X(915) X(916) X(917) X(918) X(919) \ + X(920) X(921) X(922) X(923) X(924) X(925) X(926) X(927) \ + X(928) X(929) X(930) X(931) X(932) X(933) X(934) X(935) \ + X(936) X(937) X(938) X(939) X(940) X(941) X(942) X(943) \ + X(944) X(945) X(946) X(947) X(948) X(949) X(950) X(951) \ + X(952) X(953) X(954) X(955) X(956) X(957) X(958) X(959) \ + X(960) X(961) X(962) X(963) X(964) X(965) X(966) X(967) \ + X(968) X(969) X(970) X(971) X(972) X(973) X(974) X(975) \ + X(976) X(977) X(978) X(979) X(980) X(981) X(982) X(983) \ + X(984) X(985) X(986) X(987) X(988) X(989) X(990) X(991) \ + X(992) X(993) X(994) X(995) X(996) X(997) X(998) X(999) \ + X(1000) X(1001) X(1002) X(1003) X(1004) X(1005) X(1006) X(1007) \ + X(1008) X(1009) X(1010) X(1011) X(1012) X(1013) X(1014) X(1015) \ + X(1016) X(1017) X(1018) X(1019) X(1020) X(1021) X(1022) X(1023) #define X(N) static test_result_t rfl_thunk_##N(void) { return run_rfl_at(N); } RFL_THUNKS(X) @@ -606,6 +670,8 @@ static int rfl_scan_at(const char* base_root, const char* cur_dir) { size_t nlen = strlen(ent->d_name); if (nlen < 4 || strcmp(ent->d_name + nlen - 4, ".rfl") != 0) continue; + _Static_assert(sizeof(rfl_thunks) / sizeof(*rfl_thunks) == RFL_THUNK_CAPACITY, + "one thunk per .rfl slot: extend RFL_THUNKS with RFL_THUNK_CAPACITY"); if (g_rfl_count >= RFL_THUNK_CAPACITY) { fprintf(stderr, "test driver: more than %d .rfl files — raise RFL_THUNK_CAPACITY\n", RFL_THUNK_CAPACITY); diff --git a/test/rfl/group/radix_first_seen_order.rfl b/test/rfl/group/radix_first_seen_order.rfl new file mode 100644 index 00000000..71aaad7a --- /dev/null +++ b/test/rfl/group/radix_first_seen_order.rfl @@ -0,0 +1,10 @@ +;; The radix full path emits groups in first-seen order regardless of the +;; number of partitions used. With v equal to the row index, "first-seen" +;; is exactly ascending (first v). + +(set n 300000) +(set t (table [k j v] (list (% (* (til n) 104729) 200003) (% (til n) 7) (til n)))) +(set r (select {from: t f: (first v) c: (count v) by: {k: k j: j}})) +(> (count r) 200000) -- true +(all (== (at r 'f) (asc (at r 'f)))) -- true +(sum (at r 'c)) -- 300000 From 9f94d44978947d54795330cfc8dc2dd9abf63dfd Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 12:47:04 +0200 Subject: [PATCH 14/23] perf(group): serve unordered take on the dense task-local path A bounded group emit no longer excludes the task-local dense strategy: its finish selects the N groups with the smallest first rows (an N-sized heap over the occupied slots) and emits them in first-seen order, exactly the prefix the radix bounded emit produced. 10M rows, 100k-key count take 10: 1 core 106 -> 17 ms, 28 cores 17 -> 5.3 ms. --- src/ops/agg_engine.c | 78 ++++++++++++++++++++++--- test/rfl/group/take_unordered_dense.rfl | 21 +++++++ test/test_agg_contract.c | 30 ++++++++++ 3 files changed, 120 insertions(+), 9 deletions(-) create mode 100644 test/rfl/group/take_unordered_dense.rfl diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index 8ff23149..1962d75c 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -1846,7 +1846,7 @@ static ray_t* agg_dense_finish(ray_t** key_cols, int64_t* key_syms, ray_op_ext_t ray_pool_t* pool, const agg_vo_t* vo, const agg_desc_t* d, int64_t total_slots, char* gstates, int64_t* gfirst, const dense_plan_t* key_plan, int64_t key_part_slots, uint32_t key_bits, - const ray_group_emit_filter_t* ef) { + const ray_group_emit_filter_t* ef, int64_t group_limit) { uint32_t n_keys = ext->n_keys, n_aggs = ext->n_aggs; const agg_vtable_t** vts = vo->vts; const size_t* off = vo->off; @@ -1875,6 +1875,62 @@ static ray_t* agg_dense_finish(ray_t** key_cols, int64_t* key_syms, ray_op_ext_t occupied_slot[i] = s; i++; } } + /* Bounded emit (HEAD(GROUP) hint): the N groups with the smallest first + * row ARE the first N groups in first-seen order. Select them with an + * N-sized max-heap over the occupied slots and emit them ascending by + * first row, byte-identical to trimming a first-seen-ordered result. */ + if (group_limit > 0 && first_row_ordered && ng > group_limit) { + int64_t n_keep = group_limit; + int64_t* hkey = ray_alloc_raw((size_t)n_keep * sizeof(int64_t)); + int64_t* hslot = ray_alloc_raw((size_t)n_keep * sizeof(int64_t)); + if (hkey && hslot) { + int64_t hn = 0; + for (int64_t i = 0; i < ng; i++) { + int64_t first = first_row_ordered[i], slot = occupied_slot[i]; + if (hn < n_keep) { + int64_t j = hn++; + hkey[j] = first; hslot[j] = slot; + while (j > 0) { + int64_t par = (j - 1) / 2; + if (hkey[par] >= hkey[j]) break; + int64_t tk = hkey[par]; hkey[par] = hkey[j]; hkey[j] = tk; + int64_t ts = hslot[par]; hslot[par] = hslot[j]; hslot[j] = ts; + j = par; + } + } else if (first < hkey[0]) { + hkey[0] = first; hslot[0] = slot; + int64_t j = 0; + for (;;) { + int64_t l = 2 * j + 1, rr = l + 1, m = j; + if (l < n_keep && hkey[l] > hkey[m]) m = l; + if (rr < n_keep && hkey[rr] > hkey[m]) m = rr; + if (m == j) break; + int64_t tk = hkey[m]; hkey[m] = hkey[j]; hkey[j] = tk; + int64_t ts = hslot[m]; hslot[m] = hslot[j]; hslot[j] = ts; + j = m; + } + } + } + /* heap sort ascending by first row */ + for (int64_t end = hn - 1; end > 0; end--) { + int64_t tk = hkey[0]; hkey[0] = hkey[end]; hkey[end] = tk; + int64_t ts = hslot[0]; hslot[0] = hslot[end]; hslot[end] = ts; + int64_t j = 0; + for (;;) { + int64_t l = 2 * j + 1, rr = l + 1, m = j; + if (l < end && hkey[l] > hkey[m]) m = l; + if (rr < end && hkey[rr] > hkey[m]) m = rr; + if (m == j) break; + tk = hkey[m]; hkey[m] = hkey[j]; hkey[j] = tk; + ts = hslot[m]; hslot[m] = hslot[j]; hslot[j] = ts; + j = m; + } + } + for (int64_t i = 0; i < hn; i++) { occupied_slot[i] = hslot[i]; first_row_ordered[i] = hkey[i]; } + ng = hn; + } + ray_free_raw(hkey); ray_free_raw(hslot); + } /* Top-N emit filter: keep only the groups the filter would keep, in the * same slot order. Nothing below allocates for the dropped groups. */ if (ef && ng > 0) { @@ -2332,7 +2388,7 @@ static ray_t* agg_dense_partitioned(ray_t** key_cols, int64_t* key_syms, ray_op_ route_stats.dense_strategy = AGG_DENSE_PARTITIONED; route_stats.dense_tasks = c.n_tasks; route_stats.dense_local_slots = slots + extra_slots; - return agg_dense_finish(key_cols, key_syms, ext, pool, vo, d, slots, c.states, c.first, plan, part_slots, bits, ef); + return agg_dense_finish(key_cols, key_syms, ext, pool, vo, d, slots, c.states, c.first, plan, part_slots, bits, ef, 0); failed: ray_free_raw(c.counts); ray_free_raw(c.gids); ray_free_raw(c.tasks); ray_free_raw(c.payload); ray_free_raw(c.value_offsets); ray_release(c.selection_indices); ray_free_raw(c.states); ray_free_raw(c.first); @@ -2432,7 +2488,7 @@ static ray_t* agg_dense_shared(ray_t** key_cols, int64_t* key_syms, ray_op_ext_t route_stats.dense_strategy = AGG_DENSE_SHARED; route_stats.dense_tasks = ray_pool_total_workers(pool); route_stats.dense_local_slots = slots; - return agg_dense_finish(key_cols, key_syms, ext, pool, vo, d, slots, c.states, c.first, plan, slots, 0, ef); + return agg_dense_finish(key_cols, key_syms, ext, pool, vo, d, slots, c.states, c.first, plan, slots, 0, ef, 0); failed: ray_free_raw(c.states); ray_free_raw(c.first); ray_free_raw(c.occupied); return ray_error(agg_cancelled() ? "cancel" : "oom", NULL); @@ -2443,7 +2499,7 @@ static ray_t* exec_group_v2_parallel_dense( ray_t** key_cols, int64_t* key_syms, ray_op_ext_t* ext, int64_t nrows, ray_pool_t* pool, const dense_plan_t* dp, uint32_t nw, agg_dense_strategy_t strategy, ray_t* sel, const int64_t* sel_prefix, int64_t n_sel, - const ray_group_emit_filter_t* efp) { + const ray_group_emit_filter_t* efp, int64_t group_limit) { uint32_t n_keys = ext->n_keys, n_aggs = ext->n_aggs; int64_t total_slots = dp->total_slots; @@ -2575,7 +2631,7 @@ static ray_t* exec_group_v2_parallel_dense( ray_free_raw(locals); ray_t* result = agg_dense_finish(key_cols, key_syms, ext, pool, &vo, &d, - total_slots, gstates, gfirst, NULL, 0, 0, efp); + total_slots, gstates, gfirst, NULL, 0, 0, efp, group_limit); agg_vo_free(&vo); agg_desc_free(&d); return result; } @@ -4986,7 +5042,7 @@ static ray_t* exec_group_v2_run_inner(ray_graph_t* g, ray_op_t* op, ray_t* tbl, route_stats.dense_worker_budget = false; agg_route_record(AGG_ROUTE_V2_DENSE); ray_t* result = exec_group_v2_parallel_dense(g, op, tbl, key_cols, key_syms, ext, - nrows, pool, &dp, dense_workers, AGG_DENSE_SHARED, sel, sel_prefix, n_sel, efp); + nrows, pool, &dp, dense_workers, AGG_DENSE_SHARED, sel, sel_prefix, n_sel, efp, 0); agg_vo_free(&vo); agg_dense_plan_free(&dp); scratch_free(kc_hdr); return result; } @@ -5032,7 +5088,7 @@ static ray_t* exec_group_v2_run_inner(ray_graph_t* g, ray_op_t* op, ray_t* tbl, route_stats.dense_worker_budget = false; agg_route_record(AGG_ROUTE_V2_DENSE); ray_t* result = exec_group_v2_parallel_dense(g, op, tbl, key_cols, key_syms, ext, - nrows, pool, &dp, dense_workers, AGG_DENSE_PARTITIONED, sel, sel_prefix, n_sel, efp); + nrows, pool, &dp, dense_workers, AGG_DENSE_PARTITIONED, sel, sel_prefix, n_sel, efp, 0); agg_vo_free(&vo); agg_dense_plan_free(&dp); scratch_free(kc_hdr); return result; } @@ -5042,7 +5098,11 @@ static ray_t* exec_group_v2_run_inner(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (fit < dense_workers) dense_workers = fit >= 2 ? (uint32_t)fit : 0; if (cache_tasks < dense_workers) dense_workers = cache_tasks >= 2 ? cache_tasks : 0; } - bool dense_par_ok = dp.ok && group_limit <= 0 && dense_workers > 0; + /* A bounded group emit (unordered take: N) selects the first N groups + * in first-seen order. Task-local slabs keep a true first row per + * slot, so their finish can select those groups; the shared and + * partition strategies above keep no first rows and stay gated. */ + bool dense_par_ok = dp.ok && dense_workers > 0; /* Allocation size alone misses repeated wide-range worker updates. * Estimate touched slot traffic from evenly spaced key samples in each * worker-sized input range. Prefer radix when duplicated state traffic @@ -5104,7 +5164,7 @@ static ray_t* exec_group_v2_run_inner(ray_graph_t* g, ray_op_t* op, ray_t* tbl, route_stats.dense_local_slots += dp.total_slots; agg_route_record(AGG_ROUTE_V2_DENSE); ray_t* r = exec_group_v2_parallel_dense(g, op, tbl, key_cols, key_syms, ext, nrows, pool, &dp, dense_workers, AGG_DENSE_TASK_LOCAL, - sel, sel_prefix, n_sel, efp); + sel, sel_prefix, n_sel, efp, group_limit); agg_vo_free(&vo); agg_dense_plan_free(&dp); scratch_free(kc_hdr); return r; } if (keys_intsym) { diff --git a/test/rfl/group/take_unordered_dense.rfl b/test/rfl/group/take_unordered_dense.rfl new file mode 100644 index 00000000..7a842aea --- /dev/null +++ b/test/rfl/group/take_unordered_dense.rfl @@ -0,0 +1,21 @@ +;; An unordered `take: N` on a grouped select returns the first N groups in +;; first-seen order. Bounded-domain keys used to be excluded from the dense +;; plan for this shape and fell to the radix bounded emit (105 ms serial for +;; 10M rows where the dense count took 17 ms). The task-local dense path +;; keeps a true first row per slot, so it selects the N smallest first rows. + +(set n 200000) +(set t (table [k v] (list (as 'I32 (% (* (til n) 7919) 50000)) (til n)))) +(set r (select {from: t f: (first v) c: (count v) by: {k: k} take: 10})) +(count r) -- 10 +;; the first 10 groups in first-seen order have the 10 smallest first rows, +;; and they come out in that order +(all (== (at r 'f) (take (asc (at (select {from: t f: (first v) by: {k: k}}) 'f)) 10))) -- true +(all (> (at r 'c) 0)) -- true +;; take larger than the group count returns everything, still ordered +(set r2 (select {from: t f: (first v) by: {k: k} take: 100000})) +(count r2) -- 50000 +;; a composite bounded key takes the same path +(set r3 (select {from: t f: (first v) by: {k: k j: (% v 3)} take: 7})) +(count r3) -- 7 +(all (== (at r3 'f) (asc (at r3 'f)))) -- true diff --git a/test/test_agg_contract.c b/test/test_agg_contract.c index adc42cd0..8743c0b1 100644 --- a/test/test_agg_contract.c +++ b/test/test_agg_contract.c @@ -1868,6 +1868,35 @@ static test_result_t test_dense_native_topn(void) { PASS(); } +/* Unordered take: N on a bounded key stays on the dense task-local path and + * emits the first N groups in first-seen order. */ +static test_result_t test_dense_unordered_take(void) { + ray_pool_destroy(); + TEST_ASSERT_EQ_I(ray_pool_init_total(8), RAY_OK); + ray_t* setup = ray_eval_str( + "(set ut_i (til 400000)) " + "(set ut_t (table [k v] (list (as 'I32 (% (* ut_i 7919) 50000)) ut_i)))"); + TEST_ASSERT_NOT_NULL(setup); TEST_ASSERT_FALSE(RAY_IS_ERR(setup)); ray_release(setup); + agg_route_reset(); + ray_t* r = ray_eval_str("(select {from:ut_t c:(count v) by:k take:10})"); + TEST_ASSERT_NOT_NULL(r); TEST_ASSERT_FALSE(RAY_IS_ERR(r)); + agg_route_stats_t stats = agg_route_stats(); + TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_V2_DENSE], 1); + TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_V2_RADIX], 0); + TEST_ASSERT_EQ_I(stats.dense_strategy, AGG_DENSE_TASK_LOCAL); + TEST_ASSERT_EQ_I(ray_table_nrows(r), 10); + /* rows 0..9 start groups (i*7919) % 50000, emitted in that order */ + const int32_t* k = (const int32_t*)ray_data(ray_table_get_col_idx(r, 0)); + const int64_t* c = (const int64_t*)ray_data(ray_table_get_col_idx(r, 1)); + for (int64_t i = 0; i < 10; i++) { + TEST_ASSERT_EQ_I(k[i], (int32_t)((i * 7919) % 50000)); + TEST_ASSERT_EQ_I(c[i], 8); + } + ray_release(r); + ray_release(ray_eval_str("(set ut_t 0) (set ut_i 0)")); + PASS(); +} + const test_entry_t agg_contract_entries[] = { { "agg_contract/empty_inference_errors", test_empty_inference_errors, contract_setup, contract_teardown }, { "agg_contract/nth_bounds", test_nth_bounds, contract_setup, contract_teardown }, @@ -1886,6 +1915,7 @@ const test_entry_t agg_contract_entries[] = { { "agg_contract/dense_composite_compaction", test_dense_composite_compaction, contract_setup, contract_teardown }, { "agg_contract/radix_native_topn", test_radix_native_topn, contract_setup, contract_teardown }, { "agg_contract/dense_native_topn", test_dense_native_topn, contract_setup, contract_teardown }, + { "agg_contract/dense_unordered_take", test_dense_unordered_take, contract_setup, contract_teardown }, { "agg_contract/rank_widths_nulls_slices", test_rank_widths_nulls_and_slices, contract_setup, contract_teardown }, { "agg_contract/nullable_differential", test_nullable_differential, contract_setup, contract_teardown }, { "agg_contract/wide_key_routes", test_wide_key_routes, contract_setup, contract_teardown }, From 432cfe7711a8e2493b998c796ae23c38b49efefd Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 12:56:56 +0200 Subject: [PATCH 15/23] docs(group): census of shapes still served by the legacy grouping ladder Adds a --census PATH flag to the test driver that records, per evaluated .rfl line, grouped selects that took the legacy route with their admission reason; the document tallies the corpus (340 hits in five classes) and what v2 needs to close each. --- docs/grouping-legacy-census.md | 56 ++++++++++++++++++++++++++++++++++ test/main.c | 45 ++++++++++++++++++++++++--- 2 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 docs/grouping-legacy-census.md diff --git a/docs/grouping-legacy-census.md b/docs/grouping-legacy-census.md new file mode 100644 index 00000000..a221bf4d --- /dev/null +++ b/docs/grouping-legacy-census.md @@ -0,0 +1,56 @@ +# Grouping: census of shapes still served by the legacy ladder + +Status: recorded 2026-09-19 after the core-scaling follow-up. The parallel +grouping engine ("v2") now owns every top-N emit filter, unordered bounded +emit, multi-key arithmetic over aggregates, and composite symbol keys. This +census lists what still reaches the legacy grouping ladder in +`exec_group_run`, as the factual starting point for retiring it. + +## How it was measured + +The test driver records, for every evaluated `.rfl` line, whether the +grouping executor took the legacy route and the admission reason it was +given: + +``` +make test # builds ./rayforce.test +RAYFORCE_CORES=2 ./rayforce.test --census census.tsv -f rfl/ +cut -f1 census.tsv | sort | uniq -c +``` + +Each census line is `reasonfile:linesource`. Run it with the +suite's worker count: one admission reason (`parallel_wide`) depends on the +pool size, and several ordering assertions in the corpus assume two workers. + +## Results (514 `.rfl` files, 340 legacy hits) + +| Reason | Lines | With `by:` | What it means | What v2 needs | +|---|---:|---:|---|---| +| `shape` | 200 | 5 | 195 are scalar aggregations with no keys (`select {s: (sum v) from: T}`), mostly over partitioned stores; the 5 grouped ones are nested selects whose outer scalar select carries the reason, plus one `by:` on a missing column | A keyless (single-group) path in v2, or routing scalar aggregation to the vector aggregators; the grouped 5 are already served | +| `agg_expression` | 63 | 63 | Aggregate over an expression: `(sum (strlen s))` ×35, `(sum (at ...))` ×15, `(first (at ...))` ×14, `(avg (strlen s))` ×11, `(count (select ...))` ×7 | The expression-input path (`exec_group_v2_exprs`) materializes inputs it can evaluate as a full column; string-length and indexed/nested inputs are declined today | +| `key_expression` | 31 | 27 | Computed keys: `(+ k 1)` ×10, `(xbar ts N)` ×5, `(substr ref 0 3)` ×2, `(minute EventTime)`, dotted temporal accessors `ts.date` | Materialize computed keys as scan columns before admission, the way expression aggregate inputs are; temporal accessors need the same | +| `parallel_wide` | 30 | 30 | Float keys with 2 to 8 workers and no indexed aggregate (`by: {f: f}` on F64) — a measured choice to prefer the ladder's directory over v2's replicated float hash | Make the v2 float route competitive at small pools (or drop the pool-size gate once measured) | +| `admitted` | 16 | 15 | Admitted by v2 but executed on the ladder: partitioned-store sources (`from: Pmc by: date`) where the per-partition executor merges partial groups on the legacy path, and one derived-key symbol-domain case | A partition-aware merge in v2 (streaming states merge; buffered ones need the ladder's row slices) | + +No line reached the ladder because of an emit filter, a bounded emit, or a +multi-key compound expression; those classes are closed. + +## Reading the table + +- Two classes are not grouping at all: keyless scalar aggregation (195 lines) + and the outer scalar select of a nested query. They keep the ladder alive + for reasons unrelated to group-by scaling and can be moved independently. +- The grouped remainder is 140 lines in four classes. Three of them + (`agg_expression`, `key_expression`, `admitted` on partitioned sources) are + admission gaps that a materialize-then-scan step closes without a new + strategy; `parallel_wide` is a performance gate that needs a measurement, + not a feature. +- The benchmark query files add two shapes the corpus lacks: a computed key + through several nested string functions, and a `where:` filter on the + grouped result (`(> c 100000)`), which the emit filter already expresses. + +## Next step + +Retiring the ladder is its own plan: close the four grouped classes above, +route keyless aggregation to the vector aggregators, then delete +`exec_group_run` and the consumers it alone uses. diff --git a/test/main.c b/test/main.c index 54904c56..64a3a968 100644 --- a/test/main.c +++ b/test/main.c @@ -34,6 +34,7 @@ #define _POSIX_C_SOURCE 200809L #include "test.h" +#include "ops/agg_engine.h" /* --census: agg_route_stats */ #include "test_rfl.h" #include @@ -330,6 +331,38 @@ static void rfl_rewrite_goldens(const char* path, const int* lns, char (*vals)[5 free(s); fclose(o); } +/* --census PATH: after every evaluated .rfl line, record grouped selects + * that reached the legacy grouping ladder (route counters are per thread + * and reset before each line). One tab-separated line per hit: + * reason, file:line, source. Diagnostic only; off unless the flag is set. */ +static FILE* g_census = NULL; +static const char* census_reason_name(agg_v2_reason_t r) { + switch (r) { + case AGG_V2_ADMITTED: return "admitted"; + case AGG_V2_SHAPE: return "shape"; + case AGG_V2_KEY_EXPRESSION: return "key_expression"; + case AGG_V2_KEY_TYPE: return "key_type"; + case AGG_V2_AGG_EXPRESSION: return "agg_expression"; + case AGG_V2_AGG_TYPE: return "agg_type"; + case AGG_V2_BUFFERED: return "buffered"; + case AGG_V2_PARAMETER: return "parameter"; + case AGG_V2_DISABLED: return "disabled"; + case AGG_V2_EMIT_FILTER: return "emit_filter"; + case AGG_V2_PARALLEL_WIDE: return "parallel_wide"; + } + return "unknown"; +} +static ray_t* rfl_eval(const char* src, const char* path, int line_no) { + if (g_census) agg_route_reset(); + ray_t* v = ray_eval_str(src); + if (g_census) { + agg_route_stats_t st = agg_route_stats(); + if (st.routes[AGG_ROUTE_LEGACY] > 0) + fprintf(g_census, "%s\t%s:%d\t%s\n", census_reason_name(st.last_v2_reason), path, line_no, src); + } + return v; +} + static test_result_t run_rfl_file(const char* path) { FILE* f = fopen(path, "rb"); if (!f) FAILF("cannot open %s", path); @@ -380,7 +413,7 @@ static test_result_t run_rfl_file(const char* path) { *eq = '\0'; char* lhs = start; char* rhs = eq + 4; - ray_t* le = ray_eval_str(lhs); + ray_t* le = rfl_eval(lhs, path, line_no); if (RAY_IS_ERR(le)) { char buf[512]; fmt_into(le, buf, sizeof buf); snprintf(ray_test_fail_buf, sizeof ray_test_fail_buf, @@ -424,7 +457,7 @@ static test_result_t run_rfl_file(const char* path) { *er = '\0'; char* expr = start; char* substr = er + 4; - ray_t* ev = ray_eval_str(expr); + ray_t* ev = rfl_eval(expr, path, line_no); if (!RAY_IS_ERR(ev)) { /* ev is a value here — we expected an error but got one. */ char buf[512]; fmt_into(ev, buf, sizeof buf); @@ -451,7 +484,7 @@ static test_result_t run_rfl_file(const char* path) { ray_error_free(ev); } else { /* Raw Rayfall code — eval; error is a test failure. */ - ray_t* ev = ray_eval_str(start); + ray_t* ev = rfl_eval(start, path, line_no); if (ev && RAY_IS_ERR(ev)) { char buf[512]; fmt_into(ev, buf, sizeof buf); snprintf(ray_test_fail_buf, sizeof ray_test_fail_buf, @@ -825,9 +858,13 @@ int main(int argc, char** argv) { if ((strcmp(argv[i], "--filter") == 0 || strcmp(argv[i], "-f") == 0) && i + 1 < argc) { filter = argv[++i]; + } else if (strcmp(argv[i], "--census") == 0 && i + 1 < argc) { + g_census = fopen(argv[++i], "w"); + if (!g_census) { fprintf(stderr, "cannot open census file\n"); return 2; } } else if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) { - printf("Usage: %s [-f SUBSTR]\n", argv[0]); + printf("Usage: %s [-f SUBSTR] [--census PATH]\n", argv[0]); printf(" -f, --filter SUBSTR Only run tests whose name contains SUBSTR.\n"); + printf(" --census PATH Record .rfl lines whose grouping ran on the legacy ladder.\n"); return 0; } else { fprintf(stderr, "unknown argument: %s\n", argv[i]); From c7069a354cf094d9a646305c4744ba0df292fe59 Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 13:00:09 +0200 Subject: [PATCH 16/23] docs(group): results and mechanics of the top-N, bounded-emit and radix ordering work --- docs/docs/architecture/pipeline.md | 4 +++- docs/grouping-engine-scaling-plan.md | 31 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/docs/architecture/pipeline.md b/docs/docs/architecture/pipeline.md index ca35ce74..a18921ce 100644 --- a/docs/docs/architecture/pipeline.md +++ b/docs/docs/architecture/pipeline.md @@ -221,7 +221,9 @@ Grouped aggregates run on the parallel aggregation engine, which picks a strateg - **Radix** — unbounded integer or symbol keys (many-million-group inputs) are hash-partitioned and reduced per partition. - **Shared directory** — float, string, GUID, and list keys use a shared parallel key directory. -Arithmetic over aggregates (`(- (max v1) (min v2))`, `(pow (pearson_corr a b) 2)`) is decomposed at compile time: the aggregates run as hidden slots inside the same group pass, and the outer expression is evaluated once over the grouped result. This applies to any number of keys and to binary aggregates. Ordered top-N clauses (`desc: c take: 10`) run the full parallel grouping and trim the result to the top-N superset before the final sort; only unbounded key domains still use the older ordered-emit path. +Arithmetic over aggregates (`(- (max v1) (min v2))`, `(pow (pearson_corr a b) 2)`) is decomposed at compile time: the aggregates run as hidden slots inside the same group pass, and the outer expression is evaluated once over the grouped result. This applies to any number of keys and to binary aggregates. + +Ordered top-N clauses (`desc: c take: 10`, `asc: m take: 5`) are selected inside the strategies: radix partitions and dense finishes finalize the ordering aggregate to one value per group in parallel, keep a bounded candidate set each, take the threshold from the union, and emit only the kept groups (ties included), so a 10M-group query never materializes 10M output rows. Routes that emit every group trim afterwards with the same decision. An unordered `take: N` emits the first N groups in first-seen order; on the dense task-local path that is a selection of the N smallest first rows. The radix full path restores first-seen order with a scatter, count and compaction that are all dispatched across the pool. ### Per-Thread Heaps diff --git a/docs/grouping-engine-scaling-plan.md b/docs/grouping-engine-scaling-plan.md index af940ceb..9e83ac57 100644 --- a/docs/grouping-engine-scaling-plan.md +++ b/docs/grouping-engine-scaling-plan.md @@ -298,3 +298,34 @@ restore first-seen order, then emission) scale poorly on many-million-group keys; the legacy ladder remains faster for those top-N shapes. An unordered `take:` on a grouped select still uses radix's bounded emit rather than the dense plan. + +## Follow-up results: top-N, bounded emit and radix ordering + +Five more changes closed the shapes the previous follow-up left open. The +parallel engine now owns the top-N emit filter: radix partitions and dense +finishes finalize the ordering aggregate per group in parallel, keep bounded +candidate heaps, take one threshold from their union and emit only the kept +superset; every other route trims its full result with the same decision, +and the legacy ladder's carve-outs are gone. The radix full path's first-seen +ordering was dispatched by element grain over partitions and chunks, which +produced a single task; it is dispatched by task count and compacts out of +place. An unordered `take: N` on a bounded key selects the N smallest first +rows on the dense task-local path instead of the radix bounded emit. A test +driver flag records which `.rfl` lines still reach the legacy ladder +(`docs/grouping-legacy-census.md`). + +Measured on 10M rows (min of five warm runs, ms; before = the branch point, +after = this follow-up): + +| Query | 1 core | 8 cores | 28 cores | +|---|---|---|---| +| three-key count, desc take 10 (10M groups) | 962 → 285 | 46 → 54 | 46 → 44 | +| three aggregates by 100k key, desc take 10 | 77 → 41 | 12.8 → 11.0 | 75 → 10.4 | +| where + count by 15-value key, desc take 10 | 451 → 51 | 59 → 9.6 | 23.5 → 6.3 | +| count by 100k key, unordered take 10 | 107 → 17 | 21 → 4.0 | 17.8 → 5.4 | +| count by 100k key, desc take 10 | 28 → 17 | 4.9 → 4.0 | 4.8 → 5.5 | +| sum + count by six keys, no take (10M groups) | 1543 → 1592 | 245 → 201 | 187 → 137 | +| pow(pearson) by two keys | 1840 → 68 | 993 → 12 | 936 → 9.2 | + +The remaining legacy routes are enumerated in the census; none of them is a +top-N, bounded-emit or compound-expression shape. From 082c60a36c8ac5f559944f0ea092d14b696df731 Mon Sep 17 00:00:00 2001 From: Serhii Savchuk Date: Sat, 19 Sep 2026 14:10:01 +0300 Subject: [PATCH 17/23] perf(group): chunk length derived from the morsel budget, test seam instead of an env knob The chunk of the per-distinct-symbol key evaluation is 256 dispatch rounds of morsels (RAY_MORSEL_ELEMS * RAY_DISPATCH_MORSELS * 256, 2M values) rather than a bare power of two; the comment says why. The RAY_DERIVED_KEY_CHUNK environment variable is gone: a DEBUG-only ray_derived_key_chunk_set_for_test replaces it, so release builds carry no knob and no getenv per call. The several-chunks case moves to test_lang.c (lang/select/ derived_key_file_chunks): a 414-value vocabulary through 100-value and 7-value chunks against the in-memory oracle, the seam reset around each evaluation. The rfl test keeps the default-chunk FILE-domain checks. Co-Authored-By: Claude Fable 5.1 --- src/lang/internal.h | 5 ++ src/ops/query.c | 24 ++++++--- test/rfl/group/derived_key_sym_domain.rfl | 9 ++-- test/test_lang.c | 62 +++++++++++++++++++++++ 4 files changed, 86 insertions(+), 14 deletions(-) diff --git a/src/lang/internal.h b/src/lang/internal.h index 09a3b5d1..71a4e8b0 100644 --- a/src/lang/internal.h +++ b/src/lang/internal.h @@ -704,6 +704,11 @@ ray_t* ray_within_fn(ray_t* vals, ray_t* range); /* Query bridge builtins (formerly in eval.c, now in ops/query.c) */ ray_t* ray_select_fn(ray_t** args, int64_t n); +#ifdef DEBUG +/* Chunk length of the per-distinct-symbol key evaluation over a FILE + * domain (ops/query.c derived_key_str_chunks); 0 restores the default. */ +void ray_derived_key_chunk_set_for_test(int64_t rows); +#endif ray_t* ray_window_fn(ray_t** args, int64_t n); ray_t* ray_try_count_select_expr(ray_t* expr, int* handled); ray_t* ray_update_fn(ray_t** args, int64_t n); diff --git a/src/ops/query.c b/src/ops/query.c index 3f32ca45..ddbcfde2 100644 --- a/src/ops/query.c +++ b/src/ops/query.c @@ -2902,15 +2902,23 @@ static int64_t derived_key_name(ray_t* by_expr) { * (so the interned result has the type the caller expects) and the * column's vocabulary is readable through the raw snapshot. Any other * shape returns NULL and the caller takes the one-shot SYM evaluation. */ -#define DERIVED_KEY_CHUNK (1LL << 21) -/* Chunk length; RAY_DERIVED_KEY_CHUNK overrides it so a test can drive - * several chunks through a small vocabulary. */ +/* Chunk length in distinct values: 256 dispatch rounds of morsels (2M + * rows). Enough rows for the pooled string ops to spread over the + * workers, while every chunk-scoped STR intermediate stays in the + * low hundreds of MB at URL-sized strings. */ +#define DERIVED_KEY_CHUNK ((int64_t)RAY_MORSEL_ELEMS * RAY_DISPATCH_MORSELS * 256) +#ifdef DEBUG +static int64_t g_derived_key_chunk_test = 0; +/* Test seam: a positive value replaces the chunk length until reset to 0, + * so a test can drive several chunks through a small vocabulary. */ +void ray_derived_key_chunk_set_for_test(int64_t rows) { + g_derived_key_chunk_test = rows > 0 ? rows : 0; +} +#endif static int64_t derived_key_chunk_rows(void) { - const char* env = getenv("RAY_DERIVED_KEY_CHUNK"); - if (env && *env) { - long v = strtol(env, NULL, 10); - if (v > 0) return (int64_t)v; - } +#ifdef DEBUG + if (g_derived_key_chunk_test > 0) return g_derived_key_chunk_test; +#endif return DERIVED_KEY_CHUNK; } static ray_t* derived_key_str_chunks(ray_t* by_expr, int64_t col_sym, ray_t* dom_vec, diff --git a/test/rfl/group/derived_key_sym_domain.rfl b/test/rfl/group/derived_key_sym_domain.rfl index 6521305b..be9f6df1 100644 --- a/test/rfl/group/derived_key_sym_domain.rfl +++ b/test/rfl/group/derived_key_sym_domain.rfl @@ -88,8 +88,9 @@ ;; ── the column's ids in a FILE domain (a splayed table read back) ────── ;; The per-symbol evaluation then runs over the vocabulary's bytes, a ;; chunk of distinct values at a time, and interns only the final key. -;; Same groups and aggregates as the in-memory table; a chunk far smaller -;; than the vocabulary drives the expression through several chunks. +;; Same groups and aggregates as the in-memory table. (Several chunks +;; over a small vocabulary: lang/select/derived_key_file_chunks in +;; test/test_lang.c, through the DEBUG-only chunk-length seam.) (.sys.exec "rm -rf /tmp/rfl_dkey_file/") -- 0 (.db.splayed.set "/tmp/rfl_dkey_file/" T) (set F (.db.splayed.get "/tmp/rfl_dkey_file/")) @@ -99,15 +100,11 @@ (count RF) -- 37 (all (== (FP RF) (FP O))) -- true (== (at (cols RF) 0) 'p) -- true -(== (.os.setenv "RAY_DERIVED_KEY_CHUNK" "100") "100") -- true -(set RC (KEYQ F)) -(all (== (FP RC) (FP O))) -- true (all (== (FP2 (select {from: F by: (if (== (str-find ref "www.") 0) (substr ref 4 -1) ref) c: (count ref)})) (FP2 O2))) -- true ;; a key whose result is not a symbol keeps the one-shot evaluation (set RL (select {from: F by: (strlen ref) c: (count ref)})) (set OL (select {from: T by: (strlen ref) c: (count ref)})) (all (== (FP2 RL) (FP2 OL))) -- true -(== (.os.setenv "RAY_DERIVED_KEY_CHUNK" "") "") -- true ;; nulls in the file-backed column (.sys.exec "rm -rf /tmp/rfl_dkey_filen/") -- 0 (.db.splayed.set "/tmp/rfl_dkey_filen/" TN) diff --git a/test/test_lang.c b/test/test_lang.c index 3ca76856..7dfb6a7e 100644 --- a/test/test_lang.c +++ b/test/test_lang.c @@ -8698,6 +8698,67 @@ static test_result_t test_builtin_group_guid_rfl(void) { /* ── builtins.c coverage: ray_group_indices_fn empty and list ─────────────── * Covers empty vector and RAY_LIST paths in ray_group_indices_fn. */ +/* ---- Test: derived symbol key over a FILE domain, several chunks ---- + * The per-distinct-symbol key evaluation over a file-backed column runs + * the expression over the vocabulary bytes a chunk at a time. With the + * chunk shrunk to 100 values, a 200-value vocabulary crosses chunk + * boundaries; the groups and aggregates must equal the in-memory answer. */ +static test_result_t test_select_derived_key_file_chunks(void) { + ray_t* r = ray_eval_str( + "(do (set __dk_i (til 20000)) " + " (set __dk_hosts (map (fn [k] (format \"h%.example.com\" k)) (til 37))) " + " (set __dk_mk (fn [k] ((fn [j] (if (== 0 (% j 11)) \"\" (if (== 0 (% j 7)) " + " (format \"https://www.%/p/%\" (at __dk_hosts (% (* 31 j) 37)) (% j 13)) " + " (if (== 0 (% j 5)) (at __dk_hosts (% (* 31 j) 37)) " + " (format \"http://%/a/%\" (at __dk_hosts (% (* 31 j) 37)) (% j 17)))))) (% k 500)))) " + " (set __dk_T (table [ref v] (list (as 'SYMBOL (map __dk_mk __dk_i)) (as 'F64 (% (* __dk_i 7) 101))))) " + " (.sys.exec \"rm -rf /tmp/rf_test_dk_chunks/\") " + " (.db.splayed.set \"/tmp/rf_test_dk_chunks/\" __dk_T) " + " (set __dk_F (.db.splayed.get \"/tmp/rf_test_dk_chunks/\")) " + " (set __dk_keyq (fn [t] (select {from: t by: (let p (str-find ref \"://\") (let s (substr ref (+ p 4) -1) " + " (let r (if (== (str-find s \"www.\") 0) (substr s 5 -1) s) (let sl (str-find r \"/\") " + " (if (and (within p [4 5]) (== (substr ref 1 4) \"http\") (not (nil? sl))) (substr r 1 sl) ref))))) " + " c: (count ref) sv: (sum v) mn: (min ref) where: (!= ref \"\")}))) " + " (set __dk_ora (fn [t] (select {from: (select {from: t p: (let p (str-find ref \"://\") (let s (substr ref (+ p 4) -1) " + " (let r (if (== (str-find s \"www.\") 0) (substr s 5 -1) s) (let sl (str-find r \"/\") " + " (if (and (within p [4 5]) (== (substr ref 1 4) \"http\") (not (nil? sl))) (substr r 1 sl) ref))))) " + " ref: ref v: v where: (!= ref \"\")}) by: p c: (count ref) sv: (sum v) mn: (min ref)}))) " + " (set __dk_fp (fn [r] (ser (xasc (xasc (table [k c sv mn] (list (at (value r) 0) (at r 'c) (at r 'sv) (at r 'mn))) 'mn) 'c)))) " + " (count (distinct (at __dk_F 'ref))))"); + TEST_ASSERT_NOT_NULL(r); + TEST_ASSERT_FALSE(RAY_IS_ERR(r)); + TEST_ASSERT_EQ_I(r->i64, 414); /* vocabulary of 414 values */ + ray_release(r); + + ray_derived_key_chunk_set_for_test(100); /* five chunks over the vocabulary */ + ray_t* same = ray_eval_str("(all (== (__dk_fp (__dk_keyq __dk_F)) (__dk_fp (__dk_ora __dk_T))))"); + ray_t* cols = ray_eval_str("(at (cols (__dk_keyq __dk_F)) 0)"); + ray_derived_key_chunk_set_for_test(0); + + ray_t* seven = ray_eval_str("(do (set __dk_r7 (__dk_keyq __dk_F)) 0)"); + ray_release(seven); + ray_derived_key_chunk_set_for_test(7); /* 60 chunks, the last one short */ + ray_t* same7 = ray_eval_str("(all (== (__dk_fp (__dk_keyq __dk_F)) (__dk_fp __dk_r7)))"); + ray_derived_key_chunk_set_for_test(0); + ray_t* cleanup = ray_eval_str("(.sys.exec \"rm -rf /tmp/rf_test_dk_chunks/\")"); + ray_release(cleanup); + + TEST_ASSERT_NOT_NULL(same); + TEST_ASSERT_FALSE(RAY_IS_ERR(same)); + TEST_ASSERT_EQ_I(same->type, -RAY_BOOL); + TEST_ASSERT_TRUE(same->b8); + TEST_ASSERT_NOT_NULL(cols); + TEST_ASSERT_FALSE(RAY_IS_ERR(cols)); + TEST_ASSERT_EQ_I(cols->i64, ray_sym_intern("p", 1)); + TEST_ASSERT_NOT_NULL(same7); + TEST_ASSERT_FALSE(RAY_IS_ERR(same7)); + TEST_ASSERT_TRUE(same7->b8); + ray_release(same); + ray_release(cols); + ray_release(same7); + PASS(); +} + static test_result_t test_builtin_group_empty_and_list(void) { /* Empty group */ ASSERT_EQ("(count (key (group [])))", "0"); @@ -9376,6 +9437,7 @@ const test_entry_t lang_entries[] = { { "lang/builtin/idiv_rfl", test_builtin_idiv_rfl, lang_setup, lang_teardown }, { "lang/builtin/group_guid_rfl", test_builtin_group_guid_rfl, lang_setup, lang_teardown }, { "lang/builtin/group_empty_list", test_builtin_group_empty_and_list, lang_setup, lang_teardown }, + { "lang/select/derived_key_file_chunks", test_select_derived_key_file_chunks, lang_setup, lang_teardown }, { "lang/temporal/extract_builtins_fn", test_temporal_extract_builtins_fn, lang_setup, lang_teardown }, { "lang/temporal/extract_time_atom", test_temporal_extract_time_atom, lang_setup, lang_teardown }, { "lang/temporal/extract_time_vector", test_temporal_extract_time_vector, lang_setup, lang_teardown }, From 56ee1bad086721cc468d9ee61578b166bc1d365b Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 13:21:46 +0200 Subject: [PATCH 18/23] fix(group): keep null-valued groups where the sort ranks them; restore ranges on compaction failure - top-N value view maps nulls/NaN to -INFINITY in both directions, matching sort_nulls_first (first ascending, last descending); the trimmed route applies the same mapping, so every v2 route returns the same set - agg_dense_plan_compact publishes shrunken key ranges only when every remap table exists; an allocation failure restores the raw ranges - drop the unused dense-plan probe and its thread-local cache --- src/ops/agg_engine.c | 79 ++++++------------------- src/ops/agg_engine.h | 10 +--- test/rfl/group/emit_filter_v2_route.rfl | 21 +++++++ test/test_agg_engine.c | 14 ++--- 4 files changed, 49 insertions(+), 75 deletions(-) diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index 1962d75c..3c165ad5 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -198,50 +198,6 @@ bool agg_v2_can_handle(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { * the unbounded radix route still pays a large serial ordering/emission * tail on many-million-group inputs. Costs one parallel min/max prescan * of the non-SYM keys; SYM keys resolve from their domain bounds. */ -/* The probe's plan is kept for the run that follows on this thread so the - * key prescan is not paid twice (4 ms serial on a 10M-row I64 key). The - * run consumes it only when the same key columns and row count are seen. */ -typedef struct { - bool valid; - uint32_t n_keys; - int64_t nrows; - ray_t* key_cols[16]; - dense_plan_t dp; -} agg_dense_plan_cache_t; -static _Thread_local agg_dense_plan_cache_t g_dense_plan_cache; - -static bool agg_dense_plan_cached(ray_t** key_cols, uint32_t n_keys, int64_t nrows, - dense_plan_t* out) { - agg_dense_plan_cache_t* c = &g_dense_plan_cache; - bool hit = c->valid && c->n_keys == n_keys && c->nrows == nrows; - for (uint32_t k = 0; hit && k < n_keys; k++) hit = c->key_cols[k] == key_cols[k]; - if (c->valid && !hit) agg_dense_plan_free(&c->dp); - c->valid = false; /* one run per probe */ - if (!hit) return false; - *out = c->dp; /* compaction tables transfer to the run */ - return true; -} - -bool agg_v2_dense_plan_available(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { - if (!g || !op || !tbl) return false; - ray_op_ext_t* ext = find_ext(g, op->id); - if (!ext || ext->n_keys < 1 || ext->n_keys > 16) return false; - agg_dense_plan_cache_t* c = &g_dense_plan_cache; - if (c->valid) agg_dense_plan_free(&c->dp); - c->valid = false; - for (uint32_t k = 0; k < ext->n_keys; k++) { - ray_op_t* key = op_node(g, ext->keys[k]); - ray_op_ext_t* kext = key ? find_ext(g, key->id) : NULL; - c->key_cols[k] = kext ? ray_table_get_col(tbl, kext->sym) : NULL; - if (!c->key_cols[k]) return false; - } - c->n_keys = ext->n_keys; - c->nrows = ray_table_nrows(tbl); - bool ok = agg_dense_plan(c->key_cols, c->n_keys, NULL, 0, c->nrows, &c->dp); - c->valid = ok; - return ok; -} - /* ── Dense grouping eligibility selector (mirrors group.c DA path) ──────── * Decides whether the key tuple packs into a bounded direct-index slot space * (gid = sum_k (key_k - min_k)*strides[k]) so grouping can skip hashing. @@ -480,10 +436,15 @@ static bool agg_dense_plan_compact(ray_t** key_cols, uint32_t n_keys, int64_t nr if (fits) total *= rng; } if (!fits) { ray_free_raw(bits); ray_free_raw(task_bits); return false; } - /* Build remap/inverse for every candidate key that actually shrank. */ + /* Build remap/inverse for every candidate key that actually shrank. The + * raw ranges are restored on any allocation failure so a plan that is + * still accepted (the non-overflow entry) never pairs a shrunken range + * with the raw slot computation. */ + int64_t raw_ranges[16]; + memcpy(raw_ranges, out->ranges, sizeof(raw_ranges)); for (uint32_t k = 0; k < n_keys; k++) { if (!candidate[k]) continue; - int64_t raw = out->ranges[k] - out->nullable[k]; + int64_t raw = raw_ranges[k] - out->nullable[k]; if (used[k] >= raw) continue; /* every code used: raw range is already dense */ int32_t* remap = ray_alloc_raw((size_t)raw * sizeof(int32_t)); int64_t* inverse = ray_alloc_raw((size_t)(used[k] > 0 ? used[k] : 1) * sizeof(int64_t)); @@ -491,6 +452,7 @@ static bool agg_dense_plan_compact(ray_t** key_cols, uint32_t n_keys, int64_t nr ray_free_raw(remap); ray_free_raw(inverse); ray_free_raw(bits); ray_free_raw(task_bits); agg_dense_plan_free(out); + memcpy(out->ranges, raw_ranges, sizeof(raw_ranges)); return false; } const uint64_t* acc = task_bits[k]; @@ -1479,7 +1441,7 @@ static void agg_topn_candidates(const double* vals, int64_t n, int64_t cap, uint bool agg_group_values_f64(const agg_vtable_t* vt, const char* states, size_t stride, size_t off, const int64_t* slots, - int64_t n, int64_t param, uint8_t desc, double* out) { + int64_t n, int64_t param, double* out) { int8_t t = vt->out_type; switch (t) { case RAY_F64: case RAY_F32: case RAY_I64: case RAY_TIMESTAMP: @@ -1487,7 +1449,11 @@ bool agg_group_values_f64(const agg_vtable_t* vt, const char* states, case RAY_U8: case RAY_BOOL: break; default: return false; } - double null_sink = desc ? -INFINITY : INFINITY; + /* The sort downstream ranks nulls FIRST ascending and LAST descending + * (sort_nulls_first is !desc), i.e. below every value either way: an + * asc take keeps an all-null group at rank 1, a desc take drops it. + * -INFINITY reproduces exactly that in agg_topn_mark for both directions. */ + const double null_sink = -INFINITY; ray_t* cell = ray_vec_new(t, 1); if (!cell || RAY_IS_ERR(cell)) { if (cell) ray_error_free(cell); return false; } cell->len = 1; @@ -1782,7 +1748,7 @@ static void agg_slots_vals_fn(void* raw, uint32_t wid, int64_t start, int64_t en int64_t b, e; agg_slots_topn_range(c, task, &b, &e); if (!agg_group_values_f64(c->vt, c->states, c->block, c->off, c->slots + b, - e - b, c->param, c->ef->desc, c->vals + b)) { + e - b, c->param, c->vals + b)) { atomic_store_explicit(&c->fail, 1, memory_order_relaxed); continue; } @@ -3662,7 +3628,7 @@ static void agg_radix_vals_fn(void* raw, uint32_t wid, int64_t start, int64_t en for (int64_t p = start; p < end; p++) { double* v = c->vals + c->base[p]; if (!agg_group_values_f64(c->vt, c->parts[p].states, c->block, c->off, NULL, - c->parts[p].ng, c->param, c->ef->desc, v)) { + c->parts[p].ng, c->param, v)) { atomic_store_explicit(&c->fail, 1, memory_order_relaxed); continue; } @@ -4849,12 +4815,13 @@ ray_t* agg_emit_filter_trim(ray_t* result, uint32_t n_keys, uint32_t n_aggs, if (idx && RAY_IS_ERR(idx)) ray_error_free(idx); else ray_release(idx); return result; } + /* Same null placement as agg_group_values_f64: below every value. */ if (vcol->type == RAY_F64) { const double* vf = (const double*)ray_data(vcol); - for (int64_t r = 0; r < nrows; r++) vals[r] = vf[r]; + for (int64_t r = 0; r < nrows; r++) vals[r] = vf[r] != vf[r] ? -INFINITY : vf[r]; } else { const int64_t* vi = (const int64_t*)ray_data(vcol); - for (int64_t r = 0; r < nrows; r++) vals[r] = (double)vi[r]; + for (int64_t r = 0; r < nrows; r++) vals[r] = vi[r] == NULL_I64 ? -INFINITY : (double)vi[r]; } int64_t kept = agg_topn_keep(vals, nrows, ef, keep); int64_t* ix = (int64_t*)ray_data(idx); @@ -4957,16 +4924,8 @@ static ray_t* exec_group_v2_run_inner(ray_graph_t* g, ray_op_t* op, ray_t* tbl, * not O(nrows)) — both correct (grouped keys ARE the selected rows) and cheap * for high-selectivity filters, and it can pull a sparse-but-low-card selected * key set into the dense path that the full-table range would have rejected. */ - /* Every run consumes the probe's cached plan (agg_dense_plan_cached - * invalidates it), so a plan can never outlive the query it was probed - * for and match a recycled column pointer later. Selected runs plan - * over the selected rows instead and discard the cached full-table plan. */ dense_plan_t dp; - dense_plan_t probed; - bool probed_hit = agg_dense_plan_cached(key_cols, ext->n_keys, nrows, &probed); - if (probed_hit && sel) { agg_dense_plan_free(&probed); probed_hit = false; } bool dense = sel ? agg_dense_plan_sel(key_cols, ext->n_keys, n_sel, sel, sel_prefix, &dp) - : probed_hit ? (dp = probed, true) : agg_dense_plan(key_cols, ext->n_keys, vts, ext->n_aggs, nrows, &dp); route_stats.dense_plan_available = dense; diff --git a/src/ops/agg_engine.h b/src/ops/agg_engine.h index 662b948d..0ae3abf3 100644 --- a/src/ops/agg_engine.h +++ b/src/ops/agg_engine.h @@ -76,10 +76,6 @@ void agg_route_reason(agg_v2_reason_t reason); * Conservative: any uncertainty → false → caller uses the existing engine. */ bool agg_v2_can_handle(ray_graph_t* g, ray_op_t* op, ray_t* tbl); -/* True when v2 would group this node through a bounded dense plan (see the - * definition for the strategy-prediction contract). */ -bool agg_v2_dense_plan_available(ray_graph_t* g, ray_op_t* op, ray_t* tbl); - /* Top-N keep decision shared by every strategy: keep[i] = 1 when group i * passes min_count_exclusive and (when top_count_take > 0) lies within the * top-N by value in the filter's direction, ties included (a superset of N; @@ -96,12 +92,12 @@ int64_t agg_topn_mark(const double* vals, int64_t n, const ray_group_emit_filter bool have_thr, double thr, uint8_t* keep); /* Double view of aggregate `vt` for n groups: group i's state is at - * states + (slots ? slots[i] : i) * stride + off. Nulls (and NaN) sink to - * the far end of the keep direction (`desc`) so they never enter a top-N. + * states + (slots ? slots[i] : i) * stride + off. Nulls (and NaN) map to + * -INFINITY, where the sort places them (first ascending, last descending). * Returns false for an out_type without a scalar order (LIST, STR, ...). */ bool agg_group_values_f64(const agg_vtable_t* vt, const char* states, size_t stride, size_t off, const int64_t* slots, - int64_t n, int64_t param, uint8_t desc, double* out); + int64_t n, int64_t param, double* out); /* Trim a finished group result to the emit filter's kept superset (row order * preserved; consumes `result`). Used by routes that emit every group. */ diff --git a/test/rfl/group/emit_filter_v2_route.rfl b/test/rfl/group/emit_filter_v2_route.rfl index 57996ec4..a80935e5 100644 --- a/test/rfl/group/emit_filter_v2_route.rfl +++ b/test/rfl/group/emit_filter_v2_route.rfl @@ -73,3 +73,24 @@ (count a9) -- 4 (all (== (at a9 'c) (at b9 'c))) -- true (all (== (at a9 'm) (at b9 'm))) -- true + +;; null-valued groups: the sort ranks nulls first ascending and last +;; descending, so an all-null group is rank 1 of an asc take and absent from +;; a desc take -- on the native selections and on the trimmed routes alike +(set tn (update {from: (table [k v] (list (% i 6) (% (* i 7) 50))) v: 0N where: (== k 2)})) +(set a10 (select {from: tn by: k m: (min v) asc: m take: 2})) +(count a10) -- 2 +(at (at a10 'k) 0) -- 2 +(nil? (at (at a10 'm) 0)) -- true +(set a11 (select {from: tn by: k m: (min v) desc: m take: 2})) +(count a11) -- 2 +(all (!= (at a11 'k) 2)) -- true +(set tf (update {from: (table [k v] (list (% i 6) (as 'F64 (% (* i 7) 50)))) v: 0Nf where: (== k 3)})) +(set a12 (select {from: tf by: k m: (max v) asc: m take: 2})) +(at (at a12 'k) 0) -- 3 +(set a13 (select {from: tf by: k m: (max v) desc: m take: 2})) +(all (!= (at a13 'k) 3)) -- true +;; the same shape with a string key runs the trimmed route +(set tsn (table [s v] (list (at ["a" "b" "c" "d" "e" "f"] (% i 6)) (at tn 'v)))) +(set a14 (select {from: tsn by: s m: (min v) asc: m take: 2})) +(at (at a14 's) 0) -- "c" diff --git a/test/test_agg_engine.c b/test/test_agg_engine.c index 804002a5..ebd94475 100644 --- a/test/test_agg_engine.c +++ b/test/test_agg_engine.c @@ -2078,14 +2078,14 @@ static test_result_t test_group_values_f64(void) { ray_valid_t valid = { vals, RAY_I64, false }; vt->update_batch(states, STRIDE, gids, vals, &valid, 6, NULL); double out[4]; - TEST_ASSERT_TRUE(agg_group_values_f64(vt, states, STRIDE, 0, NULL, 3, 0, 1, out)); + TEST_ASSERT_TRUE(agg_group_values_f64(vt, states, STRIDE, 0, NULL, 3, 0, out)); TEST_ASSERT_TRUE(out[0] == 10 && out[1] == 3 && out[2] == 15); /* slot list picks groups 2 and 0 */ int64_t slots[] = {2, 0}; - TEST_ASSERT_TRUE(agg_group_values_f64(vt, states, STRIDE, 0, slots, 2, 0, 1, out)); + TEST_ASSERT_TRUE(agg_group_values_f64(vt, states, STRIDE, 0, slots, 2, 0, out)); TEST_ASSERT_TRUE(out[0] == 15 && out[1] == 10); - /* a group whose only input is null finalizes to a null minimum: it sinks - * below every value for desc and above every value for asc */ + /* a group whose only input is null finalizes to a null minimum: it maps + * below every value, where the sort ranks nulls (first asc, last desc) */ const agg_vtable_t* mn = agg_resolve(OP_MIN, RAY_I64); TEST_ASSERT_NOT_NULL(mn); for (int i = 0; i < 4; i++) mn->init(states + i * STRIDE); @@ -2093,13 +2093,11 @@ static test_result_t test_group_values_f64(void) { int64_t mvals[] = {7, 2, NULL_I64}; ray_valid_t mvalid = { mvals, RAY_I64, true }; mn->update_batch(states, STRIDE, mgids, mvals, &mvalid, 3, NULL); - TEST_ASSERT_TRUE(agg_group_values_f64(mn, states, STRIDE, 0, NULL, 4, 0, 1, out)); + TEST_ASSERT_TRUE(agg_group_values_f64(mn, states, STRIDE, 0, NULL, 4, 0, out)); TEST_ASSERT_TRUE(out[0] == 7 && out[1] == 2 && out[3] < out[1]); - TEST_ASSERT_TRUE(agg_group_values_f64(mn, states, STRIDE, 0, NULL, 4, 0, 0, out)); - TEST_ASSERT_TRUE(out[3] > out[0]); /* a list-valued aggregate has no scalar view */ const agg_vtable_t* top = agg_resolve(OP_TOP_N, RAY_I64); - if (top) TEST_ASSERT_FALSE(agg_group_values_f64(top, states, STRIDE, 0, NULL, 1, 3, 1, out)); + if (top) TEST_ASSERT_FALSE(agg_group_values_f64(top, states, STRIDE, 0, NULL, 1, 3, out)); ray_sym_destroy(); ray_heap_destroy(); PASS(); From 2ca56f9e74d184eb0ca5764774a45099a2055f03 Mon Sep 17 00:00:00 2001 From: Serhii Savchuk Date: Sat, 19 Sep 2026 14:46:59 +0300 Subject: [PATCH 19/23] perf(sort): bounded-heap ordering compares FILE-domain symbols by raw bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The asc:/desc: with take: path resolved a SYM sort key's two strings through ray_sym_domain_str on every compare. On a FILE domain that materialises an atom per vocabulary entry under the domain lock, so the first such query in a process paid for the whole vocabulary before it compared anything — seconds on a column with millions of distinct values, while the ordering itself takes milliseconds. A FILE-domain key now pins the raw vocabulary snapshot for the duration of the query and compares the entries' bytes in ray_str_cmp order (common prefix, then length); runtime-domain keys keep the borrowed string snapshot. The rows taken are identical. Test: test/rfl/symbol/file_domain_topk.rfl — one key ascending and descending, ties broken by a second key in both directions, the symbol as the second key behind an integer, the empty symbol taking part, a take past the row count, all against the same ordering over the strings and past the parallel threshold; the in-memory column as a control. Co-Authored-By: Claude Fable 5.1 --- src/ops/fused_topk.c | 59 ++++++++++++++++++++-------- test/rfl/symbol/file_domain_topk.rfl | 40 +++++++++++++++++++ 2 files changed, 82 insertions(+), 17 deletions(-) create mode 100644 test/rfl/symbol/file_domain_topk.rfl diff --git a/src/ops/fused_topk.c b/src/ops/fused_topk.c index 3985b754..ddf2ddfc 100644 --- a/src/ops/fused_topk.c +++ b/src/ops/fused_topk.c @@ -86,10 +86,18 @@ typedef struct { * (ctx.sym_strings). 0 routes the compare through the column's * own domain (ray_sym_domain_str). */ uint8_t dom_runtime; + /* SYM keys on a FILE domain: the mapped prefix pinned for the + * dispatch — compares read raw bytes, no atom per symbol. */ + uint8_t raw_ok; + ray_sym_domain_raw_t raw; int64_t sym; const void* base; ray_t* col; /* for ray_vec_is_null when has_nulls */ } fpk_keyspec_t; +static void fpk_unpin_keys(fpk_keyspec_t* keys, uint32_t n) { + for (uint32_t i = 0; i < n; i++) + if (keys[i].raw_ok) { ray_sym_domain_raw_unpin(ray_sym_vec_domain(keys[i].col)); keys[i].raw_ok = 0; } +} typedef struct { fp_pred_t pred; @@ -160,19 +168,29 @@ static inline int fpk_cmp(const fpk_par_ctx_t* c, int64_t row_a, int64_t row_b) uint32_t ia = (uint32_t)read_by_esz(ks->base, row_a, ks->esz); uint32_t ib = (uint32_t)read_by_esz(ks->base, row_b, ks->esz); if (ia == ib) continue; - ray_t* sa; - ray_t* sb; - if (ks->dom_runtime) { - if (ia >= c->sym_count || ib >= c->sym_count) continue; - sa = c->sym_strings[ia]; - sb = c->sym_strings[ib]; + if (ks->raw_ok && (int64_t)ia < ks->raw.count && (int64_t)ib < ks->raw.count) { + /* ray_str_cmp order: bytes of the common prefix, then length. */ + size_t la, lb; + const char* pa = ray_sym_domain_raw_str(&ks->raw, (int64_t)ia, &la); + const char* pb = ray_sym_domain_raw_str(&ks->raw, (int64_t)ib, &lb); + size_t ml = la < lb ? la : lb; + cmp = ml ? memcmp(pa, pb, ml) : 0; + if (cmp == 0) cmp = la < lb ? -1 : (la > lb ? 1 : 0); } else { - struct ray_sym_domain_s* dom = ray_sym_vec_domain(ks->col); - sa = ray_sym_domain_str(dom, (int64_t)ia); - sb = ray_sym_domain_str(dom, (int64_t)ib); + ray_t* sa; + ray_t* sb; + if (ks->dom_runtime) { + if (ia >= c->sym_count || ib >= c->sym_count) continue; + sa = c->sym_strings[ia]; + sb = c->sym_strings[ib]; + } else { + struct ray_sym_domain_s* dom = ray_sym_vec_domain(ks->col); + sa = ray_sym_domain_str(dom, (int64_t)ia); + sb = ray_sym_domain_str(dom, (int64_t)ib); + } + if (!sa || !sb) continue; + cmp = ray_str_cmp(sa, sb); } - if (!sa || !sb) continue; - cmp = ray_str_cmp(sa, sb); } else if (ks->type == RAY_STR) { /* Variable-length STR key: compare the actual strings (dict * codes, if any, are first-occurrence order — not sorted). @@ -332,14 +350,14 @@ ray_t* ray_fused_topk_select(ray_t* tbl, int sym_needed = 0; for (uint32_t i = 0; i < n_sort_keys; i++) { ray_t* col = ray_table_get_col(tbl, sort_key_syms[i]); - if (!col) return NULL; + if (!col) { fpk_unpin_keys(ctx.keys, i); return NULL; } int8_t kt = col->type; - if (RAY_IS_PARTED(kt) || kt == RAY_MAPCOMMON) return NULL; + if (RAY_IS_PARTED(kt) || kt == RAY_MAPCOMMON) { fpk_unpin_keys(ctx.keys, i); return NULL; } if (kt != RAY_SYM && kt != RAY_STR && kt != RAY_BOOL && kt != RAY_U8 && kt != RAY_I16 && kt != RAY_I32 && kt != RAY_I64 && kt != RAY_DATE && kt != RAY_TIME && kt != RAY_TIMESTAMP && kt != RAY_F32 && kt != RAY_F64 && kt != RAY_GUID) - return NULL; + { fpk_unpin_keys(ctx.keys, i); return NULL; } ctx.keys[i].type = kt; ctx.keys[i].attrs = col->attrs; ctx.keys[i].esz = ray_sym_elem_size(kt, col->attrs); @@ -352,7 +370,10 @@ ray_t* ray_fused_topk_select(ray_t* tbl, ctx.keys[i].sym = sort_key_syms[i]; ctx.keys[i].base = ray_data(col); ctx.keys[i].col = col; + ctx.keys[i].raw_ok = 0; if (ctx.keys[i].dom_runtime) sym_needed = 1; + else if (kt == RAY_SYM) + ctx.keys[i].raw_ok = ray_sym_domain_raw_pin(ray_sym_vec_domain(col), &ctx.keys[i].raw) ? 1 : 0; } ctx.n_keys = n_sort_keys; ctx.k = k; @@ -360,18 +381,19 @@ ray_t* ray_fused_topk_select(ray_t* tbl, /* Compile the predicate via a temp graph just for the WHERE clause. */ ray_graph_t* g = ray_graph_new(tbl); - if (!g) return NULL; + if (!g) { fpk_unpin_keys(ctx.keys, n_sort_keys); return NULL; } ray_op_t* pred_dag = compile_expr_dag(g, where_expr); - if (!pred_dag) { ray_graph_free(g); return NULL; } + if (!pred_dag) { ray_graph_free(g); fpk_unpin_keys(ctx.keys, n_sort_keys); return NULL; } if (fp_compile_pred(g, pred_dag, tbl, &ctx.pred) != 0) { fp_pred_cleanup(&ctx.pred); ray_graph_free(g); + fpk_unpin_keys(ctx.keys, n_sort_keys); return NULL; } if (sym_needed) { ray_sym_strings_borrow(&ctx.sym_strings, &ctx.sym_count); - if (!ctx.sym_strings) { ray_graph_free(g); return NULL; } + if (!ctx.sym_strings) { ray_graph_free(g); fpk_unpin_keys(ctx.keys, n_sort_keys); return NULL; } } atomic_store_explicit(&ctx.oom, 0, memory_order_relaxed); @@ -389,6 +411,7 @@ ray_t* ray_fused_topk_select(ray_t* tbl, if (hn_hdr) scratch_free(hn_hdr); fp_pred_cleanup(&ctx.pred); ray_graph_free(g); + fpk_unpin_keys(ctx.keys, n_sort_keys); return NULL; } @@ -399,6 +422,7 @@ ray_t* ray_fused_topk_select(ray_t* tbl, scratch_free(idx_hdr); scratch_free(hn_hdr); fp_pred_cleanup(&ctx.pred); ray_graph_free(g); + fpk_unpin_keys(ctx.keys, n_sort_keys); return NULL; } @@ -430,6 +454,7 @@ ray_t* ray_fused_topk_select(ray_t* tbl, scratch_free(hn_hdr); fpk_sort_final(&ctx, global_idx, global_n); + fpk_unpin_keys(ctx.keys, n_sort_keys); /* no more compares past here */ /* Materialize n_out output columns by gathering rows[global_idx]. */ ray_t* result = ray_table_new(n_out); diff --git a/test/rfl/symbol/file_domain_topk.rfl b/test/rfl/symbol/file_domain_topk.rfl new file mode 100644 index 00000000..15c11e2a --- /dev/null +++ b/test/rfl/symbol/file_domain_topk.rfl @@ -0,0 +1,40 @@ +;; Bounded-heap ordering (asc:/desc: with take:) keyed by a SYM column whose +;; ids live in a FILE domain (a splayed table read back from disk). The +;; compare reads the entries' bytes straight off the mapped vocabulary +;; instead of materialising an atom per symbol; every answer must equal the +;; same ordering over the strings themselves. Sizes past the parallel +;; threshold (65536) so the pooled heap runs; a second key breaks ties, and +;; the empty symbol (a null cell) takes part on both sides. +(.sys.exec "rm -rf /tmp/rfl_fdom_topk/") -- 0 +(set N 120000) +(set i (til N)) +(set hosts (map (fn [k] (format "h%.example.com" k)) (til 300))) +(set u (as 'SYMBOL (map (fn [k] (if (== 0 (% k 41)) (format "http://%/google/%" (at hosts (% k 300)) (% k 977)) (format "http://%/p/%/%" (at hosts (% (* 31 k) 300)) (% k 977) (% k 7)))) i))) +(set sp (as 'SYMBOL (map (fn [k] (if (== 0 (% k 3)) "" (format "phrase % %" (% (* k 7) 5000) (% k 13)))) i))) +(set g (as 'I64 (% (* i 7919) 500))) +(set T (table [u sp g] (list u sp g))) +(.db.splayed.set "/tmp/rfl_fdom_topk/" T) +(set F (.db.splayed.get "/tmp/rfl_fdom_topk/")) +(count F) -- 120000 +(== (type (at F 'u)) 'SYM) -- true +;; string oracle: the same values as plain strings, same row order +(set TS (table [us ss g] (list (as 'STR (at F 'u)) (as 'STR (at F 'sp)) (at F 'g)))) +(set S (fn [v] (as 'STR v))) + +;; one key, ascending and descending, with the empty symbol filtered out +(all (== (S (at (select {from: F sp: sp where: (!= sp "") asc: sp take: 10}) 'sp)) (at (select {from: TS ss: ss where: (!= ss "") asc: ss take: 10}) 'ss))) -- true +(all (== (S (at (select {from: F sp: sp where: (!= sp "") desc: sp take: 10}) 'sp)) (at (select {from: TS ss: ss where: (!= ss "") desc: ss take: 10}) 'ss))) -- true +;; ties on the symbol broken by a second key, both directions +(all (== (S (at (select {from: F u: u g: g where: (!= sp "") asc: [u g] take: 25}) 'u)) (at (select {from: TS us: us g: g where: (!= ss "") asc: [us g] take: 25}) 'us))) -- true +(all (== (at (select {from: F u: u g: g where: (!= sp "") asc: [u g] take: 25}) 'g) (at (select {from: TS us: us g: g where: (!= ss "") asc: [us g] take: 25}) 'g))) -- true +(all (== (S (at (select {from: F u: u g: g where: (!= sp "") desc: [u g] take: 25}) 'u)) (at (select {from: TS us: us g: g where: (!= ss "") desc: [us g] take: 25}) 'us))) -- true +;; the symbol as the second key behind an integer +(all (== (S (at (select {from: F g: g sp: sp where: (!= sp "") asc: [g sp] take: 40}) 'sp)) (at (select {from: TS g: g ss: ss where: (!= ss "") asc: [g ss] take: 40}) 'ss))) -- true +;; the empty symbol left in: it orders as the empty string on both sides +(all (== (S (at (select {from: F sp: sp asc: sp take: 10}) 'sp)) (at (select {from: TS ss: ss asc: ss take: 10}) 'ss))) -- true +(all (== (S (at (select {from: F sp: sp desc: sp take: 10}) 'sp)) (at (select {from: TS ss: ss desc: ss take: 10}) 'ss))) -- true +;; a full take (no bound below the row count) keeps the same order +(all (== (S (at (select {from: F u: u where: (== g 7) asc: u take: 1000}) 'u)) (at (select {from: TS us: us where: (== g 7) asc: us take: 1000}) 'us))) -- true +;; in-memory SYM (runtime domain) still answers the same +(all (== (S (at (select {from: T sp: sp where: (!= sp "") asc: sp take: 10}) 'sp)) (at (select {from: TS ss: ss where: (!= ss "") asc: ss take: 10}) 'ss))) -- true +(.sys.exec "rm -rf /tmp/rfl_fdom_topk/") -- 0 From 74c1b913903f934402956cc6bb33a89ba1a7c3e7 Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 14:13:44 +0200 Subject: [PATCH 20/23] fix(group): free compaction tables on the indexed route; select top-N below the parallel threshold - agg_indexed_run frees the dense plan after the key build (tables leaked on every buffered aggregate over a compacted composite key) - the candidate-union threshold handles a single selection task, whose union holds exactly N values: the worst candidate is the threshold, so small dense top-N queries are selected natively instead of emitting every group; route stats record the kept count and the contract tests pin it - the partition-vs-task-local comparison uses the uncapped replication so the choice no longer moves with the machine's cache size (macOS runner) --- src/ops/agg_engine.c | 28 +++++++++++++++++++++++++--- src/ops/agg_engine.h | 1 + test/main.c | 1 + test/test_agg_contract.c | 4 ++++ 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index 3c165ad5..5576a279 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -1405,6 +1405,22 @@ int64_t agg_topn_keep(const double* vals, int64_t n, return agg_topn_mark(vals, n, ef, have_thr, thr, keep); } +/* Threshold of a top-N from the union of per-range candidate sets. The + * union of every range's N best contains the global N best, so when it + * holds more than N values the N-th in keep direction is the threshold; when + * it holds exactly N (one range) the threshold is the worst candidate. No + * threshold when the whole population fits in N. */ +static bool agg_topn_union_threshold(const double* cand, int64_t nc, int64_t total, + const ray_group_emit_filter_t* ef, double* thr) { + if (ef->top_count_take <= 0 || total <= ef->top_count_take || nc <= 0) return false; + if (nc > ef->top_count_take) return agg_topn_threshold(cand, nc, ef, thr); + double worst = cand[0]; + for (int64_t i = 1; i < nc; i++) + if (ef->desc ? cand[i] < worst : cand[i] > worst) worst = cand[i]; + *thr = worst; + return true; +} + /* Bounded candidate heap: the N best values of a range in the keep * direction. For desc a min-heap of the N largest; for asc a max-heap of * the N smallest. Every partition contributes one such set, and the @@ -1794,7 +1810,7 @@ static int64_t agg_slots_topn_select(ray_pool_t* pool, const agg_vtable_t* vt, memmove(cand + nc, cand + (size_t)t * cap, (size_t)cand_n[t] * sizeof(double)); nc += cand_n[t]; } - c.have_thr = agg_topn_threshold(cand, nc, ef, &c.thr); + c.have_thr = agg_topn_union_threshold(cand, nc, n, ef, &c.thr); } if (tasks > 1) ray_pool_dispatch_n(pool, agg_slots_mark_fn, &c, tasks); else agg_slots_mark_fn(&c, 0, 0, 1); @@ -1913,6 +1929,7 @@ static ray_t* agg_dense_finish(ray_t** key_cols, int64_t* key_syms, ray_op_ext_t } ng = kept; route_stats.topn_native = true; + route_stats.topn_kept = kept; } ray_free_raw(keep); ray_profile_tick("dense: selected top-N groups"); @@ -3687,7 +3704,7 @@ agg_radix_select_topn(ray_pool_t* pool, const agg_radix_part_t* parts, uint32_t memmove(cand + nc, cand + (size_t)p * cap, (size_t)cand_n[p] * sizeof(double)); nc += cand_n[p]; } - c.have_thr = agg_topn_threshold(cand, nc, ef, &c.thr); + c.have_thr = agg_topn_union_threshold(cand, nc, ng, ef, &c.thr); } if (pool) ray_pool_dispatch_n(pool, agg_radix_mark_fn, &c, n_parts); else agg_radix_mark_fn(&c, 0, 0, n_parts); @@ -3925,6 +3942,7 @@ static ray_t* exec_group_v2_parallel_radix( pairs = sel_pairs; n_emit = kept; route_stats.topn_native = true; + route_stats.topn_kept = kept; } else if (rc == 1) { agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); @@ -4635,6 +4653,7 @@ static ray_t* agg_indexed_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, route_stats.dense_plan_available = dense; int rc = dense ? agg_group_keys_dense(keys, nrows, &dp, &groups) : agg_group_keys(keys, ext->n_keys, nrows, &groups); + agg_dense_plan_free(&dp); /* compaction tables serve the key build only */ if (rc) return ray_error(agg_cancelled() ? "cancel" : "oom", NULL); if (agg_cancelled()) { agg_groups_free(&groups); return ray_error("cancel", NULL); } int64_t ng = groups.ngroups; @@ -5029,8 +5048,11 @@ static ray_t* exec_group_v2_run_inner(ray_graph_t* g, ray_op_t* op, ray_t* tbl, * pass. Prefer partition ownership only once replicated state * traffic exceeds its row traffic; large pools/ranges still use * bounded shared storage. */ + /* Compare the UNCAPPED replication so the choice between the two + * strategies does not move with the machine's cache size; the + * cache bound applies inside the task-local branch below. */ uint32_t local_tasks = cache_tasks < dense_workers ? cache_tasks : dense_workers; - double local_traffic = local_tasks * slab_bytes; + double local_traffic = dense_workers * slab_bytes; double partition_traffic = (double)eff_n * (sizeof(uint32_t) + record_size); /* Compare the complete partition allocation against radix's * payload plus its worst-case per-row group state. A payload-only diff --git a/src/ops/agg_engine.h b/src/ops/agg_engine.h index 0ae3abf3..b914e49c 100644 --- a/src/ops/agg_engine.h +++ b/src/ops/agg_engine.h @@ -65,6 +65,7 @@ typedef struct { uint32_t dense_tasks; /* local/partition tasks; worker count for shared updates */ uint64_t key_domain_evals; /* computed keys evaluated once per distinct symbol */ bool topn_native; /* last v2 run selected the emit filter's top-N itself */ + int64_t topn_kept; /* groups kept by that selection (ties included) */ } agg_route_stats_t; void agg_route_reset(void); void agg_route_note_key_domain(void); diff --git a/test/main.c b/test/main.c index 64a3a968..6cf59894 100644 --- a/test/main.c +++ b/test/main.c @@ -872,6 +872,7 @@ int main(int argc, char** argv) { } } + if (g_census) setvbuf(g_census, NULL, _IOLBF, 0); /* closed by the OS at exit; lines land as written */ const char* rfl_root = getenv("RFL_ROOT"); if (!rfl_root || !*rfl_root) rfl_root = "test/rfl"; if (rfl_scan(rfl_root) < 0) return 2; diff --git a/test/test_agg_contract.c b/test/test_agg_contract.c index 8743c0b1..2fd9d320 100644 --- a/test/test_agg_contract.c +++ b/test/test_agg_contract.c @@ -1815,6 +1815,7 @@ static test_result_t test_radix_native_topn(void) { TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_LEGACY], 0); TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_V2_RADIX], 1); TEST_ASSERT_TRUE(stats.topn_native); + TEST_ASSERT_TRUE(stats.topn_kept >= 10 && stats.topn_kept < 100000); TEST_ASSERT_EQ_I(ray_table_nrows(r), 10); ray_t* check = ray_eval_str( "(== (at (at (select {from:rt_t by:[k j] c:(count v) desc:c take:10}) 'c) 0) " @@ -1846,6 +1847,9 @@ static test_result_t test_dense_native_topn(void) { TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_V2_DENSE], 1); TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_LEGACY], 0); TEST_ASSERT_TRUE(stats.topn_native); + /* 50k groups sit below the parallel threshold (one selection task): the + * native selection must still cut the group set down to N plus ties. */ + TEST_ASSERT_TRUE(stats.topn_kept >= 5 && stats.topn_kept < 1000); TEST_ASSERT_EQ_I(ray_table_nrows(r), 5); ray_t* check = ray_eval_str( "(all (== (at (select {from:dt_t by:k s:(sum v) c:(count v) desc:s take:5}) 's) " From fb497793735675776d8b12a7fcbca0d0176f285e Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 14:17:22 +0200 Subject: [PATCH 21/23] test(group): pin the native top-N kept counts to the exact tie sets --- test/test_agg_contract.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/test_agg_contract.c b/test/test_agg_contract.c index 2fd9d320..c05e53ea 100644 --- a/test/test_agg_contract.c +++ b/test/test_agg_contract.c @@ -1815,7 +1815,9 @@ static test_result_t test_radix_native_topn(void) { TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_LEGACY], 0); TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_V2_RADIX], 1); TEST_ASSERT_TRUE(stats.topn_native); - TEST_ASSERT_TRUE(stats.topn_kept >= 10 && stats.topn_kept < 100000); + /* counts are 1 or 2: the top-10 superset is every count-2 group (500,000 + * of 1,500,000), not the full group set */ + TEST_ASSERT_EQ_I(stats.topn_kept, 500000); TEST_ASSERT_EQ_I(ray_table_nrows(r), 10); ray_t* check = ray_eval_str( "(== (at (at (select {from:rt_t by:[k j] c:(count v) desc:c take:10}) 'c) 0) " @@ -1848,8 +1850,9 @@ static test_result_t test_dense_native_topn(void) { TEST_ASSERT_EQ_I(stats.routes[AGG_ROUTE_LEGACY], 0); TEST_ASSERT_TRUE(stats.topn_native); /* 50k groups sit below the parallel threshold (one selection task): the - * native selection must still cut the group set down to N plus ties. */ - TEST_ASSERT_TRUE(stats.topn_kept >= 5 && stats.topn_kept < 1000); + * native selection must still cut the group set down to N plus ties — + * here every group whose sum equals the maximum (126), 3,846 of 50,000. */ + TEST_ASSERT_EQ_I(stats.topn_kept, 3846); TEST_ASSERT_EQ_I(ray_table_nrows(r), 5); ray_t* check = ray_eval_str( "(all (== (at (select {from:dt_t by:k s:(sum v) c:(count v) desc:s take:5}) 's) " From f8e643bbcf72e955295cd31c6c8bac9fde4241ef Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 14:38:50 +0200 Subject: [PATCH 22/23] fix(group): honor the emit filter only on the node it was armed for The filter is thread-local and stays armed across every grouped select nested in the matched select's from: expression; v2 now requires the slot to hold the operation the filter names (count when unset). Also: a bounded emit that cannot allocate its selection fails instead of emitting the wrong prefix; groups within the limit are emitted in first-seen order too; the compaction prescan bounds its transient bitmap footprint. --- src/ops/agg_engine.c | 32 +++++++++++++++++++------ test/rfl/group/emit_filter_v2_route.rfl | 13 ++++++++++ 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index 5576a279..cbd8750b 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -346,6 +346,7 @@ static void agg_key_bounds_parallel(ray_t* key, int64_t rows, bool nullable, * bounded (AGG_COMPACT_MAX_RANGE codes) are compacted. */ enum { AGG_COMPACT_MAX_RANGE = 1 << 22, /* largest per-key remap table (codes) */ AGG_COMPACT_TRY_SLOTS = 1 << 16 }; /* raw products above this try compaction */ +#define AGG_COMPACT_BITMAP_BUDGET ((size_t)32 << 20) /* transient per-task bitmaps */ typedef struct { ray_t** key_cols; @@ -397,6 +398,9 @@ static bool agg_dense_plan_compact(ray_t** key_cols, uint32_t n_keys, int64_t nr size_t words_total = 0; for (uint32_t k = 0; k < n_keys; k++) if (candidate[k]) words_total += (size_t)((out->ranges[k] - out->nullable[k] + 63) / 64); + /* Private bitmaps are words_total * tasks; keep the transient footprint + * within AGG_COMPACT_BITMAP_BUDGET by using fewer, longer tasks. */ + while (tasks > 1 && words_total * tasks * sizeof(uint64_t) > AGG_COMPACT_BITMAP_BUDGET) tasks /= 2; uint64_t* bits = ray_calloc_raw(words_total * tasks * sizeof(uint64_t)); uint64_t** task_bits = ray_calloc_raw((size_t)tasks * n_keys * sizeof(uint64_t*)); if (!bits || !task_bits) { ray_free_raw(bits); ray_free_raw(task_bits); return false; } @@ -1861,11 +1865,20 @@ static ray_t* agg_dense_finish(ray_t** key_cols, int64_t* key_syms, ray_op_ext_t * row ARE the first N groups in first-seen order. Select them with an * N-sized max-heap over the occupied slots and emit them ascending by * first row, byte-identical to trimming a first-seen-ordered result. */ - if (group_limit > 0 && first_row_ordered && ng > group_limit) { - int64_t n_keep = group_limit; + if (group_limit > 0 && first_row_ordered && ng > 0) { + int64_t n_keep = ng < group_limit ? ng : group_limit; int64_t* hkey = ray_alloc_raw((size_t)n_keep * sizeof(int64_t)); int64_t* hslot = ray_alloc_raw((size_t)n_keep * sizeof(int64_t)); - if (hkey && hslot) { + if (!hkey || !hslot) { + /* Emitting slot order here would hand the caller's head the wrong + * prefix; fail the query instead of answering it differently. */ + ray_free_raw(hkey); ray_free_raw(hslot); + ray_free_raw(occupied_slot); ray_free_raw(first_row_ordered); + agg_dense_slab_destroy_states(gstates, total_slots, vts, off, block, n_aggs); + ray_free_raw(gstates); ray_free_raw(gfirst); + return ray_error("oom", NULL); + } + { int64_t hn = 0; for (int64_t i = 0; i < ng; i++) { int64_t first = first_row_ordered[i], slot = occupied_slot[i]; @@ -4885,10 +4898,15 @@ static ray_t* exec_group_v2_run_inner(ray_graph_t* g, ray_op_t* op, ray_t* tbl, route_stats.dense_worker_budget = false; route_stats.dense_tasks = 0; ray_op_ext_t* ext = find_ext(g, op->id); - /* The top-N emit filter names an aggregate slot; an out-of-range slot or a - * filter armed for a different node means "no filter" here (the caller's - * sort+take still produces the final answer from the full result). */ - if (efp && (!efp->enabled || efp->agg_index >= ext->n_aggs)) efp = NULL; + /* The emit filter is thread-local and stays armed while the whole + * `from:` expression of the select that matched it evaluates, so any + * grouped select nested inside sees it too. Honor it only when it was + * armed for THIS node: the slot exists and holds the operation the + * filter names (an unset op means count, per the filter's contract). + * Anything else is "no filter": the outer sort+take still finalizes. */ + if (efp && (!efp->enabled || efp->agg_index >= ext->n_aggs || + (efp->agg_op ? efp->agg_op : OP_COUNT) != ext->agg_ops[efp->agg_index])) + efp = NULL; /* Exact-size carve for the per-key column pointers + syms (one block, both * 8-byte): unbounded key count, freed at every exit of this function diff --git a/test/rfl/group/emit_filter_v2_route.rfl b/test/rfl/group/emit_filter_v2_route.rfl index a80935e5..adef3319 100644 --- a/test/rfl/group/emit_filter_v2_route.rfl +++ b/test/rfl/group/emit_filter_v2_route.rfl @@ -94,3 +94,16 @@ (set tsn (table [s v] (list (at ["a" "b" "c" "d" "e" "f"] (% i 6)) (at tn 'v)))) (set a14 (select {from: tsn by: s m: (min v) asc: m take: 2})) (at (at a14 's) 0) -- "c" + +;; the emit filter is armed while the matched select's from: evaluates; a +;; grouped select nested inside must not be trimmed by it (its aggregate slot +;; holds a different operation than the filter names) +(set inner (select {from: t by: k s: (sum v)})) +(set nested (select {from: (select {from: (select {from: t by: k s: (sum v)}) by: s c: (count s)}) where: (> c 5)})) +(set ref (select {from: (select {from: inner by: s c: (count s)}) where: (> c 5)})) +(== (count nested) (count ref)) -- true +(== (sum (at nested 'c)) (sum (at ref 'c))) -- true +;; the same nesting with the inner select ordered by the filtered op keeps its full group set +(set nested2 (select {from: (select {from: (select {from: t by: k c: (count v)}) by: c n: (count c)}) where: (> n 2)})) +(set ref2 (select {from: (select {from: (select {from: t by: k c: (count v)}) by: c n: (count c)})})) +(== (count nested2) (count (select {from: ref2 where: (> n 2)}))) -- true From 14680d1979da701230331a4f0f4d3fd6639d8aa8 Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 19 Sep 2026 15:00:26 +0200 Subject: [PATCH 23/23] fix(group): bind the emit filter to the evaluation depth of its target select The filter is thread-local and stays armed while the arming select's from: evaluates, so a grouped select nested two levels down was filtered by its grandparent's having threshold (pre-existing: count-of-count under where (> n N) returned nothing when the inner counts fell below N). Each arming site now records the target depth (the direct child for the having shape, itself for take/desc and the count-distinct rewrite) and both engines read the filter through ray_group_emit_filter_active(), which returns it only at that depth. The nested-select test now exercises data an erroneous trim would empty. --- src/core/platform.c | 1 + src/ops/agg_engine.c | 45 ++++++++++++------------- src/ops/agg_engine.h | 2 +- src/ops/group.c | 12 +++++-- src/ops/internal.h | 10 ++++++ src/ops/query.c | 7 +++- test/rfl/group/emit_filter_v2_route.rfl | 24 +++++++------ 7 files changed, 61 insertions(+), 40 deletions(-) diff --git a/src/core/platform.c b/src/core/platform.c index 3879fea6..89852918 100644 --- a/src/core/platform.c +++ b/src/core/platform.c @@ -469,6 +469,7 @@ void ray_sem_signal(ray_sem_t* s) { #define WIN32_LEAN_AND_MEAN #endif #include +#include "mem/sys.h" /* -------------------------------------------------------------------------- * Virtual memory diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index cbd8750b..e030b13c 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -445,7 +445,7 @@ static bool agg_dense_plan_compact(ray_t** key_cols, uint32_t n_keys, int64_t nr * still accepted (the non-overflow entry) never pairs a shrunken range * with the raw slot computation. */ int64_t raw_ranges[16]; - memcpy(raw_ranges, out->ranges, sizeof(raw_ranges)); + memcpy(raw_ranges, out->ranges, (size_t)n_keys * sizeof(int64_t)); for (uint32_t k = 0; k < n_keys; k++) { if (!candidate[k]) continue; int64_t raw = raw_ranges[k] - out->nullable[k]; @@ -456,7 +456,7 @@ static bool agg_dense_plan_compact(ray_t** key_cols, uint32_t n_keys, int64_t nr ray_free_raw(remap); ray_free_raw(inverse); ray_free_raw(bits); ray_free_raw(task_bits); agg_dense_plan_free(out); - memcpy(out->ranges, raw_ranges, sizeof(raw_ranges)); + memcpy(out->ranges, raw_ranges, (size_t)n_keys * sizeof(int64_t)); return false; } const uint64_t* acc = task_bits[k]; @@ -3929,13 +3929,6 @@ static ray_t* exec_group_v2_parallel_radix( * first-seen order without a comparison sort or a routing threshold. */ int64_t input_count = sel ? n_sel : nrows; - /* Bounded emit under a HEAD(GROUP) limit hint: when only the first - * `group_limit` groups are wanted, selecting them directly is O(ng) with a - * `group_limit`-sized heap, versus the full path's O(input_count) order map - * (an 80MB alloc + memset + scatter + compact on a 10M-row input) followed - * by a full key-unpack and finalize of every group. The N groups with the - * SMALLEST first_row ARE the first N groups in first-seen order, so the - * emitted prefix is byte-identical to trimming the full result to N. */ /* Bounded emit under a HEAD(GROUP) limit hint: when only the first * `group_limit` groups are wanted, selecting them directly is O(ng) with a * `group_limit`-sized heap, versus the full path's O(input_count) order map @@ -4832,9 +4825,11 @@ static ray_t* exec_group_v2_run_inner(ray_graph_t* g, ray_op_t* op, ray_t* tbl, * order is preserved. Consumes `result`; a failure keeps the full result, * which is still a valid answer for the sort+take downstream. */ ray_t* agg_emit_filter_trim(ray_t* result, uint32_t n_keys, uint32_t n_aggs, - const ray_group_emit_filter_t* ef) { + const uint16_t* agg_ops, const ray_group_emit_filter_t* ef) { if (!result || RAY_IS_ERR(result) || result->type != RAY_TABLE) return result; if (ef->agg_index >= n_aggs) return result; + /* second line of defence: never trim by a slot the filter did not name */ + if (agg_ops && (ef->agg_op ? ef->agg_op : OP_COUNT) != agg_ops[ef->agg_index]) return result; int64_t nrows = ray_table_nrows(result); if (nrows <= 0) return result; ray_t* vcol = ray_table_get_col_idx(result, (int64_t)n_keys + ef->agg_index); @@ -4872,17 +4867,26 @@ ray_t* agg_emit_filter_trim(ray_t* result, uint32_t n_keys, uint32_t n_aggs, /* Every v2 strategy returns through here: a route that could not select * the emit filter's top-N itself hands back its full result and is trimmed * to the kept superset, so callers see one contract regardless of route. */ +/* True when the thread-local emit filter was armed for THIS node: the slot + * exists and holds the operation the filter names (count when unset). The + * filter stays armed while the matched select's whole `from:` evaluates, so + * grouped selects nested inside see it too; for them this is false. */ +static bool agg_emit_filter_targets(const ray_group_emit_filter_t* ef, const ray_op_ext_t* ext) { + return ef && ef->enabled && ext && ef->agg_index < ext->n_aggs && + (ef->agg_op ? ef->agg_op : OP_COUNT) == ext->agg_ops[ef->agg_index]; +} + static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t nrows, ray_t* sel, const int64_t* sel_prefix, int64_t n_sel, int64_t group_limit, const ray_group_emit_filter_t* efp) { + ray_op_ext_t* ext = find_ext(g, op->id); + if (!agg_emit_filter_targets(efp, ext)) efp = NULL; /* one decision for routing AND trim */ ray_t* r = exec_group_v2_run_inner(g, op, tbl, nrows, sel, sel_prefix, n_sel, group_limit, efp); - if (efp && r && !RAY_IS_ERR(r) && !route_stats.topn_native) { - ray_op_ext_t* ext = find_ext(g, op->id); - if (ext) r = agg_emit_filter_trim(r, ext->n_keys, ext->n_aggs, efp); - } + if (efp && r && !RAY_IS_ERR(r) && !route_stats.topn_native) + r = agg_emit_filter_trim(r, ext->n_keys, ext->n_aggs, ext->agg_ops, efp); return r; } @@ -4898,15 +4902,8 @@ static ray_t* exec_group_v2_run_inner(ray_graph_t* g, ray_op_t* op, ray_t* tbl, route_stats.dense_worker_budget = false; route_stats.dense_tasks = 0; ray_op_ext_t* ext = find_ext(g, op->id); - /* The emit filter is thread-local and stays armed while the whole - * `from:` expression of the select that matched it evaluates, so any - * grouped select nested inside sees it too. Honor it only when it was - * armed for THIS node: the slot exists and holds the operation the - * filter names (an unset op means count, per the filter's contract). - * Anything else is "no filter": the outer sort+take still finalizes. */ - if (efp && (!efp->enabled || efp->agg_index >= ext->n_aggs || - (efp->agg_op ? efp->agg_op : OP_COUNT) != ext->agg_ops[efp->agg_index])) - efp = NULL; + /* efp was already gated on this node by exec_group_v2_run; the compact + * fallback recursion re-enters through that wrapper too. */ /* Exact-size carve for the per-key column pointers + syms (one block, both * 8-byte): unbounded key count, freed at every exit of this function @@ -5322,7 +5319,7 @@ ray_t* exec_group_v2(ray_graph_t* g, ray_op_t* op, ray_t* tbl, /* The top-N emit filter (`desc: AGG take: N`) is read once here and * handed to every strategy: radix and the dense finishes select the kept * groups themselves; the other routes trim their full result. */ - ray_group_emit_filter_t ef = ray_group_emit_filter_get(); + ray_group_emit_filter_t ef = ray_group_emit_filter_active(); const ray_group_emit_filter_t* efp = ef.enabled ? &ef : NULL; if (!g || !g->selection) return exec_group_v2_run(g, op, tbl, ray_table_nrows(tbl), NULL, NULL, 0, diff --git a/src/ops/agg_engine.h b/src/ops/agg_engine.h index b914e49c..e31af91a 100644 --- a/src/ops/agg_engine.h +++ b/src/ops/agg_engine.h @@ -103,7 +103,7 @@ bool agg_group_values_f64(const agg_vtable_t* vt, const char* states, /* Trim a finished group result to the emit filter's kept superset (row order * preserved; consumes `result`). Used by routes that emit every group. */ ray_t* agg_emit_filter_trim(ray_t* result, uint32_t n_keys, uint32_t n_aggs, - const ray_group_emit_filter_t* ef); + const uint16_t* agg_ops, const ray_group_emit_filter_t* ef); /* Precondition: agg_v2_can_handle(g, op, tbl) returned true. * `group_limit` is the HEAD(GROUP) row-limit HINT (0 = no limit): when diff --git a/src/ops/group.c b/src/ops/group.c index ad5d3fe3..8ebc0418 100644 --- a/src/ops/group.c +++ b/src/ops/group.c @@ -7901,6 +7901,12 @@ ray_group_emit_filter_t ray_group_emit_filter_get(void) { void ray_group_emit_filter_set(ray_group_emit_filter_t filter) { tl_group_emit_filter = filter; } +ray_group_emit_filter_t ray_group_emit_filter_active(void) { + ray_group_emit_filter_t f = ray_group_emit_filter_get(); + if (!f.enabled) return f; + if (__VM && __VM->eval_depth != f.target_depth) { ray_group_emit_filter_t off = {0}; return off; } + return f; +} static int64_t da_count_emit_keep_min(const int64_t* counts, uint32_t n_slots, uint32_t group_count, @@ -10391,7 +10397,7 @@ static bool sg_shape_eligible(ray_graph_t* g, ray_op_t* op, ray_t* tbl, * is simply ignored here (staying eligible keeps where+by+take shapes on * the slice kernel instead of dropping them to the generic ladder). */ if (group_limit < 0) return false; - if (ray_group_emit_filter_get().enabled) return false; + if (ray_group_emit_filter_active().enabled) return false; ray_op_ext_t* ext = find_ext(g, op->id); if (!ext || ext->n_keys != 1 || ext->n_aggs < 1 || ext->n_aggs > 16) return false; @@ -11333,7 +11339,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, * `_e{a}_{op}` output names the legacy expression emit produced. * Any ineligibility falls through to the legacy path unchanged. */ if (ray_agg_engine_v2 && group_limit >= 0 - && !ray_group_emit_filter_get().enabled) { + && !ray_group_emit_filter_active().enabled) { ray_t* r = exec_group_v2_exprs(g, op, tbl, group_limit); if (r) return r; } @@ -11723,7 +11729,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, key_attrs[k] = 0; } } - ray_group_emit_filter_t emit_filter = ray_group_emit_filter_get(); + ray_group_emit_filter_t emit_filter = ray_group_emit_filter_active(); /* Historical: enabled only for OP_COUNT (the min_count_exclusive * heavy-hitter filter and the top_count_take heap). The * top_count_take heap path now also accepts SUM/MIN/MAX — those diff --git a/src/ops/internal.h b/src/ops/internal.h index 6465b0a1..4ff40059 100644 --- a/src/ops/internal.h +++ b/src/ops/internal.h @@ -1504,8 +1504,18 @@ typedef struct { * more — consumers take .desc at face value, and the desc-only * keep-min trims are gated off entirely when it is 0. */ uint8_t desc; + /* Evaluation depth (__VM->eval_depth) of the select whose group node + * this filter targets. The filter is thread-local and stays armed while + * an arming select evaluates its `from:` (the "having" shape arms it for + * the DIRECT child select), so grouped selects nested deeper would see + * it too; consumers read it through ray_group_emit_filter_active(), + * which returns it only at this depth. */ + int32_t target_depth; } ray_group_emit_filter_t; ray_group_emit_filter_t ray_group_emit_filter_get(void); +/* The armed filter when the calling thread's eval depth is its target + * depth (or no VM is bound); a disabled filter otherwise. */ +ray_group_emit_filter_t ray_group_emit_filter_active(void); void ray_group_emit_filter_set(ray_group_emit_filter_t filter); /* Hash-aggregate rows [start, end) into ht. * diff --git a/src/ops/query.c b/src/ops/query.c index 5f23f2d2..6f7d2d3b 100644 --- a/src/ops/query.c +++ b/src/ops/query.c @@ -4219,6 +4219,7 @@ static ray_t* try_count_distinct_v2_rewrite( * so set it explicitly instead of leaning on a zero default that * used to be coerced to desc inside group.c. */ emit_f.desc = 1; + emit_f.target_depth = __VM ? __VM->eval_depth : 0; /* executed right here */ ray_group_emit_filter_set(emit_f); emit_set = 1; } @@ -6087,8 +6088,11 @@ ray_t* ray_select(ray_t** args, int64_t n) { ray_group_emit_filter_t emit_filter = {0}; bool emit_filter_set = match_group_count_emit_filter( from_expr, where_expr, &emit_filter); - if (emit_filter_set) + if (emit_filter_set) { + /* armed for the DIRECT child select evaluated by the from: below */ + emit_filter.target_depth = (__VM ? __VM->eval_depth : 0) + 1; ray_group_emit_filter_set(emit_filter); + } /* Projection pushdown: publish the columns this select references so an * IMMEDIATE nested `select {by:}` distinct in `from:` (eval_depth+1) carries * only those. Stack buffer stays live across the from: eval; save/restore @@ -10496,6 +10500,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { bool self_emit_set = false; if (pre_top_emit_matched) { prev_self_emit = ray_group_emit_filter_get(); + pre_top_emit.target_depth = __VM ? __VM->eval_depth : 0; /* our own group node */ ray_group_emit_filter_set(pre_top_emit); self_emit_set = true; } diff --git a/test/rfl/group/emit_filter_v2_route.rfl b/test/rfl/group/emit_filter_v2_route.rfl index adef3319..54447a99 100644 --- a/test/rfl/group/emit_filter_v2_route.rfl +++ b/test/rfl/group/emit_filter_v2_route.rfl @@ -96,14 +96,16 @@ (at (at a14 's) 0) -- "c" ;; the emit filter is armed while the matched select's from: evaluates; a -;; grouped select nested inside must not be trimmed by it (its aggregate slot -;; holds a different operation than the filter names) -(set inner (select {from: t by: k s: (sum v)})) -(set nested (select {from: (select {from: (select {from: t by: k s: (sum v)}) by: s c: (count s)}) where: (> c 5)})) -(set ref (select {from: (select {from: inner by: s c: (count s)}) where: (> c 5)})) -(== (count nested) (count ref)) -- true -(== (sum (at nested 'c)) (sum (at ref 'c))) -- true -;; the same nesting with the inner select ordered by the filtered op keeps its full group set -(set nested2 (select {from: (select {from: (select {from: t by: k c: (count v)}) by: c n: (count c)}) where: (> n 2)})) -(set ref2 (select {from: (select {from: (select {from: t by: k c: (count v)}) by: c n: (count c)})})) -(== (count nested2) (count (select {from: ref2 where: (> n 2)}))) -- true +;; grouped select nested inside must not be trimmed by it. Every inner sum +;; is 0 here, so an erroneous trim (sum > 5) would empty the inner result and +;; the outer count would drop from 1 to 0. +(set tz (table [k v] (list (% i 1000) (* 0 i)))) +(set nested (select {from: (select {from: (select {from: tz by: k s: (sum v)}) by: s c: (count s)}) where: (> c 5)})) +(count nested) -- 1 +(at (at nested 'c) 0) -- 1000 +;; inner node whose slot 0 IS a count, but a different node than the matched +;; one: its counts (30 each) all fail the outer threshold (> 100), so a leaked +;; trim would empty it; the outer result must still see all 1000 inner groups +(set nested2 (select {from: (select {from: (select {from: tz by: k c: (count v)}) by: c n: (count c)}) where: (> n 100)})) +(count nested2) -- 1 +(at (at nested2 'n) 0) -- 1000