Add a full Pinot result for c6a.4xlarge (2026-09-20)

pinot/benchmark.sh

@@ -4,6 +4,29 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-tsv"
 # inside one JVM and takes longer than the lib's 300 s default to be
 # query-ready on a cold instance. 900 s clears the observed cold start.
 export BENCH_CHECK_TIMEOUT=900
+# Pinot QuickStart's -dataDir persistence is broken in 1.5.0/1.5.1: the
+# second QuickStart invocation against an existing -dataDir always dies
+# with IllegalStateException from Quickstart.java's
+# Preconditions.checkState(quickstartRunnerDir.mkdirs()) — mkdirs() returns
+# false because the directory from the first run already exists (verified
+# by reproducing this locally in Docker and reading the Pinot source).
+# A stop/start cycle therefore always wipes the in-memory table/segment
+# registration and there is no supported way to bring the daemon back up
+# against its previous state directory.
+#
+# What *does* survive the cycle is the segment tar files QuickStart wrote
+# to a plain local directory (batch/hits/segments/, not process memory —
+# verified locally: kill -9 the JVM, restart it, the .tar.gz files are
+# still on disk). BENCH_DURABLE=no makes the shared driver re-run ./load
+# after every restart; our ./load exploits this by re-pushing those
+# existing tars straight to the controller's segment-upload API
+# (https://docs.pinot.apache.org/build-with-pinot/ingestion/batch-ingestion/segment-upload)
+# instead of re-parsing hits.tsv, so the "reload" on cold tries 2-129
+# (43 queries x 3 tries) takes seconds instead of the ~78 min a real
+# from-source reload would need. This keeps the real stop/start/
+# drop_caches cycle other systems get, rather than skipping it via
+# BENCH_RESTARTABLE=no.
+export BENCH_DURABLE=no
 # Skip the pre-snapshot ./stop+./start cycle: the loaded
 # state lives only in the daemon's process memory (in-process
 # DataFrame, JVM heap caches) and stopping wipes it. The

pinot/check

@@ -1,10 +1,28 @@
 #!/bin/bash
 set -e
 
-# Pinot is responsive once both the controller and the broker accept queries.
-RES=$(curl -sf -o /dev/null -w '%{http_code}' \
+# The QuickStart process only registers its clean-shutdown hook after
+# ./start's launched process finishes starting every component AND
+# bootstrapping every sample table (including waiting for each table's
+# async minion segment-generation-and-push task to finish). If ./stop
+# sends SIGTERM any earlier than that, either shutdown-hook registration
+# itself throws (component startup still in flight) or an in-flight
+# minion task gets killed (bootstrap still in flight) -- either way the
+# process dies without a clean stop and corrupts the next ./start.
+#
+# Broker readiness alone is not a safe proxy for "fully done starting":
+# it comes up long before bootstrap finishes. The only reliable signal is
+# the "Quick start setup complete" line QuickStart itself prints right
+# after runner.bootstrapTable() + waitForBootstrapToComplete() return
+# (see pinot-tools Quickstart.java, method execute()). Since pinot.log is
+# appended across restarts, only look at what was appended after the
+# offset ./start recorded for this invocation.
+BROKER_RES=$(curl -sf -o /dev/null -w '%{http_code}' \
     -XPOST -H'Content-Type: application/json' \
     http://localhost:8000/query/sql/ \
-    -d '{"sql":"SELECT 1"}')
+    -d '{"sql":"SELECT 1"}' 2>/dev/null || echo "000")
 
-[ "$RES" = "200" ]
+OFFSET=$(cat pinot.log.offset 2>/dev/null || echo 0)
+SETUP_DONE=$(tail -c +"$((OFFSET + 1))" pinot.log 2>/dev/null | grep -c 'Quick start setup complete' || true)
+
+[ "$BROKER_RES" = "200" ] && [ "${SETUP_DONE:-0}" -ge 1 ]

pinot/load

@@ -3,6 +3,7 @@ set -e
 
 PINOT_VERSION=1.5.1
 PINOT_DIR="apache-pinot-$PINOT_VERSION-bin"
+SEGMENTS_DIR="batch/hits/segments"
 
 # Wait for the controller's REST API to be live. `start` only waits for
 # the broker; the controller can lag, and AddTable below talks to the
@@ -32,15 +33,68 @@ for _ in $(seq 1 60); do
     sleep 1
 done
 
-# Register the schema and table. Previously trailed `|| true`, which
-# masked genuine failures (controller not up, bad JSON, etc.) and let
-# LaunchDataIngestionJob run against a missing table — the symptom
-# was a misleading "Failed to fetch hits/schema" deep in the ingestion
-# job. Fail loudly here.
-"./${PINOT_DIR}/bin/pinot-admin.sh" AddTable \
+# Register the schema and table. AddTable fails with "already exists" on
+# every call after the first (BENCH_DURABLE=no re-runs ./load on every
+# cold cycle; QuickStart's own restart wipes the table registration but
+# not the segment tar files on disk — see the fast path below). Treat
+# that specific failure as success; anything else should still fail loudly
+# as before.
+add_table_out=$("./${PINOT_DIR}/bin/pinot-admin.sh" AddTable \
     -tableConfigFile offline_table.json \
-    -schemaFile schema.json -exec
+    -schemaFile schema.json -exec 2>&1) && add_table_rc=0 || add_table_rc=$?
+if [ "$add_table_rc" -ne 0 ]; then
+    if ! printf '%s\n' "$add_table_out" | grep -q 'already exists'; then
+        printf '%s\n' "$add_table_out" >&2
+        exit "$add_table_rc"
+    fi
+fi
+
+# Fast path: segment tar files from a previous ./load already exist on
+# disk. QuickStart -type batch writes them under $SEGMENTS_DIR (a plain
+# local directory, not process memory) and a stop/start cycle wipes only
+# the in-memory table/segment registration, not these files (verified by
+# reproducing this locally in Docker: kill -9 the QuickStart JVM, restart
+# it, curl POST each existing .tar.gz straight to the controller's
+# /segments endpoint — official "tar push" upload API, see
+# https://docs.pinot.apache.org/build-with-pinot/ingestion/batch-ingestion/segment-upload
+# — and the table is fully queryable again, star-tree index included,
+# without re-parsing hits.tsv). Re-pushing ~100 existing tars takes
+# seconds; re-running LaunchDataIngestionJob from hits.tsv takes ~78 min.
+if [ -d "$SEGMENTS_DIR" ] && ls "$SEGMENTS_DIR"/*.tar.gz >/dev/null 2>&1; then
+    for f in "$SEGMENTS_DIR"/*.tar.gz; do
+        curl -sS -X POST -H 'Content-Type: multipart/form-data' \
+            -F "file=@${f}" \
+            'http://localhost:9000/segments?tableName=hits&tableType=OFFLINE' \
+            | grep -q 'Successfully uploaded' || {
+                echo "bench: failed to re-push ${f}" >&2
+                exit 1
+            }
+    done
+
+    # Re-pushed segments take a few seconds to show up in the external
+    # view (Helix propagation), during which queries return a transient
+    # "N segments unavailable" error. Poll until every expected segment
+    # is ONLINE instead of racing the very first query against it
+    # (observed locally: up to ~20s for 5 segments; sized generously
+    # for ~100).
+    expected=$(ls "$SEGMENTS_DIR"/*.tar.gz | wc -l | tr -d ' ')
+    for _ in $(seq 1 120); do
+        online=$(curl -sf http://localhost:9000/tables/hits/externalview 2>/dev/null \
+            | python3 -c 'import json,sys
+d=json.load(sys.stdin)
+ov=d.get("OFFLINE") or {}
+print(sum(1 for st in ov.values() if list(st.values()) == ["ONLINE"]))' \
+            2>/dev/null || echo 0)
+        if [ "$online" -ge "$expected" ]; then
+            break
+        fi
+        sleep 1
+    done
 
+    exit 0
+fi
+
+# Slow path (first ./load only): generate segments from hits.tsv.
 # Pinot was unable to load data as a single file without errors. Split.
 split -d --additional-suffix .tsv -n l/100 hits.tsv parts
 
@@ -51,7 +105,42 @@ sed parts93.tsv -e 's/"tatuirovarki_redmond/tatuirovarki_redmond/g' -i
 sed splitted.yaml -e "s|PWD_DIR_PLACEHOLDER|$PWD|g" -i
 sed local.yaml    -e "s|PWD_DIR_PLACEHOLDER|$PWD|g" -i
 
+# pinot-admin.sh's own default is -Xms4G with NO -Xmx (falls back to the
+# JVM's default of ~25% of physical RAM). On a real ~74GB/100-segment
+# ClickBench hits.tsv this OOMed mid-job around segment 60/100 on a
+# 30GB-RAM c6a.4xlarge (OutOfMemoryError in SegmentDictionaryCreator) —
+# and LaunchDataIngestionJob still exited 0, silently leaving only 60/100
+# segments on disk.
+#
+# A fixed -Xmx20G (this PR's original fix) instead broke every machine
+# smaller than c6a.4xlarge — see ./start for the full explanation and the
+# same 64%-of-RAM/20G-cap/no-4G-floor rationale (anchored so c6a.4xlarge,
+# the only end-to-end-verified machine, gets back exactly its proven 20G).
+# Keep this in sync with ./start; both invoke pinot-admin.sh and both need
+# the same ceiling.
+RAM_MB=$(awk '/MemTotal/{ printf "%d", $2 / 1024 }' /proc/meminfo)
+XMX_MB=$(( RAM_MB * 64 / 100 ))
+[ "$XMX_MB" -gt 20480 ] && XMX_MB=20480
+XMS_MB=$XMX_MB
+[ "$XMS_MB" -gt 4096 ] && XMS_MB=4096
+export JAVA_OPTS="-Xms${XMS_MB}m -Xmx${XMX_MB}m"
+
 "./${PINOT_DIR}/bin/pinot-admin.sh" LaunchDataIngestionJob -jobSpecFile splitted.yaml
 
+# Defense against the exact failure this instance just hit: Pinot's own
+# ingestion job runner can swallow a per-file OutOfMemoryError (or other
+# per-task exception) and still exit 0, leaving fewer segment tars on
+# disk than input splits. Fail loudly instead of letting a partial load
+# silently pass bench_load's >5GB data-size check downstream.
+actual_segments=$(ls "$SEGMENTS_DIR"/*.tar.gz 2>/dev/null | wc -l | tr -d ' ')
+expected_segments=$(ls parts*.tsv 2>/dev/null | wc -l | tr -d ' ')
+if [ "$expected_segments" -gt 0 ] && [ "$actual_segments" -ne "$expected_segments" ]; then
+    echo "bench: LaunchDataIngestionJob produced ${actual_segments}/${expected_segments} segments — partial load (likely OOM swallowed per-task)" >&2
+    exit 1
+fi
+
+# Keep hits.tsv's derived parts only as long as needed; the source file
+# itself and the split parts are no longer needed once segments exist in
+# $SEGMENTS_DIR (used by the fast path above on every later cold cycle).
 rm -f hits.tsv parts*.tsv
 sync

pinot/queries.sql

@@ -16,7 +16,7 @@ SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase
 SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;
 SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;
 SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;
-SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;
+SELECT UserID, extract(minute FROM CAST(EventTime AS TIMESTAMP)) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;
 SELECT UserID FROM hits WHERE UserID = 435090932899640449;
 SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';
 SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;
@@ -40,4 +40,4 @@ SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >
 SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;
 SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;
 SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;
-SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;
+SELECT DATE_TRUNC('minute', CAST(EventTime AS TIMESTAMP)) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', CAST(EventTime AS TIMESTAMP)) ORDER BY DATE_TRUNC('minute', CAST(EventTime AS TIMESTAMP)) LIMIT 10 OFFSET 1000;

pinot/query

@@ -12,7 +12,7 @@ query=$(printf '%s' "$query" | tr -d ';')
 req=$(printf '%s' "$query" | python3 -c '
 import json, sys
 q = sys.stdin.read()
-print(json.dumps({"sql": q + "  option(timeoutMs=300000)"}))
+print(json.dumps({"sql": q + "  option(timeoutMs=300000,autoRewriteAggregationType=true)"}))
 ')
 
 resp=$(curl -sS -XPOST -H'Content-Type: application/json' \

pinot/results/20260920/c8g.metal-48xl.json

@@ -0,0 +1,60 @@
+{
+      "system": "Pinot",
+      "date": "2026-09-20",
+      "machine": "c8g.metal-48xl",
+      "cluster_size": 1,
+      "proprietary": "no",
+      "hardware": "cpu",
+      "tuned": "no",
+      "tags": ["Java","column-oriented"],
+      "load_time": 1505,
+      "data_size": 9157412094,
+      "concurrent_qps": 1.372,
+      "concurrent_error_ratio": 0,
+      "result": [
+        [300.066, 0.072, 0.015],
+        [288.551, 0.12, 0.033],
+        [284.689, 0.192, 0.187],
+        [284.271, 0.191, 0.1],
+        [273.266, 1.2, 1.439],
+        [284.503, 2.441, 2.989],
+        [280.345, 0.111, 0.103],
+        [280.847, 0.143, 0.035],
+        [281.654, 0.385, 0.251],
+        [281.513, 0.376, 0.265],
+        [298.725, 1.457, 1.421],
+        [295.127, 1.512, 1.208],
+        [297.565, 4.568, 3.427],
+        [290.355, 5.201, 4.274],
+        [285.071, 4.139, 3.345],
+        [281.256, 6.944, 5.595],
+        [292.33, 6.141, 8.877],
+        [284.002, 0.59, 0.426],
+        [290.633, 8.089, 8.388],
+        [286.35, 0.105, 0.101],
+        [278.944, 4.287, 4.102],
+        [290.626, 1.987, 1.763],
+        [276.847, 1.924, 1.737],
+        [275.681, 1.321, 1.176],
+        [300.172, 0.273, 0.222],
+        [285.248, 0.277, 0.223],
+        [284.661, 0.468, 0.449],
+        [287.354, 1.154, 0.771],
+        [290.595, 2.321, 2.332],
+        [271.899, 0.911, 0.898],
+        [295.926, 6.813, 7.448],
+        [299.353, 8.174, 9.238],
+        [282.683, 7.797, 8.087],
+        [294.62, 8.817, 7.506],
+        [300.904, 7.794, 8.323],
+        [288.837, 10.026, 6.499],
+        [286.527, 0.278, 0.179],
+        [284.732, 0.13, 0.057],
+        [276.827, 0.129, 0.034],
+        [287.396, 0.456, 0.411],
+        [294.71, 0.135, 0.124],
+        [285.092, 0.131, 0.034],
+        [293.298, 0.182, 0.096]
+]
+  } 
+

pinot/results/20260921/c6a.2xlarge.json

@@ -0,0 +1,60 @@
+{
+      "system": "Pinot",
+      "date": "2026-09-21",
+      "machine": "c6a.2xlarge",
+      "cluster_size": 1,
+      "proprietary": "no",
+      "hardware": "cpu",
+      "tuned": "no",
+      "tags": ["Java","column-oriented"],
+      "load_time": 4959,
+      "data_size": 9157431566,
+      "concurrent_qps": 0.03,
+      "concurrent_error_ratio": 0.471,
+      "result": [
+        [395.267, 0.012, 0.005],
+        [398.901, 0.06, 0.047],
+        [397.537, 0.255, 0.267],
+        [398.716, 0.201, 0.191],
+        [398.63, 1.813, 1.743],
+        [426.737, 2.866, 3.013],
+        [395.903, 0.012, 0.006],
+        [401.898, 0.056, 0.052],
+        [399.892, 1.572, 1.507],
+        [403.206, 1.954, 1.774],
+        [402.81, 1.935, 1.917],
+        [401.486, 1.959, 1.926],
+        [422.512, 3.003, 2.944],
+        [425.476, 6.115, 90.906],
+        [422.02, 3.165, 3.356],
+        [401.815, 4.161, 4.001],
+        [419.785, 4.933, 4.667],
+        [423.769, 1.224, 1.194],
+        [427.693, 9.554, 62.492],
+        [396.638, 0.016, 0.008],
+        [463.544, 56.659, 73.484],
+        [463.226, 62.363, 63.91],
+        [522.005, 144.074, 139.253],
+        [414.91, 2.416, 2.293],
+        [402.172, 0.21, 0.203],
+        [401.485, 0.358, 0.333],
+        [407.382, 0.801, 0.74],
+        [459.526, 64.327, 64.955],
+        [463.65, 83.448, 174.006],
+        [396.891, 2.135, 2.108],
+        [406.753, 4.673, 4.342],
+        [407.023, 5.026, 4.888],
+        [407.092, 5.019, 4.949],
+        [451.543, 47.354, 88.609],
+        [452.001, 47.293, 55.999],
+        [407.655, 6.073, 6.221],
+        [401.964, 0.22, 0.183],
+        [401.094, 0.088, 0.066],
+        [397.46, 0.047, 0.03],
+        [398.813, 0.394, 0.373],
+        [400.037, 0.067, 0.037],
+        [399.697, 0.046, 0.031],
+        [396.677, 0.107, 0.081]
+]
+  } 
+

pinot/results/20260921/c6a.4xlarge.json

@@ -0,0 +1,60 @@
+{
+      "system": "Pinot",
+      "date": "2026-09-21",
+      "machine": "c6a.4xlarge",
+      "cluster_size": 1,
+      "proprietary": "no",
+      "hardware": "cpu",
+      "tuned": "no",
+      "tags": ["Java","column-oriented"],
+      "load_time": 2355,
+      "data_size": 9157431366,
+      "concurrent_qps": 0.057,
+      "concurrent_error_ratio": 0.15,
+      "result": [
+        [343.95, 0.015, 0.013],
+        [341.614, 0.039, 0.034],
+        [342.787, 0.144, 0.132],
+        [342.765, 0.122, 0.103],
+        [343.377, 1.746, 1.659],
+        [360.756, 2.524, 2.549],
+        [341.239, 0.018, 0.006],
+        [343.111, 0.037, 0.039],
+        [343.683, 0.819, 0.813],
+        [344.637, 0.942, 0.915],
+        [342.371, 1.88, 1.792],
+        [344.455, 1.816, 1.879],
+        [367.568, 2.781, 2.744],
+        [363.317, 4.575, 5.022],
+        [363.134, 2.969, 2.887],
+        [349.008, 4.504, 4.254],
+        [363.736, 4.583, 4.278],
+        [369.891, 0.782, 0.787],
+        [366.571, 6.587, 6.692],
+        [342.459, 0.02, 0.008],
+        [398.09, 5.079, 4.845],
+        [399.966, 4.263, 4.276],
+        [445.66, 75.758, 13.258],
+        [353.809, 1.365, 1.366],
+        [344.476, 0.223, 0.227],
+        [345.165, 0.266, 0.268],
+        [345.743, 0.476, 0.438],
+        [389.992, 6.2, 2.506],
+        [398.81, 8.321, 7.806],
+        [343.297, 1.09, 1.082],
+        [346.582, 4.251, 4.224],
+        [348.705, 4.83, 4.767],
+        [347.677, 4.793, 4.912],
+        [381.262, 4.652, 4.748],
+        [387.239, 5.118, 5.553],
+        [347.87, 4.867, 4.656],
+        [343.282, 0.224, 0.168],
+        [342.496, 0.083, 0.058],
+        [342.529, 0.043, 0.029],
+        [345.006, 0.42, 0.357],
+        [342.65, 0.052, 0.039],
+        [344.575, 0.095, 0.035],
+        [342.597, 0.091, 0.086]
+]
+  } 
+