diff --git a/pinot/benchmark.sh b/pinot/benchmark.sh index e5db3195ea..f0515a7577 100755 --- a/pinot/benchmark.sh +++ b/pinot/benchmark.sh @@ -4,6 +4,29 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-tsv" # inside one JVM and takes longer than the lib's 300 s default to be # query-ready on a cold instance. 900 s clears the observed cold start. export BENCH_CHECK_TIMEOUT=900 +# Pinot QuickStart's -dataDir persistence is broken in 1.5.0/1.5.1: the +# second QuickStart invocation against an existing -dataDir always dies +# with IllegalStateException from Quickstart.java's +# Preconditions.checkState(quickstartRunnerDir.mkdirs()) — mkdirs() returns +# false because the directory from the first run already exists (verified +# by reproducing this locally in Docker and reading the Pinot source). +# A stop/start cycle therefore always wipes the in-memory table/segment +# registration and there is no supported way to bring the daemon back up +# against its previous state directory. +# +# What *does* survive the cycle is the segment tar files QuickStart wrote +# to a plain local directory (batch/hits/segments/, not process memory — +# verified locally: kill -9 the JVM, restart it, the .tar.gz files are +# still on disk). BENCH_DURABLE=no makes the shared driver re-run ./load +# after every restart; our ./load exploits this by re-pushing those +# existing tars straight to the controller's segment-upload API +# (https://docs.pinot.apache.org/build-with-pinot/ingestion/batch-ingestion/segment-upload) +# instead of re-parsing hits.tsv, so the "reload" on cold tries 2-129 +# (43 queries x 3 tries) takes seconds instead of the ~78 min a real +# from-source reload would need. This keeps the real stop/start/ +# drop_caches cycle other systems get, rather than skipping it via +# BENCH_RESTARTABLE=no. +export BENCH_DURABLE=no # Skip the pre-snapshot ./stop+./start cycle: the loaded # state lives only in the daemon's process memory (in-process # DataFrame, JVM heap caches) and stopping wipes it. The diff --git a/pinot/check b/pinot/check index 3bfe104c3f..ac6b50fc09 100755 --- a/pinot/check +++ b/pinot/check @@ -1,10 +1,28 @@ #!/bin/bash set -e -# Pinot is responsive once both the controller and the broker accept queries. -RES=$(curl -sf -o /dev/null -w '%{http_code}' \ +# The QuickStart process only registers its clean-shutdown hook after +# ./start's launched process finishes starting every component AND +# bootstrapping every sample table (including waiting for each table's +# async minion segment-generation-and-push task to finish). If ./stop +# sends SIGTERM any earlier than that, either shutdown-hook registration +# itself throws (component startup still in flight) or an in-flight +# minion task gets killed (bootstrap still in flight) -- either way the +# process dies without a clean stop and corrupts the next ./start. +# +# Broker readiness alone is not a safe proxy for "fully done starting": +# it comes up long before bootstrap finishes. The only reliable signal is +# the "Quick start setup complete" line QuickStart itself prints right +# after runner.bootstrapTable() + waitForBootstrapToComplete() return +# (see pinot-tools Quickstart.java, method execute()). Since pinot.log is +# appended across restarts, only look at what was appended after the +# offset ./start recorded for this invocation. +BROKER_RES=$(curl -sf -o /dev/null -w '%{http_code}' \ -XPOST -H'Content-Type: application/json' \ http://localhost:8000/query/sql/ \ - -d '{"sql":"SELECT 1"}') + -d '{"sql":"SELECT 1"}' 2>/dev/null || echo "000") -[ "$RES" = "200" ] +OFFSET=$(cat pinot.log.offset 2>/dev/null || echo 0) +SETUP_DONE=$(tail -c +"$((OFFSET + 1))" pinot.log 2>/dev/null | grep -c 'Quick start setup complete' || true) + +[ "$BROKER_RES" = "200" ] && [ "${SETUP_DONE:-0}" -ge 1 ] diff --git a/pinot/load b/pinot/load index 7229254546..cf35df3e38 100755 --- a/pinot/load +++ b/pinot/load @@ -3,6 +3,7 @@ set -e PINOT_VERSION=1.5.1 PINOT_DIR="apache-pinot-$PINOT_VERSION-bin" +SEGMENTS_DIR="batch/hits/segments" # Wait for the controller's REST API to be live. `start` only waits for # the broker; the controller can lag, and AddTable below talks to the @@ -32,15 +33,68 @@ for _ in $(seq 1 60); do sleep 1 done -# Register the schema and table. Previously trailed `|| true`, which -# masked genuine failures (controller not up, bad JSON, etc.) and let -# LaunchDataIngestionJob run against a missing table — the symptom -# was a misleading "Failed to fetch hits/schema" deep in the ingestion -# job. Fail loudly here. -"./${PINOT_DIR}/bin/pinot-admin.sh" AddTable \ +# Register the schema and table. AddTable fails with "already exists" on +# every call after the first (BENCH_DURABLE=no re-runs ./load on every +# cold cycle; QuickStart's own restart wipes the table registration but +# not the segment tar files on disk — see the fast path below). Treat +# that specific failure as success; anything else should still fail loudly +# as before. +add_table_out=$("./${PINOT_DIR}/bin/pinot-admin.sh" AddTable \ -tableConfigFile offline_table.json \ - -schemaFile schema.json -exec + -schemaFile schema.json -exec 2>&1) && add_table_rc=0 || add_table_rc=$? +if [ "$add_table_rc" -ne 0 ]; then + if ! printf '%s\n' "$add_table_out" | grep -q 'already exists'; then + printf '%s\n' "$add_table_out" >&2 + exit "$add_table_rc" + fi +fi + +# Fast path: segment tar files from a previous ./load already exist on +# disk. QuickStart -type batch writes them under $SEGMENTS_DIR (a plain +# local directory, not process memory) and a stop/start cycle wipes only +# the in-memory table/segment registration, not these files (verified by +# reproducing this locally in Docker: kill -9 the QuickStart JVM, restart +# it, curl POST each existing .tar.gz straight to the controller's +# /segments endpoint — official "tar push" upload API, see +# https://docs.pinot.apache.org/build-with-pinot/ingestion/batch-ingestion/segment-upload +# — and the table is fully queryable again, star-tree index included, +# without re-parsing hits.tsv). Re-pushing ~100 existing tars takes +# seconds; re-running LaunchDataIngestionJob from hits.tsv takes ~78 min. +if [ -d "$SEGMENTS_DIR" ] && ls "$SEGMENTS_DIR"/*.tar.gz >/dev/null 2>&1; then + for f in "$SEGMENTS_DIR"/*.tar.gz; do + curl -sS -X POST -H 'Content-Type: multipart/form-data' \ + -F "file=@${f}" \ + 'http://localhost:9000/segments?tableName=hits&tableType=OFFLINE' \ + | grep -q 'Successfully uploaded' || { + echo "bench: failed to re-push ${f}" >&2 + exit 1 + } + done + + # Re-pushed segments take a few seconds to show up in the external + # view (Helix propagation), during which queries return a transient + # "N segments unavailable" error. Poll until every expected segment + # is ONLINE instead of racing the very first query against it + # (observed locally: up to ~20s for 5 segments; sized generously + # for ~100). + expected=$(ls "$SEGMENTS_DIR"/*.tar.gz | wc -l | tr -d ' ') + for _ in $(seq 1 120); do + online=$(curl -sf http://localhost:9000/tables/hits/externalview 2>/dev/null \ + | python3 -c 'import json,sys +d=json.load(sys.stdin) +ov=d.get("OFFLINE") or {} +print(sum(1 for st in ov.values() if list(st.values()) == ["ONLINE"]))' \ + 2>/dev/null || echo 0) + if [ "$online" -ge "$expected" ]; then + break + fi + sleep 1 + done + exit 0 +fi + +# Slow path (first ./load only): generate segments from hits.tsv. # Pinot was unable to load data as a single file without errors. Split. split -d --additional-suffix .tsv -n l/100 hits.tsv parts @@ -51,7 +105,42 @@ sed parts93.tsv -e 's/"tatuirovarki_redmond/tatuirovarki_redmond/g' -i sed splitted.yaml -e "s|PWD_DIR_PLACEHOLDER|$PWD|g" -i sed local.yaml -e "s|PWD_DIR_PLACEHOLDER|$PWD|g" -i +# pinot-admin.sh's own default is -Xms4G with NO -Xmx (falls back to the +# JVM's default of ~25% of physical RAM). On a real ~74GB/100-segment +# ClickBench hits.tsv this OOMed mid-job around segment 60/100 on a +# 30GB-RAM c6a.4xlarge (OutOfMemoryError in SegmentDictionaryCreator) — +# and LaunchDataIngestionJob still exited 0, silently leaving only 60/100 +# segments on disk. +# +# A fixed -Xmx20G (this PR's original fix) instead broke every machine +# smaller than c6a.4xlarge — see ./start for the full explanation and the +# same 64%-of-RAM/20G-cap/no-4G-floor rationale (anchored so c6a.4xlarge, +# the only end-to-end-verified machine, gets back exactly its proven 20G). +# Keep this in sync with ./start; both invoke pinot-admin.sh and both need +# the same ceiling. +RAM_MB=$(awk '/MemTotal/{ printf "%d", $2 / 1024 }' /proc/meminfo) +XMX_MB=$(( RAM_MB * 64 / 100 )) +[ "$XMX_MB" -gt 20480 ] && XMX_MB=20480 +XMS_MB=$XMX_MB +[ "$XMS_MB" -gt 4096 ] && XMS_MB=4096 +export JAVA_OPTS="-Xms${XMS_MB}m -Xmx${XMX_MB}m" + "./${PINOT_DIR}/bin/pinot-admin.sh" LaunchDataIngestionJob -jobSpecFile splitted.yaml +# Defense against the exact failure this instance just hit: Pinot's own +# ingestion job runner can swallow a per-file OutOfMemoryError (or other +# per-task exception) and still exit 0, leaving fewer segment tars on +# disk than input splits. Fail loudly instead of letting a partial load +# silently pass bench_load's >5GB data-size check downstream. +actual_segments=$(ls "$SEGMENTS_DIR"/*.tar.gz 2>/dev/null | wc -l | tr -d ' ') +expected_segments=$(ls parts*.tsv 2>/dev/null | wc -l | tr -d ' ') +if [ "$expected_segments" -gt 0 ] && [ "$actual_segments" -ne "$expected_segments" ]; then + echo "bench: LaunchDataIngestionJob produced ${actual_segments}/${expected_segments} segments — partial load (likely OOM swallowed per-task)" >&2 + exit 1 +fi + +# Keep hits.tsv's derived parts only as long as needed; the source file +# itself and the split parts are no longer needed once segments exist in +# $SEGMENTS_DIR (used by the fast path above on every later cold cycle). rm -f hits.tsv parts*.tsv sync diff --git a/pinot/queries.sql b/pinot/queries.sql index 31f65fc898..47114dee7f 100644 --- a/pinot/queries.sql +++ b/pinot/queries.sql @@ -16,7 +16,7 @@ SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; -SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, extract(minute FROM CAST(EventTime AS TIMESTAMP)) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; SELECT UserID FROM hits WHERE UserID = 435090932899640449; SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; @@ -40,4 +40,4 @@ SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate > SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; -SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000; +SELECT DATE_TRUNC('minute', CAST(EventTime AS TIMESTAMP)) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', CAST(EventTime AS TIMESTAMP)) ORDER BY DATE_TRUNC('minute', CAST(EventTime AS TIMESTAMP)) LIMIT 10 OFFSET 1000; diff --git a/pinot/query b/pinot/query index 337792437a..d8b068644a 100755 --- a/pinot/query +++ b/pinot/query @@ -12,7 +12,7 @@ query=$(printf '%s' "$query" | tr -d ';') req=$(printf '%s' "$query" | python3 -c ' import json, sys q = sys.stdin.read() -print(json.dumps({"sql": q + " option(timeoutMs=300000)"})) +print(json.dumps({"sql": q + " option(timeoutMs=300000,autoRewriteAggregationType=true)"})) ') resp=$(curl -sS -XPOST -H'Content-Type: application/json' \ diff --git a/pinot/results/20260920/c8g.metal-48xl.json b/pinot/results/20260920/c8g.metal-48xl.json new file mode 100644 index 0000000000..718d12b34c --- /dev/null +++ b/pinot/results/20260920/c8g.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Pinot", + "date": "2026-09-20", + "machine": "c8g.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Java","column-oriented"], + "load_time": 1505, + "data_size": 9157412094, + "concurrent_qps": 1.372, + "concurrent_error_ratio": 0, + "result": [ + [300.066, 0.072, 0.015], + [288.551, 0.12, 0.033], + [284.689, 0.192, 0.187], + [284.271, 0.191, 0.1], + [273.266, 1.2, 1.439], + [284.503, 2.441, 2.989], + [280.345, 0.111, 0.103], + [280.847, 0.143, 0.035], + [281.654, 0.385, 0.251], + [281.513, 0.376, 0.265], + [298.725, 1.457, 1.421], + [295.127, 1.512, 1.208], + [297.565, 4.568, 3.427], + [290.355, 5.201, 4.274], + [285.071, 4.139, 3.345], + [281.256, 6.944, 5.595], + [292.33, 6.141, 8.877], + [284.002, 0.59, 0.426], + [290.633, 8.089, 8.388], + [286.35, 0.105, 0.101], + [278.944, 4.287, 4.102], + [290.626, 1.987, 1.763], + [276.847, 1.924, 1.737], + [275.681, 1.321, 1.176], + [300.172, 0.273, 0.222], + [285.248, 0.277, 0.223], + [284.661, 0.468, 0.449], + [287.354, 1.154, 0.771], + [290.595, 2.321, 2.332], + [271.899, 0.911, 0.898], + [295.926, 6.813, 7.448], + [299.353, 8.174, 9.238], + [282.683, 7.797, 8.087], + [294.62, 8.817, 7.506], + [300.904, 7.794, 8.323], + [288.837, 10.026, 6.499], + [286.527, 0.278, 0.179], + [284.732, 0.13, 0.057], + [276.827, 0.129, 0.034], + [287.396, 0.456, 0.411], + [294.71, 0.135, 0.124], + [285.092, 0.131, 0.034], + [293.298, 0.182, 0.096] +] + } + \ No newline at end of file diff --git a/pinot/results/20260921/c6a.2xlarge.json b/pinot/results/20260921/c6a.2xlarge.json new file mode 100644 index 0000000000..108af64e88 --- /dev/null +++ b/pinot/results/20260921/c6a.2xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Pinot", + "date": "2026-09-21", + "machine": "c6a.2xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Java","column-oriented"], + "load_time": 4959, + "data_size": 9157431566, + "concurrent_qps": 0.03, + "concurrent_error_ratio": 0.471, + "result": [ + [395.267, 0.012, 0.005], + [398.901, 0.06, 0.047], + [397.537, 0.255, 0.267], + [398.716, 0.201, 0.191], + [398.63, 1.813, 1.743], + [426.737, 2.866, 3.013], + [395.903, 0.012, 0.006], + [401.898, 0.056, 0.052], + [399.892, 1.572, 1.507], + [403.206, 1.954, 1.774], + [402.81, 1.935, 1.917], + [401.486, 1.959, 1.926], + [422.512, 3.003, 2.944], + [425.476, 6.115, 90.906], + [422.02, 3.165, 3.356], + [401.815, 4.161, 4.001], + [419.785, 4.933, 4.667], + [423.769, 1.224, 1.194], + [427.693, 9.554, 62.492], + [396.638, 0.016, 0.008], + [463.544, 56.659, 73.484], + [463.226, 62.363, 63.91], + [522.005, 144.074, 139.253], + [414.91, 2.416, 2.293], + [402.172, 0.21, 0.203], + [401.485, 0.358, 0.333], + [407.382, 0.801, 0.74], + [459.526, 64.327, 64.955], + [463.65, 83.448, 174.006], + [396.891, 2.135, 2.108], + [406.753, 4.673, 4.342], + [407.023, 5.026, 4.888], + [407.092, 5.019, 4.949], + [451.543, 47.354, 88.609], + [452.001, 47.293, 55.999], + [407.655, 6.073, 6.221], + [401.964, 0.22, 0.183], + [401.094, 0.088, 0.066], + [397.46, 0.047, 0.03], + [398.813, 0.394, 0.373], + [400.037, 0.067, 0.037], + [399.697, 0.046, 0.031], + [396.677, 0.107, 0.081] +] + } + \ No newline at end of file diff --git a/pinot/results/20260921/c6a.4xlarge.json b/pinot/results/20260921/c6a.4xlarge.json new file mode 100644 index 0000000000..da57ec317b --- /dev/null +++ b/pinot/results/20260921/c6a.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Pinot", + "date": "2026-09-21", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Java","column-oriented"], + "load_time": 2355, + "data_size": 9157431366, + "concurrent_qps": 0.057, + "concurrent_error_ratio": 0.15, + "result": [ + [343.95, 0.015, 0.013], + [341.614, 0.039, 0.034], + [342.787, 0.144, 0.132], + [342.765, 0.122, 0.103], + [343.377, 1.746, 1.659], + [360.756, 2.524, 2.549], + [341.239, 0.018, 0.006], + [343.111, 0.037, 0.039], + [343.683, 0.819, 0.813], + [344.637, 0.942, 0.915], + [342.371, 1.88, 1.792], + [344.455, 1.816, 1.879], + [367.568, 2.781, 2.744], + [363.317, 4.575, 5.022], + [363.134, 2.969, 2.887], + [349.008, 4.504, 4.254], + [363.736, 4.583, 4.278], + [369.891, 0.782, 0.787], + [366.571, 6.587, 6.692], + [342.459, 0.02, 0.008], + [398.09, 5.079, 4.845], + [399.966, 4.263, 4.276], + [445.66, 75.758, 13.258], + [353.809, 1.365, 1.366], + [344.476, 0.223, 0.227], + [345.165, 0.266, 0.268], + [345.743, 0.476, 0.438], + [389.992, 6.2, 2.506], + [398.81, 8.321, 7.806], + [343.297, 1.09, 1.082], + [346.582, 4.251, 4.224], + [348.705, 4.83, 4.767], + [347.677, 4.793, 4.912], + [381.262, 4.652, 4.748], + [387.239, 5.118, 5.553], + [347.87, 4.867, 4.656], + [343.282, 0.224, 0.168], + [342.496, 0.083, 0.058], + [342.529, 0.043, 0.029], + [345.006, 0.42, 0.357], + [342.65, 0.052, 0.039], + [344.575, 0.095, 0.035], + [342.597, 0.091, 0.086] +] + } + \ No newline at end of file diff --git a/pinot/results/20260921/c6a.metal.json b/pinot/results/20260921/c6a.metal.json new file mode 100644 index 0000000000..435b2cfd89 --- /dev/null +++ b/pinot/results/20260921/c6a.metal.json @@ -0,0 +1,60 @@ +{ + "system": "Pinot", + "date": "2026-09-21", + "machine": "c6a.metal", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Java","column-oriented"], + "load_time": 1512, + "data_size": 9157431474, + "concurrent_qps": 1.5, + "concurrent_error_ratio": 0, + "result": [ + [298.226, 0.166, 0.15], + [296.63, 0.193, 0.16], + [301.715, 0.269, 0.115], + [299.478, 0.246, 0.221], + [307.085, 2.495, 2.04], + [309.26, 3.426, 4.179], + [302.562, 0.16, 0.14], + [297.681, 0.184, 0.048], + [293.919, 0.842, 0.756], + [298.595, 0.829, 0.8], + [301.072, 3.323, 3.126], + [300.321, 2.765, 2.463], + [306.943, 4.935, 4.802], + [313.183, 6.525, 5.754], + [311.656, 5.029, 4.809], + [312.686, 7.759, 7.155], + [307.912, 7.677, 7.65], + [306.388, 0.661, 0.582], + [305.079, 9.157, 9.069], + [306.933, 0.103, 0.14], + [303.141, 4.386, 4.373], + [300.46, 2.2, 2.106], + [301.686, 2.365, 2.51], + [305.992, 1.356, 1.309], + [297.663, 0.313, 0.323], + [307.186, 0.42, 0.251], + [296.401, 0.602, 0.579], + [298.807, 1.581, 1.007], + [298.579, 3.709, 3.221], + [303.279, 1.059, 0.902], + [315.065, 7.368, 7.42], + [304.456, 8.713, 8.133], + [313.675, 9.01, 8.729], + [309.557, 6.944, 7.464], + [310.383, 8.248, 7.948], + [311.436, 7.925, 6.928], + [300.153, 0.435, 0.393], + [299.798, 0.257, 0.078], + [299.6, 0.199, 0.052], + [300.92, 0.614, 0.539], + [295.857, 0.216, 0.053], + [295.506, 0.193, 0.049], + [298.593, 0.28, 0.092] +] + } + \ No newline at end of file diff --git a/pinot/results/20260921/c8g.4xlarge.json b/pinot/results/20260921/c8g.4xlarge.json new file mode 100644 index 0000000000..0181a1d665 --- /dev/null +++ b/pinot/results/20260921/c8g.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Pinot", + "date": "2026-09-21", + "machine": "c8g.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Java","column-oriented"], + "load_time": 2182, + "data_size": 9157411858, + "concurrent_qps": 0.06, + "concurrent_error_ratio": 0.1, + "result": [ + [334.371, 0.016, 0.005], + [334.227, 0.036, 0.025], + [334.987, 0.141, 0.139], + [335.219, 0.116, 0.105], + [334.312, 1.237, 1.198], + [356.166, 2.626, 2.72], + [335.091, 0.016, 0.006], + [333.547, 0.03, 0.029], + [337.044, 0.529, 0.515], + [338.089, 0.647, 0.612], + [335.976, 1.119, 1.057], + [335.768, 1.121, 1.058], + [356.335, 3.661, 3.576], + [358.765, 4.05, 4.192], + [356.522, 5.011, 3.902], + [340.32, 5.472, 5.493], + [359.258, 7.635, 5.343], + [358.512, 1.171, 0.612], + [361.222, 6.676, 6.316], + [332.812, 0.019, 0.008], + [393.491, 4.673, 4.458], + [394.312, 2.989, 2.975], + [438.911, 93.804, 11.416], + [351.037, 1.265, 1.248], + [336.666, 0.242, 0.212], + [341.955, 0.266, 0.239], + [345.304, 0.399, 0.385], + [388.73, 2.446, 2.525], + [397.876, 5.256, 5.01], + [335.396, 0.947, 0.937], + [342.642, 5.097, 4.821], + [344.5, 5.385, 5.277], + [343.789, 5.346, 5.5], + [383.582, 5.443, 5.507], + [387.308, 4.66, 5.627], + [341.106, 5.246, 5.17], + [338.734, 0.261, 0.242], + [335.922, 0.066, 0.06], + [335.69, 0.042, 0.037], + [341.97, 0.429, 0.435], + [334.402, 0.058, 0.05], + [334.965, 0.037, 0.035], + [334.254, 0.105, 0.083] +] + } + \ No newline at end of file diff --git a/pinot/start b/pinot/start index bd189e8077..54b35ad3c3 100755 --- a/pinot/start +++ b/pinot/start @@ -4,7 +4,9 @@ set -e PINOT_VERSION=1.5.1 PINOT_DIR="apache-pinot-$PINOT_VERSION-bin" -# Idempotent: if broker query endpoint is up, do nothing. +# Idempotent: if broker query endpoint is up, do nothing (leave the +# existing pinot.log offset marker as-is; ./check will find the earlier +# "setup complete" marker in the untouched log). if curl -sf -o /dev/null -w '%{http_code}' \ -XPOST -H'Content-Type: application/json' \ http://localhost:8000/query/sql/ \ @@ -12,6 +14,42 @@ if curl -sf -o /dev/null -w '%{http_code}' \ exit 0 fi +# Record the current end of pinot.log so ./check can look only at what +# this invocation appends, not a "setup complete" line left over from an +# earlier run of the log-appending (>>) QuickStart process. +touch pinot.log +wc -c < pinot.log | tr -d ' ' > pinot.log.offset + +# pinot-admin.sh's own default is -Xms4G with NO -Xmx (falls back to the +# JVM's default of ~25% of physical RAM). On a 100-segment/~74GB ClickBench +# run this OOMed mid-LaunchDataIngestionJob around segment 60/100 on a +# 30GB-RAM c6a.4xlarge (observed: OutOfMemoryError in +# SegmentDictionaryCreator, job silently reported success anyway). +# +# A fixed -Xmx20G (this PR's original fix) instead broke every machine +# smaller than c6a.4xlarge: on c6a.large (4GiB)/t3a.small (2GiB)/ +# c6a.xlarge (8GiB), the JVM can't even reserve the requested heap and +# QuickStart fails to start at all (confirmed via ClickHouse/ClickBench's +# own machine:all CI run on PR #2126). Size the heap off actual physical +# RAM instead of hard-coding one instance's value. +# +# 64% of RAM, capped at 20G — chosen so a 32GiB c6a.4xlarge (this fix's +# only end-to-end-verified machine: 43/43 queries passed 3/3 tries) gets +# back exactly the already-proven 20G, while smaller machines scale down +# instead of over-requesting. Not Presto's 70%/install pattern: Pinot +# keeps segments memory-mapped/off-heap, so a heap that large would fight +# the OS page cache on machines above 32GiB instead of helping — the 20G +# cap keeps larger machines at the proven-safe ceiling rather than +# growing further. No 4G floor on -Xms either (a 4G floor is what killed +# t3a.small in the first place); -Xms only reaches 4G once the machine +# can actually spare it. +RAM_MB=$(awk '/MemTotal/{ printf "%d", $2 / 1024 }' /proc/meminfo) +XMX_MB=$(( RAM_MB * 64 / 100 )) +[ "$XMX_MB" -gt 20480 ] && XMX_MB=20480 +XMS_MB=$XMX_MB +[ "$XMS_MB" -gt 4096 ] && XMS_MB=4096 +export JAVA_OPTS="-Xms${XMS_MB}m -Xmx${XMX_MB}m" + nohup "./${PINOT_DIR}/bin/pinot-admin.sh" QuickStart -type batch \ >> pinot.log 2>&1 < /dev/null & disown diff --git a/pinot/stop b/pinot/stop index 140a589693..cbfab09ec6 100755 --- a/pinot/stop +++ b/pinot/stop @@ -3,5 +3,26 @@ pkill -f 'pinot-admin' 2>/dev/null || true pkill -f 'pinot.tools.admin' 2>/dev/null || true pkill -f 'org.apache.pinot' 2>/dev/null || true -sleep 2 + +# pkill only sends SIGTERM; the actual teardown (JVM shutdown hook +# stopping controller/broker/server, then the embedded Zookeeper +# releasing its port) is asynchronous and takes several seconds. Two +# things were previously observed to still be alive well after a fixed +# short sleep or even after the broker itself stopped responding: +# - the broker itself (was seen answering 200 for ~7-8s after SIGTERM) +# - the embedded Zookeeper's port 2123, which a fixed sleep or a +# broker-only check does not account for; the next ./start's fresh +# Zookeeper can then fail to bind/connect on the still-held port, +# surfacing as "ZkTimeoutException: Unable to connect to zookeeper +# server within timeout" in the next start's controller. +# The only reliable signal that teardown is actually finished is that +# the pinot-admin JVM process itself is gone. Poll for that (up to 30s) +# instead of guessing a delay or checking a single port. +for _ in $(seq 1 30); do + if ! pgrep -f 'org.apache.pinot.tools.admin.PinotAdministrator' > /dev/null 2>&1; then + break + fi + sleep 1 +done + exit 0