From ae76d76b00ce3f3c50feac0f4c4f3a9ce0047d48 Mon Sep 17 00:00:00 2001 From: CarlosFelipeOR Date: Mon, 24 Aug 2026 11:22:46 -0300 Subject: [PATCH 1/2] Wait for export completion in part_log instead of fixed sleeps in export_merge_tree_part tests Signed-off-by: CarlosFelipeOR --- .../03572_export_merge_tree_part_basic.sh | 25 +++++++++++--- ..._export_merge_tree_part_special_columns.sh | 21 +++++++++++- .../03604_export_merge_tree_partition.sh | 33 ++++++++++++++++--- ...export_merge_tree_part_filename_pattern.sh | 25 ++++++++++++-- 4 files changed, 92 insertions(+), 12 deletions(-) diff --git a/tests/queries/0_stateless/03572_export_merge_tree_part_basic.sh b/tests/queries/0_stateless/03572_export_merge_tree_part_basic.sh index fc5df9b541da..fbf4f3979afb 100755 --- a/tests/queries/0_stateless/03572_export_merge_tree_part_basic.sh +++ b/tests/queries/0_stateless/03572_export_merge_tree_part_basic.sh @@ -17,6 +17,25 @@ query() { $CLICKHOUSE_CLIENT --query "$1" } +# EXPORT PART runs in the background, so wait for it instead of sleeping. +wait_for_exports() { + local expected="$1" + local timeout="${2:-120}" + local start + start=$(date +%s) + local n=0 + while [ $(( $(date +%s) - start )) -lt "$timeout" ]; do + query "SYSTEM FLUSH LOGS" > /dev/null 2>&1 || true + n=$(query "SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' AND database = currentDatabase()" | tr -d '\n') + if [ "$n" -ge "$expected" ]; then + return 0 + fi + sleep 0.5 + done + echo "Timeout waiting for $expected export(s) to complete (got $n)" + return 1 +} + query "DROP TABLE IF EXISTS $mt_table, $s3_table, $mt_table_roundtrip, $s3_table_wildcard, $s3_table_wildcard_partition_expression_with_function, $mt_table_partition_expression_with_function" # Create all tables @@ -46,8 +65,7 @@ echo "---- Export 3: Export 2020_1_1_0 and 2021_2_2_0 to wildcard table with par query "ALTER TABLE $mt_table_partition_expression_with_function EXPORT PART 'cb217c742dc7d143b61583011996a160_1_1_0' TO TABLE $s3_table_wildcard_partition_expression_with_function SETTINGS allow_experimental_export_merge_tree_part = 1" query "ALTER TABLE $mt_table_partition_expression_with_function EXPORT PART '3be6d49ecf9749a383964bc6fab22d10_2_2_0' TO TABLE $s3_table_wildcard_partition_expression_with_function SETTINGS allow_experimental_export_merge_tree_part = 1" -# below exports are using parts that were exported in export 1 and export 2, so we need to wait for them to complete -sleep 5 +wait_for_exports 6 echo "---- Export 4: Export the same part again, it should be idempotent" query "ALTER TABLE $mt_table EXPORT PART '2020_1_1_0' TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" @@ -55,8 +73,7 @@ query "ALTER TABLE $mt_table EXPORT PART '2020_1_1_0' TO TABLE $s3_table SETTING echo "---- Export 5: Export the same part again to wildcard, it should be idempotent" query "ALTER TABLE $mt_table EXPORT PART '2022_3_3_0' TO TABLE $s3_table_wildcard SETTINGS allow_experimental_export_merge_tree_part = 1" -# ONE BIG SLEEP after all exports -sleep 15 +wait_for_exports 8 # ============================================================================ # ALL SELECTS/VERIFICATIONS HAPPEN HERE diff --git a/tests/queries/0_stateless/03572_export_merge_tree_part_special_columns.sh b/tests/queries/0_stateless/03572_export_merge_tree_part_special_columns.sh index 0164dd70c4e0..e4f7d1e9ee8f 100755 --- a/tests/queries/0_stateless/03572_export_merge_tree_part_special_columns.sh +++ b/tests/queries/0_stateless/03572_export_merge_tree_part_special_columns.sh @@ -22,6 +22,25 @@ query() { $CLICKHOUSE_CLIENT --query "$1" } +# EXPORT PART runs in the background, so wait for it instead of sleeping. +wait_for_exports() { + local expected="$1" + local timeout="${2:-120}" + local start + start=$(date +%s) + local n=0 + while [ $(( $(date +%s) - start )) -lt "$timeout" ]; do + query "SYSTEM FLUSH LOGS" > /dev/null 2>&1 || true + n=$(query "SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' AND database = currentDatabase()" | tr -d '\n') + if [ "$n" -ge "$expected" ]; then + return 0 + fi + sleep 0.5 + done + echo "Timeout waiting for $expected export(s) to complete (got $n)" + return 1 +} + query "DROP TABLE IF EXISTS $mt_alias, $mt_materialized, $s3_alias_export, $s3_materialized_export, $mt_mixed, $s3_mixed_export, $mt_complex_expr, $s3_complex_expr_export, $mt_ephemeral, $s3_ephemeral_export" # Create all tables @@ -112,7 +131,7 @@ query "ALTER TABLE $mt_mixed EXPORT PART '$mixed_part_2' TO TABLE FUNCTION s3(s3 query "ALTER TABLE $mt_complex_expr EXPORT PART '$complex_expr_part' TO TABLE $s3_complex_expr_export SETTINGS allow_experimental_export_merge_tree_part = 1" # ONE BIG SLEEP after all exports -sleep 20 +wait_for_exports 6 # ============================================================================ # ALL SELECTS/VERIFICATIONS HAPPEN HERE diff --git a/tests/queries/0_stateless/03604_export_merge_tree_partition.sh b/tests/queries/0_stateless/03604_export_merge_tree_partition.sh index 87503112aadb..e51f0ef62052 100755 --- a/tests/queries/0_stateless/03604_export_merge_tree_partition.sh +++ b/tests/queries/0_stateless/03604_export_merge_tree_partition.sh @@ -13,6 +13,29 @@ query() { $CLICKHOUSE_CLIENT --query "$1" } +# EXPORT PART runs in the background, so wait for it instead of sleeping. +active_parts_in_partition() { + query "SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = '$rmt_table' AND partition = '$1' AND active" | tr -d '\n' +} + +wait_for_exports() { + local expected="$1" + local timeout="${2:-120}" + local start + start=$(date +%s) + local n=0 + while [ $(( $(date +%s) - start )) -lt "$timeout" ]; do + query "SYSTEM FLUSH LOGS" > /dev/null 2>&1 || true + n=$(query "SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' AND database = currentDatabase()" | tr -d '\n') + if [ "$n" -ge "$expected" ]; then + return 0 + fi + sleep 0.5 + done + echo "Timeout waiting for $expected export(s) to complete (got $n)" + return 1 +} + query "DROP TABLE IF EXISTS $rmt_table, $s3_table, $rmt_table_roundtrip" query "CREATE TABLE $rmt_table (id UInt64, year UInt16) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/$rmt_table', 'replica1') PARTITION BY year ORDER BY tuple()" @@ -27,12 +50,13 @@ query "INSERT INTO $rmt_table VALUES (6, 2022), (7, 2022)" # sync replicas query "SYSTEM SYNC REPLICA $rmt_table" +expected_exports=$(( $(active_parts_in_partition 2020) + $(active_parts_in_partition 2021) )) + query "ALTER TABLE $rmt_table EXPORT PARTITION ID '2020' TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" query "ALTER TABLE $rmt_table EXPORT PARTITION ID '2021' TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" -# todo poll some kind of status -sleep 15 +wait_for_exports "$expected_exports" echo "Select from source table" query "SELECT * FROM $rmt_table ORDER BY id" @@ -41,10 +65,11 @@ echo "Select from destination table" query "SELECT * FROM $s3_table ORDER BY id" echo "Export partition 2022" +expected_exports=$(( expected_exports + $(active_parts_in_partition 2022) )) + query "ALTER TABLE $rmt_table EXPORT PARTITION ID '2022' TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" -# todo poll some kind of status -sleep 5 +wait_for_exports "$expected_exports" echo "Select from destination table again" query "SELECT * FROM $s3_table ORDER BY id" diff --git a/tests/queries/0_stateless/03608_export_merge_tree_part_filename_pattern.sh b/tests/queries/0_stateless/03608_export_merge_tree_part_filename_pattern.sh index 12b47f4f2664..9f9f251b625d 100755 --- a/tests/queries/0_stateless/03608_export_merge_tree_part_filename_pattern.sh +++ b/tests/queries/0_stateless/03608_export_merge_tree_part_filename_pattern.sh @@ -16,6 +16,25 @@ query() { $CLICKHOUSE_CLIENT --query "$1" } +# EXPORT PART runs in the background, so wait for it instead of sleeping. +wait_for_exports() { + local expected="$1" + local timeout="${2:-120}" + local start + start=$(date +%s) + local n=0 + while [ $(( $(date +%s) - start )) -lt "$timeout" ]; do + query "SYSTEM FLUSH LOGS" > /dev/null 2>&1 || true + n=$(query "SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' AND database = currentDatabase()" | tr -d '\n') + if [ "$n" -ge "$expected" ]; then + return 0 + fi + sleep 0.5 + done + echo "Timeout waiting for $expected export(s) to complete (got $n)" + return 1 +} + query "DROP TABLE IF EXISTS $mt, $dest1, $dest2, $dest3" query "CREATE TABLE $mt (id UInt64, year UInt16) ENGINE = MergeTree() PARTITION BY year ORDER BY tuple()" @@ -27,21 +46,21 @@ query "CREATE TABLE $dest3 (id UInt64, year UInt16) ENGINE = S3(s3_conn, filenam echo "---- Test: Default pattern {part_name}_{checksum}" query "ALTER TABLE $mt EXPORT PART '2020_1_1_0' TO TABLE $dest1 SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_filename_pattern = '{part_name}_{checksum}'" -sleep 3 +wait_for_exports 1 query "SELECT * FROM $dest1 ORDER BY id" echo "---- Verify filename matches 2020_1_1_0_*.1.parquet" query "SELECT count() FROM s3(s3_conn, filename='$dest1/**/2020_1_1_0_*.1.parquet', format='One')" echo "---- Test: Custom prefix pattern" query "ALTER TABLE $mt EXPORT PART '2021_2_2_0' TO TABLE $dest2 SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_filename_pattern = 'myprefix_{part_name}'" -sleep 3 +wait_for_exports 2 query "SELECT * FROM $dest2 ORDER BY id" echo "---- Verify filename matches myprefix_2021_2_2_0.1.parquet" query "SELECT count() FROM s3(s3_conn, filename='$dest2/**/myprefix_2021_2_2_0.1.parquet', format='One')" echo "---- Test: Pattern with macros" query "ALTER TABLE $mt EXPORT PART '2020_1_1_0' TO TABLE $dest3 SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_filename_pattern = '{database}_{table}_{part_name}'" -sleep 3 +wait_for_exports 3 query "SELECT * FROM $dest3 ORDER BY id" echo "---- Verify macros expanded (no literal braces in parquet filenames, that's the best we can do for stateless tests)" query "SELECT count() = 0 FROM s3(s3_conn, filename='$dest3/**/*.1.parquet', format='One') WHERE _file LIKE '%{%'" From ff4c6490cd32625b9dc028c9bf220f1b0430e85a Mon Sep 17 00:00:00 2001 From: CarlosFelipeOR Date: Mon, 24 Aug 2026 16:17:49 -0300 Subject: [PATCH 2/2] Move the export wait helpers to export_part.lib and use replicated_partition_exports for partitions Signed-off-by: CarlosFelipeOR --- .../03572_export_merge_tree_part_basic.sh | 24 ++------ ..._export_merge_tree_part_special_columns.sh | 23 ++------ .../03604_export_merge_tree_partition.sh | 36 +++--------- ...export_merge_tree_part_filename_pattern.sh | 25 ++------- tests/queries/0_stateless/export_part.lib | 56 +++++++++++++++++++ 5 files changed, 78 insertions(+), 86 deletions(-) create mode 100644 tests/queries/0_stateless/export_part.lib diff --git a/tests/queries/0_stateless/03572_export_merge_tree_part_basic.sh b/tests/queries/0_stateless/03572_export_merge_tree_part_basic.sh index fbf4f3979afb..804b1849d316 100755 --- a/tests/queries/0_stateless/03572_export_merge_tree_part_basic.sh +++ b/tests/queries/0_stateless/03572_export_merge_tree_part_basic.sh @@ -6,6 +6,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./export_part.lib +. "$CURDIR"/export_part.lib + mt_table="mt_table_${RANDOM}" mt_table_partition_expression_with_function="mt_table_partition_expression_with_function_${RANDOM}" s3_table="s3_table_${RANDOM}" @@ -17,25 +20,6 @@ query() { $CLICKHOUSE_CLIENT --query "$1" } -# EXPORT PART runs in the background, so wait for it instead of sleeping. -wait_for_exports() { - local expected="$1" - local timeout="${2:-120}" - local start - start=$(date +%s) - local n=0 - while [ $(( $(date +%s) - start )) -lt "$timeout" ]; do - query "SYSTEM FLUSH LOGS" > /dev/null 2>&1 || true - n=$(query "SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' AND database = currentDatabase()" | tr -d '\n') - if [ "$n" -ge "$expected" ]; then - return 0 - fi - sleep 0.5 - done - echo "Timeout waiting for $expected export(s) to complete (got $n)" - return 1 -} - query "DROP TABLE IF EXISTS $mt_table, $s3_table, $mt_table_roundtrip, $s3_table_wildcard, $s3_table_wildcard_partition_expression_with_function, $mt_table_partition_expression_with_function" # Create all tables @@ -65,6 +49,7 @@ echo "---- Export 3: Export 2020_1_1_0 and 2021_2_2_0 to wildcard table with par query "ALTER TABLE $mt_table_partition_expression_with_function EXPORT PART 'cb217c742dc7d143b61583011996a160_1_1_0' TO TABLE $s3_table_wildcard_partition_expression_with_function SETTINGS allow_experimental_export_merge_tree_part = 1" query "ALTER TABLE $mt_table_partition_expression_with_function EXPORT PART '3be6d49ecf9749a383964bc6fab22d10_2_2_0' TO TABLE $s3_table_wildcard_partition_expression_with_function SETTINGS allow_experimental_export_merge_tree_part = 1" +# wait until part_log has 6 ExportPart records wait_for_exports 6 echo "---- Export 4: Export the same part again, it should be idempotent" @@ -73,6 +58,7 @@ query "ALTER TABLE $mt_table EXPORT PART '2020_1_1_0' TO TABLE $s3_table SETTING echo "---- Export 5: Export the same part again to wildcard, it should be idempotent" query "ALTER TABLE $mt_table EXPORT PART '2022_3_3_0' TO TABLE $s3_table_wildcard SETTINGS allow_experimental_export_merge_tree_part = 1" +# wait until part_log has 8 ExportPart records (6 from the previous exports) wait_for_exports 8 # ============================================================================ diff --git a/tests/queries/0_stateless/03572_export_merge_tree_part_special_columns.sh b/tests/queries/0_stateless/03572_export_merge_tree_part_special_columns.sh index e4f7d1e9ee8f..1dc66f49e66b 100755 --- a/tests/queries/0_stateless/03572_export_merge_tree_part_special_columns.sh +++ b/tests/queries/0_stateless/03572_export_merge_tree_part_special_columns.sh @@ -6,6 +6,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./export_part.lib +. "$CURDIR"/export_part.lib + mt_alias="mt_alias_${RANDOM}" mt_materialized="mt_materialized_${RANDOM}" s3_alias_export="s3_alias_export_${RANDOM}" @@ -22,25 +25,6 @@ query() { $CLICKHOUSE_CLIENT --query "$1" } -# EXPORT PART runs in the background, so wait for it instead of sleeping. -wait_for_exports() { - local expected="$1" - local timeout="${2:-120}" - local start - start=$(date +%s) - local n=0 - while [ $(( $(date +%s) - start )) -lt "$timeout" ]; do - query "SYSTEM FLUSH LOGS" > /dev/null 2>&1 || true - n=$(query "SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' AND database = currentDatabase()" | tr -d '\n') - if [ "$n" -ge "$expected" ]; then - return 0 - fi - sleep 0.5 - done - echo "Timeout waiting for $expected export(s) to complete (got $n)" - return 1 -} - query "DROP TABLE IF EXISTS $mt_alias, $mt_materialized, $s3_alias_export, $s3_materialized_export, $mt_mixed, $s3_mixed_export, $mt_complex_expr, $s3_complex_expr_export, $mt_ephemeral, $s3_ephemeral_export" # Create all tables @@ -131,6 +115,7 @@ query "ALTER TABLE $mt_mixed EXPORT PART '$mixed_part_2' TO TABLE FUNCTION s3(s3 query "ALTER TABLE $mt_complex_expr EXPORT PART '$complex_expr_part' TO TABLE $s3_complex_expr_export SETTINGS allow_experimental_export_merge_tree_part = 1" # ONE BIG SLEEP after all exports +# wait until part_log has 6 ExportPart records wait_for_exports 6 # ============================================================================ diff --git a/tests/queries/0_stateless/03604_export_merge_tree_partition.sh b/tests/queries/0_stateless/03604_export_merge_tree_partition.sh index e51f0ef62052..a2fc0947dc3b 100755 --- a/tests/queries/0_stateless/03604_export_merge_tree_partition.sh +++ b/tests/queries/0_stateless/03604_export_merge_tree_partition.sh @@ -5,6 +5,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./export_part.lib +. "$CURDIR"/export_part.lib + rmt_table="rmt_table_${RANDOM}" s3_table="s3_table_${RANDOM}" rmt_table_roundtrip="rmt_table_roundtrip_${RANDOM}" @@ -13,29 +16,6 @@ query() { $CLICKHOUSE_CLIENT --query "$1" } -# EXPORT PART runs in the background, so wait for it instead of sleeping. -active_parts_in_partition() { - query "SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = '$rmt_table' AND partition = '$1' AND active" | tr -d '\n' -} - -wait_for_exports() { - local expected="$1" - local timeout="${2:-120}" - local start - start=$(date +%s) - local n=0 - while [ $(( $(date +%s) - start )) -lt "$timeout" ]; do - query "SYSTEM FLUSH LOGS" > /dev/null 2>&1 || true - n=$(query "SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' AND database = currentDatabase()" | tr -d '\n') - if [ "$n" -ge "$expected" ]; then - return 0 - fi - sleep 0.5 - done - echo "Timeout waiting for $expected export(s) to complete (got $n)" - return 1 -} - query "DROP TABLE IF EXISTS $rmt_table, $s3_table, $rmt_table_roundtrip" query "CREATE TABLE $rmt_table (id UInt64, year UInt16) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/$rmt_table', 'replica1') PARTITION BY year ORDER BY tuple()" @@ -50,13 +30,12 @@ query "INSERT INTO $rmt_table VALUES (6, 2022), (7, 2022)" # sync replicas query "SYSTEM SYNC REPLICA $rmt_table" -expected_exports=$(( $(active_parts_in_partition 2020) + $(active_parts_in_partition 2021) )) - query "ALTER TABLE $rmt_table EXPORT PARTITION ID '2020' TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" query "ALTER TABLE $rmt_table EXPORT PARTITION ID '2021' TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" -wait_for_exports "$expected_exports" +# wait until 2 partition exports have completed +wait_for_partition_exports 2 echo "Select from source table" query "SELECT * FROM $rmt_table ORDER BY id" @@ -65,11 +44,10 @@ echo "Select from destination table" query "SELECT * FROM $s3_table ORDER BY id" echo "Export partition 2022" -expected_exports=$(( expected_exports + $(active_parts_in_partition 2022) )) - query "ALTER TABLE $rmt_table EXPORT PARTITION ID '2022' TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" -wait_for_exports "$expected_exports" +# wait until 3 partition exports have completed (2 from the previous exports) +wait_for_partition_exports 3 echo "Select from destination table again" query "SELECT * FROM $s3_table ORDER BY id" diff --git a/tests/queries/0_stateless/03608_export_merge_tree_part_filename_pattern.sh b/tests/queries/0_stateless/03608_export_merge_tree_part_filename_pattern.sh index 9f9f251b625d..fa8b1c7c4daf 100755 --- a/tests/queries/0_stateless/03608_export_merge_tree_part_filename_pattern.sh +++ b/tests/queries/0_stateless/03608_export_merge_tree_part_filename_pattern.sh @@ -6,6 +6,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./export_part.lib +. "$CURDIR"/export_part.lib + R=$RANDOM mt="mt_${R}" dest1="fp_dest1_${R}" @@ -16,25 +19,6 @@ query() { $CLICKHOUSE_CLIENT --query "$1" } -# EXPORT PART runs in the background, so wait for it instead of sleeping. -wait_for_exports() { - local expected="$1" - local timeout="${2:-120}" - local start - start=$(date +%s) - local n=0 - while [ $(( $(date +%s) - start )) -lt "$timeout" ]; do - query "SYSTEM FLUSH LOGS" > /dev/null 2>&1 || true - n=$(query "SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' AND database = currentDatabase()" | tr -d '\n') - if [ "$n" -ge "$expected" ]; then - return 0 - fi - sleep 0.5 - done - echo "Timeout waiting for $expected export(s) to complete (got $n)" - return 1 -} - query "DROP TABLE IF EXISTS $mt, $dest1, $dest2, $dest3" query "CREATE TABLE $mt (id UInt64, year UInt16) ENGINE = MergeTree() PARTITION BY year ORDER BY tuple()" @@ -46,6 +30,7 @@ query "CREATE TABLE $dest3 (id UInt64, year UInt16) ENGINE = S3(s3_conn, filenam echo "---- Test: Default pattern {part_name}_{checksum}" query "ALTER TABLE $mt EXPORT PART '2020_1_1_0' TO TABLE $dest1 SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_filename_pattern = '{part_name}_{checksum}'" +# wait until part_log has 1 ExportPart record wait_for_exports 1 query "SELECT * FROM $dest1 ORDER BY id" echo "---- Verify filename matches 2020_1_1_0_*.1.parquet" @@ -53,6 +38,7 @@ query "SELECT count() FROM s3(s3_conn, filename='$dest1/**/2020_1_1_0_*.1.parque echo "---- Test: Custom prefix pattern" query "ALTER TABLE $mt EXPORT PART '2021_2_2_0' TO TABLE $dest2 SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_filename_pattern = 'myprefix_{part_name}'" +# wait until part_log has 2 ExportPart records (1 from the previous export) wait_for_exports 2 query "SELECT * FROM $dest2 ORDER BY id" echo "---- Verify filename matches myprefix_2021_2_2_0.1.parquet" @@ -60,6 +46,7 @@ query "SELECT count() FROM s3(s3_conn, filename='$dest2/**/myprefix_2021_2_2_0.1 echo "---- Test: Pattern with macros" query "ALTER TABLE $mt EXPORT PART '2020_1_1_0' TO TABLE $dest3 SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_filename_pattern = '{database}_{table}_{part_name}'" +# wait until part_log has 3 ExportPart records (2 from the previous exports) wait_for_exports 3 query "SELECT * FROM $dest3 ORDER BY id" echo "---- Verify macros expanded (no literal braces in parquet filenames, that's the best we can do for stateless tests)" diff --git a/tests/queries/0_stateless/export_part.lib b/tests/queries/0_stateless/export_part.lib new file mode 100644 index 000000000000..f7564af7a123 --- /dev/null +++ b/tests/queries/0_stateless/export_part.lib @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# EXPORT PART runs in the background, so wait for it instead of sleeping. +# $1 is how many exports should have completed by this point, counted from the start of +# the test (part_log is not cleared during a run, so the number is cumulative). +function wait_for_exports() +{ + local expected=$1 + local database=${2:-$CLICKHOUSE_DATABASE} + local timeout=${3:-120} + + while [[ $timeout -gt 0 ]] + do + ${CLICKHOUSE_CLIENT} --query="SYSTEM FLUSH LOGS" > /dev/null 2>&1 + res=$(${CLICKHOUSE_CLIENT} --query="SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' AND database = '$database'") + [[ $res -ge $expected ]] && return 0 + + sleep 1 + timeout=$((timeout - 1)) + done + + echo "Timed out while waiting for $expected export(s) to complete!" >&2 + ${CLICKHOUSE_CLIENT} --query="SELECT source_table, part_name, elapsed FROM system.exports WHERE source_database = '$database'" + return 2 +} + +# EXPORT PARTITION runs in the background, so wait for it instead of sleeping. +# $1 is how many partition exports should have completed by this point, counted from the +# start of the test (the table keeps one row per export operation). +function wait_for_partition_exports() +{ + local expected=$1 + local database=${2:-$CLICKHOUSE_DATABASE} + local timeout=${3:-120} + + while [[ $timeout -gt 0 ]] + do + res=$(${CLICKHOUSE_CLIENT} --query="SELECT countIf(status = 'COMPLETED') FROM system.replicated_partition_exports WHERE source_database = '$database'") + [[ $res -ge $expected ]] && return 0 + + if [[ $(${CLICKHOUSE_CLIENT} --query="SELECT countIf(status IN ('FAILED', 'KILLED')) FROM system.replicated_partition_exports WHERE source_database = '$database'") -gt 0 ]]; then + echo "Partition export failed!" >&2 + ${CLICKHOUSE_CLIENT} --query="SELECT partition_id, status, exception_count, last_exception_per_replica FROM system.replicated_partition_exports WHERE source_database = '$database'" + return 2 + fi + + sleep 1 + timeout=$((timeout - 1)) + done + + echo "Timed out while waiting for $expected partition export(s) to complete!" >&2 + ${CLICKHOUSE_CLIENT} --query="SELECT partition_id, status, parts_count, parts_to_do FROM system.replicated_partition_exports WHERE source_database = '$database'" + return 2 +} + +# vi: ft=bash