Antalya-26.6 - Fix races in the export_merge_tree_part test family

tests/queries/0_stateless/03572_export_merge_tree_part_basic.sh

@@ -6,6 +6,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
+# shellcheck source=./export_part.lib
+. "$CURDIR"/export_part.lib
+
 mt_table="mt_table_${RANDOM}"
 mt_table_partition_expression_with_function="mt_table_partition_expression_with_function_${RANDOM}"
 s3_table="s3_table_${RANDOM}"
@@ -46,17 +49,17 @@ echo "---- Export 3: Export 2020_1_1_0 and 2021_2_2_0 to wildcard table with par
 query "ALTER TABLE $mt_table_partition_expression_with_function EXPORT PART 'cb217c742dc7d143b61583011996a160_1_1_0' TO TABLE $s3_table_wildcard_partition_expression_with_function SETTINGS allow_experimental_export_merge_tree_part = 1"
 query "ALTER TABLE $mt_table_partition_expression_with_function EXPORT PART '3be6d49ecf9749a383964bc6fab22d10_2_2_0' TO TABLE $s3_table_wildcard_partition_expression_with_function SETTINGS allow_experimental_export_merge_tree_part = 1"
 
-# below exports are using parts that were exported in export 1 and export 2, so we need to wait for them to complete
-sleep 5
+# wait until part_log has 6 ExportPart records
+wait_for_exports 6
 
 echo "---- Export 4: Export the same part again, it should be idempotent"
 query "ALTER TABLE $mt_table EXPORT PART '2020_1_1_0' TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1"
 
 echo "---- Export 5: Export the same part again to wildcard, it should be idempotent"
 query "ALTER TABLE $mt_table EXPORT PART '2022_3_3_0' TO TABLE $s3_table_wildcard SETTINGS allow_experimental_export_merge_tree_part = 1"
 
-# ONE BIG SLEEP after all exports
-sleep 15
+# wait until part_log has 8 ExportPart records (6 from the previous exports)
+wait_for_exports 8
 
 # ============================================================================
 # ALL SELECTS/VERIFICATIONS HAPPEN HERE

tests/queries/0_stateless/03572_export_merge_tree_part_special_columns.sh

@@ -6,6 +6,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
+# shellcheck source=./export_part.lib
+. "$CURDIR"/export_part.lib
+
 mt_alias="mt_alias_${RANDOM}"
 mt_materialized="mt_materialized_${RANDOM}"
 s3_alias_export="s3_alias_export_${RANDOM}"
@@ -112,7 +115,8 @@ query "ALTER TABLE $mt_mixed EXPORT PART '$mixed_part_2' TO TABLE FUNCTION s3(s3
 query "ALTER TABLE $mt_complex_expr EXPORT PART '$complex_expr_part' TO TABLE $s3_complex_expr_export SETTINGS allow_experimental_export_merge_tree_part = 1"
 
 # ONE BIG SLEEP after all exports
-sleep 20
+# wait until part_log has 6 ExportPart records
+wait_for_exports 6
 
 # ============================================================================
 # ALL SELECTS/VERIFICATIONS HAPPEN HERE

tests/queries/0_stateless/03604_export_merge_tree_partition.sh

@@ -5,6 +5,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
+# shellcheck source=./export_part.lib
+. "$CURDIR"/export_part.lib
+
 rmt_table="rmt_table_${RANDOM}"
 s3_table="s3_table_${RANDOM}"
 rmt_table_roundtrip="rmt_table_roundtrip_${RANDOM}"
@@ -31,8 +34,8 @@ query "ALTER TABLE $rmt_table EXPORT PARTITION ID '2020' TO TABLE $s3_table SETT
 
 query "ALTER TABLE $rmt_table EXPORT PARTITION ID '2021' TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1"
 
-# todo poll some kind of status
-sleep 15
+# wait until 2 partition exports have completed
+wait_for_partition_exports 2
 
 echo "Select from source table"
 query "SELECT * FROM $rmt_table ORDER BY id"
@@ -43,8 +46,8 @@ query "SELECT * FROM $s3_table ORDER BY id"
 echo "Export partition 2022"
 query "ALTER TABLE $rmt_table EXPORT PARTITION ID '2022' TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1"
 
-# todo poll some kind of status
-sleep 5
+# wait until 3 partition exports have completed (2 from the previous exports)
+wait_for_partition_exports 3
 
 echo "Select from destination table again"
 query "SELECT * FROM $s3_table ORDER BY id"

tests/queries/0_stateless/03608_export_merge_tree_part_filename_pattern.sh

@@ -6,6 +6,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
+# shellcheck source=./export_part.lib
+. "$CURDIR"/export_part.lib
+
 R=$RANDOM
 mt="mt_${R}"
 dest1="fp_dest1_${R}"
@@ -27,21 +30,24 @@ query "CREATE TABLE $dest3 (id UInt64, year UInt16) ENGINE = S3(s3_conn, filenam
 
 echo "---- Test: Default pattern {part_name}_{checksum}"
 query "ALTER TABLE $mt EXPORT PART '2020_1_1_0' TO TABLE $dest1 SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_filename_pattern = '{part_name}_{checksum}'"
-sleep 3
+# wait until part_log has 1 ExportPart record
+wait_for_exports 1
 query "SELECT * FROM $dest1 ORDER BY id"
 echo "---- Verify filename matches 2020_1_1_0_*.1.parquet"
 query "SELECT count() FROM s3(s3_conn, filename='$dest1/**/2020_1_1_0_*.1.parquet', format='One')"
 
 echo "---- Test: Custom prefix pattern"
 query "ALTER TABLE $mt EXPORT PART '2021_2_2_0' TO TABLE $dest2 SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_filename_pattern = 'myprefix_{part_name}'"
-sleep 3
+# wait until part_log has 2 ExportPart records (1 from the previous export)
+wait_for_exports 2
 query "SELECT * FROM $dest2 ORDER BY id"
 echo "---- Verify filename matches myprefix_2021_2_2_0.1.parquet"
 query "SELECT count() FROM s3(s3_conn, filename='$dest2/**/myprefix_2021_2_2_0.1.parquet', format='One')"
 
 echo "---- Test: Pattern with macros"
 query "ALTER TABLE $mt EXPORT PART '2020_1_1_0' TO TABLE $dest3 SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_filename_pattern = '{database}_{table}_{part_name}'"
-sleep 3
+# wait until part_log has 3 ExportPart records (2 from the previous exports)
+wait_for_exports 3
 query "SELECT * FROM $dest3 ORDER BY id"
 echo "---- Verify macros expanded (no literal braces in parquet filenames, that's the best we can do for stateless tests)"
 query "SELECT count() = 0 FROM s3(s3_conn, filename='$dest3/**/*.1.parquet', format='One') WHERE _file LIKE '%{%'"

tests/queries/0_stateless/export_part.lib

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+# EXPORT PART runs in the background, so wait for it instead of sleeping.
+# $1 is how many exports should have completed by this point, counted from the start of
+# the test (part_log is not cleared during a run, so the number is cumulative).
+function wait_for_exports()
+{
+    local expected=$1
+    local database=${2:-$CLICKHOUSE_DATABASE}
+    local timeout=${3:-120}
+
+    while [[ $timeout -gt 0 ]]
+    do
+        ${CLICKHOUSE_CLIENT} --query="SYSTEM FLUSH LOGS" > /dev/null 2>&1
+        res=$(${CLICKHOUSE_CLIENT} --query="SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' AND database = '$database'")
+        [[ $res -ge $expected ]] && return 0
+
+        sleep 1
+        timeout=$((timeout - 1))
+    done
+
+    echo "Timed out while waiting for $expected export(s) to complete!" >&2
+    ${CLICKHOUSE_CLIENT} --query="SELECT source_table, part_name, elapsed FROM system.exports WHERE source_database = '$database'"
+    return 2
+}
+
+# EXPORT PARTITION runs in the background, so wait for it instead of sleeping.
+# $1 is how many partition exports should have completed by this point, counted from the
+# start of the test (the table keeps one row per export operation).
+function wait_for_partition_exports()
+{
+    local expected=$1
+    local database=${2:-$CLICKHOUSE_DATABASE}
+    local timeout=${3:-120}
+
+    while [[ $timeout -gt 0 ]]
+    do
+        res=$(${CLICKHOUSE_CLIENT} --query="SELECT countIf(status = 'COMPLETED') FROM system.replicated_partition_exports WHERE source_database = '$database'")
+        [[ $res -ge $expected ]] && return 0
+
+        if [[ $(${CLICKHOUSE_CLIENT} --query="SELECT countIf(status IN ('FAILED', 'KILLED')) FROM system.replicated_partition_exports WHERE source_database = '$database'") -gt 0 ]]; then
+            echo "Partition export failed!" >&2
+            ${CLICKHOUSE_CLIENT} --query="SELECT partition_id, status, exception_count, last_exception_per_replica FROM system.replicated_partition_exports WHERE source_database = '$database'"
+            return 2
+        fi
+
+        sleep 1
+        timeout=$((timeout - 1))
+    done
+
+    echo "Timed out while waiting for $expected partition export(s) to complete!" >&2
+    ${CLICKHOUSE_CLIENT} --query="SELECT partition_id, status, parts_count, parts_to_do FROM system.replicated_partition_exports WHERE source_database = '$database'"
+    return 2
+}
+
+# vi: ft=bash