From 56c574cb12132bf70c5e18b6bfc542ad430694fa Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 1 Sep 2026 17:54:43 +0800 Subject: [PATCH 1/5] CI: count TAP test failures correctly in parse-results.pl The regex that picks failed test files out of prove's "Test Summary Report" required leading whitespace, but prove prints those lines flush left: Test Summary Report ------------------- t/019_replslot_limit.pl (Wstat: 7424 (exited 29) Tests: 9 Failed: 2) Nothing ever matched, so for the TAP suites (ic-recovery) the parser reported STATUS=failed together with FAILED_TESTS=0 and PASSED_TESTS=0, and the job summary claimed "Failed Tests | 0" while the job was red. Match the line as prove actually prints it. On the ic-recovery log from run 33169855823 the parser now reports 1 failed test (479 passed) and names t/019_replslot_limit.pl; parsing of the pg_regress logs is unchanged. Assisted-by: Claude Code Backpatch-through: REL_2_STABLE --- devops/build/automation/cloudberry/scripts/parse-results.pl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/devops/build/automation/cloudberry/scripts/parse-results.pl b/devops/build/automation/cloudberry/scripts/parse-results.pl index 2c754bcae9d..4506045619d 100755 --- a/devops/build/automation/cloudberry/scripts/parse-results.pl +++ b/devops/build/automation/cloudberry/scripts/parse-results.pl @@ -145,8 +145,10 @@ } elsif (/^Result: FAIL/) { $status = 'failed'; - # TAP individual test failure: " t/xxx.pl (Wstat: ...)" - } elsif (/^\s+(t\/\S+\.pl)\s+\(Wstat:/) { + # TAP individual test failure, as listed by prove's "Test Summary + # Report". Those lines start at column 0, not indented, e.g. + # "t/019_replslot_limit.pl (Wstat: 7424 Tests: 9 Failed: 2)". + } elsif (/^\s*(\S+\.pl)\s+\(Wstat:/) { push @failed_test_list, $1; } From 2741898334116b1d4692811255e13914220d499d Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 1 Sep 2026 21:23:23 +0800 Subject: [PATCH 2/5] Fix flaky autovacuum-analyze: force the stats flush instead of sleeping The pax and singlenode copies of autovacuum-analyze still poke the stats collector with 2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. where the isolation2 copy has already moved to gp_stat_force_next_flush(). Sleeping does not force anything: whether the INSERT's 1000 tuple modifications have reached the collector by the time the next query reads pg_stat_all_tables is a race, so n_mod_since_analyze reads either 1000 or 0. The pax expected output has that race baked in. Of its three sampling points, two were captured before the flush and one after: analyze_count | autoanalyze_count | n_mod_since_analyze 1 | 0 | 0 <- isolation2 says 1000 2 | 1 | 1000 2 | 2 | 0 <- isolation2 says 1000 so the suite fails whenever the collector happens to be quick. It did so in 6 of the last 60 pushes to main, on both opt-on and opt-off. Use gp_stat_force_next_flush() in all three copies and correct the two stale pax values. The pax input file is now byte-identical to the isolation2 one, and the only difference left between their expected outputs is relpages, which genuinely differs because pax packs pages differently. Assisted-by: Claude Code Backpatch-through: REL_2_STABLE --- .../input/autovacuum-analyze.source | 6 ++--- .../output/autovacuum-analyze.source | 25 +++++++++++++++---- .../input/autovacuum-analyze.source | 6 ++--- .../output/autovacuum-analyze.source | 21 +++++++++++++--- 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/contrib/pax_storage/src/test/isolation2/input/autovacuum-analyze.source b/contrib/pax_storage/src/test/isolation2/input/autovacuum-analyze.source index 19187b107c2..7b9b2df22d4 100644 --- a/contrib/pax_storage/src/test/isolation2/input/autovacuum-analyze.source +++ b/contrib/pax_storage/src/test/isolation2/input/autovacuum-analyze.source @@ -171,7 +171,7 @@ SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', ' -- with auto_stats, the auto-ANALYZE still trigger 2: INSERT INTO autostatstbl select i from generate_series(1, 1000) as i; -2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. +2: select gp_stat_force_next_flush(); -- auto_stats executed but auto-ANALYZE not execute yet since we suspend before finish ANALYZE. SELECT count(*) FROM pg_statistic where starelid = 'autostatstbl'::regclass; @@ -205,7 +205,7 @@ SELECT gp_inject_fault('analyze_finished_one_relation', 'skip', '', '', 'autosta SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', '', 'autostatstbl', 1, -1, 0, 1); 2: INSERT INTO autostatstbl select i from generate_series(1001, 2000) as i; -2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. +2: select gp_stat_force_next_flush(); -- auto_stats executed but auto-ANALYZE not execute yet since we suspend before finish ANALYZE. select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass; @@ -238,7 +238,7 @@ SELECT gp_inject_fault('analyze_finished_one_relation', 'skip', '', '', 'autosta SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', '', 'autostatstbl', 1, -1, 0, 1); 2: INSERT INTO autostatstbl select i from generate_series(2001, 3000) as i; -2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. +2: select gp_stat_force_next_flush(); -- auto_stats should not executed and auto-ANALYZE not execute yet since we suspend before finish ANALYZE. select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass; diff --git a/contrib/pax_storage/src/test/isolation2/output/autovacuum-analyze.source b/contrib/pax_storage/src/test/isolation2/output/autovacuum-analyze.source index be374068d33..42370fdf595 100644 --- a/contrib/pax_storage/src/test/isolation2/output/autovacuum-analyze.source +++ b/contrib/pax_storage/src/test/isolation2/output/autovacuum-analyze.source @@ -408,7 +408,12 @@ SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', ' -- with auto_stats, the auto-ANALYZE still trigger 2: INSERT INTO autostatstbl select i from generate_series(1, 1000) as i; INSERT 1000 -2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. +2: select gp_stat_force_next_flush(); + gp_stat_force_next_flush +-------------------------- + +(1 row) + -- auto_stats executed but auto-ANALYZE not execute yet since we suspend before finish ANALYZE. SELECT count(*) FROM pg_statistic where starelid = 'autostatstbl'::regclass; count @@ -425,7 +430,7 @@ select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass; select analyze_count, autoanalyze_count, n_mod_since_analyze from pg_stat_all_tables where relname = 'autostatstbl'; analyze_count | autoanalyze_count | n_mod_since_analyze ---------------+-------------------+--------------------- - 1 | 0 | 0 + 1 | 0 | 1000 (1 row) -- Wait until autovacuum is triggered @@ -493,7 +498,12 @@ SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', ' 2: INSERT INTO autostatstbl select i from generate_series(1001, 2000) as i; INSERT 1000 -2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. +2: select gp_stat_force_next_flush(); + gp_stat_force_next_flush +-------------------------- + +(1 row) + -- auto_stats executed but auto-ANALYZE not execute yet since we suspend before finish ANALYZE. select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass; relpages | reltuples @@ -572,7 +582,12 @@ SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', ' 2: INSERT INTO autostatstbl select i from generate_series(2001, 3000) as i; INSERT 1000 -2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. +2: select gp_stat_force_next_flush(); + gp_stat_force_next_flush +-------------------------- + +(1 row) + -- auto_stats should not executed and auto-ANALYZE not execute yet since we suspend before finish ANALYZE. select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass; relpages | reltuples @@ -584,7 +599,7 @@ select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass; select analyze_count, autoanalyze_count, n_mod_since_analyze from pg_stat_all_tables where relname = 'autostatstbl'; analyze_count | autoanalyze_count | n_mod_since_analyze ---------------+-------------------+--------------------- - 2 | 2 | 0 + 2 | 2 | 1000 (1 row) -- Wait until autovacuum is triggered diff --git a/src/test/singlenode_isolation2/input/autovacuum-analyze.source b/src/test/singlenode_isolation2/input/autovacuum-analyze.source index b35c3072a85..1e1fcb69f2a 100644 --- a/src/test/singlenode_isolation2/input/autovacuum-analyze.source +++ b/src/test/singlenode_isolation2/input/autovacuum-analyze.source @@ -170,7 +170,7 @@ SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', ' -- with auto_stats, the auto-ANALYZE still trigger 2: INSERT INTO autostatstbl select i from generate_series(1, 1000) as i; -2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. +2: select gp_stat_force_next_flush(); -- auto_stats executed but auto-ANALYZE not execute yet since we suspend before finish ANALYZE. SELECT count(*) FROM pg_statistic where starelid = 'autostatstbl'::regclass; @@ -204,7 +204,7 @@ SELECT gp_inject_fault('analyze_finished_one_relation', 'skip', '', '', 'autosta SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', '', 'autostatstbl', 1, -1, 0, 1); 2: INSERT INTO autostatstbl select i from generate_series(1001, 2000) as i; -2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. +2: select gp_stat_force_next_flush(); -- auto_stats executed but auto-ANALYZE not execute yet since we suspend before finish ANALYZE. select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass; @@ -237,7 +237,7 @@ SELECT gp_inject_fault('analyze_finished_one_relation', 'skip', '', '', 'autosta SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', '', 'autostatstbl', 1, -1, 0, 1); 2: INSERT INTO autostatstbl select i from generate_series(2001, 3000) as i; -2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. +2: select gp_stat_force_next_flush(); -- auto_stats should not executed and auto-ANALYZE not execute yet since we suspend before finish ANALYZE. select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass; diff --git a/src/test/singlenode_isolation2/output/autovacuum-analyze.source b/src/test/singlenode_isolation2/output/autovacuum-analyze.source index 382b443944c..9ab7afa90ee 100644 --- a/src/test/singlenode_isolation2/output/autovacuum-analyze.source +++ b/src/test/singlenode_isolation2/output/autovacuum-analyze.source @@ -403,7 +403,12 @@ SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', ' -- with auto_stats, the auto-ANALYZE still trigger 2: INSERT INTO autostatstbl select i from generate_series(1, 1000) as i; INSERT 1000 -2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. +2: select gp_stat_force_next_flush(); + gp_stat_force_next_flush +-------------------------- + +(1 row) + -- auto_stats executed but auto-ANALYZE not execute yet since we suspend before finish ANALYZE. SELECT count(*) FROM pg_statistic where starelid = 'autostatstbl'::regclass; count @@ -488,7 +493,12 @@ SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', ' 2: INSERT INTO autostatstbl select i from generate_series(1001, 2000) as i; INSERT 1000 -2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. +2: select gp_stat_force_next_flush(); + gp_stat_force_next_flush +-------------------------- + +(1 row) + -- auto_stats executed but auto-ANALYZE not execute yet since we suspend before finish ANALYZE. select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass; relpages | reltuples @@ -567,7 +577,12 @@ SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', ' 2: INSERT INTO autostatstbl select i from generate_series(2001, 3000) as i; INSERT 1000 -2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. +2: select gp_stat_force_next_flush(); + gp_stat_force_next_flush +-------------------------- + +(1 row) + -- auto_stats should not executed and auto-ANALYZE not execute yet since we suspend before finish ANALYZE. select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass; relpages | reltuples From 0b15293b9675c6bff7f0ad835ec4bb18fa264086 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 1 Sep 2026 21:23:55 +0800 Subject: [PATCH 3/5] Fix flaky resgroup_cpu_max_percent: expect the CPU share the group really gets verify_cpu_usage() checks abs(measured - expected) <= err_rate, and the first assertion asks for 90 +/- 10. But rg1_cpu_test runs with cpu_max_percent=-1 as the only busy group, so it takes essentially every core and gp_resgroup_status_per_host reports ~100, not 90 -- the true value sits on the upper edge of the window and any upward sampling jitter fails the test. The samples from the ic-resgroup-v2 job of run 33224642149 show exactly that: 99.62 99.54 96.61 106.98 99.60 -> mean 100.47, window [80,100] A single sample of 106.98 is enough, and cpu_usage can exceed 100 because convertcpuusage() divides the cgroup cpu-time delta by a separately measured wall-clock duration. The same mistake is in the two-group case: with cpu_weight 100 and 200 and both groups uncapped they split the whole machine, so the shares are ~33 and ~67, not ~30 and ~60. Measured on the same run: 32.40 and 64.93, both still inside the window but off-centre. Re-centre the three uncapped assertions on 100, 33 and 67, which turns the margins from (-0.47, 7.60, 5.07) into (9.53, 9.40, 7.93). The three capped assertions are already correct -- measured 9.93, 10.04 and 20.09 against expected 10, 10 and 20 -- and are left alone. Comments record why the numbers are what they are. Averaging all the samples instead of only the first (3fad47560e1) did not help: all six failures in the last 60 pushes to main are from after that commit. Assisted-by: Claude Code Backpatch-through: REL_2_STABLE --- .../expected/resgroup/resgroup_cpu_max_percent.out | 12 +++++++++--- .../sql/resgroup/resgroup_cpu_max_percent.sql | 12 +++++++++--- .../expected/resgroup/resgroup_cpu_max_percent.out | 12 +++++++++--- .../sql/resgroup/resgroup_cpu_max_percent.sql | 12 +++++++++--- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/contrib/pax_storage/src/test/isolation2/expected/resgroup/resgroup_cpu_max_percent.out b/contrib/pax_storage/src/test/isolation2/expected/resgroup/resgroup_cpu_max_percent.out index 1e011ecec22..78b8bc561ed 100644 --- a/contrib/pax_storage/src/test/isolation2/expected/resgroup/resgroup_cpu_max_percent.out +++ b/contrib/pax_storage/src/test/isolation2/expected/resgroup/resgroup_cpu_max_percent.out @@ -220,7 +220,11 @@ SELECT pg_sleep(1.7); (1 row) -- end_ignore -SELECT verify_cpu_usage('rg1_cpu_test', 90, 10); +-- rg1_cpu_test is uncapped (cpu_max_percent=-1) and it is the only busy +-- group, so it takes essentially every core: gp_resgroup_status reports +-- ~100, not 90. Expecting 90 put the real value on the upper edge of the +-- +/- err_rate window, so any upward sampling jitter failed the test. +SELECT verify_cpu_usage('rg1_cpu_test', 100, 10); verify_cpu_usage ------------------ t @@ -395,12 +399,14 @@ SELECT pg_sleep(1.7); (1 row) -- end_ignore -SELECT verify_cpu_usage('rg1_cpu_test', 30, 10); +-- Both groups are uncapped, so they share the whole machine in proportion +-- to cpu_weight (100 and 200): ~33 and ~67, not ~30 and ~60. +SELECT verify_cpu_usage('rg1_cpu_test', 33, 10); verify_cpu_usage ------------------ t (1 row) -SELECT verify_cpu_usage('rg2_cpu_test', 60, 10); +SELECT verify_cpu_usage('rg2_cpu_test', 67, 10); verify_cpu_usage ------------------ t diff --git a/contrib/pax_storage/src/test/isolation2/sql/resgroup/resgroup_cpu_max_percent.sql b/contrib/pax_storage/src/test/isolation2/sql/resgroup/resgroup_cpu_max_percent.sql index a014108bb52..7c249c1e6c8 100644 --- a/contrib/pax_storage/src/test/isolation2/sql/resgroup/resgroup_cpu_max_percent.sql +++ b/contrib/pax_storage/src/test/isolation2/sql/resgroup/resgroup_cpu_max_percent.sql @@ -141,7 +141,11 @@ SELECT fetch_sample(); SELECT pg_sleep(1.7); -- end_ignore -SELECT verify_cpu_usage('rg1_cpu_test', 90, 10); +-- rg1_cpu_test is uncapped (cpu_max_percent=-1) and it is the only busy +-- group, so it takes essentially every core: gp_resgroup_status reports +-- ~100, not 90. Expecting 90 put the real value on the upper edge of the +-- +/- err_rate window, so any upward sampling jitter failed the test. +SELECT verify_cpu_usage('rg1_cpu_test', 100, 10); -- start_ignore SELECT * FROM cancel_all; @@ -212,8 +216,10 @@ SELECT fetch_sample(); SELECT pg_sleep(1.7); -- end_ignore -SELECT verify_cpu_usage('rg1_cpu_test', 30, 10); -SELECT verify_cpu_usage('rg2_cpu_test', 60, 10); +-- Both groups are uncapped, so they share the whole machine in proportion +-- to cpu_weight (100 and 200): ~33 and ~67, not ~30 and ~60. +SELECT verify_cpu_usage('rg1_cpu_test', 33, 10); +SELECT verify_cpu_usage('rg2_cpu_test', 67, 10); -- start_ignore SELECT * FROM cancel_all; diff --git a/src/test/isolation2/expected/resgroup/resgroup_cpu_max_percent.out b/src/test/isolation2/expected/resgroup/resgroup_cpu_max_percent.out index 44969ba7eea..c4706bbc9f9 100644 --- a/src/test/isolation2/expected/resgroup/resgroup_cpu_max_percent.out +++ b/src/test/isolation2/expected/resgroup/resgroup_cpu_max_percent.out @@ -220,7 +220,11 @@ SELECT pg_sleep(1.7); (1 row) -- end_ignore -SELECT verify_cpu_usage('rg1_cpu_test', 90, 10); +-- rg1_cpu_test is uncapped (cpu_max_percent=-1) and it is the only busy +-- group, so it takes essentially every core: gp_resgroup_status reports +-- ~100, not 90. Expecting 90 put the real value on the upper edge of the +-- +/- err_rate window, so any upward sampling jitter failed the test. +SELECT verify_cpu_usage('rg1_cpu_test', 100, 10); verify_cpu_usage ------------------ t @@ -395,13 +399,15 @@ SELECT pg_sleep(1.7); (1 row) -- end_ignore -SELECT verify_cpu_usage('rg1_cpu_test', 30, 10); +-- Both groups are uncapped, so they share the whole machine in proportion +-- to cpu_weight (100 and 200): ~33 and ~67, not ~30 and ~60. +SELECT verify_cpu_usage('rg1_cpu_test', 33, 10); verify_cpu_usage ------------------ t (1 row) -- start_ignore -SELECT verify_cpu_usage('rg2_cpu_test', 60, 10); +SELECT verify_cpu_usage('rg2_cpu_test', 67, 10); verify_cpu_usage ------------------ t diff --git a/src/test/isolation2/sql/resgroup/resgroup_cpu_max_percent.sql b/src/test/isolation2/sql/resgroup/resgroup_cpu_max_percent.sql index 89fa523cc88..164fdcd8199 100644 --- a/src/test/isolation2/sql/resgroup/resgroup_cpu_max_percent.sql +++ b/src/test/isolation2/sql/resgroup/resgroup_cpu_max_percent.sql @@ -141,7 +141,11 @@ SELECT fetch_sample(); SELECT pg_sleep(1.7); -- end_ignore -SELECT verify_cpu_usage('rg1_cpu_test', 90, 10); +-- rg1_cpu_test is uncapped (cpu_max_percent=-1) and it is the only busy +-- group, so it takes essentially every core: gp_resgroup_status reports +-- ~100, not 90. Expecting 90 put the real value on the upper edge of the +-- +/- err_rate window, so any upward sampling jitter failed the test. +SELECT verify_cpu_usage('rg1_cpu_test', 100, 10); -- start_ignore SELECT * FROM cancel_all; @@ -212,9 +216,11 @@ SELECT fetch_sample(); SELECT pg_sleep(1.7); -- end_ignore -SELECT verify_cpu_usage('rg1_cpu_test', 30, 10); +-- Both groups are uncapped, so they share the whole machine in proportion +-- to cpu_weight (100 and 200): ~33 and ~67, not ~30 and ~60. +SELECT verify_cpu_usage('rg1_cpu_test', 33, 10); -- start_ignore -SELECT verify_cpu_usage('rg2_cpu_test', 60, 10); +SELECT verify_cpu_usage('rg2_cpu_test', 67, 10); SELECT * FROM cancel_all; From 0848a663d00ae237567d0414e06485d2f3e19464 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 1 Sep 2026 23:09:19 +0800 Subject: [PATCH 4/5] Fix flaky vacuum_progress_row/column: wait for the walsender to park Both tests park the walsender of seg1's primary so that the compact phase blocks in syncrep, stop the mirror, and then expect the post-cleanup phase to be taken over by a new vacuum worker: 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) ... 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) ... Injecting the fault only arms it. The walsender parks in it the next time round WalSndLoop(), and on an otherwise idle cluster that takes a while - measured on a demo cluster, six consecutive times: 3866 ms 7869 ms 7870 ms 8070 ms 7865 ms 7870 ms The very next statement resumes the vacuum. If the compact phase commits inside that window the walsender is still streaming, syncrep is satisfied by the live mirror, the whole vacuum finishes on the same gang, ao_vacuum_rel() never sees a NULL vacrelstats in a fresh backend, and vacuum_worker_changed never fires. Both gp_wait_until_triggered_fault() calls that follow then burn their full ten minute ceiling, the test spends twenty minutes and fails, and the progress views it queries afterwards return no rows because the vacuum is long gone. That is what the ic-isolation2 failures look like: 11 of the 57 failed test jobs in the last 60 pushes to main, up to 27 Aug. Wait until the walsender has really been caught by the fault before letting the vacuum go on. gp_wait_until_triggered_fault('wal_sender_loop') is the same pattern pg_rewind_fail_missing_xlog already uses. Verified on a three-primary demo cluster: three consecutive runs of both tests pass, in their usual 63 s and 125 s rather than the twenty minutes a timed-out run takes. Assisted-by: Claude Code Backpatch-through: REL_2_STABLE --- .../expected/vacuum_progress_column.out | 11 ++++++++++ .../expected/vacuum_progress_row.out | 22 +++++++++++++++++++ .../isolation2/sql/vacuum_progress_column.sql | 7 ++++++ .../isolation2/sql/vacuum_progress_row.sql | 14 ++++++++++++ 4 files changed, 54 insertions(+) diff --git a/src/test/isolation2/expected/vacuum_progress_column.out b/src/test/isolation2/expected/vacuum_progress_column.out index 12a3bceb78f..8f3f5f1d033 100644 --- a/src/test/isolation2/expected/vacuum_progress_column.out +++ b/src/test/isolation2/expected/vacuum_progress_column.out @@ -305,11 +305,22 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he (1 row) -- Resume execution of compact phase and block at syncrep on one segment. +-- The suspend only takes effect the next time the walsender goes round its +-- loop, so wait until it is really parked before letting the vacuum go on. +-- Otherwise the compact phase can commit while the walsender is still +-- streaming, syncrep is satisfied by the live mirror, the vacuum runs to +-- completion on the same gang, no new vacuum worker ever takes over and the +-- gp_wait_until_triggered_fault() below times out after ten minutes. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; gp_inject_fault_infinite -------------------------- Success: (1 row) +2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- diff --git a/src/test/isolation2/expected/vacuum_progress_row.out b/src/test/isolation2/expected/vacuum_progress_row.out index ef39d8edaf3..a7b713d689e 100644 --- a/src/test/isolation2/expected/vacuum_progress_row.out +++ b/src/test/isolation2/expected/vacuum_progress_row.out @@ -359,11 +359,22 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he (1 row) -- Resume execution of compact phase and block at syncrep. +-- The suspend only takes effect the next time the walsender goes round its +-- loop, so wait until it is really parked before letting the vacuum go on. +-- Otherwise the compact phase can commit while the walsender is still +-- streaming, syncrep is satisfied by the live mirror, the vacuum runs to +-- completion on the same gang, no new vacuum worker ever takes over and the +-- gp_wait_until_triggered_fault() below times out after ten minutes. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; gp_inject_fault_infinite -------------------------- Success: (1 row) +2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- @@ -563,11 +574,22 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he (1 row) -- Resume execution of compact phase and block at syncrep. +-- The suspend only takes effect the next time the walsender goes round its +-- loop, so wait until it is really parked before letting the vacuum go on. +-- Otherwise the compact phase can commit while the walsender is still +-- streaming, syncrep is satisfied by the live mirror, the vacuum runs to +-- completion on the same gang, no new vacuum worker ever takes over and the +-- gp_wait_until_triggered_fault() below times out after ten minutes. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; gp_inject_fault_infinite -------------------------- Success: (1 row) +2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- diff --git a/src/test/isolation2/sql/vacuum_progress_column.sql b/src/test/isolation2/sql/vacuum_progress_column.sql index 60250368b46..42f20659224 100644 --- a/src/test/isolation2/sql/vacuum_progress_column.sql +++ b/src/test/isolation2/sql/vacuum_progress_column.sql @@ -131,7 +131,14 @@ select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_b select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; -- Resume execution of compact phase and block at syncrep on one segment. +-- The suspend only takes effect the next time the walsender goes round its +-- loop, so wait until it is really parked before letting the vacuum go on. +-- Otherwise the compact phase can commit while the walsender is still +-- streaming, syncrep is satisfied by the live mirror, the vacuum runs to +-- completion on the same gang, no new vacuum worker ever takes over and the +-- gp_wait_until_triggered_fault() below times out after ten minutes. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; +2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- stop the mirror should turn off syncrep 2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content = 1 AND role = 'm'; diff --git a/src/test/isolation2/sql/vacuum_progress_row.sql b/src/test/isolation2/sql/vacuum_progress_row.sql index c832a1fd0df..acf9d0635cf 100644 --- a/src/test/isolation2/sql/vacuum_progress_row.sql +++ b/src/test/isolation2/sql/vacuum_progress_row.sql @@ -136,7 +136,14 @@ select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_b select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; -- Resume execution of compact phase and block at syncrep. +-- The suspend only takes effect the next time the walsender goes round its +-- loop, so wait until it is really parked before letting the vacuum go on. +-- Otherwise the compact phase can commit while the walsender is still +-- streaming, syncrep is satisfied by the live mirror, the vacuum runs to +-- completion on the same gang, no new vacuum worker ever takes over and the +-- gp_wait_until_triggered_fault() below times out after ten minutes. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; +2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- stop the mirror should turn off syncrep 2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content=1 AND role = 'm'; @@ -210,7 +217,14 @@ select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_b select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; -- Resume execution of compact phase and block at syncrep. +-- The suspend only takes effect the next time the walsender goes round its +-- loop, so wait until it is really parked before letting the vacuum go on. +-- Otherwise the compact phase can commit while the walsender is still +-- streaming, syncrep is satisfied by the live mirror, the vacuum runs to +-- completion on the same gang, no new vacuum worker ever takes over and the +-- gp_wait_until_triggered_fault() below times out after ten minutes. 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; +2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- stop the mirror should turn off syncrep 2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content=1 AND role = 'm'; From 784c44f2c32d008d3d6c00699270a3cdd7c11954 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Thu, 17 Sep 2026 11:08:10 +0800 Subject: [PATCH 5/5] Fix flaky task/oid_wraparound: stop the parse-only tasks from running task.sql creates four tasks just to check that second-based schedule strings are accepted: create task valid_task_1 schedule '1 second' as 'select 1'; ... Nothing ever asserts on their execution, but they are created active, so the scheduler starts firing valid_task_1 once a second and keeps going until the drops at the end of the file. Two failures follow from that, and they always appear together. Every run draws a run id from the cluster-wide Oid counter: task->runId = NextRunId(); (pg_cron.c) runId = GetNewOidWithIndex(pg_task_run_history, (job_metadata.c) TaskRunHistoryRunIdIndexId, ...); NextRunId() is called unconditionally, not under task_log_run. So a task left running consumes one Oid per second for the rest of the regression run, and oid_wraparound, which sets the counter and then asserts on exact values, drifts by one: SELECT gp_get_next_oid_master(); - 16387 + 16388 DROP TASK also deletes the task's pg_task_run_history rows, and the scheduler updates those same rows as a run moves from starting to succeeded. When the delete scan meets a row the scheduler has just updated, simple_heap_delete() returns TM_Updated: drop task valid_task_1; +ERROR: tuple concurrently updated (heapam.c:3112) That aborts the drop, the task survives, and it goes on consuming Oids -- which is why the oid_wraparound failure only shows up in jobs where task failed first. The exposure is the window between the creates and the drops. On CI the task test takes 3.7 to 5.8 seconds, so valid_task_1 fires three to five times inside it; on an idle machine the test finishes in 0.5 s and the scheduler never gets a turn, which is why this only fails under load. Reproduced by holding that window open for five seconds, the length CI shows: original 5 scheduler runs per iteration, 25 over five iterations deactivated 0 Deactivating each task right after it is created keeps the parser coverage and takes the scheduler out of the picture. ALTER TASK ... NOT ACTIVE is honoured before a task is ever launched -- ScheduleCronJobs() skips inactive tasks ("if (!task->isActive) continue") rather than cancelling them afterwards -- so no run is started at all. Verified on a three-primary demo cluster: five consecutive rounds of task followed by oid_wraparound all pass, leaving no rows in pg_task and no rows in pg_task_run_history. With a task deliberately left running, oid_wraparound fails the same way CI does, twice in twelve runs, against zero failures in six runs with none running. The race between DROP TASK and a running job is a real defect in its own right and is not addressed here; this only stops the regression suite from walking into it. Assisted-by: Claude Code Backpatch-through: REL_2_STABLE --- .../pax_storage/src/test/regress/expected/task.out | 13 +++++++++++++ contrib/pax_storage/src/test/regress/sql/task.sql | 13 +++++++++++++ src/test/regress/expected/task.out | 13 +++++++++++++ src/test/regress/sql/task.sql | 13 +++++++++++++ 4 files changed, 52 insertions(+) diff --git a/contrib/pax_storage/src/test/regress/expected/task.out b/contrib/pax_storage/src/test/regress/expected/task.out index 2f052fdad1a..3f4cf9c14ed 100644 --- a/contrib/pax_storage/src/test/regress/expected/task.out +++ b/contrib/pax_storage/src/test/regress/expected/task.out @@ -69,10 +69,23 @@ ERROR: User task_cron does not have CONNECT privilege on task_dbno alter task vacuum_db user hopedoesnotexist; ERROR: role "hopedoesnotexist" does not exist -- valid interval tasks +-- +-- These four only exercise the schedule parser, they are never meant to run. +-- Deactivate each one as soon as it exists: while a second-based task stays +-- active the scheduler keeps firing it, and every run draws a run id from the +-- cluster-wide Oid counter (NextRunId -> GetNewOidWithIndex). A task still +-- firing later in the same regression run shifts the counter that +-- oid_wraparound hardcodes, and DROP TASK, which also deletes the task's +-- pg_task_run_history rows, fails with "tuple concurrently updated" when it +-- races the run the scheduler is in the middle of recording. create task valid_task_1 schedule '1 second' as 'select 1'; +alter task valid_task_1 not active; create task valid_task_2 schedule ' 30 sEcOnDs ' as 'select 1'; +alter task valid_task_2 not active; create task valid_task_3 schedule '59 seconds' as 'select 1'; +alter task valid_task_3 not active; create task valid_task_4 schedule '17 seconds ' as 'select 1'; +alter task valid_task_4 not active; -- task in function DO $$ BEGIN diff --git a/contrib/pax_storage/src/test/regress/sql/task.sql b/contrib/pax_storage/src/test/regress/sql/task.sql index bf12909c761..3e29c46adad 100644 --- a/contrib/pax_storage/src/test/regress/sql/task.sql +++ b/contrib/pax_storage/src/test/regress/sql/task.sql @@ -49,10 +49,23 @@ alter task vacuum_db database task_dbno user task_cron; alter task vacuum_db user hopedoesnotexist; -- valid interval tasks +-- +-- These four only exercise the schedule parser, they are never meant to run. +-- Deactivate each one as soon as it exists: while a second-based task stays +-- active the scheduler keeps firing it, and every run draws a run id from the +-- cluster-wide Oid counter (NextRunId -> GetNewOidWithIndex). A task still +-- firing later in the same regression run shifts the counter that +-- oid_wraparound hardcodes, and DROP TASK, which also deletes the task's +-- pg_task_run_history rows, fails with "tuple concurrently updated" when it +-- races the run the scheduler is in the middle of recording. create task valid_task_1 schedule '1 second' as 'select 1'; +alter task valid_task_1 not active; create task valid_task_2 schedule ' 30 sEcOnDs ' as 'select 1'; +alter task valid_task_2 not active; create task valid_task_3 schedule '59 seconds' as 'select 1'; +alter task valid_task_3 not active; create task valid_task_4 schedule '17 seconds ' as 'select 1'; +alter task valid_task_4 not active; -- task in function DO $$ diff --git a/src/test/regress/expected/task.out b/src/test/regress/expected/task.out index 2f052fdad1a..3f4cf9c14ed 100644 --- a/src/test/regress/expected/task.out +++ b/src/test/regress/expected/task.out @@ -69,10 +69,23 @@ ERROR: User task_cron does not have CONNECT privilege on task_dbno alter task vacuum_db user hopedoesnotexist; ERROR: role "hopedoesnotexist" does not exist -- valid interval tasks +-- +-- These four only exercise the schedule parser, they are never meant to run. +-- Deactivate each one as soon as it exists: while a second-based task stays +-- active the scheduler keeps firing it, and every run draws a run id from the +-- cluster-wide Oid counter (NextRunId -> GetNewOidWithIndex). A task still +-- firing later in the same regression run shifts the counter that +-- oid_wraparound hardcodes, and DROP TASK, which also deletes the task's +-- pg_task_run_history rows, fails with "tuple concurrently updated" when it +-- races the run the scheduler is in the middle of recording. create task valid_task_1 schedule '1 second' as 'select 1'; +alter task valid_task_1 not active; create task valid_task_2 schedule ' 30 sEcOnDs ' as 'select 1'; +alter task valid_task_2 not active; create task valid_task_3 schedule '59 seconds' as 'select 1'; +alter task valid_task_3 not active; create task valid_task_4 schedule '17 seconds ' as 'select 1'; +alter task valid_task_4 not active; -- task in function DO $$ BEGIN diff --git a/src/test/regress/sql/task.sql b/src/test/regress/sql/task.sql index bf12909c761..3e29c46adad 100644 --- a/src/test/regress/sql/task.sql +++ b/src/test/regress/sql/task.sql @@ -49,10 +49,23 @@ alter task vacuum_db database task_dbno user task_cron; alter task vacuum_db user hopedoesnotexist; -- valid interval tasks +-- +-- These four only exercise the schedule parser, they are never meant to run. +-- Deactivate each one as soon as it exists: while a second-based task stays +-- active the scheduler keeps firing it, and every run draws a run id from the +-- cluster-wide Oid counter (NextRunId -> GetNewOidWithIndex). A task still +-- firing later in the same regression run shifts the counter that +-- oid_wraparound hardcodes, and DROP TASK, which also deletes the task's +-- pg_task_run_history rows, fails with "tuple concurrently updated" when it +-- races the run the scheduler is in the middle of recording. create task valid_task_1 schedule '1 second' as 'select 1'; +alter task valid_task_1 not active; create task valid_task_2 schedule ' 30 sEcOnDs ' as 'select 1'; +alter task valid_task_2 not active; create task valid_task_3 schedule '59 seconds' as 'select 1'; +alter task valid_task_3 not active; create task valid_task_4 schedule '17 seconds ' as 'select 1'; +alter task valid_task_4 not active; -- task in function DO $$