Cc/fix flaky tests

contrib/pax_storage/src/test/isolation2/expected/resgroup/resgroup_cpu_max_percent.out

@@ -220,7 +220,11 @@ SELECT pg_sleep(1.7);
 (1 row)
 -- end_ignore
 
-SELECT verify_cpu_usage('rg1_cpu_test', 90, 10);
+-- rg1_cpu_test is uncapped (cpu_max_percent=-1) and it is the only busy
+-- group, so it takes essentially every core: gp_resgroup_status reports
+-- ~100, not 90.  Expecting 90 put the real value on the upper edge of the
+-- +/- err_rate window, so any upward sampling jitter failed the test.
+SELECT verify_cpu_usage('rg1_cpu_test', 100, 10);
  verify_cpu_usage 
 ------------------
  t                
@@ -395,12 +399,14 @@ SELECT pg_sleep(1.7);
 (1 row)
 -- end_ignore
 
-SELECT verify_cpu_usage('rg1_cpu_test', 30, 10);
+-- Both groups are uncapped, so they share the whole machine in proportion
+-- to cpu_weight (100 and 200): ~33 and ~67, not ~30 and ~60.
+SELECT verify_cpu_usage('rg1_cpu_test', 33, 10);
  verify_cpu_usage 
 ------------------
  t                
 (1 row)
-SELECT verify_cpu_usage('rg2_cpu_test', 60, 10);
+SELECT verify_cpu_usage('rg2_cpu_test', 67, 10);
  verify_cpu_usage 
 ------------------
  t                

contrib/pax_storage/src/test/isolation2/input/autovacuum-analyze.source

@@ -171,7 +171,7 @@ SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', '
 
 -- with auto_stats, the auto-ANALYZE still trigger
 2: INSERT INTO autostatstbl select i from generate_series(1, 1000) as i;
-2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat.
+2: select gp_stat_force_next_flush();
 
 -- auto_stats executed but auto-ANALYZE not execute yet since we suspend before finish ANALYZE.
 SELECT count(*) FROM pg_statistic where starelid = 'autostatstbl'::regclass;
@@ -205,7 +205,7 @@ SELECT gp_inject_fault('analyze_finished_one_relation', 'skip', '', '', 'autosta
 SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', '', 'autostatstbl', 1, -1, 0, 1);
 
 2: INSERT INTO autostatstbl select i from generate_series(1001, 2000) as i;
-2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat.
+2: select gp_stat_force_next_flush();
 
 -- auto_stats executed but auto-ANALYZE not execute yet since we suspend before finish ANALYZE.
 select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass;
@@ -238,7 +238,7 @@ SELECT gp_inject_fault('analyze_finished_one_relation', 'skip', '', '', 'autosta
 SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', '', 'autostatstbl', 1, -1, 0, 1);
 
 2: INSERT INTO autostatstbl select i from generate_series(2001, 3000) as i;
-2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat.
+2: select gp_stat_force_next_flush();
 
 -- auto_stats should not executed and auto-ANALYZE not execute yet since we suspend before finish ANALYZE.
 select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass;

contrib/pax_storage/src/test/isolation2/output/autovacuum-analyze.source

@@ -408,7 +408,12 @@ SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', '
 -- with auto_stats, the auto-ANALYZE still trigger
 2: INSERT INTO autostatstbl select i from generate_series(1, 1000) as i;
 INSERT 1000
-2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. 
+2: select gp_stat_force_next_flush();
+ gp_stat_force_next_flush 
+--------------------------
+
+(1 row)
+
 -- auto_stats executed but auto-ANALYZE not execute yet since we suspend before finish ANALYZE.
 SELECT count(*) FROM pg_statistic where starelid = 'autostatstbl'::regclass;
  count 
@@ -425,7 +430,7 @@ select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass;
 select analyze_count, autoanalyze_count, n_mod_since_analyze from pg_stat_all_tables where relname = 'autostatstbl';
  analyze_count | autoanalyze_count | n_mod_since_analyze 
 ---------------+-------------------+---------------------
- 1             | 0                 | 0                   
+ 1             | 0                 | 1000                
 (1 row)
 
 -- Wait until autovacuum is triggered
@@ -493,7 +498,12 @@ SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', '
 
 2: INSERT INTO autostatstbl select i from generate_series(1001, 2000) as i;
 INSERT 1000
-2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. 
+2: select gp_stat_force_next_flush();
+ gp_stat_force_next_flush 
+--------------------------
+
+(1 row)
+
 -- auto_stats executed but auto-ANALYZE not execute yet since we suspend before finish ANALYZE.
 select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass;
  relpages | reltuples 
@@ -572,7 +582,12 @@ SELECT gp_inject_fault('auto_vac_worker_after_report_activity', 'suspend', '', '
 
 2: INSERT INTO autostatstbl select i from generate_series(2001, 3000) as i;
 INSERT 1000
-2: select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. 
+2: select gp_stat_force_next_flush();
+ gp_stat_force_next_flush 
+--------------------------
+
+(1 row)
+
 -- auto_stats should not executed and auto-ANALYZE not execute yet since we suspend before finish ANALYZE.
 select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass;
  relpages | reltuples 
@@ -584,7 +599,7 @@ select relpages, reltuples from pg_class where oid = 'autostatstbl'::regclass;
 select analyze_count, autoanalyze_count, n_mod_since_analyze from pg_stat_all_tables where relname = 'autostatstbl';
  analyze_count | autoanalyze_count | n_mod_since_analyze 
 ---------------+-------------------+---------------------
- 2             | 2                 | 0                   
+ 2             | 2                 | 1000                
 (1 row)
 
 -- Wait until autovacuum is triggered

contrib/pax_storage/src/test/isolation2/sql/resgroup/resgroup_cpu_max_percent.sql

@@ -141,7 +141,11 @@ SELECT fetch_sample();
 SELECT pg_sleep(1.7);
 -- end_ignore
 
-SELECT verify_cpu_usage('rg1_cpu_test', 90, 10);
+-- rg1_cpu_test is uncapped (cpu_max_percent=-1) and it is the only busy
+-- group, so it takes essentially every core: gp_resgroup_status reports
+-- ~100, not 90.  Expecting 90 put the real value on the upper edge of the
+-- +/- err_rate window, so any upward sampling jitter failed the test.
+SELECT verify_cpu_usage('rg1_cpu_test', 100, 10);
 
 -- start_ignore
 SELECT * FROM cancel_all;
@@ -212,8 +216,10 @@ SELECT fetch_sample();
 SELECT pg_sleep(1.7);
 -- end_ignore
 
-SELECT verify_cpu_usage('rg1_cpu_test', 30, 10);
-SELECT verify_cpu_usage('rg2_cpu_test', 60, 10);
+-- Both groups are uncapped, so they share the whole machine in proportion
+-- to cpu_weight (100 and 200): ~33 and ~67, not ~30 and ~60.
+SELECT verify_cpu_usage('rg1_cpu_test', 33, 10);
+SELECT verify_cpu_usage('rg2_cpu_test', 67, 10);
 
 -- start_ignore
 SELECT * FROM cancel_all;

contrib/pax_storage/src/test/regress/expected/task.out

@@ -69,10 +69,23 @@ ERROR:  User task_cron does not have CONNECT privilege on task_dbno
 alter task vacuum_db user hopedoesnotexist;
 ERROR:  role "hopedoesnotexist" does not exist
 -- valid interval tasks
+--
+-- These four only exercise the schedule parser, they are never meant to run.
+-- Deactivate each one as soon as it exists: while a second-based task stays
+-- active the scheduler keeps firing it, and every run draws a run id from the
+-- cluster-wide Oid counter (NextRunId -> GetNewOidWithIndex).  A task still
+-- firing later in the same regression run shifts the counter that
+-- oid_wraparound hardcodes, and DROP TASK, which also deletes the task's
+-- pg_task_run_history rows, fails with "tuple concurrently updated" when it
+-- races the run the scheduler is in the middle of recording.
 create task valid_task_1 schedule '1 second' as 'select 1';
+alter task valid_task_1 not active;
 create task valid_task_2 schedule ' 30 sEcOnDs ' as 'select 1';
+alter task valid_task_2 not active;
 create task valid_task_3 schedule '59 seconds' as 'select 1';
+alter task valid_task_3 not active;
 create task valid_task_4 schedule '17  seconds ' as 'select 1';
+alter task valid_task_4 not active;
 -- task in function
 DO $$
 BEGIN

contrib/pax_storage/src/test/regress/sql/task.sql

@@ -49,10 +49,23 @@ alter task vacuum_db database task_dbno user task_cron;
 alter task vacuum_db user hopedoesnotexist;
 
 -- valid interval tasks
+--
+-- These four only exercise the schedule parser, they are never meant to run.
+-- Deactivate each one as soon as it exists: while a second-based task stays
+-- active the scheduler keeps firing it, and every run draws a run id from the
+-- cluster-wide Oid counter (NextRunId -> GetNewOidWithIndex).  A task still
+-- firing later in the same regression run shifts the counter that
+-- oid_wraparound hardcodes, and DROP TASK, which also deletes the task's
+-- pg_task_run_history rows, fails with "tuple concurrently updated" when it
+-- races the run the scheduler is in the middle of recording.
 create task valid_task_1 schedule '1 second' as 'select 1';
+alter task valid_task_1 not active;
 create task valid_task_2 schedule ' 30 sEcOnDs ' as 'select 1';
+alter task valid_task_2 not active;
 create task valid_task_3 schedule '59 seconds' as 'select 1';
+alter task valid_task_3 not active;
 create task valid_task_4 schedule '17  seconds ' as 'select 1';
+alter task valid_task_4 not active;
 
 -- task in function
 DO $$

devops/build/automation/cloudberry/scripts/parse-results.pl

@@ -145,8 +145,10 @@
     } elsif (/^Result: FAIL/) {
         $status = 'failed';
 
-    # TAP individual test failure: "  t/xxx.pl (Wstat: ...)"
-    } elsif (/^\s+(t\/\S+\.pl)\s+\(Wstat:/) {
+    # TAP individual test failure, as listed by prove's "Test Summary
+    # Report".  Those lines start at column 0, not indented, e.g.
+    # "t/019_replslot_limit.pl  (Wstat: 7424 Tests: 9 Failed: 2)".
+    } elsif (/^\s*(\S+\.pl)\s+\(Wstat:/) {
         push @failed_test_list, $1;
     }
 

src/test/isolation2/expected/resgroup/resgroup_cpu_max_percent.out

@@ -220,7 +220,11 @@ SELECT pg_sleep(1.7);
 (1 row)
 -- end_ignore
 
-SELECT verify_cpu_usage('rg1_cpu_test', 90, 10);
+-- rg1_cpu_test is uncapped (cpu_max_percent=-1) and it is the only busy
+-- group, so it takes essentially every core: gp_resgroup_status reports
+-- ~100, not 90.  Expecting 90 put the real value on the upper edge of the
+-- +/- err_rate window, so any upward sampling jitter failed the test.
+SELECT verify_cpu_usage('rg1_cpu_test', 100, 10);
  verify_cpu_usage 
 ------------------
  t                
@@ -395,13 +399,15 @@ SELECT pg_sleep(1.7);
 (1 row)
 -- end_ignore
 
-SELECT verify_cpu_usage('rg1_cpu_test', 30, 10);
+-- Both groups are uncapped, so they share the whole machine in proportion
+-- to cpu_weight (100 and 200): ~33 and ~67, not ~30 and ~60.
+SELECT verify_cpu_usage('rg1_cpu_test', 33, 10);
  verify_cpu_usage 
 ------------------
  t                
 (1 row)
 -- start_ignore
-SELECT verify_cpu_usage('rg2_cpu_test', 60, 10);
+SELECT verify_cpu_usage('rg2_cpu_test', 67, 10);
  verify_cpu_usage 
 ------------------
  t                

src/test/isolation2/expected/vacuum_progress_column.out

@@ -305,11 +305,22 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he
 (1 row)
 
 -- Resume execution of compact phase and block at syncrep on one segment.
+-- The suspend only takes effect the next time the walsender goes round its
+-- loop, so wait until it is really parked before letting the vacuum go on.
+-- Otherwise the compact phase can commit while the walsender is still
+-- streaming, syncrep is satisfied by the live mirror, the vacuum runs to
+-- completion on the same gang, no new vacuum worker ever takes over and the
+-- gp_wait_until_triggered_fault() below times out after ten minutes.
 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
  gp_inject_fault_infinite 
 --------------------------
  Success:                 
 (1 row)
+2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
+ gp_wait_until_triggered_fault 
+-------------------------------
+ Success:                      
+(1 row)
 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p';
  gp_inject_fault 
 -----------------

src/test/isolation2/expected/vacuum_progress_row.out

@@ -359,11 +359,22 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he
 (1 row)
 
 -- Resume execution of compact phase and block at syncrep.
+-- The suspend only takes effect the next time the walsender goes round its
+-- loop, so wait until it is really parked before letting the vacuum go on.
+-- Otherwise the compact phase can commit while the walsender is still
+-- streaming, syncrep is satisfied by the live mirror, the vacuum runs to
+-- completion on the same gang, no new vacuum worker ever takes over and the
+-- gp_wait_until_triggered_fault() below times out after ten minutes.
 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
  gp_inject_fault_infinite 
 --------------------------
  Success:                 
 (1 row)
+2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
+ gp_wait_until_triggered_fault 
+-------------------------------
+ Success:                      
+(1 row)
 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p';
  gp_inject_fault 
 -----------------
@@ -563,11 +574,22 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he
 (1 row)
 
 -- Resume execution of compact phase and block at syncrep.
+-- The suspend only takes effect the next time the walsender goes round its
+-- loop, so wait until it is really parked before letting the vacuum go on.
+-- Otherwise the compact phase can commit while the walsender is still
+-- streaming, syncrep is satisfied by the live mirror, the vacuum runs to
+-- completion on the same gang, no new vacuum worker ever takes over and the
+-- gp_wait_until_triggered_fault() below times out after ten minutes.
 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
  gp_inject_fault_infinite 
 --------------------------
  Success:                 
 (1 row)
+2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
+ gp_wait_until_triggered_fault 
+-------------------------------
+ Success:                      
+(1 row)
 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p';
  gp_inject_fault 
 -----------------

src/test/isolation2/sql/resgroup/resgroup_cpu_max_percent.sql

@@ -141,7 +141,11 @@ SELECT fetch_sample();
 SELECT pg_sleep(1.7);
 -- end_ignore
 
-SELECT verify_cpu_usage('rg1_cpu_test', 90, 10);
+-- rg1_cpu_test is uncapped (cpu_max_percent=-1) and it is the only busy
+-- group, so it takes essentially every core: gp_resgroup_status reports
+-- ~100, not 90.  Expecting 90 put the real value on the upper edge of the
+-- +/- err_rate window, so any upward sampling jitter failed the test.
+SELECT verify_cpu_usage('rg1_cpu_test', 100, 10);
 
 -- start_ignore
 SELECT * FROM cancel_all;
@@ -212,9 +216,11 @@ SELECT fetch_sample();
 SELECT pg_sleep(1.7);
 -- end_ignore
 
-SELECT verify_cpu_usage('rg1_cpu_test', 30, 10);
+-- Both groups are uncapped, so they share the whole machine in proportion
+-- to cpu_weight (100 and 200): ~33 and ~67, not ~30 and ~60.
+SELECT verify_cpu_usage('rg1_cpu_test', 33, 10);
 -- start_ignore
-SELECT verify_cpu_usage('rg2_cpu_test', 60, 10);
+SELECT verify_cpu_usage('rg2_cpu_test', 67, 10);
 
 SELECT * FROM cancel_all;
 

src/test/isolation2/sql/vacuum_progress_column.sql

@@ -131,7 +131,14 @@ select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_b
 select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary;
 
 -- Resume execution of compact phase and block at syncrep on one segment.
+-- The suspend only takes effect the next time the walsender goes round its
+-- loop, so wait until it is really parked before letting the vacuum go on.
+-- Otherwise the compact phase can commit while the walsender is still
+-- streaming, syncrep is satisfied by the live mirror, the vacuum runs to
+-- completion on the same gang, no new vacuum worker ever takes over and the
+-- gp_wait_until_triggered_fault() below times out after ten minutes.
 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
+2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p';
 -- stop the mirror should turn off syncrep
 2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content = 1 AND role = 'm';

src/test/isolation2/sql/vacuum_progress_row.sql

@@ -136,7 +136,14 @@ select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_b
 select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary;
 
 -- Resume execution of compact phase and block at syncrep.
+-- The suspend only takes effect the next time the walsender goes round its
+-- loop, so wait until it is really parked before letting the vacuum go on.
+-- Otherwise the compact phase can commit while the walsender is still
+-- streaming, syncrep is satisfied by the live mirror, the vacuum runs to
+-- completion on the same gang, no new vacuum worker ever takes over and the
+-- gp_wait_until_triggered_fault() below times out after ten minutes.
 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
+2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p';
 -- stop the mirror should turn off syncrep
 2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content=1 AND role = 'm';
@@ -210,7 +217,14 @@ select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_b
 select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary;
 
 -- Resume execution of compact phase and block at syncrep.
+-- The suspend only takes effect the next time the walsender goes round its
+-- loop, so wait until it is really parked before letting the vacuum go on.
+-- Otherwise the compact phase can commit while the walsender is still
+-- streaming, syncrep is satisfied by the live mirror, the vacuum runs to
+-- completion on the same gang, no new vacuum worker ever takes over and the
+-- gp_wait_until_triggered_fault() below times out after ten minutes.
 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
+2: SELECT gp_wait_until_triggered_fault('wal_sender_loop', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1;
 2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p';
 -- stop the mirror should turn off syncrep
 2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content=1 AND role = 'm';