From 4bb63a4202f38f167a529572cc82726423abb376 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 2 Sep 2026 13:08:56 -0400 Subject: [PATCH 1/2] ci: take gpu-l40s out of rotation and seed frontier10202 as excluded Two GPU resources accounted for most of the non-code failures in run 33553417354. gpu-l40s has been failing jobs for weeks, and it is also what the partition selector kept choosing and then starving on: four Phoenix lanes submitted, never started, and were cancelled after 3-5.5h. Two causes, both fixed here. The selector counted "mix" nodes as available, but a mix node is partially allocated and may have no free GPU -- it picked gpu-l40s on "1 idle/mix nodes" and queued until timeout -- so only fully idle nodes count now. And l40s is out of the priority list, fallback included: leaving the fallback pointed at it sent jobs back there whenever nothing was idle. frontier10202 produced all 183 GPU memory-access faults in that run, across 43 distinct tests, three of which exhausted their retries; the same lanes passed on eight other g1 nodes with zero faults. It is seeded into node_exclude the way phoenix already seeds its two nodes, so the preflight from #1797 can still add more at run time. Its faults are intermittent (379 of 382 tests passed there), which is why syscheck can clear it and a seed is still worth having. Drop it once OLCF has looked at the node. Claude-Session: https://claude.ai/code/session_01XZjDp6ch827LVJRrvVb8hC --- .github/scripts/select-gpu-partition.sh | 18 +++++++++++++----- .github/scripts/submit-slurm-job.sh | 6 ++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/scripts/select-gpu-partition.sh b/.github/scripts/select-gpu-partition.sh index c812c000a9..c40c65eead 100644 --- a/.github/scripts/select-gpu-partition.sh +++ b/.github/scripts/select-gpu-partition.sh @@ -4,7 +4,15 @@ # # Priority order prefers partitions most likely to have availability. # V100 is last due to slower performance near the test time limit. -# Falls back to gpu-l40s if no partition meets the idle node threshold. +# Falls back to gpu-a100 if no partition meets the idle node threshold. +# +# gpu-l40s is out of rotation: it has been failing jobs for weeks, and it was +# also the partition CI kept selecting and then starving on. +# +# Only fully idle nodes count. A "mix" node is partially allocated and may have +# no free GPU, so counting it overstates availability: run 33553417354 picked +# gpu-l40s on "1 idle/mix nodes", then sat in the queue until the 3-5.5h job +# timeout without ever starting. # RTX 6000 nodes are excluded (too slow for the test suite time limit). # # Optional: set GPU_PARTITION_MIN_NODES before sourcing to require a minimum @@ -12,16 +20,16 @@ # # Usage: source .github/scripts/select-gpu-partition.sh -_GPU_PARTITION_PRIORITY="gpu-l40s gpu-h200 gpu-h100 gpu-a100 gpu-v100" -_GPU_PARTITION_FALLBACK="gpu-l40s" +_GPU_PARTITION_PRIORITY="gpu-h200 gpu-h100 gpu-a100 gpu-v100" +_GPU_PARTITION_FALLBACK="gpu-a100" _GPU_PARTITION_MIN_NODES="${GPU_PARTITION_MIN_NODES:-1}" SELECTED_GPU_PARTITION="" for _part in $_GPU_PARTITION_PRIORITY; do - _idle=$(sinfo -p "$_part" --noheader -o "%t" 2>/dev/null | grep -cE "^(idle|mix)" || true) + _idle=$(sinfo -p "$_part" --noheader -o "%t" 2>/dev/null | grep -cE "^idle" || true) if [ "${_idle:-0}" -ge "$_GPU_PARTITION_MIN_NODES" ]; then SELECTED_GPU_PARTITION="$_part" - echo "Selected GPU partition: $SELECTED_GPU_PARTITION ($_idle idle/mix nodes)" + echo "Selected GPU partition: $SELECTED_GPU_PARTITION ($_idle idle nodes)" break fi done diff --git a/.github/scripts/submit-slurm-job.sh b/.github/scripts/submit-slurm-job.sh index f5480d176f..bb4e79e520 100755 --- a/.github/scripts/submit-slurm-job.sh +++ b/.github/scripts/submit-slurm-job.sh @@ -150,6 +150,12 @@ elif [ "$device" = "gpu" ]; then sbatch_device_opts="\ #SBATCH -n 8 #SBATCH -p g1" + # Seed, same as phoenix above: the preflight adds nodes to this at + # run time. frontier10202 produced all 183 GPU memory-access faults + # in run 33553417354 (43 distinct tests) while the same lanes passed + # on eight other g1 nodes with none. Its faults are intermittent -- + # 379 of 382 tests still passed there -- so syscheck can clear it. + node_exclude="frontier10202" ;; esac else From 1341e445aa0bfdaa222470e7fbe0b6349921250f Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 2 Sep 2026 17:25:41 -0400 Subject: [PATCH 2/2] ci: anchor the idle match and correct the MIN_NODES doc sinfo's %t suffixes a state to flag it: "*" not responding, "$" reserved for maintenance, "~" powered down. Frontier emits them right now (drain*, down*, alloc$, drain$ are all live), so idle* and idle$ are reachable states and the unanchored "^idle" would have counted them as available -- reintroducing on unusable nodes exactly the starvation this PR removes. Anchor both ends. The GPU_PARTITION_MIN_NODES usage note still said the threshold counted "idle/mix" nodes, which stopped being true in the previous commit. Both from review on #1804. Claude-Session: https://claude.ai/code/session_01XZjDp6ch827LVJRrvVb8hC --- .github/scripts/select-gpu-partition.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/scripts/select-gpu-partition.sh b/.github/scripts/select-gpu-partition.sh index c40c65eead..b86c030982 100644 --- a/.github/scripts/select-gpu-partition.sh +++ b/.github/scripts/select-gpu-partition.sh @@ -13,10 +13,16 @@ # no free GPU, so counting it overstates availability: run 33553417354 picked # gpu-l40s on "1 idle/mix nodes", then sat in the queue until the 3-5.5h job # timeout without ever starting. +# +# The match is anchored at both ends. sinfo's %t suffixes a state to flag it -- +# "*" not responding, "$" reserved for maintenance, "~" powered down -- and this +# cluster does emit them (drain*, down*, alloc$, drain$ are all live right now). +# A bare "^idle" would count idle* and idle$ as available and starve the job on +# nodes that cannot take it. # RTX 6000 nodes are excluded (too slow for the test suite time limit). # # Optional: set GPU_PARTITION_MIN_NODES before sourcing to require a minimum -# number of idle/mix nodes (e.g. GPU_PARTITION_MIN_NODES=2 for parallel bench jobs). +# number of idle nodes (e.g. GPU_PARTITION_MIN_NODES=2 for parallel bench jobs). # # Usage: source .github/scripts/select-gpu-partition.sh @@ -26,7 +32,7 @@ _GPU_PARTITION_MIN_NODES="${GPU_PARTITION_MIN_NODES:-1}" SELECTED_GPU_PARTITION="" for _part in $_GPU_PARTITION_PRIORITY; do - _idle=$(sinfo -p "$_part" --noheader -o "%t" 2>/dev/null | grep -cE "^idle" || true) + _idle=$(sinfo -p "$_part" --noheader -o "%t" 2>/dev/null | grep -cE "^idle$" || true) if [ "${_idle:-0}" -ge "$_GPU_PARTITION_MIN_NODES" ]; then SELECTED_GPU_PARTITION="$_part" echo "Selected GPU partition: $SELECTED_GPU_PARTITION ($_idle idle nodes)"