diff --git a/.github/scripts/select-gpu-partition.sh b/.github/scripts/select-gpu-partition.sh index c812c000a..b86c03098 100644 --- a/.github/scripts/select-gpu-partition.sh +++ b/.github/scripts/select-gpu-partition.sh @@ -4,24 +4,38 @@ # # Priority order prefers partitions most likely to have availability. # V100 is last due to slower performance near the test time limit. -# Falls back to gpu-l40s if no partition meets the idle node threshold. +# Falls back to gpu-a100 if no partition meets the idle node threshold. +# +# gpu-l40s is out of rotation: it has been failing jobs for weeks, and it was +# also the partition CI kept selecting and then starving on. +# +# Only fully idle nodes count. A "mix" node is partially allocated and may have +# no free GPU, so counting it overstates availability: run 33553417354 picked +# gpu-l40s on "1 idle/mix nodes", then sat in the queue until the 3-5.5h job +# timeout without ever starting. +# +# The match is anchored at both ends. sinfo's %t suffixes a state to flag it -- +# "*" not responding, "$" reserved for maintenance, "~" powered down -- and this +# cluster does emit them (drain*, down*, alloc$, drain$ are all live right now). +# A bare "^idle" would count idle* and idle$ as available and starve the job on +# nodes that cannot take it. # RTX 6000 nodes are excluded (too slow for the test suite time limit). # # Optional: set GPU_PARTITION_MIN_NODES before sourcing to require a minimum -# number of idle/mix nodes (e.g. GPU_PARTITION_MIN_NODES=2 for parallel bench jobs). +# number of idle nodes (e.g. GPU_PARTITION_MIN_NODES=2 for parallel bench jobs). # # Usage: source .github/scripts/select-gpu-partition.sh -_GPU_PARTITION_PRIORITY="gpu-l40s gpu-h200 gpu-h100 gpu-a100 gpu-v100" -_GPU_PARTITION_FALLBACK="gpu-l40s" +_GPU_PARTITION_PRIORITY="gpu-h200 gpu-h100 gpu-a100 gpu-v100" +_GPU_PARTITION_FALLBACK="gpu-a100" _GPU_PARTITION_MIN_NODES="${GPU_PARTITION_MIN_NODES:-1}" SELECTED_GPU_PARTITION="" for _part in $_GPU_PARTITION_PRIORITY; do - _idle=$(sinfo -p "$_part" --noheader -o "%t" 2>/dev/null | grep -cE "^(idle|mix)" || true) + _idle=$(sinfo -p "$_part" --noheader -o "%t" 2>/dev/null | grep -cE "^idle$" || true) if [ "${_idle:-0}" -ge "$_GPU_PARTITION_MIN_NODES" ]; then SELECTED_GPU_PARTITION="$_part" - echo "Selected GPU partition: $SELECTED_GPU_PARTITION ($_idle idle/mix nodes)" + echo "Selected GPU partition: $SELECTED_GPU_PARTITION ($_idle idle nodes)" break fi done diff --git a/.github/scripts/submit-slurm-job.sh b/.github/scripts/submit-slurm-job.sh index f5480d176..bb4e79e52 100755 --- a/.github/scripts/submit-slurm-job.sh +++ b/.github/scripts/submit-slurm-job.sh @@ -150,6 +150,12 @@ elif [ "$device" = "gpu" ]; then sbatch_device_opts="\ #SBATCH -n 8 #SBATCH -p g1" + # Seed, same as phoenix above: the preflight adds nodes to this at + # run time. frontier10202 produced all 183 GPU memory-access faults + # in run 33553417354 (43 distinct tests) while the same lanes passed + # on eight other g1 nodes with none. Its faults are intermittent -- + # 379 of 382 tests still passed there -- so syscheck can clear it. + node_exclude="frontier10202" ;; esac else