ci: fail fast on bad nodes and cluster outages

.github/scripts/ci-outage.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+# Per-cluster circuit breaker for environment-wide outages.
+#
+# Some failures are not the node's fault and not the code's fault: pypi.org
+# unreachable from a login node, a module tree mid-upgrade, a full project
+# filesystem. Requeuing elsewhere cannot help, and every job that starts pays
+# the same discovery cost -- on 2026-08-28, 17 Frontier jobs each spent ~33
+# minutes learning that PyPI was down.
+#
+# The first job to notice records a marker on the shared filesystem (every
+# self-hosted runner for a cluster shares $HOME); later jobs check it and exit
+# immediately instead of submitting a SLURM job that is going to fail.
+#
+# The breaker is deliberately self-healing. A marker expires after
+# MFC_CI_OUTAGE_TTL_SECONDS, and a marker that cannot be parsed is ignored, so
+# neither a stale file nor a truncated write can wedge CI. Only jobs that
+# actually observe the outage re-mark it, so once the outage clears the breaker
+# closes on its own.
+#
+# Usage:
+#   ci-outage.sh mark  <cluster> <reason>   record an outage
+#   ci-outage.sh check <cluster>            exit 0 = clear, 1 = outage active
+#   ci-outage.sh clear <cluster>            reset the breaker
+#
+# Env:
+#   MFC_CI_STATE_DIR             where markers live (default ~/.mfc-ci-state)
+#   MFC_CI_OUTAGE_TTL_SECONDS    marker lifetime in seconds (default 1200)
+
+set -uo pipefail
+
+STATE_DIR="${MFC_CI_STATE_DIR:-$HOME/.mfc-ci-state}"
+TTL="${MFC_CI_OUTAGE_TTL_SECONDS:-1200}"
+
+# A non-numeric TTL would make the age comparison below emit "integer expression
+# expected" and exit with a code the caller reads as neither clear nor tripped.
+# Fall back to the default rather than letting a typo gate CI.
+case "$TTL" in
+    ''|*[!0-9]*)
+        echo "Ignoring non-numeric MFC_CI_OUTAGE_TTL_SECONDS='$TTL'; using 1200." >&2
+        TTL=1200
+        ;;
+esac
+
+EXIT_CLEAR=0
+EXIT_TRIPPED=1
+EXIT_USAGE=2
+
+usage() {
+    echo "Usage: $0 {mark <cluster> <reason>|check <cluster>|clear <cluster>}" >&2
+}
+
+# Keep the marker name filesystem-safe regardless of what the caller passes.
+marker_for() {
+    local cluster
+    cluster=$(printf '%s' "$1" | tr -c 'A-Za-z0-9_.-' '_')
+    printf '%s/outage-%s' "$STATE_DIR" "$cluster"
+}
+
+cmd="${1:-}"
+cluster="${2:-}"
+
+if [ -z "$cmd" ] || [ -z "$cluster" ]; then
+    usage
+    exit $EXIT_USAGE
+fi
+
+marker=$(marker_for "$cluster")
+
+case "$cmd" in
+    mark)
+        reason="${3:-unspecified}"
+        mkdir -p "$STATE_DIR" || exit $EXIT_USAGE
+        # Write to a temporary file and rename so a concurrent `check` never
+        # observes a half-written marker.
+        tmp="${marker}.$$.tmp"
+        {
+            date +%s
+            printf '%s\n' "$reason"
+        } > "$tmp" && mv -f "$tmp" "$marker"
+        echo "Recorded $cluster outage: $reason"
+        echo "  marker: $marker (expires after ${TTL}s)"
+        ;;
+
+    check)
+        [ -f "$marker" ] || exit $EXIT_CLEAR
+
+        stamp=$(head -n1 "$marker" 2>/dev/null)
+        reason=$(tail -n +2 "$marker" 2>/dev/null)
+
+        # A marker we cannot parse is treated as absent: an unreadable breaker
+        # must never be an un-clearable one.
+        case "$stamp" in
+            ''|*[!0-9]*)
+                echo "Ignoring unparseable outage marker $marker"
+                exit $EXIT_CLEAR
+                ;;
+        esac
+
+        age=$(( $(date +%s) - stamp ))
+        if [ "$age" -ge "$TTL" ] || [ "$age" -lt 0 ]; then
+            exit $EXIT_CLEAR
+        fi
+
+        echo "::warning::Skipping: known $cluster outage recorded ${age}s ago: ${reason:-unspecified}"
+        echo "Clear it early by deleting $marker"
+        exit $EXIT_TRIPPED
+        ;;
+
+    clear)
+        rm -f "$marker"
+        echo "Cleared any $cluster outage marker ($marker)"
+        ;;
+
+    *)
+        usage
+        exit $EXIT_USAGE
+        ;;
+esac

.github/scripts/classify-build-failure.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Decide whether a failed build was a cluster-wide dependency outage.
+#
+# MFC bootstraps its Python toolchain into build/venv on the first ./mfc.sh call
+# of a job, pulling from pypi.org. On Phoenix clean_build has just moved build/
+# aside, so that happens every time; on Frontier it happens in the login-node
+# "Fetch Dependencies" step. When the index is unreachable the build fails for a
+# reason no other node improves on, so it is worth recording once and skipping
+# the rest of the matrix rather than having each job spend ~33 minutes
+# rediscovering it (17 Frontier jobs did exactly that on 2026-08-28).
+#
+# Usage: classify-build-failure.sh <logfile> <cluster>
+#
+# Exit codes:
+#   78  cluster-wide dependency outage; it has been recorded
+#   0   ordinary build failure, caller should keep its own exit code
+
+set -uo pipefail
+
+log="${1:-}"
+cluster="${2:-}"
+
+if [ -z "$log" ] || [ -z "$cluster" ]; then
+    echo "Usage: $0 <logfile> <cluster>" >&2
+    exit 0
+fi
+
+# No log means nothing to classify. Never claim an outage on absent evidence.
+[ -f "$log" ] || exit 0
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# The URL may or may not be wrapped (uv quotes it in backticks, plain pip does
+# not), so do not require a character between the colon and the scheme.
+if grep -qE "Failed to fetch:[^h]*https?://pypi|uv install failed|\(venv\) Installation failed" "$log"; then
+    bash "$SCRIPT_DIR/ci-outage.sh" mark "$cluster" \
+        "PyPI/uv dependency install failed during build"
+    exit 78
+fi
+
+exit 0

.github/scripts/monitor_slurm_job.sh

@@ -262,6 +262,23 @@ if [ -z "$exit_code" ]; then
   exit 1
 fi
 
+# Infrastructure verdicts from the in-allocation preflight come back as the
+# job's own exit code. Relay them verbatim: flattening them to 1 would leave the
+# submit wrapper unable to tell "this node is unusable" (exclude it and try
+# again) from "the tests failed" (report it).
+case "$exit_code" in
+  77:*)
+    echo "Job $job_id failed preflight: the node is unusable — signaling caller to exclude it and resubmit."
+    monitor_success=1
+    exit 77
+    ;;
+  78:*)
+    echo "Job $job_id skipped: a cluster-wide outage is already recorded."
+    monitor_success=1
+    exit 78
+    ;;
+esac
+
 # Check if job succeeded
 if [ "$exit_code" != "0:0" ]; then
   echo "ERROR: Job $job_id failed with exit code $exit_code"

.github/scripts/node-exclude.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Bookkeeping for the sbatch --exclude list used when a node fails preflight.
+#
+# The in-allocation preflight prints "MFC_FAULT_NODE=<name>" into the job's
+# output file when it finds the node unusable (dead GPU, SIGILL from a binary
+# built for another microarchitecture). The submit wrapper reads that back, adds
+# the node to --exclude and resubmits, so the retry lands elsewhere.
+#
+# Bad nodes are concentrated rather than scattered -- over 2026-08-18..31 one
+# Phoenix node accounted for 25 of 29 ECC failures -- which is why excluding the
+# offender is worth doing and why it was previously a hand-edited constant.
+#
+# Usage:
+#   node-exclude.sh node-from <output-file>   print the faulted node, if any
+#   node-exclude.sh merge <csv> <node>        print <csv> with <node> added once
+
+set -uo pipefail
+
+usage() {
+    echo "Usage: $0 {node-from <output-file>|merge <csv> <node>}" >&2
+}
+
+case "${1:-}" in
+    node-from)
+        file="${2:-}"
+        [ -f "$file" ] || exit 0
+        # Last marker wins: one output path is reused across resubmits, so an
+        # earlier attempt's marker can still be sitting above the current one.
+        sed -n 's/.*MFC_FAULT_NODE=\([A-Za-z0-9._-]\{1,\}\).*/\1/p' "$file" | tail -n1
+        ;;
+
+    merge)
+        csv="${2-}"
+        node="${3-}"
+        if [ -z "$node" ]; then
+            printf '%s\n' "$csv"
+            exit 0
+        fi
+        if [ -z "$csv" ]; then
+            printf '%s\n' "$node"
+            exit 0
+        fi
+        # Wrapping both sides in commas compares whole fields, so a shorter name
+        # that happens to be a prefix of the new one is not mistaken for a match.
+        case ",$csv," in
+            *",$node,"*) printf '%s\n' "$csv" ;;
+            *)           printf '%s,%s\n' "$csv" "$node" ;;
+        esac
+        ;;
+
+    *)
+        usage
+        exit 2
+        ;;
+esac

.github/scripts/prebuild-case-optimization.sh

@@ -90,6 +90,18 @@ if [ -n "$shard" ] && [ "$shard_count" -gt 1 ]; then
     fi
 fi
 
+# Probe this node before spending the allocation on it. Placed after the build,
+# not in the sbatch template: these scripts nuke and rebuild build/ themselves
+# (Phoenix does so precisely because its compute nodes are heterogeneous), so a
+# probe running earlier would test a stale binary from a previous job -- likely
+# built for another microarchitecture -- and a SIGILL there would be reported as
+# a bad node, excluding a healthy one.
+preflight_rc=0
+bash .github/scripts/preflight.sh "$job_cluster" "${job_device:-gpu}" || preflight_rc=$?
+if [ "$preflight_rc" -ne 0 ]; then
+    exit "$preflight_rc"
+fi
+
 idx=0
 for case in "${benchmarks[@]}"; do
     idx=$((idx + 1))

.github/scripts/preflight.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+# Prove this node can run MFC before spending an allocation on it.
+#
+# Runs syscheck -- the same binary the suite already builds -- as the first
+# thing inside the allocation that will execute the tests. syscheck initialises
+# MPI, creates a device context, launches a kernel and reads the result back, so
+# a node with a dead GPU, a broken MPI layer, or a binary built for a different
+# microarchitecture fails here in seconds instead of after the build.
+#
+# Why at the top of the *test* allocation: over 2026-08-18..31, 48 of 58
+# measurable jobs built on one node and tested on another, and in every ECC
+# failure the build node was healthy while the tests landed on a bad one. A
+# probe placed after the build checks the wrong machine. (Phoenix's combined
+# build-and-test allocation avoids the split; Frontier still submits the two
+# separately and lands elsewhere ~90% of the time.)
+#
+# Usage: preflight.sh <cluster> <device>
+#
+# Exit codes:
+#   0   node looks healthy, carry on
+#   77  node-local fault -- caller should exclude this node and resubmit
+#   78  cluster-wide outage already recorded -- caller should skip, not requeue
+
+set -uo pipefail
+
+cluster="${1:-}"
+device="${2:-}"
+
+if [ -z "$cluster" ] || [ -z "$device" ]; then
+    echo "Usage: $0 <cluster> <device>" >&2
+    exit 2
+fi
+
+EXIT_HEALTHY=0
+EXIT_NODE_FAULT=77
+EXIT_OUTAGE=78
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+node="${SLURMD_NODENAME:-$(hostname -s 2>/dev/null || hostname)}"
+
+# Only judge a node from inside its own allocation. `mfc.sh load` is also used
+# for building on login nodes -- bench.yml and frontier/build.sh both load the
+# GPU module set there -- and a login node has no GPU to probe. Reporting a node
+# fault in that context would have the wrapper exclude a login node and requeue,
+# which is both wrong and hard to diagnose. Nothing calls the probe from a login
+# node today; this makes that a property of the probe rather than a convention
+# every future caller has to remember.
+if [ -z "${SLURM_JOB_ID:-}" ]; then
+    echo "Preflight: not inside a SLURM allocation; skipping the node probe."
+    exit $EXIT_HEALTHY
+fi
+
+# --- Cluster-wide outage: requeuing cannot help, so skip rather than retry ---
+outage_rc=0
+bash "$SCRIPT_DIR/ci-outage.sh" check "$cluster" || outage_rc=$?
+if [ "$outage_rc" -eq 1 ]; then
+    echo "Preflight: skipping on $node because $cluster is known to be down."
+    exit $EXIT_OUTAGE
+elif [ "$outage_rc" -ne 0 ]; then
+    # Only exit 1 means "tripped". Anything else means the breaker could not be
+    # read at all (missing script, unreadable state dir), which says nothing
+    # about the cluster -- treating it as an outage would halt CI on a bug here.
+    echo "Preflight: could not read the outage breaker (exit $outage_rc); continuing."
+fi
+
+# --- Node health ---
+# Pick the *newest* install matching this job's device (build/install is named
+# e.g. gpu-acc-<hash>, gpu-mp-<hash>). Both halves matter: the device filter
+# avoids probing another variant's binary, and newest-wins avoids probing a
+# leftover from an earlier job. Not every caller nukes build/ first -- bench.sh
+# only does so on Phoenix -- and a stale binary compiled for a different
+# microarchitecture dies with SIGILL, which would be reported as a bad node and
+# get a perfectly healthy one excluded.
+newest_syscheck() {
+    find "$@" -name syscheck -type f -printf '%T@ %p\n' 2>/dev/null \
+        | sort -rn | head -1 | cut -d' ' -f2-
+}
+
+syscheck_bin=$(newest_syscheck build/install -path "*${device}*")
+if [ -z "$syscheck_bin" ]; then
+    syscheck_bin=$(newest_syscheck build/install)
+fi
+
+if [ -z "$syscheck_bin" ]; then
+    # Nothing to probe with. A missing binary is a build problem, not a bad
+    # node: requeuing would land somewhere healthy and fail the same way, so
+    # let the build or test step report it instead.
+    echo "Preflight: no syscheck binary under build/install; skipping node probe on $node."
+    exit $EXIT_HEALTHY
+fi
+
+echo "Preflight: probing $node with $syscheck_bin"
+
+# Launch the probe the way this cluster launches everything else. Phoenix uses
+# mpirun -- its openmpi predates the PMIx that shipped with its Slurm upgrade,
+# so a bare MPI binary misreads the environment and aborts in MPI_Init. Frontier
+# and frontier_amd use srun and Cray MPICH ships no mpirun at all, so running
+# one there fails 127 no matter how healthy the node is. See
+# toolchain/templates/{phoenix,frontier,frontier_amd}.mako.
+case "$cluster" in
+    phoenix)               launcher=(mpirun -np 1) ;;
+    frontier|frontier_amd) launcher=(srun -n1) ;;
+    *)                     launcher=() ;;
+esac
+
+# A launcher missing from PATH says nothing about the node. Probing bare is a
+# weaker test, but calling a healthy node bad is far worse: it costs three
+# allocations and blacklists three good nodes before giving up.
+if [ "${#launcher[@]}" -gt 0 ] && ! command -v "${launcher[0]}" >/dev/null 2>&1; then
+    echo "Preflight: ${launcher[0]} is not on PATH; probing without a launcher."
+    launcher=()
+fi
+
+# Output goes to the log verbatim. Only the exit status decides the verdict:
+# PMIX_ERR_NO_PERMISSIONS and friends from dstore_base.c are benign and appear
+# in more passing jobs than failing ones, so matching on log text would fail
+# healthy nodes.
+probe_rc=0
+if [ "${#launcher[@]}" -eq 0 ]; then
+    "$syscheck_bin" 2>&1 || probe_rc=$?
+else
+    "${launcher[@]}" "$syscheck_bin" 2>&1 || probe_rc=$?
+fi
+
+if [ "$probe_rc" -eq 0 ]; then
+    echo "Preflight: $node passed."
+    exit $EXIT_HEALTHY
+fi
+
+echo "::error::Preflight failed on $node: syscheck could not run MFC here."
+echo "This is an INFRASTRUCTURE fault, not a code or test failure."
+echo "MFC_FAULT_NODE=$node"
+exit $EXIT_NODE_FAULT