DO NOT MERGE (fault injected): name the kernel behind a GPU memory fault, on the first failure

.github/scripts/run_case_optimization.sh

@@ -102,6 +102,23 @@ for case in "${benchmarks[@]}"; do
     # its run is sharded across concurrent jobs sharing one workspace, so a
     # fallback rebuild would race on the shared install paths (the collision the
     # --no-build guard above prevents).
+    # The same offload diagnostics the test harness sets. These cases run on
+    # GPUs, and a memory fault here previously surfaced as a bare device
+    # address with nothing to act on. Both variables are inert until a fault;
+    # the debug agent is what gives CCE a faulting kernel at all, and is set
+    # only where its library is actually reachable.
+    # OFFLOAD_TRACK_ALLOCATION_TRACES / _NUM_KERNEL_LAUNCH_TRACES are deliberately
+    # NOT set: measured on an MI210 with amdflang, either one alone turns a
+    # 5.94 s test into a >400 s timeout, because they instrument every
+    # allocation and every kernel launch. See toolchain/mfc/gpu_diagnostics.py.
+    # Skipped when the caller already chose a tool, or is collecting a GPU core
+    # dump -- the agent is mutually exclusive with one, so loading it anyway
+    # would leave them with no dump and no reason why.
+    if [ -z "${HSA_TOOLS_LIB:-}" ] && [ -z "${HSA_ENABLE_DEBUG:-}" ] \
+       && [ -n "${ROCM_PATH:-}" ] && [ -f "$ROCM_PATH/lib/librocm-debug-agent.so.2" ]; then
+        export HSA_TOOLS_LIB=librocm-debug-agent.so.2
+    fi
+
     run_log="$(mktemp)"
     ./mfc.sh run "$case" --case-optimization $gpu_opts $build_opts -n "$ngpus" -j 8 -c "$job_cluster" -- --gbpp 1 --steps 10 2>&1 | tee "$run_log"
     run_rc=${PIPESTATUS[0]}
@@ -118,6 +135,14 @@ for case in "${benchmarks[@]}"; do
     else
         run_ok=0
     fi
+
+    # A fault's agent report runs to tens of thousands of lines and the useful
+    # part is in the middle, so re-print a bounded summary at the end where a
+    # reader will actually find it. Silent when the log has no agent report.
+    if [ "$run_ok" = 0 ]; then
+        build/venv/bin/python3 .github/scripts/summarize_gpu_fault.py "$run_log" || true
+    fi
+
     rm -f "$run_log"
 
     if [ "$run_ok" = 1 ]; then

.github/scripts/summarize_gpu_fault.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+"""Print a bounded summary of a GPU memory fault in a run log.
+
+For callers that are shell scripts. The ROCm debug agent emits tens of
+thousands of lines per fault -- one disassembly and register dump repeated per
+faulting wave -- and the part worth reading (the faulting kernel, the fault
+reason, the stop-PC distribution) is buried in the middle, so `tail` cannot
+find it.
+
+Exits 0 having printed a summary, or 1 having printed nothing when the log has
+no agent report, which lets the caller fall back to whatever it did before.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "toolchain"))
+
+from mfc.gpu_diagnostics import summarize_rocm_debug_agent  # noqa: E402
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        print(f"usage: {sys.argv[0]} <run log>", file=sys.stderr)
+        return 2
+
+    try:
+        with open(sys.argv[1], "r", encoding="utf-8", errors="replace") as log:
+            summary = summarize_rocm_debug_agent(log.read())
+    except OSError as exc:
+        print(f"could not read {sys.argv[1]}: {exc}", file=sys.stderr)
+        return 1
+
+    if not summary:
+        return 1
+
+    print(summary)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/simulation/m_time_steppers.fpp

@@ -492,6 +492,30 @@ contains
                                 q_cons_ts(stor)%vf(i)%sf(j, k, l) = q_cons_ts(1)%vf(i)%sf(j, k, l)
                             end if
                             if (igr) then
+                                ! ############ DO NOT MERGE ############
+                                ! Deliberate out-of-bounds device write, to make
+                                ! CI produce a real GPU memory fault so the retry
+                                ! diagnostics can be seen end to end in a job log.
+                                ! Scoped to igr so only a handful of tests fault
+                                ! instead of the whole GPU matrix.
+                                ! Revert with: git revert <this commit>
+                                ! GPU builds only: GPU_PARALLEL_LOOP emits nothing
+                                ! on CPU, so without this gate the same statement
+                                ! is an out-of-bounds *host* write -- undefined
+                                ! behaviour, and not the clean device fault this
+                                ! is meant to produce.
+#ifdef MFC_GPU
+                                ! 2e9 elements (16 GB) past the base. Measured
+                                ! on an MI210: 1e8 (762 MB) does NOT fault once
+                                ! several GB of device arrays exist -- it lands
+                                ! inside a neighbouring allocation and silently
+                                ! corrupts, which is what the first attempt did.
+                                ! 1e11 overshoots into
+                                ! HSA_STATUS_ERROR_MEMORY_APERTURE_VIOLATION, a
+                                ! different error the detector does not match.
+                                q_cons_ts(1)%vf(i)%sf(j + 2000000000, k, l) = 1._wp
+#endif
+                                ! ######################################
                                 q_cons_ts(1)%vf(i)%sf(j, k, l) = (rk_coef(s, 1)*q_cons_ts(1)%vf(i)%sf(j, k, l) + rk_coef(s, &
                                           & 2)*q_cons_ts(stor)%vf(i)%sf(j, k, l) + rk_coef(s, 3)*rhs_vf(i)%sf(j, k, &
                                           & l))/rk_coef(s, 4)

toolchain/mfc/bench.py

@@ -12,6 +12,7 @@
 
 from .build import DEFAULT_TARGETS, SIMULATION, get_targets
 from .common import MFC_BENCH_FILEPATH, MFC_BUILD_DIR, MFCException, console_safe, create_directory, file_dump_yaml, file_load_yaml, format_list_to_string, log_tail, system
+from .gpu_diagnostics import fault_diagnostic_env, summarize_rocm_debug_agent
 from .printer import cons
 from .state import ARG, CFG
 
@@ -23,6 +24,25 @@ class BenchCase:
     args: typing.List[str]
 
 
+def bench_failure_report(log_filepath: str) -> str:
+    """What to show for a failed benchmark case.
+
+    A GPU memory fault under the ROCm debug agent runs to tens of thousands of
+    lines, nearly all of it one disassembly and register dump repeated per wave.
+    A fixed tail is not merely long here, it is wrong: measured on a real
+    report, the last 80 lines are a single wave's registers and the kernel name
+    -- the only part worth having -- is not among them. Fall back to the tail
+    only when there is no agent report to summarize.
+    """
+    try:
+        with open(log_filepath, "r", encoding="utf-8", errors="replace") as log_file:
+            summary = summarize_rocm_debug_agent(log_file.read())
+    except OSError:
+        return log_tail(log_filepath)
+
+    return summary or log_tail(log_filepath)
+
+
 def bench(targets=None):
     if targets is None:
         targets = ARG("targets")
@@ -76,6 +96,10 @@ def bench(targets=None):
                                 ["./mfc.sh", "run", case.path] + ["--targets"] + [t.name for t in targets] + ["--output-summary", summary_filepath] + case.args + ["--", "--gbpp", str(ARG("mem"))],
                                 stdout=log_file,
                                 stderr=subprocess.STDOUT,
+                                # Same offload diagnostics the test harness uses:
+                                # these cases run on GPUs too, and a fault here
+                                # was previously reported as a bare address.
+                                env=fault_diagnostic_env(dict(os.environ)),
                             )
 
                         # Check return code (handle CompletedProcess or int defensively)
@@ -89,7 +113,7 @@ def bench(targets=None):
                             cons.print(f"[bold red]ERROR[/bold red]: Case {case.slug} failed with exit code {rc}")
                             # Print the log, not just its path: this file lives
                             # on the cluster and no artifact upload collects it.
-                            cons.print(console_safe(log_tail(log_filepath)))
+                            cons.print(console_safe(bench_failure_report(log_filepath)))
                             failed_cases.append(case.slug)
                             break
 
@@ -101,7 +125,7 @@ def bench(targets=None):
                                 time.sleep(5)
                                 continue
                             cons.print(f"[bold red]ERROR[/bold red]: Summary file not created for {case.slug}")
-                            cons.print(console_safe(log_tail(log_filepath)))
+                            cons.print(console_safe(bench_failure_report(log_filepath)))
                             cons.print(f"[bold red]      Expected: {summary_filepath}[/bold red]")
                             failed_cases.append(case.slug)
                             break

toolchain/mfc/gpu_diagnostics.py

@@ -0,0 +1,200 @@
+"""Offload-runtime diagnostics for GPU memory faults.
+
+Shared by the test harness, the benchmark runner and the case-optimization CI
+script -- all three run GPU cases and all three need the same answer when one
+faults. Kept out of test/ because bench.py depending on the test module to
+explain a crash would be the wrong way round.
+"""
+
+import collections
+import os
+import re
+import typing
+
+# The marker _handle_case attaches to the exception it raises, so that
+# classify_error can tell a GPU memory fault from any other execution failure.
+# Two constraints, both learned the hard way:
+#
+#   * It must be one of the signatures below verbatim, because classify_error
+#     recognises it by running the same matcher over the message. An earlier
+#     version wrote "[gpu-memory-fault]" while the reader searched for "memory
+#     access fault by gpu", so the two never matched and the feature was dead
+#     while seven source-inspecting tests passed.
+#   * No square brackets. main.py renders these messages through Rich, which
+#     parses "[...]" as a style tag and deletes it -- which is why a CI log
+#     showed a bare "Failed to execute MFC. " with the marker missing.
+GPU_FAULT_MARKER = "(memory access fault by GPU)"
+
+GPU_FAULT_SIGNATURES = (
+    # AMD/HSA -- Frontier, both CCE and AFAR builds.
+    "memory access fault by gpu",
+    "offload error: memory access fault",
+    # NVHPC -- Phoenix. Worded nothing like the AMD ones, so matching only the
+    # above meant 189 faults on a Phoenix gpu-acc shard were never recognised.
+    # Only the specific error: NVHPC prefixes unrelated failures with
+    # "Accelerator Fatal Error" too, including "call to cuMemAlloc returned
+    # error 2: Out of memory", which is not a memory fault and must not be
+    # classified as one.
+    "cuda_error_illegal_address",
+)
+
+
+def is_gpu_memory_fault(text: str) -> bool:
+    """Whether output shows a GPU memory fault, as opposed to any other failure.
+
+    Deliberately narrow. PMIX_ERR_NO_PERMISSIONS and friends appear in 16% of
+    *passing* self-hosted jobs, so anything broader would fire constantly.
+    """
+    lowered = (text or "").lower()
+
+    return any(sig in lowered for sig in GPU_FAULT_SIGNATURES)
+
+
+def fault_diagnostic_env(base: dict) -> dict:
+    """`base` plus the one diagnostic cheap enough to leave on.
+
+    Set on every run rather than on a retry: the ROCm debug agent writes nothing
+    until the runtime is already aborting on a memory fault, so a first failure
+    is explained without spending a second run to reproduce it.
+
+    Two variables that used to live here were removed after measurement -- see
+    below. What is left is the agent, which is what names the faulting kernel.
+    """
+    env = dict(base)
+
+    # OFFLOAD_TRACK_ALLOCATION_TRACES and OFFLOAD_TRACK_NUM_KERNEL_LAUNCH_TRACES
+    # were set here and had to be removed. They instrument every allocation and
+    # every kernel launch, so a healthy run pays continuously: on an MI210 with
+    # amdflang, test AFBCBDFA takes 5.94 s with neither and times out past 400 s
+    # with either one alone -- enough, against the 1-hour test timeout, to turn
+    # a fault into a timeout and hide what they exist to explain.
+    #
+    # They looked free only because the A/B that cleared them ran on CCE, whose
+    # offload runtime ignores libomptarget variables entirely. What they added,
+    # one line on whether the address was ever a real allocation, the agent's
+    # kernel name and source line subsume.
+
+    # Skipped when the caller is already debugging by hand: they chose a tool,
+    # or they set HSA_ENABLE_DEBUG to collect a GPU core dump, which the agent
+    # is mutually exclusive with. Loading it anyway would leave them with
+    # "Failed to enable debug interface" and no dump. An attached rocgdb trips
+    # the same path.
+    #
+    # Cost, Frontier CCE --gpu mp over four interleaved pairs: no effect
+    # detected on a healthy run (resolution ~0.8%), no output at all until
+    # something faults, and +0.387 s on a faulting run -- 0.011% of the test
+    # timeout. ~3-4% on an MI210 (n=2). It does not supersede libomptarget's own
+    # report on the AFAR lane; it is exclusive with ROCr core dumps only.
+    if "HSA_TOOLS_LIB" not in env and not env.get("HSA_ENABLE_DEBUG") and rocm_debug_agent_path() is not None:
+        env["HSA_TOOLS_LIB"] = ROCM_DEBUG_AGENT
+
+    return env
+
+
+ROCM_DEBUG_AGENT = "librocm-debug-agent.so.2"
+
+
+def rocm_debug_agent_path() -> typing.Optional[str]:
+    """Where the ROCm debug agent lives, or None if it is not reachable.
+
+    MUST be evaluated at call time, never cached at import. On Frontier the
+    library is on disk the whole time, but /opt/rocm-*/lib only reaches
+    LD_LIBRARY_PATH once `mfc.sh load` runs. A gate evaluated at import decides
+    "absent" on the one machine this exists for, and does it indistinguishably
+    from the Phoenix case where the library really is missing.
+
+    Probes for the file rather than dlopen'ing it: ctypes.CDLL would load a
+    debug agent into the test harness's own process to answer a question about
+    the subprocess.
+    """
+    rocm_path = os.environ.get("ROCM_PATH", "")
+    search = [os.path.join(rocm_path, "lib")] if rocm_path else []
+    search += os.environ.get("LD_LIBRARY_PATH", "").split(os.pathsep)
+
+    for directory in search:
+        if directory and os.path.isfile(os.path.join(directory, ROCM_DEBUG_AGENT)):
+            return os.path.join(directory, ROCM_DEBUG_AGENT)
+
+    return None
+
+
+def summarize_rocm_debug_agent(out: str, max_disasm: int = 14) -> str:
+    """Collapse librocm-debug-agent output to a bounded, informative summary.
+
+    The agent repeats an identical disassembly block and a 115-line register
+    dump per faulting wave -- 125 waves produced 14,635 lines on a 49x39 case.
+    Only the kernel name, fault reason, stop-PC distribution and one
+    representative wave carry information; the rest is duplicated.
+
+    A fixed tail cannot substitute. Measured on that log: the first 80 lines are
+    one wave's registers and the last 80 are another's, and the kernel name --
+    the entire point -- appears in neither. The stop-PC histogram is kept
+    because the waves halted at four distinct PCs whose modal one is a load
+    while the injected fault is a write, so quoting a single PC without the
+    distribution hands the reader the wrong instruction.
+
+    Returns '' when there is no agent report, so callers fall back to the raw
+    output.
+
+    The format is NOT stable across ROCm versions, and the failure is silent --
+    no wave match means an empty summary and a fallback to tens of thousands of
+    raw lines, with nothing saying why. Measured between two versions:
+
+        6.3.1  wave_124: pc=0x7ff77e253408 (stopped, reason: MEMORY_VIOLATION)
+        7.2.0  wave_250: pc=0x7ff734dcbf3c (kernel_code_entry=0x... <...>,
+                         kernargs=0x...) (stopped, reason: MEMORY_VIOLATION)
+
+        6.3.1  Memory access fault by GPU node-4 (Agent handle: ...) on address
+        7.2.0  OFFLOAD ERROR: memory access fault by GPU 4 (agent ...) at ...
+
+    An earlier version required pc= and "(stopped, reason:" to be adjacent and
+    matched the fault line case-sensitively on "Memory". It returned nothing at
+    all for 65,210 lines of real 7.2.0 output. Hence the tolerant separator, and
+    reusing is_gpu_memory_fault rather than hardcoding one version's wording.
+    Both formats are pinned by fixtures below.
+
+    Validated against three real reports, not one:
+
+        CCE  acc  ROCm 6.3.1   14,635 lines -> 37
+        CCE  mp   ROCm 6.3.1   13,826 lines -> 35
+        AFAR mp   ROCm 7.2.0   65,210 lines -> 36
+
+    and output from a run with no agent loaded still yields '', so the fallback
+    is intact. The stop-PC histogram earns its place most on the CCE OpenMP
+    lane, which halts at seven distinct PCs (62/21/19/10/10/2/1) against four
+    for CCE OpenACC and one for AFAR: quoting a single PC would be wrong there
+    six times in seven.
+    """
+    waves = re.findall(r"^wave_\d+: pc=(0x[0-9a-f]+).*?\(stopped, reason: (\w+)\)", out, re.M)
+    if not waves:
+        return ""
+
+    lines = out.splitlines()
+    fault = next((line for line in lines if is_gpu_memory_fault(line)), None)
+    kernels = sorted({m.group(1) for m in re.finditer(r"^Disassembly for function (.+):$", out, re.M)})
+    pcs = collections.Counter(pc for pc, _ in waves)
+    reasons = collections.Counter(reason for _, reason in waves)
+
+    summary = [f"=== GPU fault summary (rocm-debug-agent, {len(lines)} lines collapsed) ==="]
+    if fault:
+        summary.append(fault.strip())
+    summary.append("faulting kernel(s): " + (", ".join(kernels) or "<none reported>"))
+    summary.append(f"faulting waves:     {len(waves)}  [" + ", ".join(f"{r} x{n}" for r, n in reasons.most_common()) + "]")
+    summary.append("stop PCs:           " + ", ".join(f"{pc} x{n}" for pc, n in pcs.most_common()))
+    summary.append("NOTE: waves halt on fault detection, so the PC is near -- not necessarily at -- the offending instruction.")
+
+    disasm_starts = [n for n, line in enumerate(lines) if line.startswith("Disassembly for function")]
+    if disasm_starts:
+        start = disasm_starts[0]
+        end = next((n for n, line in enumerate(lines[start:], start) if line.startswith("End of disassembly")), start + max_disasm)
+        summary += ["", f"--- disassembly (1 of {len(disasm_starts)} identical blocks) ---"]
+        summary += lines[start : min(end + 1, start + max_disasm)]
+
+    modal_pc = pcs.most_common(1)[0][0]
+    start = next((n for n, line in enumerate(lines) if re.match(r"^wave_\d+: pc=" + re.escape(modal_pc) + r"(?![0-9a-f])", line)), None)
+    if start is not None:
+        summary += ["", f"--- representative wave (modal PC {modal_pc}, {pcs[modal_pc]} of {len(waves)} waves) ---"]
+        summary += lines[start : start + max_disasm]
+        summary.append(f"    ... (registers for {len(waves) - 1} further waves suppressed)")
+
+    return "\n".join(summary)