From 1cec3a7a8038d9256625f1896ec62fdce20bff86 Mon Sep 17 00:00:00 2001 From: Daniele Bagni <30289343+dannybaths@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:14:47 +0200 Subject: [PATCH] Fix profiler and TAU harness checks Make profiler and TAU tests assert real artifacts instead of matching diagnostic text, fail TAU builds explicitly, detect MPI by querying the wrapper that will actually be used, and check the rocprof-compute dependency pins before a long profiling run rather than after it. --- HIP/jacobi/Makefile | 16 ++- .../rocm-compute-profiler/single_process.sh | 33 ++++++ tests/CMakeLists.txt | 12 +- tests/rocprof-compute_analyze_check.sh | 8 ++ tests/rocprof-compute_profile_check.sh | 8 ++ tests/rocprof-compute_roofline_check.sh | 17 +++ tests/rocprof-sys_stream_overlap.sh | 25 ++++- tests/tau_exec_check.sh | 103 +++++++++++++++++- tests/tau_rccl_pc_check.sh | 16 ++- 9 files changed, 222 insertions(+), 16 deletions(-) diff --git a/HIP/jacobi/Makefile b/HIP/jacobi/Makefile index 18c830e4..bd74257b 100644 --- a/HIP/jacobi/Makefile +++ b/HIP/jacobi/Makefile @@ -27,7 +27,13 @@ ifneq ($(USE_CRAYPE),) MPICC=CC endif -IS_OMPI=$(shell which ompi_info 2>/dev/null) +# Ask the wrapper we are about to invoke. `which ompi_info` reports that some +# OpenMPI is installed, which is not the question: with MPICH wrappers in front +# (any MPICH-built module can pull them in) this branch asked MPICH's mpic++ for +# --showme:compile, got the flag forwarded to the compiler as an unknown +# argument, and left both flag sets empty -- so the link had no MPI library and +# produced a page of undefined symbols. +IS_OMPI=$(shell ${MPICC} --showme:version >/dev/null 2>&1 && echo openmpi) # MPICH detection. The Cray cc/CC wrapper prints "CrayPE is loaded" (matched by # 'Cray' below). The standalone mpich-wrappers MPICH on a Cray, however, reports # the underlying compiler from `--version` (e.g. "AMD clang"), so grepping the @@ -58,6 +64,14 @@ else $(error Unknown MPI version! Currently can detect mpich or openmpi) endif +# MPILD is hipcc, not the MPI wrapper, so the MPI library only reaches the link +# line through MPILDFLAGS. Empty flags therefore produce a page of undefined MPI +# symbols that names no cause. Fail here instead, while the reason is still +# visible. +ifeq ($(strip $(MPILDFLAGS)),) + $(error MPI detected but $(strip ${MPICC}) returned no link flags. Two MPIs on PATH is the usual cause: check that mpic++ and any MPI-linked module you loaded are the same MPI) +endif + # Flags CFLAGS=-O3 -g -ggdb -fPIC -std=c++11 -march=native -Wall CFLAGS+=-I$(ROCM_PATH)/roctracer/include -I${ROCM_PATH}/include/roctracer diff --git a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/single_process.sh b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/single_process.sh index ec2b12b1..01ef0dbd 100755 --- a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/single_process.sh +++ b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/single_process.sh @@ -7,6 +7,39 @@ PROFILER_TOP_DIR="$(dirname "$(dirname "$(readlink -fm "$0")")")" # Call the software set up script: source ${PROFILER_TOP_DIR}/setup.sh +# Dependency precondition. `analyze` enforces the exact pins in its own +# requirements.txt and this run ends in `analyze`, so verify them before +# spending around 25 minutes profiling. Fail fast rather than time out. +_rpc_bin="$(command -v rocprof-compute 2>/dev/null)" +if [ -n "${_rpc_bin}" ]; then + _rpc_req="$(dirname "$(dirname "${_rpc_bin}")")/libexec/rocprofiler-compute/requirements.txt" + if [ -f "${_rpc_req}" ] && ! python3 - "${_rpc_req}" <<'PINS' +import sys +from importlib.metadata import version, PackageNotFoundError +bad = [] +for raw in open(sys.argv[1]): + raw = raw.split('#')[0].strip() + if not raw or '==' not in raw: + continue + name, want = raw.split('==', 1) + try: + have = version(name) + except PackageNotFoundError: + bad.append('%s: absent (needs %s)' % (name, want)); continue + if have != want: + bad.append('%s: %s installed, needs %s' % (name, have, want)) +for b in bad: + print(' ' + b) +sys.exit(1 if bad else 0) +PINS + then + echo "rocprof-compute dependency pins are unsatisfied under $(python3 -V 2>&1)." + echo "'analyze' cannot run, so skipping the profile run instead of timing out." + echo "See ${_rpc_req} for the required versions." + exit 1 + fi +fi + export NPROCS=1 pushd ${PROFILER_TOP_DIR} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9a2302fa..cf5cc3d7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1696,11 +1696,11 @@ set_property(TEST Rocprof-sys_ROCm_Run_Check PROPERTY PASS_REGULAR_EXPRESSION "T set_property(TEST Rocprof-sys_ROCm_Run_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider") add_test(NAME Rocprof-sys_ROCm_Stream_Overlap COMMAND ../rocprof-sys_stream_overlap.sh ) -set_property(TEST Rocprof-sys_ROCm_Stream_Overlap PROPERTY PASS_REGULAR_EXPRESSION "proto") +set_property(TEST Rocprof-sys_ROCm_Stream_Overlap PROPERTY PASS_REGULAR_EXPRESSION "ROCPROFSYS_STREAM_OVERLAP_RESULT: PASS") set_property(TEST Rocprof-sys_ROCm_Stream_Overlap PROPERTY SKIP_REGULAR_EXPRESSION "module spider") add_test(NAME Rocprof-compute_ROCm_Roofline_Check COMMAND ../rocprof-compute_roofline_check.sh) -set_property(TEST Rocprof-compute_ROCm_Roofline_Check PROPERTY PASS_REGULAR_EXPRESSION "Empirical Roofline .* saved!") +set_property(TEST Rocprof-compute_ROCm_Roofline_Check PROPERTY PASS_REGULAR_EXPRESSION "ROOFLINE_RESULT: PASS") set_property(TEST Rocprof-compute_ROCm_Roofline_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider" "disabled on MI300") set_tests_properties(Rocprof-compute_ROCm_Roofline_Check PROPERTIES TIMEOUT 600) @@ -1756,19 +1756,19 @@ set_property(TEST JAX_Mnist PROPERTY SKIP_REGULAR_EXPRESSION "module spider" "Un # TAU Tests add_test(NAME TAU_Trace_Check COMMAND ../tau_exec_check.sh --tau-trace) -set_property(TEST TAU_Trace_Check PROPERTY PASS_REGULAR_EXPRESSION "tautrace.0") +set_property(TEST TAU_Trace_Check PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: trace artifact PRESENT") set_property(TEST TAU_Trace_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'") add_test(NAME TAU_Profile_Check COMMAND ../tau_exec_check.sh --tau-profile) -set_property(TEST TAU_Profile_Check PROPERTY PASS_REGULAR_EXPRESSION "profile.0") +set_property(TEST TAU_Profile_Check PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: profile artifact PRESENT") set_property(TEST TAU_Profile_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'") add_test(NAME TAU_Check_HIP_Profile COMMAND ../tau_exec_check.sh --tau-profile) -set_property(TEST TAU_Check_HIP_Profile PROPERTY PASS_REGULAR_EXPRESSION "hipMemcpy") +set_property(TEST TAU_Check_HIP_Profile PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: HIP routines PRESENT in profile") set_property(TEST TAU_Check_HIP_Profile PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'") add_test(NAME TAU_Check_MPI_Profile COMMAND ../tau_exec_check.sh --tau-profile) -set_property(TEST TAU_Check_MPI_Profile PROPERTY PASS_REGULAR_EXPRESSION "MPI_Allreduce()") +set_property(TEST TAU_Check_MPI_Profile PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: MPI routines PRESENT in profile") set_property(TEST TAU_Check_MPI_Profile PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'") set_property(TEST TAU_Check_MPI_Profile PROPERTY TIMEOUT 600) diff --git a/tests/rocprof-compute_analyze_check.sh b/tests/rocprof-compute_analyze_check.sh index c88cfe42..8e79e83c 100755 --- a/tests/rocprof-compute_analyze_check.sh +++ b/tests/rocprof-compute_analyze_check.sh @@ -22,5 +22,13 @@ cmake .. make export HSA_XNACK=1 +# rocprof-compute needs Python >= 3.10; native_tool_finder.py uses PEP 604 +# unions. Fail here, naming the reason, rather than later with a SyntaxError +# from inside the tool that names no cause. +if ! python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3,10) else 1)' 2>/dev/null; then + echo "ERROR: rocprof-compute needs Python >= 3.10, but python3 is $(python3 -V 2>&1)." + echo "ERROR: load a newer Python before running this test." + exit 1 +fi rocprof-compute profile -n v1 --no-roof -- ./saxpy rocprof-compute analyze -p workloads/v1/* --block 7.1.0 7.1.1 7.1.2 7.1.0: Grid size 7.1.1: Workgroup size 7.1.2: Total Wavefronts diff --git a/tests/rocprof-compute_profile_check.sh b/tests/rocprof-compute_profile_check.sh index 795261e4..ec793317 100755 --- a/tests/rocprof-compute_profile_check.sh +++ b/tests/rocprof-compute_profile_check.sh @@ -23,4 +23,12 @@ cmake .. make export HSA_XNACK=1 +# rocprof-compute needs Python >= 3.10; native_tool_finder.py uses PEP 604 +# unions. Fail here, naming the reason, rather than later with a SyntaxError +# from inside the tool that names no cause. +if ! python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3,10) else 1)' 2>/dev/null; then + echo "ERROR: rocprof-compute needs Python >= 3.10, but python3 is $(python3 -V 2>&1)." + echo "ERROR: load a newer Python before running this test." + exit 1 +fi rocprof-compute profile -n v1 --no-roof -- ./saxpy diff --git a/tests/rocprof-compute_roofline_check.sh b/tests/rocprof-compute_roofline_check.sh index 0572bacb..819cf857 100755 --- a/tests/rocprof-compute_roofline_check.sh +++ b/tests/rocprof-compute_roofline_check.sh @@ -30,4 +30,21 @@ cmake .. make export HSA_XNACK=1 +# rocprof-compute needs Python >= 3.10; native_tool_finder.py uses PEP 604 +# unions. Fail here, naming the reason, rather than later with a SyntaxError +# from inside the tool that names no cause. +if ! python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3,10) else 1)' 2>/dev/null; then + echo "ERROR: rocprof-compute needs Python >= 3.10, but python3 is $(python3 -V 2>&1)." + echo "ERROR: load a newer Python before running this test." + exit 1 +fi rocprof-compute profile -n rooflines_PDF --roof-only -- ./saxpy + +# Assert the roofline artifact exists rather than matching the tool's log +# wording, which changed when omniperf became rocprofiler-compute. +_roofline_csv="$(find . -name roofline.csv -size +0 2>/dev/null | head -1)" +if [ -n "${_roofline_csv}" ]; then + echo "ROOFLINE_RESULT: PASS roofline data at ${_roofline_csv}" +else + echo "ROOFLINE_RESULT: FAIL no non-empty roofline.csv produced" +fi diff --git a/tests/rocprof-sys_stream_overlap.sh b/tests/rocprof-sys_stream_overlap.sh index 382f9548..31464427 100755 --- a/tests/rocprof-sys_stream_overlap.sh +++ b/tests/rocprof-sys_stream_overlap.sh @@ -126,9 +126,28 @@ ${TOOL_COMMAND}-avail -G $PWD/.configure.cfg export ${TOOL_CONFIG}_CONFIG_FILE=$PWD/.configure.cfg ${TOOL_COMMAND}-instrument -o compute_comm_overlap.inst -- compute_comm_overlap ${TOOL_COMMAND}-run -- ./compute_comm_overlap.inst 2 -cd ${TOOL_OUTPUT}-compute_comm_overlap.inst-output/ -ls * - +# Assert a real trace artifact. The tool writes +# ${TOOL_NAME}-compute_comm_overlap.inst-output (rocprofiler-systems-... on +# ROCm > 6.2.9, omnitrace-... before it), NOT ${TOOL_OUTPUT}-... Glob for the +# suffix so both eras work, and require a real .proto inside it. +assert_proto_output() { + local outdir n + outdir="$(ls -d ./*-compute_comm_overlap.inst-output 2>/dev/null | head -1)" + if [[ -z "${outdir}" || ! -d "${outdir}" ]]; then + echo "ROCPROFSYS_STREAM_OVERLAP_RESULT: FAIL no *-compute_comm_overlap.inst-output directory produced" + return 1 + fi + # The tool nests a timestamped subdirectory, so recurse. + n="$(find "${outdir}" -type f -name '*.proto' | wc -l)" + find "${outdir}" -type f | sed 's/^/ /' | head -20 + if [[ "${n}" -gt 0 ]]; then + echo "ROCPROFSYS_STREAM_OVERLAP_RESULT: PASS ${n} proto file(s) under ${outdir}" + else + echo "ROCPROFSYS_STREAM_OVERLAP_RESULT: FAIL 0 proto files under ${outdir}" + return 1 + fi +} +assert_proto_output cd .. rm -rf ${BUILD_DIR} diff --git a/tests/tau_exec_check.sh b/tests/tau_exec_check.sh index 7fa8cc93..1b493919 100755 --- a/tests/tau_exec_check.sh +++ b/tests/tau_exec_check.sh @@ -84,10 +84,23 @@ export TAU_PROFILE=${TAU_PROFILE} export TAU_TRACE=${TAU_TRACE} WORKDIR=$(mktemp -d -p ${SRCDIR} build_XXXXXX) +# Install the cleanup trap before the build gate below, which can exit early. +trap 'cd "${SRCDIR}" && rm -rf "${WORKDIR}"' EXIT cp ${SRCDIR}/*.hip ${SRCDIR}/*.hpp ${SRCDIR}/*.h ${SRCDIR}/Makefile ${SRCDIR}/input.txt ${WORKDIR}/ cd ${WORKDIR} -make +# A failed build has to end the test. Without this gate make's status was +# ignored, tau_exec profiled a binary that was never produced, and the script +# still exited 0 because cleanup was the last command -- leaving the whole +# verdict to the pass regex. +if ! make; then + echo "TAU_CHECK: build FAILED, so nothing was profiled" + exit 1 +fi +if [ ! -x ./Jacobi_hip ]; then + echo "TAU_CHECK: build reported success but ./Jacobi_hip is missing" + exit 1 +fi ROCM_VERSION=`cat ${ROCM_PATH}/.info/version | head -1 | cut -f1 -d'-' ` @@ -117,10 +130,55 @@ elif [ -n "${MPI_BINDIR}" ] && [ -x "${MPI_BINDIR}/mpiexec" ]; then else MPI_LAUNCH="srun -n 2" fi -if command -v ompi_info >/dev/null 2>&1; then +# Ask the wrapper we are about to invoke, not the PATH. `command -v ompi_info` +# answers "is some OpenMPI installed", which is a different question. An +# MPICH-built TAU module makes the two disagree: loading it pulls in +# mpich-wrappers, so mpicc/mpic++/mpiexec become MPICH's while ompi_info still +# reports the OpenMPI that is no longer in front. --showme:version succeeds only +# on OpenMPI and -compile_info only on MPICH, so each wrapper answers for itself. +MPI_FAMILY=unknown +if [ -n "${MPI_BINDIR}" ]; then + if "${MPI_BINDIR}/mpicc" --showme:version >/dev/null 2>&1; then + MPI_FAMILY=openmpi + elif "${MPI_BINDIR}/mpicc" -compile_info >/dev/null 2>&1; then + MPI_FAMILY=mpich + fi +fi +echo "TAU_CHECK: launching with ${MPI_FAMILY} wrappers from ${MPI_BINDIR:-}" +if [ "${MPI_FAMILY}" = "openmpi" ]; then MPI_LAUNCH="${MPI_LAUNCH} --oversubscribe" fi +# Keep both ranks on the node that owns the build directory. Jacobi is compiled +# into a mktemp directory inside this checkout, so if the checkout lives on +# node-local storage rather than a shared filesystem, a multi-node allocation +# lets the launcher place rank 1 on a node where that path does not exist. The +# run then dies before the test body starts: +# +# error: couldn't chdir to `/tmp/.../HIP/jacobi/build_ewLCPv': No such file or +# directory: going to /tmp instead +# [proxy:1@node2] launch_procs: unable to change wdir to /tmp/.../build_ewLCPv +# +# The test wants two ranks, not two nodes -- Jacobi's "-g 2 1" topology on two +# GPUs of one node -- so pinning costs no coverage. Hydra takes -hosts; OpenMPI +# takes --host with a slot count. Measured on two nodes: bare mpirun aborts as +# above, both pinned forms put rank 0 and rank 1 on the same node. +# +# The srun fallback is left alone deliberately: it is reached only for a bare +# MPICH with no co-located launcher, and the obvious addition there +# (--nodes=1 --nodelist=...) needs the step's GRES respecified or Slurm rejects +# the step outright, so a blind flag would trade a rare failure for a common one. +MPI_HOST_LOCAL="$(hostname -s)" +case "${MPI_LAUNCH}" in + srun*) ;; + *) case "${MPI_FAMILY}" in + openmpi) MPI_LAUNCH="${MPI_LAUNCH} --host ${MPI_HOST_LOCAL}:2" ;; + mpich) MPI_LAUNCH="${MPI_LAUNCH} -hosts ${MPI_HOST_LOCAL}" ;; + *) echo "TAU_CHECK: unknown MPI family, not pinning ranks to ${MPI_HOST_LOCAL}" ;; + esac ;; +esac +echo "TAU_CHECK: launcher is ${MPI_LAUNCH}" + # Use a 1024x1024 local mesh so the TAU trace buffer fits in GPU memory # (default 4096x4096 caused "HIP failure: 'out of memory'" during trace finalization). if [[ "${result}" ]]; then @@ -130,8 +188,43 @@ else fi ls -pprof +pprof 2>&1 | tee pprof.out + +# Assert the artifacts. Matching the tool's own vocabulary is what made these +# tests unable to fail: "profile.0" is a substring of "Could not open +# profile.0.0.0". Assert the artifact and the profiled routines instead, in +# strings the failure paths cannot produce, and let CMake match these. +tau_have() { ls $1 >/dev/null 2>&1; } +tau_status=0 + +if [ "${TAU_PROFILE}" = "1" ]; then + if tau_have 'profile.*'; then + echo "TAU_CHECK: profile artifact PRESENT" + else + echo "TAU_CHECK: profile artifact absent" + tau_status=1 + fi + # Reported but not fatal on their own: each is a different test's subject, so + # the per-test expression decides rather than failing all of them together. + if grep -q 'MPI_Allreduce' pprof.out; then + echo "TAU_CHECK: MPI routines PRESENT in profile" + else + echo "TAU_CHECK: MPI routines absent from profile" + fi + if grep -q 'hipMemcpy' pprof.out; then + echo "TAU_CHECK: HIP routines PRESENT in profile" + else + echo "TAU_CHECK: HIP routines absent from profile" + fi +fi -cd .. -rm -rf ${WORKDIR} +if [ "${TAU_TRACE}" = "1" ]; then + if tau_have 'tautrace.*'; then + echo "TAU_CHECK: trace artifact PRESENT" + else + echo "TAU_CHECK: trace artifact absent" + tau_status=1 + fi +fi +exit ${tau_status} diff --git a/tests/tau_rccl_pc_check.sh b/tests/tau_rccl_pc_check.sh index 4e8dcbe2..88d3a335 100755 --- a/tests/tau_rccl_pc_check.sh +++ b/tests/tau_rccl_pc_check.sh @@ -113,7 +113,21 @@ elif [ -n "${MPI_BINDIR}" ] && [ -x "${MPI_BINDIR}/mpiexec" ]; then else MPI_LAUNCH="srun -n 2" fi -if command -v ompi_info >/dev/null 2>&1; then +# Ask the wrapper we are about to invoke; see tau_exec_check.sh for the detail. +# Testing whether ompi_info exists added OpenMPI's --oversubscribe to MPICH's +# hydra mpiexec, which rejects it. Note this does not by itself make the test +# pass where TAU and the prebuilt rccl-tests binary come from different MPIs: +# an MPICH-built TAU cannot instrument an OpenMPI-linked benchmark. +MPI_FAMILY=unknown +if [ -n "${MPI_BINDIR}" ]; then + if "${MPI_BINDIR}/mpicc" --showme:version >/dev/null 2>&1; then + MPI_FAMILY=openmpi + elif "${MPI_BINDIR}/mpicc" -compile_info >/dev/null 2>&1; then + MPI_FAMILY=mpich + fi +fi +echo "MPI family of ${MPI_BINDIR:-}: ${MPI_FAMILY}" +if [ "${MPI_FAMILY}" = "openmpi" ]; then MPI_LAUNCH="${MPI_LAUNCH} --oversubscribe" fi