diff --git a/HIP/jacobi/Makefile b/HIP/jacobi/Makefile index 18c830e4..bd74257b 100644 --- a/HIP/jacobi/Makefile +++ b/HIP/jacobi/Makefile @@ -27,7 +27,13 @@ ifneq ($(USE_CRAYPE),) MPICC=CC endif -IS_OMPI=$(shell which ompi_info 2>/dev/null) +# Ask the wrapper we are about to invoke. `which ompi_info` reports that some +# OpenMPI is installed, which is not the question: with MPICH wrappers in front +# (any MPICH-built module can pull them in) this branch asked MPICH's mpic++ for +# --showme:compile, got the flag forwarded to the compiler as an unknown +# argument, and left both flag sets empty -- so the link had no MPI library and +# produced a page of undefined symbols. +IS_OMPI=$(shell ${MPICC} --showme:version >/dev/null 2>&1 && echo openmpi) # MPICH detection. The Cray cc/CC wrapper prints "CrayPE is loaded" (matched by # 'Cray' below). The standalone mpich-wrappers MPICH on a Cray, however, reports # the underlying compiler from `--version` (e.g. "AMD clang"), so grepping the @@ -58,6 +64,14 @@ else $(error Unknown MPI version! Currently can detect mpich or openmpi) endif +# MPILD is hipcc, not the MPI wrapper, so the MPI library only reaches the link +# line through MPILDFLAGS. Empty flags therefore produce a page of undefined MPI +# symbols that names no cause. Fail here instead, while the reason is still +# visible. +ifeq ($(strip $(MPILDFLAGS)),) + $(error MPI detected but $(strip ${MPICC}) returned no link flags. Two MPIs on PATH is the usual cause: check that mpic++ and any MPI-linked module you loaded are the same MPI) +endif + # Flags CFLAGS=-O3 -g -ggdb -fPIC -std=c++11 -march=native -Wall CFLAGS+=-I$(ROCM_PATH)/roctracer/include -I${ROCM_PATH}/include/roctracer diff --git a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/single_process.sh b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/single_process.sh index ec2b12b1..01ef0dbd 100755 --- a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/single_process.sh +++ b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/single_process.sh @@ -7,6 +7,39 @@ PROFILER_TOP_DIR="$(dirname "$(dirname "$(readlink -fm "$0")")")" # Call the software set up script: source ${PROFILER_TOP_DIR}/setup.sh +# Dependency precondition. `analyze` enforces the exact pins in its own +# requirements.txt and this run ends in `analyze`, so verify them before +# spending around 25 minutes profiling. Fail fast rather than time out. +_rpc_bin="$(command -v rocprof-compute 2>/dev/null)" +if [ -n "${_rpc_bin}" ]; then + _rpc_req="$(dirname "$(dirname "${_rpc_bin}")")/libexec/rocprofiler-compute/requirements.txt" + if [ -f "${_rpc_req}" ] && ! python3 - "${_rpc_req}" <<'PINS' +import sys +from importlib.metadata import version, PackageNotFoundError +bad = [] +for raw in open(sys.argv[1]): + raw = raw.split('#')[0].strip() + if not raw or '==' not in raw: + continue + name, want = raw.split('==', 1) + try: + have = version(name) + except PackageNotFoundError: + bad.append('%s: absent (needs %s)' % (name, want)); continue + if have != want: + bad.append('%s: %s installed, needs %s' % (name, have, want)) +for b in bad: + print(' ' + b) +sys.exit(1 if bad else 0) +PINS + then + echo "rocprof-compute dependency pins are unsatisfied under $(python3 -V 2>&1)." + echo "'analyze' cannot run, so skipping the profile run instead of timing out." + echo "See ${_rpc_req} for the required versions." + exit 1 + fi +fi + export NPROCS=1 pushd ${PROFILER_TOP_DIR} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9a2302fa..cf5cc3d7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1696,11 +1696,11 @@ set_property(TEST Rocprof-sys_ROCm_Run_Check PROPERTY PASS_REGULAR_EXPRESSION "T set_property(TEST Rocprof-sys_ROCm_Run_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider") add_test(NAME Rocprof-sys_ROCm_Stream_Overlap COMMAND ../rocprof-sys_stream_overlap.sh ) -set_property(TEST Rocprof-sys_ROCm_Stream_Overlap PROPERTY PASS_REGULAR_EXPRESSION "proto") +set_property(TEST Rocprof-sys_ROCm_Stream_Overlap PROPERTY PASS_REGULAR_EXPRESSION "ROCPROFSYS_STREAM_OVERLAP_RESULT: PASS") set_property(TEST Rocprof-sys_ROCm_Stream_Overlap PROPERTY SKIP_REGULAR_EXPRESSION "module spider") add_test(NAME Rocprof-compute_ROCm_Roofline_Check COMMAND ../rocprof-compute_roofline_check.sh) -set_property(TEST Rocprof-compute_ROCm_Roofline_Check PROPERTY PASS_REGULAR_EXPRESSION "Empirical Roofline .* saved!") +set_property(TEST Rocprof-compute_ROCm_Roofline_Check PROPERTY PASS_REGULAR_EXPRESSION "ROOFLINE_RESULT: PASS") set_property(TEST Rocprof-compute_ROCm_Roofline_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider" "disabled on MI300") set_tests_properties(Rocprof-compute_ROCm_Roofline_Check PROPERTIES TIMEOUT 600) @@ -1756,19 +1756,19 @@ set_property(TEST JAX_Mnist PROPERTY SKIP_REGULAR_EXPRESSION "module spider" "Un # TAU Tests add_test(NAME TAU_Trace_Check COMMAND ../tau_exec_check.sh --tau-trace) -set_property(TEST TAU_Trace_Check PROPERTY PASS_REGULAR_EXPRESSION "tautrace.0") +set_property(TEST TAU_Trace_Check PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: trace artifact PRESENT") set_property(TEST TAU_Trace_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'") add_test(NAME TAU_Profile_Check COMMAND ../tau_exec_check.sh --tau-profile) -set_property(TEST TAU_Profile_Check PROPERTY PASS_REGULAR_EXPRESSION "profile.0") +set_property(TEST TAU_Profile_Check PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: profile artifact PRESENT") set_property(TEST TAU_Profile_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'") add_test(NAME TAU_Check_HIP_Profile COMMAND ../tau_exec_check.sh --tau-profile) -set_property(TEST TAU_Check_HIP_Profile PROPERTY PASS_REGULAR_EXPRESSION "hipMemcpy") +set_property(TEST TAU_Check_HIP_Profile PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: HIP routines PRESENT in profile") set_property(TEST TAU_Check_HIP_Profile PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'") add_test(NAME TAU_Check_MPI_Profile COMMAND ../tau_exec_check.sh --tau-profile) -set_property(TEST TAU_Check_MPI_Profile PROPERTY PASS_REGULAR_EXPRESSION "MPI_Allreduce()") +set_property(TEST TAU_Check_MPI_Profile PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: MPI routines PRESENT in profile") set_property(TEST TAU_Check_MPI_Profile PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'") set_property(TEST TAU_Check_MPI_Profile PROPERTY TIMEOUT 600) diff --git a/tests/rocprof-compute_analyze_check.sh b/tests/rocprof-compute_analyze_check.sh index c88cfe42..8e79e83c 100755 --- a/tests/rocprof-compute_analyze_check.sh +++ b/tests/rocprof-compute_analyze_check.sh @@ -22,5 +22,13 @@ cmake .. make export HSA_XNACK=1 +# rocprof-compute needs Python >= 3.10; native_tool_finder.py uses PEP 604 +# unions. Fail here, naming the reason, rather than later with a SyntaxError +# from inside the tool that names no cause. +if ! python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3,10) else 1)' 2>/dev/null; then + echo "ERROR: rocprof-compute needs Python >= 3.10, but python3 is $(python3 -V 2>&1)." + echo "ERROR: load a newer Python before running this test." + exit 1 +fi rocprof-compute profile -n v1 --no-roof -- ./saxpy rocprof-compute analyze -p workloads/v1/* --block 7.1.0 7.1.1 7.1.2 7.1.0: Grid size 7.1.1: Workgroup size 7.1.2: Total Wavefronts diff --git a/tests/rocprof-compute_profile_check.sh b/tests/rocprof-compute_profile_check.sh index 795261e4..ec793317 100755 --- a/tests/rocprof-compute_profile_check.sh +++ b/tests/rocprof-compute_profile_check.sh @@ -23,4 +23,12 @@ cmake .. make export HSA_XNACK=1 +# rocprof-compute needs Python >= 3.10; native_tool_finder.py uses PEP 604 +# unions. Fail here, naming the reason, rather than later with a SyntaxError +# from inside the tool that names no cause. +if ! python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3,10) else 1)' 2>/dev/null; then + echo "ERROR: rocprof-compute needs Python >= 3.10, but python3 is $(python3 -V 2>&1)." + echo "ERROR: load a newer Python before running this test." + exit 1 +fi rocprof-compute profile -n v1 --no-roof -- ./saxpy diff --git a/tests/rocprof-compute_roofline_check.sh b/tests/rocprof-compute_roofline_check.sh index 0572bacb..819cf857 100755 --- a/tests/rocprof-compute_roofline_check.sh +++ b/tests/rocprof-compute_roofline_check.sh @@ -30,4 +30,21 @@ cmake .. make export HSA_XNACK=1 +# rocprof-compute needs Python >= 3.10; native_tool_finder.py uses PEP 604 +# unions. Fail here, naming the reason, rather than later with a SyntaxError +# from inside the tool that names no cause. +if ! python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3,10) else 1)' 2>/dev/null; then + echo "ERROR: rocprof-compute needs Python >= 3.10, but python3 is $(python3 -V 2>&1)." + echo "ERROR: load a newer Python before running this test." + exit 1 +fi rocprof-compute profile -n rooflines_PDF --roof-only -- ./saxpy + +# Assert the roofline artifact exists rather than matching the tool's log +# wording, which changed when omniperf became rocprofiler-compute. +_roofline_csv="$(find . -name roofline.csv -size +0 2>/dev/null | head -1)" +if [ -n "${_roofline_csv}" ]; then + echo "ROOFLINE_RESULT: PASS roofline data at ${_roofline_csv}" +else + echo "ROOFLINE_RESULT: FAIL no non-empty roofline.csv produced" +fi diff --git a/tests/rocprof-sys_stream_overlap.sh b/tests/rocprof-sys_stream_overlap.sh index 382f9548..31464427 100755 --- a/tests/rocprof-sys_stream_overlap.sh +++ b/tests/rocprof-sys_stream_overlap.sh @@ -126,9 +126,28 @@ ${TOOL_COMMAND}-avail -G $PWD/.configure.cfg export ${TOOL_CONFIG}_CONFIG_FILE=$PWD/.configure.cfg ${TOOL_COMMAND}-instrument -o compute_comm_overlap.inst -- compute_comm_overlap ${TOOL_COMMAND}-run -- ./compute_comm_overlap.inst 2 -cd ${TOOL_OUTPUT}-compute_comm_overlap.inst-output/ -ls * - +# Assert a real trace artifact. The tool writes +# ${TOOL_NAME}-compute_comm_overlap.inst-output (rocprofiler-systems-... on +# ROCm > 6.2.9, omnitrace-... before it), NOT ${TOOL_OUTPUT}-... Glob for the +# suffix so both eras work, and require a real .proto inside it. +assert_proto_output() { + local outdir n + outdir="$(ls -d ./*-compute_comm_overlap.inst-output 2>/dev/null | head -1)" + if [[ -z "${outdir}" || ! -d "${outdir}" ]]; then + echo "ROCPROFSYS_STREAM_OVERLAP_RESULT: FAIL no *-compute_comm_overlap.inst-output directory produced" + return 1 + fi + # The tool nests a timestamped subdirectory, so recurse. + n="$(find "${outdir}" -type f -name '*.proto' | wc -l)" + find "${outdir}" -type f | sed 's/^/ /' | head -20 + if [[ "${n}" -gt 0 ]]; then + echo "ROCPROFSYS_STREAM_OVERLAP_RESULT: PASS ${n} proto file(s) under ${outdir}" + else + echo "ROCPROFSYS_STREAM_OVERLAP_RESULT: FAIL 0 proto files under ${outdir}" + return 1 + fi +} +assert_proto_output cd .. rm -rf ${BUILD_DIR} diff --git a/tests/tau_exec_check.sh b/tests/tau_exec_check.sh index 7fa8cc93..1b493919 100755 --- a/tests/tau_exec_check.sh +++ b/tests/tau_exec_check.sh @@ -84,10 +84,23 @@ export TAU_PROFILE=${TAU_PROFILE} export TAU_TRACE=${TAU_TRACE} WORKDIR=$(mktemp -d -p ${SRCDIR} build_XXXXXX) +# Install the cleanup trap before the build gate below, which can exit early. +trap 'cd "${SRCDIR}" && rm -rf "${WORKDIR}"' EXIT cp ${SRCDIR}/*.hip ${SRCDIR}/*.hpp ${SRCDIR}/*.h ${SRCDIR}/Makefile ${SRCDIR}/input.txt ${WORKDIR}/ cd ${WORKDIR} -make +# A failed build has to end the test. Without this gate make's status was +# ignored, tau_exec profiled a binary that was never produced, and the script +# still exited 0 because cleanup was the last command -- leaving the whole +# verdict to the pass regex. +if ! make; then + echo "TAU_CHECK: build FAILED, so nothing was profiled" + exit 1 +fi +if [ ! -x ./Jacobi_hip ]; then + echo "TAU_CHECK: build reported success but ./Jacobi_hip is missing" + exit 1 +fi ROCM_VERSION=`cat ${ROCM_PATH}/.info/version | head -1 | cut -f1 -d'-' ` @@ -117,10 +130,55 @@ elif [ -n "${MPI_BINDIR}" ] && [ -x "${MPI_BINDIR}/mpiexec" ]; then else MPI_LAUNCH="srun -n 2" fi -if command -v ompi_info >/dev/null 2>&1; then +# Ask the wrapper we are about to invoke, not the PATH. `command -v ompi_info` +# answers "is some OpenMPI installed", which is a different question. An +# MPICH-built TAU module makes the two disagree: loading it pulls in +# mpich-wrappers, so mpicc/mpic++/mpiexec become MPICH's while ompi_info still +# reports the OpenMPI that is no longer in front. --showme:version succeeds only +# on OpenMPI and -compile_info only on MPICH, so each wrapper answers for itself. +MPI_FAMILY=unknown +if [ -n "${MPI_BINDIR}" ]; then + if "${MPI_BINDIR}/mpicc" --showme:version >/dev/null 2>&1; then + MPI_FAMILY=openmpi + elif "${MPI_BINDIR}/mpicc" -compile_info >/dev/null 2>&1; then + MPI_FAMILY=mpich + fi +fi +echo "TAU_CHECK: launching with ${MPI_FAMILY} wrappers from ${MPI_BINDIR:-}" +if [ "${MPI_FAMILY}" = "openmpi" ]; then MPI_LAUNCH="${MPI_LAUNCH} --oversubscribe" fi +# Keep both ranks on the node that owns the build directory. Jacobi is compiled +# into a mktemp directory inside this checkout, so if the checkout lives on +# node-local storage rather than a shared filesystem, a multi-node allocation +# lets the launcher place rank 1 on a node where that path does not exist. The +# run then dies before the test body starts: +# +# error: couldn't chdir to `/tmp/.../HIP/jacobi/build_ewLCPv': No such file or +# directory: going to /tmp instead +# [proxy:1@node2] launch_procs: unable to change wdir to /tmp/.../build_ewLCPv +# +# The test wants two ranks, not two nodes -- Jacobi's "-g 2 1" topology on two +# GPUs of one node -- so pinning costs no coverage. Hydra takes -hosts; OpenMPI +# takes --host with a slot count. Measured on two nodes: bare mpirun aborts as +# above, both pinned forms put rank 0 and rank 1 on the same node. +# +# The srun fallback is left alone deliberately: it is reached only for a bare +# MPICH with no co-located launcher, and the obvious addition there +# (--nodes=1 --nodelist=...) needs the step's GRES respecified or Slurm rejects +# the step outright, so a blind flag would trade a rare failure for a common one. +MPI_HOST_LOCAL="$(hostname -s)" +case "${MPI_LAUNCH}" in + srun*) ;; + *) case "${MPI_FAMILY}" in + openmpi) MPI_LAUNCH="${MPI_LAUNCH} --host ${MPI_HOST_LOCAL}:2" ;; + mpich) MPI_LAUNCH="${MPI_LAUNCH} -hosts ${MPI_HOST_LOCAL}" ;; + *) echo "TAU_CHECK: unknown MPI family, not pinning ranks to ${MPI_HOST_LOCAL}" ;; + esac ;; +esac +echo "TAU_CHECK: launcher is ${MPI_LAUNCH}" + # Use a 1024x1024 local mesh so the TAU trace buffer fits in GPU memory # (default 4096x4096 caused "HIP failure: 'out of memory'" during trace finalization). if [[ "${result}" ]]; then @@ -130,8 +188,43 @@ else fi ls -pprof +pprof 2>&1 | tee pprof.out + +# Assert the artifacts. Matching the tool's own vocabulary is what made these +# tests unable to fail: "profile.0" is a substring of "Could not open +# profile.0.0.0". Assert the artifact and the profiled routines instead, in +# strings the failure paths cannot produce, and let CMake match these. +tau_have() { ls $1 >/dev/null 2>&1; } +tau_status=0 + +if [ "${TAU_PROFILE}" = "1" ]; then + if tau_have 'profile.*'; then + echo "TAU_CHECK: profile artifact PRESENT" + else + echo "TAU_CHECK: profile artifact absent" + tau_status=1 + fi + # Reported but not fatal on their own: each is a different test's subject, so + # the per-test expression decides rather than failing all of them together. + if grep -q 'MPI_Allreduce' pprof.out; then + echo "TAU_CHECK: MPI routines PRESENT in profile" + else + echo "TAU_CHECK: MPI routines absent from profile" + fi + if grep -q 'hipMemcpy' pprof.out; then + echo "TAU_CHECK: HIP routines PRESENT in profile" + else + echo "TAU_CHECK: HIP routines absent from profile" + fi +fi -cd .. -rm -rf ${WORKDIR} +if [ "${TAU_TRACE}" = "1" ]; then + if tau_have 'tautrace.*'; then + echo "TAU_CHECK: trace artifact PRESENT" + else + echo "TAU_CHECK: trace artifact absent" + tau_status=1 + fi +fi +exit ${tau_status} diff --git a/tests/tau_rccl_pc_check.sh b/tests/tau_rccl_pc_check.sh index 4e8dcbe2..88d3a335 100755 --- a/tests/tau_rccl_pc_check.sh +++ b/tests/tau_rccl_pc_check.sh @@ -113,7 +113,21 @@ elif [ -n "${MPI_BINDIR}" ] && [ -x "${MPI_BINDIR}/mpiexec" ]; then else MPI_LAUNCH="srun -n 2" fi -if command -v ompi_info >/dev/null 2>&1; then +# Ask the wrapper we are about to invoke; see tau_exec_check.sh for the detail. +# Testing whether ompi_info exists added OpenMPI's --oversubscribe to MPICH's +# hydra mpiexec, which rejects it. Note this does not by itself make the test +# pass where TAU and the prebuilt rccl-tests binary come from different MPIs: +# an MPICH-built TAU cannot instrument an OpenMPI-linked benchmark. +MPI_FAMILY=unknown +if [ -n "${MPI_BINDIR}" ]; then + if "${MPI_BINDIR}/mpicc" --showme:version >/dev/null 2>&1; then + MPI_FAMILY=openmpi + elif "${MPI_BINDIR}/mpicc" -compile_info >/dev/null 2>&1; then + MPI_FAMILY=mpich + fi +fi +echo "MPI family of ${MPI_BINDIR:-}: ${MPI_FAMILY}" +if [ "${MPI_FAMILY}" = "openmpi" ]; then MPI_LAUNCH="${MPI_LAUNCH} --oversubscribe" fi