From 4cb2c0388adb085ecef8ecd7703f8ff5b675a226 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 12 Aug 2026 15:59:19 +0200 Subject: [PATCH 1/6] Since EB 5.4.0, the build dependencies are no longer available during the sanity check phase. This breaks our CUDA sanity check, which relies on cuobjdump --- eb_hooks.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 67b031bc..badd07cb 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -49,6 +49,7 @@ EESSI_FORCE_ATTR = 'orig_force' EESSI_SUPPORTED_MODULE_ATTR = 'eessi_supported_module' EESSI_UNSUPPORTED_MODULE_ATTR = 'eessi_unsupported_module' +EESSI_SANITYCHECK_CUDA_ATTR = 'eessi_sanitycheck_cuda_dep' SYSTEM = EASYCONFIG_CONSTANTS['SYSTEM'][0] @@ -957,6 +958,52 @@ def post_module_hook_unsupported_module(self, *args, **kwargs): del self.initial_environ[unsup_mod.envvar] +def pre_sanitycheck_hook(self, *args, **kwargs): + """Main pre-sanitycheck hook: trigger custom functions.""" + pre_sanitycheck_hook_cuda(self, *args, **kwargs) + + +def post_sanitycheck_hook(self, *args, **kwargs): + """Main post-sanitycheck hook: trigger custom functions.""" + post_sanitycheck_hook_cuda(self, *args, **kwargs) + + +def pre_sanitycheck_hook_cuda(self, *args, **kwargs): + """ + If CUDA is a build-only dependency (demoted to build dep by inject_gpu_property), + temporarily promote it to a runtime dependency so CUDA tools (cuobjdump, nvcc) + are available during the sanity check step. + This is needed since EasyBuild 5.4.0 where build deps are no longer available + during the sanity check. + """ + cudaver = get_dependency_software_version("CUDA", ec=self.cfg, check_deps=False, check_builddeps=True) + if cudaver and EASYBUILD_VERSION >= '5.4.0': + # Get the CUDA dependency info from builddependencies + build_deps = self.cfg.get_ref('builddependencies') + for dep in build_deps: + if dep['name'] == 'CUDA': + # Store the dependency for removal in post hook + setattr(self, EESSI_SANITYCHECK_CUDA_ATTR, dep) + # Add to runtime dependencies so fake module includes CUDA + self.cfg['dependencies'].append(dep) + print_msg(f"CUDA (dep) is a build-only dependency; temporarily making available for sanity check") + break + + +def post_sanitycheck_hook_cuda(self, *args, **kwargs): + """ + Reverse the temporary CUDA dependency promotion from pre_sanitycheck_hook_cuda. + """ + if hasattr(self, EESSI_SANITYCHECK_CUDA_ATTR): + cuda_dep = getattr(self, EESSI_SANITYCHECK_CUDA_ATTR) + # Remove from runtime dependencies + for dep in self.cfg['dependencies']: + if dep['name'] == 'CUDA': + self.cfg['dependencies'].remove(dep) + break + delattr(self, EESSI_SANITYCHECK_CUDA_ATTR) + + def post_easyblock_hook_copy_easybuild_subdir(self, *args, **kwargs): """ Post easyblock hook that copies the easybuild subdirectory of every installed application From d4b9721aa60984eb2c34b0f2fe574fa970d8957d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 12 Aug 2026 23:22:39 +0200 Subject: [PATCH 2/6] Simply load the module as part of the hook. Not tested yet, currently running --- eb_hooks.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index badd07cb..4d1a922b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -984,9 +984,9 @@ def pre_sanitycheck_hook_cuda(self, *args, **kwargs): if dep['name'] == 'CUDA': # Store the dependency for removal in post hook setattr(self, EESSI_SANITYCHECK_CUDA_ATTR, dep) - # Add to runtime dependencies so fake module includes CUDA - self.cfg['dependencies'].append(dep) - print_msg(f"CUDA (dep) is a build-only dependency; temporarily making available for sanity check") + # Load CUDA module + self.modules_tool.load([dep['full_mod_name']]) + print_msg(f"Loading CUDA module '{dep['full_mod_name']}', temporarily making available for the (CUDA) sanity check") break @@ -999,7 +999,8 @@ def post_sanitycheck_hook_cuda(self, *args, **kwargs): # Remove from runtime dependencies for dep in self.cfg['dependencies']: if dep['name'] == 'CUDA': - self.cfg['dependencies'].remove(dep) + self.modules_tool.unload([dep['full_mod_name']]) + print_msg(f"Unloading CUDA module '{dep['full_mod_name']}', temporarily making available for the (CUDA) sanity check") break delattr(self, EESSI_SANITYCHECK_CUDA_ATTR) From c8bac14cd45fc6da9b26cb1d3f3a84e1e854ead3 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 12 Aug 2026 23:38:25 +0200 Subject: [PATCH 3/6] Fix function description --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 4d1a922b..b22a358c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -971,7 +971,7 @@ def post_sanitycheck_hook(self, *args, **kwargs): def pre_sanitycheck_hook_cuda(self, *args, **kwargs): """ If CUDA is a build-only dependency (demoted to build dep by inject_gpu_property), - temporarily promote it to a runtime dependency so CUDA tools (cuobjdump, nvcc) + temporarily load the module so CUDA tools (cuobjdump, nvcc) are available during the sanity check step. This is needed since EasyBuild 5.4.0 where build deps are no longer available during the sanity check. From d90c39fb52be8226a2e72d44c39329d9ecb8d1a9 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 13 Aug 2026 00:01:49 +0200 Subject: [PATCH 4/6] See if we can just get the cuobjdump dir, and add it to the PATH explicitely --- eb_hooks.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index b22a358c..4e3ea388 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -986,7 +986,12 @@ def pre_sanitycheck_hook_cuda(self, *args, **kwargs): setattr(self, EESSI_SANITYCHECK_CUDA_ATTR, dep) # Load CUDA module self.modules_tool.load([dep['full_mod_name']]) - print_msg(f"Loading CUDA module '{dep['full_mod_name']}', temporarily making available for the (CUDA) sanity check") + cuobjdump_path = shutil.which('cuobjdump') + cuobjdump_dir = os.path.dirname(cuobjdump_path) + self.cuobjdump_dir = cuobjdump_dir + os.environ['PATH'] = os.environ.get('PATH', '') + os.pathsep + cuobjdump_dir + print_msg(f"Adding location of cuobjdump_dir ({cuobjdump_dir}) to the PATH so that we can execute the CUDA sanity check") + self.modules_tool.unload([dep['full_mod_name']]) break @@ -995,13 +1000,10 @@ def post_sanitycheck_hook_cuda(self, *args, **kwargs): Reverse the temporary CUDA dependency promotion from pre_sanitycheck_hook_cuda. """ if hasattr(self, EESSI_SANITYCHECK_CUDA_ATTR): - cuda_dep = getattr(self, EESSI_SANITYCHECK_CUDA_ATTR) - # Remove from runtime dependencies - for dep in self.cfg['dependencies']: - if dep['name'] == 'CUDA': - self.modules_tool.unload([dep['full_mod_name']]) - print_msg(f"Unloading CUDA module '{dep['full_mod_name']}', temporarily making available for the (CUDA) sanity check") - break + # Technically, we should probably remove the cuobjdump_dir from the path + # for now, we do nothing, let's first check this works. + # TODO: cleanup + # print_msg(f"Remove {self.cuobjdump_dir} from the PATH") delattr(self, EESSI_SANITYCHECK_CUDA_ATTR) From 4cc8edce1f7eceaf73f9c327ed15cadcf1888ed2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 13 Aug 2026 00:02:29 +0200 Subject: [PATCH 5/6] Should probably modify the path before unloading the module, just to make sure that step doesn't undo it --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 4e3ea388..9d5fd2c3 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -989,9 +989,9 @@ def pre_sanitycheck_hook_cuda(self, *args, **kwargs): cuobjdump_path = shutil.which('cuobjdump') cuobjdump_dir = os.path.dirname(cuobjdump_path) self.cuobjdump_dir = cuobjdump_dir + self.modules_tool.unload([dep['full_mod_name']]) os.environ['PATH'] = os.environ.get('PATH', '') + os.pathsep + cuobjdump_dir print_msg(f"Adding location of cuobjdump_dir ({cuobjdump_dir}) to the PATH so that we can execute the CUDA sanity check") - self.modules_tool.unload([dep['full_mod_name']]) break From 4a932f9f2695fe57f6736f6592994564a54c0eaf Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 14 Aug 2026 18:04:23 +0200 Subject: [PATCH 6/6] Need to modify self.inital_environ to make cuobjdump available --- eb_hooks.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 9d5fd2c3..33cb757c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -17,6 +17,7 @@ from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option, install_path, update_build_option +from easybuild.tools.config import ERROR from easybuild.tools.filetools import apply_regex_substitutions, copy_dir, copy_file, remove_file, symlink, which from easybuild.tools.modules import get_software_root, get_software_root_env_var_name from easybuild.tools.run import run_cmd @@ -49,7 +50,7 @@ EESSI_FORCE_ATTR = 'orig_force' EESSI_SUPPORTED_MODULE_ATTR = 'eessi_supported_module' EESSI_UNSUPPORTED_MODULE_ATTR = 'eessi_unsupported_module' -EESSI_SANITYCHECK_CUDA_ATTR = 'eessi_sanitycheck_cuda_dep' +EESSI_SANITYCHECK_CUDA_ATTR = 'eessi_sanitycheck_initial_env_path' SYSTEM = EASYCONFIG_CONSTANTS['SYSTEM'][0] @@ -971,8 +972,8 @@ def post_sanitycheck_hook(self, *args, **kwargs): def pre_sanitycheck_hook_cuda(self, *args, **kwargs): """ If CUDA is a build-only dependency (demoted to build dep by inject_gpu_property), - temporarily load the module so CUDA tools (cuobjdump, nvcc) - are available during the sanity check step. + temporarily load the module so CUDA tool cuobjdump path can be made + available during the sanity check step. This is needed since EasyBuild 5.4.0 where build deps are no longer available during the sanity check. """ @@ -982,16 +983,18 @@ def pre_sanitycheck_hook_cuda(self, *args, **kwargs): build_deps = self.cfg.get_ref('builddependencies') for dep in build_deps: if dep['name'] == 'CUDA': - # Store the dependency for removal in post hook - setattr(self, EESSI_SANITYCHECK_CUDA_ATTR, dep) # Load CUDA module self.modules_tool.load([dep['full_mod_name']]) - cuobjdump_path = shutil.which('cuobjdump') + cuobjdump_path = which('cuobjdump', on_error=ERROR) cuobjdump_dir = os.path.dirname(cuobjdump_path) self.cuobjdump_dir = cuobjdump_dir self.modules_tool.unload([dep['full_mod_name']]) - os.environ['PATH'] = os.environ.get('PATH', '') + os.pathsep + cuobjdump_dir - print_msg(f"Adding location of cuobjdump_dir ({cuobjdump_dir}) to the PATH so that we can execute the CUDA sanity check") + # Store the original PATH for restoration in post hook + original_initial_environ_path = self.initial_environ['PATH'] + setattr(self, EESSI_SANITYCHECK_CUDA_ATTR, original_initial_environ_path) + # Modify stored initial environment (restored during the sanity check), not the current environment + self.initial_environ['PATH'] = original_initial_environ_path + os.pathsep + cuobjdump_dir + print_msg(f"Add location of cuobjdump ({cuobjdump_dir}) to initial environ PATH for CUDA sanity check") break @@ -1000,10 +1003,17 @@ def post_sanitycheck_hook_cuda(self, *args, **kwargs): Reverse the temporary CUDA dependency promotion from pre_sanitycheck_hook_cuda. """ if hasattr(self, EESSI_SANITYCHECK_CUDA_ATTR): - # Technically, we should probably remove the cuobjdump_dir from the path - # for now, we do nothing, let's first check this works. - # TODO: cleanup - # print_msg(f"Remove {self.cuobjdump_dir} from the PATH") + original_path = getattr(self, EESSI_SANITYCHECK_CUDA_ATTR) + + if not isinstance(original_path, str) or os.pathsep not in original_path: + raise EasyBuildError( + f"Expected {EESSI_SANITYCHECK_CUDA_ATTR} attribute to contain a PATH-like " + f"value, but got: {original_path}" + ) + + # Restore the original initial environ PATH + print_msg(f"Restoring stored initial environ PATH after CUDA sanity check") + self.initial_environ['PATH'] = original_path delattr(self, EESSI_SANITYCHECK_CUDA_ATTR)