Add ignore_cache_subcmds configuration option

.hunspell.en.dic

@@ -1595,3 +1595,5 @@ isdir
 bizstyle
 importlib
 pythondir
+subcmds
+topCommandName

INSTALL.rst

@@ -1792,6 +1792,8 @@ installation.
 | :mconfig:`ignore_cache`           | ``0``                                        | :envvar:`MODULES_IGNORE_CACHE`,              |              |           |
 |                                   |                                              | :option:`--ignore-cache`                     |              |           |
 +-----------------------------------+----------------------------------------------+----------------------------------------------+--------------+-----------+
+| :mconfig:`ignore_cache_subcmds`   | *Empty by default*                           | :envvar:`MODULES_IGNORE_CACHE_SUBCMDS`       |              |           |
++-----------------------------------+----------------------------------------------+----------------------------------------------+--------------+-----------+
 | :mconfig:`ignore_user_rc`         | ``0``                                        | :envvar:`MODULES_IGNORE_USER_RC`,            |              |           |
 |                                   |                                              | :option:`--ignore-user-rc`                   |              |           |
 +-----------------------------------+----------------------------------------------+----------------------------------------------+--------------+-----------+

MIGRATING.rst

@@ -80,6 +80,26 @@ These measurements were obtained on a system where all modulefiles are stored
 on local flash storage. If your modulefiles reside on a shared network
 filesystem, using a `Module cache`_ is strongly recommended.
 
+A module cache file aggregates the content of all the modulercs and
+modulefiles found in a modulepath and this whole file is evaluated when a
+module search occurs, even if the search targets specific modules. On setups
+with a very large number of modulefiles, resulting in cache files of several
+megabytes, evaluating the whole cache file may take longer than walking
+through the modulepath directory to find the few requested modules. The new
+:mconfig:`ignore_cache_subcmds` configuration option defines a list of
+module sub-commands on which module cache files are ignored. For instance to
+keep cache benefit on listing sub-commands but avoid its penalty on targeted
+ones:
+
+.. parsed-literal::
+
+    :ps:`$` module config ignore_cache_subcmds load:try-load:load-any
+
+When the configuration option is changed from its default value using the
+:subcmd:`config` sub-command, the :envvar:`MODULES_IGNORE_CACHE_SUBCMDS`
+environment variable is defined accordingly. See
+:envvar:`MODULES_IGNORE_CACHE_SUBCMDS` for details.
+
 Reorder existing entries in path-like variables
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

NEWS.rst

@@ -230,6 +230,14 @@ Modules 5.7.0 (not yet released)
   :instopt:`--enable-set-pythonpath` adds this directory to
   :envvar:`PYTHONPATH` instead of the Modules ``init`` directory. (fix issue
   #643)
+* Introduce :mconfig:`ignore_cache_subcmds` configuration option that
+  defines a list of module sub-commands on which module cache files are
+  ignored. It is useful when cache files are large: a sub-command targeting
+  specific modules (like :subcmd:`load`) may find them faster by walking
+  through modulepath directories than by evaluating whole cache files. When
+  :mconfig:`ignore_cache_subcmds` is changed with :subcmd:`config`
+  sub-command, it sets the :envvar:`MODULES_IGNORE_CACHE_SUBCMDS`
+  environment variable. (fix issue #563)
 
 
 .. _5.6 release notes:

doc/source/changes.rst

@@ -1108,7 +1108,8 @@ The following environment variables appeared on Modules 5.
 |            | :envvar:`MODULES_LINKED_ENVVARS`,                               |
 |            | :envvar:`MODULES_INIT_ENVVARS`,                                 |
 |            | :envvar:`MODULES_DOMAINNAME`,                                   |
-|            | :envvar:`MODULES_MODULEPATH_IGNORE`                             |
+|            | :envvar:`MODULES_MODULEPATH_IGNORE`,                            |
+|            | :envvar:`MODULES_IGNORE_CACHE_SUBCMDS`                          |
 +------------+-----------------------------------------------------------------+
 
 Modules Specific Tcl Commands
@@ -1326,7 +1327,8 @@ The following Modules configuration option has been introduced on Modules 5.
 | 5.7        | :mconfig:`path_entry_reorder`, :mconfig:`paginate`,             |
 |            | :mconfig:`non_exportable_tags`, :mconfig:`info_extension`,      |
 |            | :mconfig:`linked_envvars`, :mconfig:`init_envvars`,             |
-|            | :mconfig:`domainname`, :mconfig:`modulepath_ignore`             |
+|            | :mconfig:`domainname`, :mconfig:`modulepath_ignore`,            |
+|            | :mconfig:`ignore_cache_subcmds`                                 |
 +------------+-----------------------------------------------------------------+
 
 :mconfig:`auto_handling`

doc/source/design/module-cache.rst

@@ -265,6 +265,21 @@ Cache files are ignored if :mconfig:`ignore_cache` configuration option is
 enabled. This option can also be enabled just for one execution with the
 :option:`--ignore-cache` command line option.
 
+Cache files are also ignored if the currently running top-level sub-command
+is part of the :mconfig:`ignore_cache_subcmds` configuration option. On
+setup with a very large number of modulefiles, resulting in cache files of
+several megabytes, evaluating the whole cache file may take longer for a
+sub-command targeting specific modules (like :subcmd:`load`) than walking
+through the modulepath directory to find these modules. This option helps
+to skip cache use for such sub-commands while preserving cache benefit on
+the others (like :subcmd:`avail`).
+
+Sub-command name aliases resolve to their target sub-command prior the
+comparison against the option value. The :command:`ml` command used without
+a sub-command name translates to the :subcmd:`load` or :subcmd:`unload`
+sub-command it is currently processing, so ``ml`` is not a value to set in
+this option.
+
 Cache files are ignored if cache expiry mechanism is enabled through the
 :mconfig:`cache_expiry_secs` configuration option. When this option is set to
 0, it means a cache file never expires. This is the default behavior. If set

doc/source/module.rst

@@ -1233,6 +1233,19 @@ Module Sub-Commands
 
      .. versionadded:: 5.3
 
+ .. mconfig:: ignore_cache_subcmds
+
+  List of module sub-commands that ignore module cache.
+
+  Default is an empty list. The :envvar:`MODULES_IGNORE_CACHE_SUBCMDS`
+  environment variable is defined by :subcmd:`config` sub-command when
+  changing this configuration option from its default value. See
+  :envvar:`MODULES_IGNORE_CACHE_SUBCMDS` description for details.
+
+  .. only:: html or latex
+
+     .. versionadded:: 5.7
+
  .. mconfig:: ignore_user_rc
 
   Skip evaluation of user-specific module rc file (:file:`$HOME/.modulerc`).
@@ -5318,6 +5331,30 @@ ENVIRONMENT
 
     .. versionadded:: 5.3
 
+.. envvar:: MODULES_IGNORE_CACHE_SUBCMDS
+
+ A colon separated list of the module sub-commands that ignore module cache.
+ When one of these sub-commands is run, modulepath directories are walked
+ through to search modules instead of relying on their cache file.
+
+ Sub-command name aliases (like ``add`` for :subcmd:`load`) resolve to their
+ target sub-command, which is the name to set in value list. The
+ :command:`ml` command used without a sub-command name translates to the
+ underlying :subcmd:`load` or :subcmd:`unload` sub-commands.
+
+ Ignoring cache may be useful for sub-commands targeting specific modules
+ (like :subcmd:`load`) when cache files are large: walking through modulepath
+ content to find the requested modules may be faster than evaluating the
+ whole cache file.
+
+ This environment variable value supersedes the default value set in the
+ :mconfig:`ignore_cache_subcmds` configuration option. It can be defined
+ with the :subcmd:`config` sub-command.
+
+ .. only:: html or latex
+
+    .. versionadded:: 5.7
+
 .. envvar:: MODULES_IGNORE_USER_RC
 
  Skip evaluation (if set to ``1``) or not (if set to ``0``) of user-specific

init/Makefile

@@ -153,7 +153,7 @@ comp_stashpop_opts := --force -f
 comp_source_opts := --auto --no-auto --force -f
 comp_path_opts := -d --delim --duplicates
 comp_rm_path_opts := -d --delim --index
-comp_config_opts := --dump-state --reset abort_on_error advanced_version_spec auto_handling avail_indepth avail_output avail_terse_output cache_buffer_bytes cache_expiry_secs collection_pin_version collection_pin_tag collection_target color colors conflict_unload contact csh_limit domainname editor extended_default extra_siteconfig hide_auto_loaded home icase ignore_cache ignore_user_rc ignored_dirs implicit_default implicit_requirement init_envvars info_extension linked_envvars list_output list_terse_output locked_configs logged_events logger mcookie_check mcookie_version_check ml modulepath_ignore nearly_forbidden_days non_exportable_tags pager paginate path_entry_reorder protected_envvars quarantine_support rcfile redirect_output require_via reset_target_state run_quarantine search_match set_shell_startup shells_with_ksh_fpath silent_shell_debug siteconfig source_cache spider_indepth spider_output spider_terse_output sticky_purge tag_abbrev tag_color_name tcl_ext_lib tcl_linter term_background term_width unique_name_loaded unload_match_order variant_shortcut verbosity wa_277
+comp_config_opts := --dump-state --reset abort_on_error advanced_version_spec auto_handling avail_indepth avail_output avail_terse_output cache_buffer_bytes cache_expiry_secs collection_pin_version collection_pin_tag collection_target color colors conflict_unload contact csh_limit domainname editor extended_default extra_siteconfig hide_auto_loaded home icase ignore_cache ignore_cache_subcmds ignore_user_rc ignored_dirs implicit_default implicit_requirement init_envvars info_extension linked_envvars list_output list_terse_output locked_configs logged_events logger mcookie_check mcookie_version_check ml modulepath_ignore nearly_forbidden_days non_exportable_tags pager paginate path_entry_reorder protected_envvars quarantine_support rcfile redirect_output require_via reset_target_state run_quarantine search_match set_shell_startup shells_with_ksh_fpath silent_shell_debug siteconfig source_cache spider_indepth spider_output spider_terse_output sticky_purge tag_abbrev tag_color_name tcl_ext_lib tcl_linter term_background term_width unique_name_loaded unload_match_order variant_shortcut verbosity wa_277
 
 define translate-in-script
 $(ECHO_GEN)

init/fish_completion

@@ -76,7 +76,7 @@ complete -c module -n '__fish_module_use_stashlist' -f -a "(module stashlist --c
    /Stash collection list\$/d; \
    /:\$/d; \
    /:ERROR:/d;')"
-complete -c module -n '__fish_module_use_config' -f -a "--dump-state --reset abort_on_error advanced_version_spec auto_handling avail_indepth avail_output avail_terse_output cache_buffer_bytes cache_expiry_secs collection_pin_version collection_pin_tag collection_target color colors conflict_unload contact csh_limit domainname editor extended_default extra_siteconfig hide_auto_loaded home icase ignore_cache ignore_user_rc ignored_dirs implicit_default implicit_requirement init_envvars info_extension linked_envvars list_output list_terse_output locked_configs logged_events logger mcookie_check mcookie_version_check ml modulepath_ignore nearly_forbidden_days non_exportable_tags pager paginate path_entry_reorder protected_envvars quarantine_support rcfile redirect_output require_via reset_target_state run_quarantine search_match set_shell_startup shells_with_ksh_fpath silent_shell_debug siteconfig source_cache spider_indepth spider_output spider_terse_output sticky_purge tag_abbrev tag_color_name tcl_ext_lib tcl_linter term_background term_width unique_name_loaded unload_match_order variant_shortcut verbosity wa_277"
+complete -c module -n '__fish_module_use_config' -f -a "--dump-state --reset abort_on_error advanced_version_spec auto_handling avail_indepth avail_output avail_terse_output cache_buffer_bytes cache_expiry_secs collection_pin_version collection_pin_tag collection_target color colors conflict_unload contact csh_limit domainname editor extended_default extra_siteconfig hide_auto_loaded home icase ignore_cache ignore_cache_subcmds ignore_user_rc ignored_dirs implicit_default implicit_requirement init_envvars info_extension linked_envvars list_output list_terse_output locked_configs logged_events logger mcookie_check mcookie_version_check ml modulepath_ignore nearly_forbidden_days non_exportable_tags pager paginate path_entry_reorder protected_envvars quarantine_support rcfile redirect_output require_via reset_target_state run_quarantine search_match set_shell_startup shells_with_ksh_fpath silent_shell_debug siteconfig source_cache spider_indepth spider_output spider_terse_output sticky_purge tag_abbrev tag_color_name tcl_ext_lib tcl_linter term_background term_width unique_name_loaded unload_match_order variant_shortcut verbosity wa_277"
 
 complete -f -n '__fish_module_no_subcommand' -c module -a 'help' --description 'Print this or modulefile(s) help info'
 complete -f -n '__fish_module_no_subcommand' -c module -a 'avail' --description 'List all or matching available modules'

init/zsh-functions/_module.in

@@ -407,7 +407,7 @@ _module() {
                _arguments \
                   '--dump-state[Report each state value of current Modules execution]' \
                   '--reset[Unset environment variable relative to configuration key]' \
-                  '1:configuration key:(abort_on_error advanced_version_spec auto_handling avail_indepth avail_output avail_terse_output cache_buffer_bytes cache_expiry_secs collection_pin_version collection_pin_tag collection_target color colors conflict_unload contact csh_limit domainname editor extended_default extra_siteconfig hide_auto_loaded home icase ignore_cache ignore_user_rc ignored_dirs implicit_default implicit_requirement init_envvars info_extension linked_envvars list_output list_terse_output locked_configs logged_events logger mcookie_check mcookie_version_check ml modulepath_ignore nearly_forbidden_days non_exportable_tags pager paginate path_entry_reorder protected_envvars quarantine_support rcfile redirect_output require_via reset_target_state run_quarantine search_match set_shell_startup shells_with_ksh_fpath silent_shell_debug siteconfig source_cache spider_indepth spider_output spider_terse_output sticky_purge tag_abbrev tag_color_name tcl_ext_lib tcl_linter term_background term_width unique_name_loaded unload_match_order variant_shortcut verbosity wa_277)' \
+                  '1:configuration key:(abort_on_error advanced_version_spec auto_handling avail_indepth avail_output avail_terse_output cache_buffer_bytes cache_expiry_secs collection_pin_version collection_pin_tag collection_target color colors conflict_unload contact csh_limit domainname editor extended_default extra_siteconfig hide_auto_loaded home icase ignore_cache ignore_cache_subcmds ignore_user_rc ignored_dirs implicit_default implicit_requirement init_envvars info_extension linked_envvars list_output list_terse_output locked_configs logged_events logger mcookie_check mcookie_version_check ml modulepath_ignore nearly_forbidden_days non_exportable_tags pager paginate path_entry_reorder protected_envvars quarantine_support rcfile redirect_output require_via reset_target_state run_quarantine search_match set_shell_startup shells_with_ksh_fpath silent_shell_debug siteconfig source_cache spider_indepth spider_output spider_terse_output sticky_purge tag_abbrev tag_color_name tcl_ext_lib tcl_linter term_background term_width unique_name_loaded unload_match_order variant_shortcut verbosity wa_277)' \
                   && ret=0
                ;;
             (edit)

tcl/cache.tcl.in

@@ -380,8 +380,10 @@ proc limited-access-directory {dir} {
 proc findModulesInCacheFile {modpath mod depthlvl fetch_mtime} {
    set cachefile [getModuleCacheFilename $modpath]
 
-   # check if cache should be ignored
-   if {[getConf ignore_cache]} {
+   # check if cache should be ignored, either globally or for currently
+   # running top-level sub-command
+   if {[getConf ignore_cache] || [topCommandName] in [getConfList\
+      ignore_cache_subcmds]} {
       return [list 0 {}]
    }
 

tcl/init.tcl.in

@@ -104,6 +104,7 @@ array set g_config_defs [list\
    home {MODULESHOME {@moduleshome@} 0 s}\
    icase {MODULES_ICASE @icase@ 0 s {never search always}}\
    ignore_cache {MODULES_IGNORE_CACHE 0 0 b {0 1}}\
+   ignore_cache_subcmds {MODULES_IGNORE_CACHE_SUBCMDS {} 0 l}\
    ignore_user_rc {MODULES_IGNORE_USER_RC 0 0 b {0 1}}\
    ignored_dirs {{} {CVS RCS SCCS .svn .git .SYNC .sos} 0 o}\
    implicit_requirement {MODULES_IMPLICIT_REQUIREMENT @implicitrequirement@ 0\

tcl/main.tcl.in

@@ -54,6 +54,16 @@ proc aboveCommandName {} {
    return [lindex [getState commandname] end-1]
 }
 
+# get top-level command name, with ml frontend translated onto the
+# underlying load or unload sub-command it currently processes
+proc topCommandName {} {
+   set subcmd [topState commandname]
+   if {$subcmd eq {ml}} {
+      set subcmd [topState eval_context]
+   }
+   return $subcmd
+}
+
 proc ongoingCommandName {commandName} {
    return [expr {[lsearch -exact [getState commandname] $commandName] != -1}]
 }

testsuite/install.00-init/010-environ.exp

@@ -168,6 +168,7 @@ unsetenv_var MODULES_RESET_TARGET_STATE
 
 # clean any cache configuration
 unsetenv_var MODULES_IGNORE_CACHE
+unsetenv_var MODULES_IGNORE_CACHE_SUBCMDS
 unsetenv_var MODULES_CACHE_BUFFER_BYTES
 unsetenv_var MODULES_CACHE_EXPIRY_SECS
 

testsuite/modules.00-init/010-environ.exp

@@ -169,6 +169,7 @@ unsetenv_var MODULES_RESET_TARGET_STATE
 
 # clean any cache configuration
 unsetenv_var MODULES_IGNORE_CACHE
+unsetenv_var MODULES_IGNORE_CACHE_SUBCMDS
 unsetenv_var MODULES_CACHE_BUFFER_BYTES
 unsetenv_var MODULES_CACHE_EXPIRY_SECS