probe external modules for missing metadata that is not provided via extermal module metadata file

easybuild/framework/easyconfig/easyconfig.py

@@ -36,6 +36,7 @@
 :author: Alan O'Cais (Juelich Supercomputing Centre)
 :author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada)
 :author: Maxime Boissonneault (Universite Laval, Calcul Quebec, Compute Canada)
+:author: Victor Holanda (CSCS, ETH Zurich)
 """
 
 import copy
@@ -61,7 +62,7 @@
 from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG, LOCAL_VAR_NAMING_CHECK_WARN
 from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme
 from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX, copy_file, decode_class_name, encode_class_name
-from easybuild.tools.filetools import find_backup_name_candidate, find_easyconfigs, read_file, write_file
+from easybuild.tools.filetools import convert_name, find_backup_name_candidate, find_easyconfigs, read_file, write_file
 from easybuild.tools.hooks import PARSE, load_hooks, run_hook
 from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
 from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
@@ -1152,19 +1153,109 @@ def _validate(self, attr, values):  # private method
         if self[attr] and self[attr] not in values:
             raise EasyBuildError("%s provided '%s' is not valid: %s", attr, self[attr], values)
 
+    def _handle_ext_module_metadata_by_probing_modules(self, dep_name, dependency=None):
+        """
+        helper function for handle_external_module_metadata
+        handles metadata for external module dependencies when there is not entry in the
+        metadata file
+
+        It should look for the pair of variables definitions in the available modules
+          1. CRAY_XXXX_PREFIX and CRAY_XXXX_VERSION
+          2. CRAY_XXXX_DIR and CRAY_XXXX_VERSION
+          2. CRAY_XXXX_ROOT and CRAY_XXXX_VERSION
+          5. XXXX_PREFIX and XXXX_VERSION
+          4. XXXX_DIR and XXXX_VERSION
+          5. XXXX_ROOT and XXXX_VERSION
+          3. XXXX_HOME and XXXX_VERSION
+
+        If neither of the pairs is found, then an empty dictionary is returned
+        """
+        if dependency is None:
+            dependency = dict()
+
+        short_ext_modname = dep_name.split('/')[0]
+        if not 'name' in dependency:
+            dependency['name'] = [short_ext_modname]
+
+        if short_ext_modname.startswith('cray-'):
+            short_ext_modname = short_ext_modname.split('cray-')[1]
+
+        short_ext_modname.replace('-', '_')
+        short_ext_modname_upper = convert_name(short_ext_modname, upper=True)
+
+        allowed_pairs = [
+            ('CRAY_%s_PREFIX', 'CRAY_%s_VERSION'),
+            ('CRAY_%s_PREFIX_DIR', 'CRAY_%s_VERSION'),
+            ('CRAY_%s_DIR', 'CRAY_%s_VERSION'),
+            ('CRAY_%s_ROOT', 'CRAY_%s_VERSION'),
+            ('%s_PREFIX', '%s_VERSION'),
+            ('%s_DIR', '%s_VERSION'),
+            ('%s_ROOT', '%s_VERSION'),
+            ('%s_HOME', '%s_VERSION'),
+        ]
+
+        for prefix, version in allowed_pairs:
+            prefix = prefix % short_ext_modname_upper
+            version = version % short_ext_modname_upper
+
+            dep_prefix = self.modules_tool.get_variable_from_modulefile(dep_name, prefix)
+            dep_version = self.modules_tool.get_variable_from_modulefile(dep_name, version)
+
+            # only update missing values with both keys are found
+            if dep_prefix and dep_version:
+                # version should hold the value, not the key
+                if 'version' not in dependency:
+                    dependency['version'] = [dep_version]
+                    self.log.info('setting external module %s version to be %s' % (dep_name, dep_version))
+                # prefix should hold the key, not the value
+                if 'prefix' not in dependency:
+                    dependency['prefix'] = prefix
+                    self.log.info('setting external module %s prefix to be %s' % (dep_name, dep_prefix))
+                break
+
+        return dependency
+
     def handle_external_module_metadata(self, dep_name):
         """
         helper function for _parse_dependency
         handles metadata for external module dependencies
         """
         dependency = {}
+        dep_name_no_version = dep_name.split('/')[0]
+        metadata_fields = ['name', 'version', 'prefix']
+        external_metadata = {}
+
         if dep_name in self.external_modules_metadata:
-            dependency['external_module_metadata'] = self.external_modules_metadata[dep_name]
-            self.log.info("Updated dependency info with available metadata for external module %s: %s",
-                          dep_name, dependency['external_module_metadata'])
+            external_metadata = self.external_modules_metadata[dep_name]
+            if not all(d in external_metadata for d in metadata_fields):
+                external_metadata = self._handle_ext_module_metadata_by_probing_modules(dep_name,
+                                                                                        dependency=external_metadata)
+            if external_metadata:
+                self.log.info("Updated dependency info with metadata from available modules for external module %s: %s",
+                              dep_name, external_metadata)
+            else:
+                self.log.info("No metadata available for external module %s.", dep_name)
+        elif dep_name_no_version in self.external_modules_metadata:
+            external_metadata = self.external_modules_metadata[dep_name_no_version]
+            if not all(d in external_metadata for d in metadata_fields):
+                external_metadata = self._handle_ext_module_metadata_by_probing_modules(dep_name_no_version,
+                                                                                        dependency=external_metadata)
+                if external_metadata:
+                    self.log.info("Updated dependency info with metadata from available modules for external module %s: %s",
+                                  dep_name, external_metadata)
+                else:
+                    self.log.info("No metadata available for external module %s.", dep_name)
         else:
-            self.log.info("No metadata available for external module %s", dep_name)
+            self.log.info("No metadata available for external module %s. Attempting to read from available modules",
+                          dep_name)
+            external_metadata = self._handle_ext_module_metadata_by_probing_modules(dep_name)
+            if external_metadata:
+                self.log.info("Updated dependency info with metadata from available modules for external module %s: %s",
+                              dep_name, external_metadata)
+            else:
+                self.log.info("No metadata available for external module %s.", dep_name)
 
+        dependency['external_module_metadata'] = external_metadata
         return dependency
 
     def handle_multi_deps(self):

easybuild/tools/modules.py

@@ -649,6 +649,15 @@ def show(self, mod_name):
 
         return ans
 
+    def get_variable_from_modulefile(self, mod_name, var_name):
+        """
+        Get info from the module file for the specified module.
+
+        :param mod_name: module name
+        :param var_name: name of the variable value to be extracted
+        """
+        pass
+
     def get_value_from_modulefile(self, mod_name, regex):
         """
         Get info from the module file for the specified module.
@@ -1126,6 +1135,21 @@ def update(self):
         """Update after new modules were added."""
         pass
 
+    def get_variable_from_modulefile(self, mod_name, var_name):
+        """
+        Get info from the module file for the specified module.
+
+        :param mod_name: module name
+        :param var_name: name of the variable value to be extracted
+        """
+        try:
+            # Tcl syntax
+            regex = re.compile(r'^setenv\s+%s\s+(?P<value>\S*)' % var_name, re.M)
+            ans = self.get_value_from_modulefile(mod_name, regex)
+        except Exception:
+            return None
+
+        return ans
 
 class EnvironmentModulesTcl(EnvironmentModulesC):
     """Interface to (Tcl) environment modules (modulecmd.tcl)."""
@@ -1390,6 +1414,21 @@ def exist(self, mod_names, skip_avail=False, maybe_partial=True):
         return super(Lmod, self).exist(mod_names, mod_exists_regex_template=r'^\s*\S*/%s.*(\.lua)?:\s*$',
                                        skip_avail=skip_avail, maybe_partial=maybe_partial)
 
+    def get_variable_from_modulefile(self, mod_name, var_name):
+        """
+        Get info from the module file for the specified module.
+
+        :param mod_name: module name
+        :param var_name: name of the variable value to be extracted
+        """
+        try:
+            # Lua syntax
+            regex = re.compile(r'^setenv\(\"%s\",\s*\"(?P<value>\S*)\"\)' % var_name, re.M)
+            ans = self.get_value_from_modulefile(mod_name, regex)
+        except Exception:
+            return None
+
+        return ans
 
 def get_software_root_env_var_name(name):
     """Return name of environment variable for software root."""