sync with develop (20250303)

easybuild/framework/easyconfig/tools.py

@@ -57,8 +57,8 @@
 from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_error, print_msg, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.environment import restore_env
-from easybuild.tools.filetools import find_easyconfigs, get_cwd, is_patch_file, locate_files
-from easybuild.tools.filetools import read_file, resolve_path, which, write_file
+from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX, get_cwd, find_easyconfigs, is_patch_file
+from easybuild.tools.filetools import locate_files, read_file, resolve_path, which, write_file
 from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO
 from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_commit
 from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data
@@ -751,7 +751,7 @@ def avail_easyblocks():
     """Return a list of all available easyblocks."""
 
     module_regexp = re.compile(r"^([^_].*)\.py$")
-    class_regex = re.compile(r"^class ([^(]*)\(", re.M)
+    class_regex = re.compile(r"^class ([^(:]*)\(", re.M)
 
     # finish initialisation of the toolchain module (ie set the TC_CONSTANT constants)
     search_toolchain('')
@@ -761,33 +761,43 @@ def avail_easyblocks():
         __import__(pkg)
 
         # determine paths for this package
-        paths = sys.modules[pkg].__path__
+        paths = [path for path in sys.modules[pkg].__path__ if os.path.exists(path)]
 
         # import all modules in these paths
         for path in paths:
-            if os.path.exists(path):
-                for fn in os.listdir(path):
-                    res = module_regexp.match(fn)
-                    if res:
-                        easyblock_mod_name = '%s.%s' % (pkg, res.group(1))
-
-                        if easyblock_mod_name not in easyblocks:
-                            __import__(easyblock_mod_name)
-                            easyblock_loc = os.path.join(path, fn)
-
-                            class_names = class_regex.findall(read_file(easyblock_loc))
-                            if len(class_names) == 1:
-                                easyblock_class = class_names[0]
-                            elif class_names:
-                                raise EasyBuildError("Found multiple class names for easyblock %s: %s",
-                                                     easyblock_loc, class_names)
-                            else:
-                                raise EasyBuildError("Failed to determine easyblock class name for %s", easyblock_loc)
-
-                            easyblocks[easyblock_mod_name] = {'class': easyblock_class, 'loc': easyblock_loc}
+            for fn in os.listdir(path):
+                res = module_regexp.match(fn)
+                if not res:
+                    continue
+                easyblock_mod_name = res.group(1)
+                easyblock_full_mod_name = '%s.%s' % (pkg, easyblock_mod_name)
+
+                if easyblock_full_mod_name in easyblocks:
+                    _log.debug("%s already imported from %s, ignoring %s",
+                               easyblock_full_mod_name, easyblocks[easyblock_full_mod_name]['loc'], path)
+                else:
+                    __import__(easyblock_full_mod_name)
+                    easyblock_loc = os.path.join(path, fn)
+
+                    class_names = class_regex.findall(read_file(easyblock_loc))
+                    if len(class_names) > 1:
+                        if pkg.endswith('.generic'):
+                            # In generic easyblocks we have e.g. ConfigureMake in configuremake.py
+                            sw_specific_class_names = [name for name in class_names
+                                                       if name.lower() == easyblock_mod_name.lower()]
                         else:
-                            _log.debug("%s already imported from %s, ignoring %s",
-                                       easyblock_mod_name, easyblocks[easyblock_mod_name]['loc'], path)
+                            # If there is exactly one software specific easyblock we use that
+                            sw_specific_class_names = [name for name in class_names
+                                                       if name.startswith(EASYBLOCK_CLASS_PREFIX)]
+                        if len(sw_specific_class_names) == 1:
+                            class_names = sw_specific_class_names
+                    if len(class_names) == 1:
+                        easyblocks[easyblock_full_mod_name] = {'class': class_names[0], 'loc': easyblock_loc}
+                    elif class_names:
+                        raise EasyBuildError("Found multiple class names for easyblock %s: %s",
+                                             easyblock_loc, class_names)
+                    else:
+                        raise EasyBuildError("Failed to determine easyblock class name for %s", easyblock_loc)
 
     return easyblocks
 

easybuild/scripts/mk_tmpl_easyblock_for.py

@@ -36,7 +36,7 @@
 import sys
 from optparse import OptionParser
 
-from easybuild.tools.filetools import encode_class_name
+from easybuild.tools.filetools import encode_class_name, EASYBLOCK_CLASS_PREFIX
 
 # parse options
 parser = OptionParser()
@@ -83,8 +83,9 @@
 # determine parent easyblock class
 parent_import = "from easybuild.framework.easyblock import EasyBlock"
 if not options.parent == "EasyBlock":
-    if options.parent.startswith('EB_'):
-        ebmod = options.parent[3:].lower()  # FIXME: here we should actually decode the encoded class name
+    if options.parent.startswith(EASYBLOCK_CLASS_PREFIX):
+        # FIXME: here we should actually decode the encoded class name
+        ebmod = options.parent[len(EASYBLOCK_CLASS_PREFIX):].lower()
     else:
         ebmod = "generic.%s" % options.parent.lower()
     parent_import = "from easybuild.easyblocks.%s import %s" % (ebmod, options.parent)

easybuild/tools/docs.py

@@ -1329,15 +1329,19 @@ def get_easyblock_classes(package_name):
     """
     Get list of all easyblock classes in specified easyblocks.* package
     """
-    easyblocks = []
+    easyblocks = set()
     modules = import_available_modules(package_name)
 
     for mod in modules:
+        easyblock_found = False
         for name, _ in inspect.getmembers(mod, inspect.isclass):
             eb_class = getattr(mod, name)
             # skip imported classes that are not easyblocks
-            if eb_class.__module__.startswith(package_name) and eb_class not in easyblocks:
-                easyblocks.append(eb_class)
+            if eb_class.__module__.startswith(package_name) and EasyBlock in inspect.getmro(eb_class):
+                easyblocks.add(eb_class)
+                easyblock_found = True
+        if not easyblock_found:
+            raise RuntimeError("No easyblocks found in module: %s", mod.__name__)
 
     return easyblocks
 

easybuild/tools/filetools.py

@@ -1695,39 +1695,52 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git=False
     return True
 
 
-def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb', on_missing_match=None):
+def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb',
+                              on_missing_match=None, match_all=False, single_line=True):
     """
     Apply specified list of regex substitutions.
 
     :param paths: list of paths to files to patch (or just a single filepath)
-    :param regex_subs: list of substitutions to apply, specified as (<regexp pattern>, <replacement string>)
+    :param regex_subs: list of substitutions to apply,
+                       specified as (<regexp pattern or regex instance>, <replacement string>)
     :param backup: create backup of original file with specified suffix (no backup if value evaluates to False)
     :param on_missing_match: Define what to do when no match was found in the file.
                              Can be 'error' to raise an error, 'warn' to print a warning or 'ignore' to do nothing
                              Defaults to the value of --strict
+    :param match_all: Expect to match all patterns in all files
+                      instead of at least one per file for error/warning reporting
+    :param single_line: Replace first match of each pattern for each line in the order of the patterns.
+                        If False the patterns are applied in order to the full text and may match line breaks.
     """
     if on_missing_match is None:
         on_missing_match = build_option('strict')
     allowed_values = (ERROR, IGNORE, WARN)
     if on_missing_match not in allowed_values:
-        raise EasyBuildError('Invalid value passed to on_missing_match: %s (allowed: %s)',
-                             on_missing_match, ', '.join(allowed_values))
+        raise ValueError('Invalid value passed to on_missing_match: %s (allowed: %s)',
+                         on_missing_match, ', '.join(allowed_values))
 
     if isinstance(paths, str):
         paths = [paths]
+    if (not isinstance(regex_subs, (list, tuple)) or
+            not all(isinstance(sub, (list, tuple)) and len(sub) == 2 for sub in regex_subs)):
+        raise ValueError('Parameter regex_subs must be a list of 2-element tuples. Got:', regex_subs)
+
+    flags = 0 if single_line else re.M
+    compiled_regex_subs = [(re.compile(regex, flags) if isinstance(regex, str) else regex, subtxt)
+                           for (regex, subtxt) in regex_subs]
 
     # only report when in 'dry run' mode
     if build_option('extended_dry_run'):
         paths_str = ', '.join(paths)
         dry_run_msg("applying regex substitutions to file(s): %s" % paths_str, silent=build_option('silent'))
-        for regex, subtxt in regex_subs:
-            dry_run_msg("  * regex pattern '%s', replacement string '%s'" % (regex, subtxt))
+        for regex, subtxt in compiled_regex_subs:
+            dry_run_msg("  * regex pattern '%s', replacement string '%s'" % (regex.pattern, subtxt))
 
     else:
-        _log.info("Applying following regex substitutions to %s: %s", paths, regex_subs)
-
-        compiled_regex_subs = [(re.compile(regex), subtxt) for (regex, subtxt) in regex_subs]
+        _log.info("Applying following regex substitutions to %s: %s",
+                  paths, [(regex.pattern, subtxt) for regex, subtxt in compiled_regex_subs])
 
+        replacement_failed_msgs = []
         for path in paths:
             try:
                 # make sure that file can be opened in text mode;
@@ -1747,32 +1760,49 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb', on_missing_m
                 if backup:
                     copy_file(path, path + backup)
                 replacement_msgs = []
+                replaced = [False] * len(compiled_regex_subs)
                 with open_file(path, 'w') as out_file:
-                    lines = txt_utf8.split('\n')
-                    del txt_utf8
-                    for line_id, line in enumerate(lines):
-                        for regex, subtxt in compiled_regex_subs:
-                            match = regex.search(line)
-                            if match:
+                    if single_line:
+                        lines = txt_utf8.split('\n')
+                        del txt_utf8
+                        for line_id, line in enumerate(lines):
+                            for i, (regex, subtxt) in enumerate(compiled_regex_subs):
+                                match = regex.search(line)
+                                if match:
+                                    origtxt = match.group(0)
+                                    replacement_msgs.append("Replaced in line %d: '%s' -> '%s'" %
+                                                            (line_id + 1, origtxt, subtxt))
+                                    replaced[i] = True
+                                    line = regex.sub(subtxt, line)
+                                    lines[line_id] = line
+                        out_file.write('\n'.join(lines))
+                    else:
+                        for i, (regex, subtxt) in enumerate(compiled_regex_subs):
+                            def do_replace(match):
                                 origtxt = match.group(0)
-                                replacement_msgs.append("Replaced in line %d: '%s' -> '%s'" %
-                                                        (line_id + 1, origtxt, subtxt))
-                                line = regex.sub(subtxt, line)
-                                lines[line_id] = line
-                    out_file.write('\n'.join(lines))
+                                # pylint: disable=cell-var-from-loop
+                                cur_subtxt = match.expand(subtxt)
+                                # pylint: disable=cell-var-from-loop
+                                replacement_msgs.append("Replaced: '%s' -> '%s'" % (origtxt, cur_subtxt))
+                                return cur_subtxt
+                            txt_utf8, replaced[i] = regex.subn(do_replace, txt_utf8)
+                        out_file.write(txt_utf8)
                 if replacement_msgs:
                     _log.info('Applied the following substitutions to %s:\n%s', path, '\n'.join(replacement_msgs))
-                else:
-                    msg = 'Nothing found to replace in %s' % path
-                    if on_missing_match == ERROR:
-                        raise EasyBuildError(msg)
-                    elif on_missing_match == WARN:
-                        _log.warning(msg)
-                    else:
-                        _log.info(msg)
-
+                if (match_all and not all(replaced)) or (not match_all and not any(replaced)):
+                    errors = ["Nothing found to replace '%s'" % regex.pattern
+                              for cur_replaced, (regex, _) in zip(replaced, compiled_regex_subs) if not cur_replaced]
+                    replacement_failed_msgs.append(', '.join(errors) + ' in ' + path)
             except (IOError, OSError) as err:
                 raise EasyBuildError("Failed to patch %s: %s", path, err)
+            if replacement_failed_msgs:
+                msg = '\n'.join(replacement_failed_msgs)
+                if on_missing_match == ERROR:
+                    raise EasyBuildError(msg)
+                elif on_missing_match == WARN:
+                    _log.warning(msg)
+                else:
+                    _log.info(msg)
 
 
 def modify_env(old, new):

easybuild/tools/include.py

@@ -37,7 +37,7 @@
 
 from easybuild.base import fancylogger
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.filetools import expand_glob_paths, read_file, symlink
+from easybuild.tools.filetools import expand_glob_paths, read_file, symlink, EASYBLOCK_CLASS_PREFIX
 # these are imported just to we can reload them later
 import easybuild.tools.module_naming_scheme
 import easybuild.toolchains
@@ -157,7 +157,8 @@ def verify_imports(pymods, pypkg, from_path):
 
 def is_software_specific_easyblock(module):
     """Determine whether Python module at specified location is a software-specific easyblock."""
-    return bool(re.search(r'^class EB_.*\(.*\):\s*$', read_file(module), re.M))
+    # All software-specific easyblocks start with the prefix and derive from another class, at least EasyBlock
+    return bool(re.search(r"^class %s[^(:]+\([^)]+\):\s*$" % EASYBLOCK_CLASS_PREFIX, read_file(module), re.M))
 
 
 def include_easyblocks(tmpdir, paths):

test/framework/docs.py

@@ -25,7 +25,6 @@
 """
 Unit tests for docs.py.
 """
-import inspect
 import os
 import re
 import sys
@@ -38,7 +37,7 @@
 from easybuild.tools.docs import list_easyblocks, list_software, list_toolchains
 from easybuild.tools.docs import md_title_and_table, rst_title_and_table
 from easybuild.tools.options import EasyBuildOptions
-from easybuild.tools.utilities import import_available_modules, mk_md_table, mk_rst_table
+from easybuild.tools.utilities import mk_md_table, mk_rst_table
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
 
 
@@ -616,7 +615,7 @@ def test_get_easyblock_classes(self):
     def test_gen_easyblocks_overview(self):
         """ Test gen_easyblocks_overview_* functions """
         gen_easyblocks_pkg = 'easybuild.easyblocks.generic'
-        modules = import_available_modules(gen_easyblocks_pkg)
+        names = [eb_class.__name__ for eb_class in get_easyblock_classes(gen_easyblocks_pkg)]
         common_params = {
             'ConfigureMake': ['configopts', 'buildopts', 'installopts'],
         }
@@ -660,15 +659,9 @@ def test_gen_easyblocks_overview(self):
         ])
 
         self.assertIn(check_configuremake, ebdoc)
-        names = []
 
-        for mod in modules:
-            for name, _ in inspect.getmembers(mod, inspect.isclass):
-                eb_class = getattr(mod, name)
-                # skip imported classes that are not easyblocks
-                if eb_class.__module__.startswith(gen_easyblocks_pkg):
-                    self.assertIn(name, ebdoc)
-                    names.append(name)
+        for name in names:
+            self.assertIn(name, ebdoc)
 
         toc = [":ref:`" + n + "`" for n in sorted(set(names))]
         pattern = " - ".join(toc)
@@ -706,17 +699,11 @@ def test_gen_easyblocks_overview(self):
         ])
 
         self.assertIn(check_configuremake, ebdoc)
-        names = []
 
-        for mod in modules:
-            for name, _ in inspect.getmembers(mod, inspect.isclass):
-                eb_class = getattr(mod, name)
-                # skip imported classes that are not easyblocks
-                if eb_class.__module__.startswith(gen_easyblocks_pkg):
-                    self.assertIn(name, ebdoc)
-                    names.append(name)
+        for name in names:
+            self.assertIn(name, ebdoc)
 
-        toc = ["\\[" + n + "\\]\\(#" + n.lower() + "\\)" for n in sorted(set(names))]
+        toc = ["\\[" + n + "\\]\\(#" + n.lower() + "\\)" for n in sorted(names)]
         pattern = " - ".join(toc)
         regex = re.compile(pattern)
         self.assertTrue(re.search(regex, ebdoc), "Pattern %s found in %s" % (regex.pattern, ebdoc))