add support for enhancing existing sanity check

easybuild/framework/easyblock.py

@@ -2409,20 +2409,36 @@ def _sanity_check_step_common(self, custom_paths, custom_commands):
             SANITY_CHECK_PATHS_DIRS: ("(non-empty) directory", lambda dp: os.path.isdir(dp) and os.listdir(dp)),
         }
 
-        # prepare sanity check paths
-        paths = self.cfg['sanity_check_paths']
-        if not paths:
+        enhance_sanity_check = self.cfg['enhance_sanity_check']
+        ec_commands = self.cfg['sanity_check_commands']
+        ec_paths = self.cfg['sanity_check_paths']
+
+        # if enhance_sanity_check is not enabled, only sanity_check_paths specified in the easyconfig file are used,
+        # the ones provided by the easyblock (via custom_paths) are ignored
+        if ec_paths and not enhance_sanity_check:
+            paths = ec_paths
+            self.log.info("Using (only) sanity check paths specified by easyconfig file: %s", paths)
+        else:
+            # if no sanity_check_paths are specified in easyconfig,
+            # we fall back to the ones provided by the easyblock via custom_paths
             if custom_paths:
                 paths = custom_paths
-                self.log.info("Using customized sanity check paths: %s" % paths)
+                self.log.info("Using customized sanity check paths: %s", paths)
+            # if custom_paths is empty, we fall back to a generic set of paths:
+            # non-empty bin/ + /lib or /lib64 directories
             else:
                 paths = {}
                 for key in path_keys_and_check:
                     paths.setdefault(key, [])
                 paths.update({SANITY_CHECK_PATHS_DIRS: ['bin', ('lib', 'lib64')]})
-                self.log.info("Using default sanity check paths: %s" % paths)
-        else:
-            self.log.info("Using specified sanity check paths: %s" % paths)
+                self.log.info("Using default sanity check paths: %s", paths)
+
+            # if enhance_sanity_check is enabled *and* sanity_check_paths are specified in the easyconfig,
+            # those paths are used to enhance the paths provided by the easyblock
+            if enhance_sanity_check and ec_paths:
+                for key in path_keys_and_check:
+                    paths[key] = paths[key] + ec_paths.get(key, [])
+                self.log.info("Enhanced sanity check paths after taking into account easyconfig file: %s", paths)
 
         ks = sorted(paths.keys())
         valnottypes = [not isinstance(x, list) for x in paths.values()]
@@ -2432,14 +2448,23 @@ def _sanity_check_step_common(self, custom_paths, custom_commands):
             raise EasyBuildError("Incorrect format for sanity_check_paths (should (only) have %s keys, "
                                  "values should be lists (at least one non-empty)).", ','.join(req_keys))
 
-        commands = self.cfg['sanity_check_commands']
-        if not commands:
+        # if enhance_sanity_check is not enabled, only sanity_check_commands specified in the easyconfig file are used,
+        # the ones provided by the easyblock (via custom_commands) are ignored
+        if ec_commands and not enhance_sanity_check:
+            commands = ec_commands
+            self.log.info("Using (only) sanity check commands specified by easyconfig file: %s", commands)
+        else:
             if custom_commands:
                 commands = custom_commands
-                self.log.info("Using customised sanity check commands: %s" % commands)
+                self.log.info("Using customised sanity check commands: %s", commands)
             else:
                 commands = []
-                self.log.info("Using specified sanity check commands: %s" % commands)
+
+            # if enhance_sanity_check is enabled, the sanity_check_commands specified in the easyconfig file
+            # are combined with those provided by the easyblock via custom_commands
+            if enhance_sanity_check and ec_commands:
+                commands = commands + ec_commands
+                self.log.info("Enhanced sanity check commands after taking into account easyconfig file: %s", commands)
 
         for i, command in enumerate(commands):
             # set command to default. This allows for config files with

easybuild/framework/easyconfig/default.py

@@ -90,6 +90,8 @@
     'easyblock': [None, "EasyBlock to use for building; if set to None, an easyblock is selected "
                         "based on the software name", BUILD],
     'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD],
+    'enhance_sanity_check': [False, "Indicate that additional sanity check commands & paths should enhance "
+                             "the existin sanity check, not replace it", BUILD],
     'fix_perl_shebang_for': [None, "List of files for which Perl shebang should be fixed "
                                    "to '#!/usr/bin/env perl' (glob patterns supported)", BUILD],
     'fix_python_shebang_for': [None, "List of files for which Python shebang should be fixed "

test/framework/sandbox/easybuild/easyblocks/t/toy.py

@@ -46,7 +46,7 @@ class EB_toy(ExtensionEasyBlock):
 
     @staticmethod
     def extra_options(extra_vars=None):
-        """Custom easyconfig parameters for toytoy."""
+        """Custom easyconfig parameters for toy."""
         if extra_vars is None:
             extra_vars = {}
 

test/framework/toy_build.py

@@ -53,7 +53,7 @@
 from easybuild.tools.filetools import adjust_permissions, mkdir, read_file, remove_dir, remove_file, which, write_file
 from easybuild.tools.module_generator import ModuleGeneratorTcl
 from easybuild.tools.modules import Lmod
-from easybuild.tools.py2vs3 import string_type
+from easybuild.tools.py2vs3 import reload, string_type
 from easybuild.tools.run import run_cmd
 from easybuild.tools.version import VERSION as EASYBUILD_VERSION
 
@@ -73,6 +73,7 @@ def setUp(self):
 
     def tearDown(self):
         """Cleanup."""
+
         super(ToyBuildTest, self).tearDown()
         # remove logs
         if os.path.exists(self.dummylogfn):
@@ -1887,6 +1888,145 @@ def test_sanity_check_paths_lib64(self):
         write_file(test_ec, ectxt)
         self.test_toy_build(ec_file=test_ec, raise_error=True)
 
+    def test_toy_build_enhanced_sanity_check(self):
+        """Test enhancing of sanity check."""
+
+        # if toy easyblock was imported, get rid of corresponding entry in sys.modules,
+        # to avoid that it messes up the use of --include-easyblocks=toy.py below...
+        if 'easybuild.easyblocks.toy' in sys.modules:
+            del sys.modules['easybuild.easyblocks.toy']
+
+        test_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)))
+        toy_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+        toy_ec_txt = read_file(toy_ec)
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+
+        # get rid of custom sanity check paths in test easyconfig
+        regex = re.compile(r'^sanity_check_paths\s*=\s*{[^}]+}', re.M)
+        test_ec_txt = regex.sub('', toy_ec_txt)
+        write_file(test_ec, test_ec_txt)
+
+        self.assertFalse('sanity_check_' in test_ec_txt)
+
+        # create custom easyblock for toy that has a custom sanity_check_step
+        toy_easyblock = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py')
+
+        toy_easyblock_txt = read_file(toy_easyblock)
+
+        toy_custom_sanity_check_step = '\n'.join([
+            '',
+            "    def sanity_check_step(self):",
+            "        paths = {",
+            "            'files': ['bin/toy'],",
+            "            'dirs': [],",
+            "        }",
+            "        cmds = ['toy']",
+            "        return super(EB_toy, self).sanity_check_step(custom_paths=paths, custom_commands=cmds)",
+        ])
+        test_toy_easyblock = os.path.join(self.test_prefix, 'toy.py')
+        write_file(test_toy_easyblock, toy_easyblock_txt + toy_custom_sanity_check_step)
+
+        eb_args = [
+            '--extended-dry-run',
+            '--include-easyblocks=%s' % test_toy_easyblock,
+        ]
+
+        # by default, sanity check commands & paths specified by easyblock are used
+        self.mock_stdout(True)
+        self.test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
+        stdout = self.get_stdout()
+        self.mock_stdout(False)
+
+        pattern_lines = [
+            r"Sanity check paths - file.*",
+            r"\s*\* bin/toy",
+            r"Sanity check paths - \(non-empty\) directory.*",
+            r"\s*\(none\)",
+            r"Sanity check commands",
+            r"\s*\* toy",
+            r'',
+        ]
+        regex = re.compile(r'\n'.join(pattern_lines), re.M)
+        self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+
+        # we need to manually wipe the entry for the included toy easyblock,
+        # to avoid trouble with subsequent EasyBuild sessions in this test
+        del sys.modules['easybuild.easyblocks.toy']
+
+        # easyconfig specifies custom sanity_check_paths & sanity_check_commands,
+        # the ones defined by the easyblock are skipped by default
+        test_ec_txt = test_ec_txt + '\n'.join([
+            '',
+            "sanity_check_paths = {",
+            "    'files': ['README'],",
+            "    'dirs': ['bin/']",
+            "}",
+            "sanity_check_commands = ['ls %(installdir)s']",
+        ])
+        write_file(test_ec, test_ec_txt)
+
+        self.mock_stdout(True)
+        self.test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
+        stdout = self.get_stdout()
+        self.mock_stdout(False)
+
+        pattern_lines = [
+            r"Sanity check paths - file.*",
+            r"\s*\* README",
+            r"Sanity check paths - \(non-empty\) directory.*",
+            r"\s*\* bin/",
+            r"Sanity check commands",
+            r"\s*\* ls .*/software/toy/0.0",
+            r'',
+        ]
+        regex = re.compile(r'\n'.join(pattern_lines), re.M)
+        self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+
+        del sys.modules['easybuild.easyblocks.toy']
+
+        # if enhance_sanity_check is enabled, then sanity check paths/commands specified in easyconfigs
+        # are used in addition to those defined in easyblock
+        test_ec_txt = test_ec_txt + '\nenhance_sanity_check = True'
+        write_file(test_ec, test_ec_txt)
+
+        self.mock_stdout(True)
+        self.test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
+        stdout = self.get_stdout()
+        self.mock_stdout(False)
+
+        # now 'bin/toy' file and 'toy' command should also be part of sanity check
+        pattern_lines = [
+            r"Sanity check paths - file.*",
+            r"\s*\* README",
+            r"\s*\* bin/toy",
+            r"Sanity check paths - \(non-empty\) directory.*",
+            r"\s*\* bin/",
+            r"Sanity check commands",
+            r"\s*\* ls .*/software/toy/0.0",
+            r"\s*\* toy",
+            r'',
+        ]
+        regex = re.compile(r'\n'.join(pattern_lines), re.M)
+        self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+
+        # kick out any paths for included easyblocks from sys.path,
+        # to avoid infected any other tests
+        for path in sys.path[:]:
+            if '/included-easyblocks' in path:
+                sys.path.remove(path)
+
+        # reload toy easyblock (and generic toy_extension easyblock that imports it) after cleaning up sys.path,
+        # to avoid trouble in other tests due to included toy easyblock that is cached somewhere
+        # (despite the cleanup in sys.modules)
+        import easybuild.easyblocks.toy
+        reload(easybuild.easyblocks.toy)
+        import easybuild.easyblocks.generic.toy_extension
+        reload(easybuild.easyblocks.generic.toy_extension)
+
+        del sys.modules['easybuild.easyblocks.toy']
+        del sys.modules['easybuild.easyblocks.generic.toy_extension']
+
     def test_toy_dumped_easyconfig(self):
         """ Test dumping of file in eb_filerepo in both .eb and .yeb format """
         filename = 'toy-0.0'