add pre/post extension hook (triggered before/after individual extension installations) (fixes #4191)

Author: boegel

easybuild/framework/easyblock.py

@@ -83,10 +83,10 @@
 from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url
 from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
 from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink
-from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
-from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
-from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
-from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook
+from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSION_INSTALL, EXTENSIONS_STEP
+from easybuild.tools.hooks import FETCH_STEP, INSTALL_STEP, MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP
+from easybuild.tools.hooks import PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP, READY_STEP
+from easybuild.tools.hooks import SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook
 from easybuild.tools.run import check_async_cmd, run_cmd
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
@@ -1851,6 +1851,8 @@ def install_extensions_sequential(self, install=True):
 
             self.log.info("Starting extension %s", ext.name)
 
+            run_hook(EXTENSION_INSTALL, self.hooks, pre_step_hook=True, args=[ext])
+
             # always go back to original work dir to avoid running stuff from a dir that no longer exists
             change_dir(self.orig_workdir)
 
@@ -1897,6 +1899,8 @@ def install_extensions_sequential(self, install=True):
 
             self.update_exts_progress_bar(progress_info, progress_size=1)
 
+            run_hook(EXTENSION_INSTALL, self.hooks, post_step_hook=True, args=[ext])
+
     def install_extensions_parallel(self, install=True):
         """
         Install extensions in parallel.

easybuild/tools/hooks.py

@@ -61,6 +61,7 @@
 
 START = 'start'
 PARSE = 'parse'
+EXTENSION_INSTALL = 'extension'
 MODULE_WRITE = 'module_write'
 END = 'end'
 
@@ -73,7 +74,31 @@
               INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP,
               PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP]
 
-HOOK_NAMES = [START, PARSE, MODULE_WRITE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]
+# hook names (in order of being triggered)
+HOOK_NAMES = [
+    START,
+    PARSE,
+] + [p + x for x in STEP_NAMES[:STEP_NAMES.index(EXTENSIONS_STEP)]
+           for p in [PRE_PREF, POST_PREF]] + [
+    # pre-extensions hook is triggered before starting installation of extensions,
+    # pre/post extension (singular) hook is triggered when installing individual extensions,
+    # post-extensions hook is triggered after installation of extensions
+    PRE_PREF + EXTENSIONS_STEP,
+    PRE_PREF + EXTENSION_INSTALL,
+    POST_PREF + EXTENSION_INSTALL,
+    POST_PREF + EXTENSIONS_STEP,
+] + [p + x for x in STEP_NAMES[STEP_NAMES.index(EXTENSIONS_STEP)+1:STEP_NAMES.index(MODULE_STEP)]
+           for p in [PRE_PREF, POST_PREF]] + [
+    # pre-module hook hook is triggered before starting module step which creates module file,
+    # module_write hook is triggered when module file has been written,
+    # post-module hook hook is triggered before after running module step
+    PRE_PREF + MODULE_STEP,
+    MODULE_WRITE,
+    POST_PREF + MODULE_STEP,
+] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:]
+           for p in [PRE_PREF, POST_PREF]] + [
+    END,
+]
 KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES]
 
 

test/framework/hooks.py

@@ -61,6 +61,9 @@ def setUp(self):
             '',
             'def pre_install_hook(self):',
             '    print("this is run before install step")',
+            '',
+            'def pre_extension_hook(ext):',
+            '    print("this is run before installing an extension")',
         ])
         write_file(self.test_hooks_pymod, test_hooks_pymod_txt)
 
@@ -71,8 +74,9 @@ def test_load_hooks(self):
 
         hooks = load_hooks(self.test_hooks_pymod)
 
-        self.assertEqual(len(hooks), 4)
-        self.assertEqual(sorted(hooks.keys()), ['parse_hook', 'post_configure_hook', 'pre_install_hook', 'start_hook'])
+        self.assertEqual(len(hooks), 5)
+        expected = ['parse_hook', 'post_configure_hook', 'pre_extension_hook', 'pre_install_hook', 'start_hook']
+        self.assertEqual(sorted(hooks.keys()), expected)
         self.assertTrue(all(callable(h) for h in hooks.values()))
 
         # test caching of hooks
@@ -101,6 +105,7 @@ def test_find_hook(self):
 
         post_configure_hook = [hooks[k] for k in hooks if k == 'post_configure_hook'][0]
         pre_install_hook = [hooks[k] for k in hooks if k == 'pre_install_hook'][0]
+        pre_extension_hook = [hooks[k] for k in hooks if k == 'pre_extension_hook'][0]
         start_hook = [hooks[k] for k in hooks if k == 'start_hook'][0]
 
         self.assertEqual(find_hook('configure', hooks), None)
@@ -111,6 +116,14 @@ def test_find_hook(self):
         self.assertEqual(find_hook('install', hooks, pre_step_hook=True), pre_install_hook)
         self.assertEqual(find_hook('install', hooks, post_step_hook=True), None)
 
+        self.assertEqual(find_hook('extension', hooks), None)
+        self.assertEqual(find_hook('extension', hooks, pre_step_hook=True), pre_extension_hook)
+        self.assertEqual(find_hook('extension', hooks, post_step_hook=True), None)
+
+        self.assertEqual(find_hook('extensions', hooks), None)
+        self.assertEqual(find_hook('extensions', hooks, pre_step_hook=True), None)
+        self.assertEqual(find_hook('extensions', hooks, post_step_hook=True), None)
+
         self.assertEqual(find_hook('build', hooks), None)
         self.assertEqual(find_hook('build', hooks, pre_step_hook=True), None)
         self.assertEqual(find_hook('build', hooks, post_step_hook=True), None)
@@ -136,6 +149,11 @@ def test_run_hook(self):
         run_hook('build', hooks, post_step_hook=True, args=[None])
         run_hook('install', hooks, pre_step_hook=True, args=[None])
         run_hook('install', hooks, post_step_hook=True, args=[None])
+        run_hook('extensions', hooks, pre_step_hook=True, args=[None])
+        for x in range(3):
+            run_hook('extension', hooks, pre_step_hook=True, args=[None])
+            run_hook('extension', hooks, post_step_hook=True, args=[None])
+        run_hook('extensions', hooks, post_step_hook=True, args=[None])
         stdout = self.get_stdout()
         stderr = self.get_stderr()
         self.mock_stdout(False)
@@ -151,6 +169,12 @@ def test_run_hook(self):
             "running foo helper method",
             "== Running pre-install hook...",
             "this is run before install step",
+            "== Running pre-extension hook...",
+            "this is run before installing an extension",
+            "== Running pre-extension hook...",
+            "this is run before installing an extension",
+            "== Running pre-extension hook...",
+            "this is run before installing an extension",
         ])
 
         self.assertEqual(stdout.strip(), expected_stdout)

test/framework/options.py

@@ -659,6 +659,66 @@ def run_test(custom=None, extra_params=[], fmt=None):
             run_test(custom='bar', extra_params=['bar_extra1', 'bar_extra2'], fmt=fmt)
             run_test(custom='EB_foofoo', extra_params=['foofoo_extra1', 'foofoo_extra2'], fmt=fmt)
 
+    def test_avail_hooks(self):
+        """
+        Test listing available hooks via --avail-hooks
+        """
+
+        self.mock_stderr(True)
+        self.mock_stdout(True)
+        self.eb_main(['--avail-hooks'], verbose=True, raise_error=True)
+        stderr, stdout = self.get_stderr(), self.get_stdout()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
+
+        self.assertFalse(stderr)
+
+        expected = '\n'.join([
+            "List of supported hooks (in order of execution):",
+            "	start_hook",
+            "	parse_hook",
+            "	pre_fetch_hook",
+            "	post_fetch_hook",
+            "	pre_ready_hook",
+            "	post_ready_hook",
+            "	pre_source_hook",
+            "	post_source_hook",
+            "	pre_patch_hook",
+            "	post_patch_hook",
+            "	pre_prepare_hook",
+            "	post_prepare_hook",
+            "	pre_configure_hook",
+            "	post_configure_hook",
+            "	pre_build_hook",
+            "	post_build_hook",
+            "	pre_test_hook",
+            "	post_test_hook",
+            "	pre_install_hook",
+            "	post_install_hook",
+            "	pre_extensions_hook",
+            "	pre_extension_hook",
+            "	post_extension_hook",
+            "	post_extensions_hook",
+            "	pre_postproc_hook",
+            "	post_postproc_hook",
+            "	pre_sanitycheck_hook",
+            "	post_sanitycheck_hook",
+            "	pre_cleanup_hook",
+            "	post_cleanup_hook",
+            "	pre_module_hook",
+            "	module_write_hook",
+            "	post_module_hook",
+            "	pre_permissions_hook",
+            "	post_permissions_hook",
+            "	pre_package_hook",
+            "	post_package_hook",
+            "	pre_testcases_hook",
+            "	post_testcases_hook",
+            "	end_hook",
+            '',
+        ])
+        self.assertEqual(stdout, expected)
+
     # double underscore to make sure it runs first, which is required to detect certain types of bugs,
     # e.g. running with non-initialized EasyBuild config (truly mimicing 'eb --list-toolchains')
     def test__list_toolchains(self):

test/framework/toy_build.py

@@ -2878,6 +2878,11 @@ def test_toy_build_trace(self):
 
     def test_toy_build_hooks(self):
         """Test use of --hooks."""
+        toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        test_ec_txt = read_file(toy_ec) + "\nexts_list = [('bar', '0.0'), ('toy', '0.0')]"
+        write_file(test_ec, test_ec_txt)
+
         hooks_file = os.path.join(self.test_prefix, 'my_hooks.py')
         hooks_file_txt = textwrap.dedent("""
             import os
@@ -2908,14 +2913,20 @@ def module_write_hook(self, module_path, module_txt):
                 print('in module-write hook hook for %s' % os.path.basename(module_path))
                 return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore')
 
+            def post_extension_hook(ext):
+                print('installing of extension %s is done!' % ext.name)
+
+            def pre_sanitycheck_hook(self):
+                print('pre_sanity_check_hook')
+
             def end_hook():
                print('end hook triggered, all done!')
         """)
         write_file(hooks_file, hooks_file_txt)
 
         self.mock_stderr(True)
         self.mock_stdout(True)
-        self.test_toy_build(extra_args=['--hooks=%s' % hooks_file], raise_error=True)
+        self.test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True)
         stderr = self.get_stderr()
         stdout = self.get_stdout()
         self.mock_stderr(False)
@@ -2927,12 +2938,16 @@ def end_hook():
             toy_mod_file += '.lua'
 
         self.assertEqual(stderr, '')
-        # There are 4 modules written:
-        # Sanitycheck for extensions and main easyblock (1 each), main and devel module
+        # parse hook is triggered 3 times: once for main install, and then again for each extension;
+        # module write hook is triggered 5 times:
+        # - before installing extensions
+        # - for fake module file being created during sanity check (triggered twice, for main + toy install)
+        # - for final module file
+        # - for devel module file
         expected_output = textwrap.dedent("""
             == Running start hook...
             start hook triggered
-            == Running parse hook for toy-0.0.eb...
+            == Running parse hook for test.eb...
             toy 0.0
             ['%(name)s-%(version)s.tar.gz']
             echo toy
@@ -2945,6 +2960,22 @@ def end_hook():
             bin, lib
             == Running module_write hook...
             in module-write hook hook for {mod_name}
+            == Running parse hook...
+            toy 0.0
+            ['%(name)s-%(version)s.tar.gz']
+            echo toy
+            == Running parse hook...
+            toy 0.0
+            ['%(name)s-%(version)s.tar.gz']
+            echo toy
+            == Running post-extension hook...
+            installing of extension bar is done!
+            == Running post-extension hook...
+            installing of extension toy is done!
+            == Running pre-sanitycheck hook...
+            pre_sanity_check_hook
+            == Running module_write hook...
+            in module-write hook hook for {mod_name}
             == Running module_write hook...
             in module-write hook hook for {mod_name}
             == Running module_write hook...