diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 6c2b43fa49..a7588f2894 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -84,9 +84,9 @@ from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP -from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP -from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP -from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook +from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP +from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP +from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook from easybuild.tools.run import check_async_cmd, run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for @@ -1851,6 +1851,8 @@ def install_extensions_sequential(self, install=True): self.log.info("Starting extension %s", ext.name) + run_hook(SINGLE_EXTENSION, self.hooks, pre_step_hook=True, args=[ext]) + # always go back to original work dir to avoid running stuff from a dir that no longer exists change_dir(self.orig_workdir) @@ -1897,6 +1899,8 @@ def install_extensions_sequential(self, install=True): self.update_exts_progress_bar(progress_info, progress_size=1) + run_hook(SINGLE_EXTENSION, self.hooks, post_step_hook=True, args=[ext]) + def install_extensions_parallel(self, install=True): """ Install extensions in parallel. diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 2861579393..1f95bacadf 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -61,6 +61,7 @@ START = 'start' PARSE = 'parse' +SINGLE_EXTENSION = 'single_extension' MODULE_WRITE = 'module_write' END = 'end' @@ -73,7 +74,31 @@ INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP, PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP] -HOOK_NAMES = [START, PARSE, MODULE_WRITE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END] +# hook names (in order of being triggered) +HOOK_NAMES = [ + START, + PARSE, +] + [p + x for x in STEP_NAMES[:STEP_NAMES.index(EXTENSIONS_STEP)] + for p in [PRE_PREF, POST_PREF]] + [ + # pre-extensions hook is triggered before starting installation of extensions, + # pre/post extension (singular) hook is triggered when installing individual extensions, + # post-extensions hook is triggered after installation of extensions + PRE_PREF + EXTENSIONS_STEP, + PRE_PREF + SINGLE_EXTENSION, + POST_PREF + SINGLE_EXTENSION, + POST_PREF + EXTENSIONS_STEP, +] + [p + x for x in STEP_NAMES[STEP_NAMES.index(EXTENSIONS_STEP)+1:STEP_NAMES.index(MODULE_STEP)] + for p in [PRE_PREF, POST_PREF]] + [ + # pre-module hook hook is triggered before starting module step which creates module file, + # module_write hook is triggered when module file has been written, + # post-module hook hook is triggered before after running module step + PRE_PREF + MODULE_STEP, + MODULE_WRITE, + POST_PREF + MODULE_STEP, +] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:] + for p in [PRE_PREF, POST_PREF]] + [ + END, +] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 5ee52a5d5f..32c0f94c36 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -61,6 +61,9 @@ def setUp(self): '', 'def pre_install_hook(self):', ' print("this is run before install step")', + '', + 'def pre_single_extension_hook(ext):', + ' print("this is run before installing an extension")', ]) write_file(self.test_hooks_pymod, test_hooks_pymod_txt) @@ -71,8 +74,15 @@ def test_load_hooks(self): hooks = load_hooks(self.test_hooks_pymod) - self.assertEqual(len(hooks), 4) - self.assertEqual(sorted(hooks.keys()), ['parse_hook', 'post_configure_hook', 'pre_install_hook', 'start_hook']) + self.assertEqual(len(hooks), 5) + expected = [ + 'parse_hook', + 'post_configure_hook', + 'pre_install_hook', + 'pre_single_extension_hook', + 'start_hook', + ] + self.assertEqual(sorted(hooks.keys()), expected) self.assertTrue(all(callable(h) for h in hooks.values())) # test caching of hooks @@ -101,6 +111,7 @@ def test_find_hook(self): post_configure_hook = [hooks[k] for k in hooks if k == 'post_configure_hook'][0] pre_install_hook = [hooks[k] for k in hooks if k == 'pre_install_hook'][0] + pre_single_extension_hook = [hooks[k] for k in hooks if k == 'pre_single_extension_hook'][0] start_hook = [hooks[k] for k in hooks if k == 'start_hook'][0] self.assertEqual(find_hook('configure', hooks), None) @@ -111,6 +122,14 @@ def test_find_hook(self): self.assertEqual(find_hook('install', hooks, pre_step_hook=True), pre_install_hook) self.assertEqual(find_hook('install', hooks, post_step_hook=True), None) + self.assertEqual(find_hook('single_extension', hooks), None) + self.assertEqual(find_hook('single_extension', hooks, pre_step_hook=True), pre_single_extension_hook) + self.assertEqual(find_hook('single_extension', hooks, post_step_hook=True), None) + + self.assertEqual(find_hook('extensions', hooks), None) + self.assertEqual(find_hook('extensions', hooks, pre_step_hook=True), None) + self.assertEqual(find_hook('extensions', hooks, post_step_hook=True), None) + self.assertEqual(find_hook('build', hooks), None) self.assertEqual(find_hook('build', hooks, pre_step_hook=True), None) self.assertEqual(find_hook('build', hooks, post_step_hook=True), None) @@ -136,6 +155,11 @@ def test_run_hook(self): run_hook('build', hooks, post_step_hook=True, args=[None]) run_hook('install', hooks, pre_step_hook=True, args=[None]) run_hook('install', hooks, post_step_hook=True, args=[None]) + run_hook('extensions', hooks, pre_step_hook=True, args=[None]) + for _ in range(3): + run_hook('single_extension', hooks, pre_step_hook=True, args=[None]) + run_hook('single_extension', hooks, post_step_hook=True, args=[None]) + run_hook('extensions', hooks, post_step_hook=True, args=[None]) stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -151,6 +175,12 @@ def test_run_hook(self): "running foo helper method", "== Running pre-install hook...", "this is run before install step", + "== Running pre-single_extension hook...", + "this is run before installing an extension", + "== Running pre-single_extension hook...", + "this is run before installing an extension", + "== Running pre-single_extension hook...", + "this is run before installing an extension", ]) self.assertEqual(stdout.strip(), expected_stdout) diff --git a/test/framework/options.py b/test/framework/options.py index ddcceceac8..dbb024fc24 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -659,6 +659,66 @@ def run_test(custom=None, extra_params=[], fmt=None): run_test(custom='bar', extra_params=['bar_extra1', 'bar_extra2'], fmt=fmt) run_test(custom='EB_foofoo', extra_params=['foofoo_extra1', 'foofoo_extra2'], fmt=fmt) + def test_avail_hooks(self): + """ + Test listing available hooks via --avail-hooks + """ + + self.mock_stderr(True) + self.mock_stdout(True) + self.eb_main(['--avail-hooks'], verbose=True, raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(stderr) + + expected = '\n'.join([ + "List of supported hooks (in order of execution):", + " start_hook", + " parse_hook", + " pre_fetch_hook", + " post_fetch_hook", + " pre_ready_hook", + " post_ready_hook", + " pre_source_hook", + " post_source_hook", + " pre_patch_hook", + " post_patch_hook", + " pre_prepare_hook", + " post_prepare_hook", + " pre_configure_hook", + " post_configure_hook", + " pre_build_hook", + " post_build_hook", + " pre_test_hook", + " post_test_hook", + " pre_install_hook", + " post_install_hook", + " pre_extensions_hook", + " pre_single_extension_hook", + " post_single_extension_hook", + " post_extensions_hook", + " pre_postproc_hook", + " post_postproc_hook", + " pre_sanitycheck_hook", + " post_sanitycheck_hook", + " pre_cleanup_hook", + " post_cleanup_hook", + " pre_module_hook", + " module_write_hook", + " post_module_hook", + " pre_permissions_hook", + " post_permissions_hook", + " pre_package_hook", + " post_package_hook", + " pre_testcases_hook", + " post_testcases_hook", + " end_hook", + '', + ]) + self.assertEqual(stdout, expected) + # double underscore to make sure it runs first, which is required to detect certain types of bugs, # e.g. running with non-initialized EasyBuild config (truly mimicing 'eb --list-toolchains') def test__list_toolchains(self): diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index ee0969a7be..fc2193be1a 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2878,6 +2878,11 @@ def test_toy_build_trace(self): def test_toy_build_hooks(self): """Test use of --hooks.""" + toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + "\nexts_list = [('bar', '0.0'), ('toy', '0.0')]" + write_file(test_ec, test_ec_txt) + hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') hooks_file_txt = textwrap.dedent(""" import os @@ -2908,6 +2913,12 @@ def module_write_hook(self, module_path, module_txt): print('in module-write hook hook for %s' % os.path.basename(module_path)) return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore') + def post_single_extension_hook(ext): + print('installing of extension %s is done!' % ext.name) + + def pre_sanitycheck_hook(self): + print('pre_sanity_check_hook') + def end_hook(): print('end hook triggered, all done!') """) @@ -2915,7 +2926,7 @@ def end_hook(): self.mock_stderr(True) self.mock_stdout(True) - self.test_toy_build(extra_args=['--hooks=%s' % hooks_file], raise_error=True) + self.test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) @@ -2927,12 +2938,16 @@ def end_hook(): toy_mod_file += '.lua' self.assertEqual(stderr, '') - # There are 4 modules written: - # Sanitycheck for extensions and main easyblock (1 each), main and devel module + # parse hook is triggered 3 times: once for main install, and then again for each extension; + # module write hook is triggered 5 times: + # - before installing extensions + # - for fake module file being created during sanity check (triggered twice, for main + toy install) + # - for final module file + # - for devel module file expected_output = textwrap.dedent(""" == Running start hook... start hook triggered - == Running parse hook for toy-0.0.eb... + == Running parse hook for test.eb... toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy @@ -2945,6 +2960,22 @@ def end_hook(): bin, lib == Running module_write hook... in module-write hook hook for {mod_name} + == Running parse hook... + toy 0.0 + ['%(name)s-%(version)s.tar.gz'] + echo toy + == Running parse hook... + toy 0.0 + ['%(name)s-%(version)s.tar.gz'] + echo toy + == Running post-single_extension hook... + installing of extension bar is done! + == Running post-single_extension hook... + installing of extension toy is done! + == Running pre-sanitycheck hook... + pre_sanity_check_hook + == Running module_write hook... + in module-write hook hook for {mod_name} == Running module_write hook... in module-write hook hook for {mod_name} == Running module_write hook...