diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c100e88d88..b9da26043a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -81,7 +81,7 @@ from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP -from easybuild.tools.hooks import load_hooks, run_hook +from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook from easybuild.tools.run import run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for @@ -144,7 +144,7 @@ def __init__(self, ec): # keep track of original working directory, so we can go back there self.orig_workdir = os.getcwd() - # list of pre- and post-step hooks + # dict of all hooks (mapping of name to function) self.hooks = load_hooks(build_option('hooks')) # list of patch/source files, along with checksums @@ -3165,6 +3165,10 @@ def make_module_step(self, fake=False): txt += self.make_module_extra() txt += self.make_module_footer() + hook_txt = run_hook(MODULE_WRITE, self.hooks, args=[self, mod_filepath, txt]) + if hook_txt is not None: + txt = hook_txt + if self.dry_run: # only report generating actual module file during dry run, don't mention temporary module files if not fake: diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 85407fb993..cb2d72c472 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -58,6 +58,7 @@ START = 'start' PARSE = 'parse' +MODULE_WRITE = 'module_write' END = 'end' PRE_PREF = 'pre_' @@ -69,7 +70,7 @@ INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP, PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP] -HOOK_NAMES = [START, PARSE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END] +HOOK_NAMES = [START, PARSE, MODULE_WRITE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] @@ -99,7 +100,7 @@ def load_hooks(hooks_path): if attr.endswith(HOOK_SUFF): hook = getattr(imported_hooks, attr) if callable(hook): - hooks.update({attr: hook}) + hooks[attr] = hook else: _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr) _log.info("Found hooks: %s", sorted(hooks.keys())) @@ -119,11 +120,8 @@ def load_hooks(hooks_path): def verify_hooks(hooks): - """Check whether list of obtained hooks only includes known hooks.""" - unknown_hooks = [] - for key in sorted(hooks): - if key not in KNOWN_HOOKS: - unknown_hooks.append(key) + """Check whether obtained hooks only includes known hooks.""" + unknown_hooks = [key for key in sorted(hooks) if key not in KNOWN_HOOKS] if unknown_hooks: error_lines = ["Found one or more unknown hooks:"] @@ -147,7 +145,7 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): Find hook with specified label. :param label: name of hook - :param hooks: list of defined hooks + :param hooks: dict of defined hooks :param pre_step_hook: indicates whether hook to run is a pre-step hook :param post_step_hook: indicates whether hook to run is a post-step hook """ @@ -162,27 +160,26 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): hook_name = hook_prefix + label + HOOK_SUFF - for key in hooks: - if key == hook_name: - _log.info("Found %s hook", hook_name) - res = hooks[key] - break + res = hooks.get(hook_name) + if res: + _log.info("Found %s hook", hook_name) return res def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None): """ - Run hook with specified label. + Run hook with specified label and return result of calling the hook or None. :param label: name of hook - :param hooks: list of defined hooks + :param hooks: dict of defined hooks :param pre_step_hook: indicates whether hook to run is a pre-step hook :param post_step_hook: indicates whether hook to run is a post-step hook :param args: arguments to pass to hook function :param msg: custom message that is printed when hook is called """ hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook) + res = None if hook: if args is None: args = [] @@ -197,4 +194,5 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, print_msg(msg) _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args) - hook(*args) + res = hook(*args) + return res diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index c339194ad3..b8d58ae64b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -39,6 +39,7 @@ import stat import sys import tempfile +import textwrap from distutils.version import LooseVersion from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from test.framework.package import mock_fpm @@ -2638,34 +2639,38 @@ def test_toy_build_trace(self): def test_toy_build_hooks(self): """Test use of --hooks.""" hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') - hooks_file_txt = '\n'.join([ - "import os", - '', - "def start_hook():", - " print('start hook triggered')", - '', - "def parse_hook(ec):", - " print('%s %s' % (ec.name, ec.version))", + hooks_file_txt = textwrap.dedent(""" + import os + + def start_hook(): + print('start hook triggered') + + def parse_hook(ec): + print('%s %s' % (ec.name, ec.version)) # print sources value to check that raw untemplated strings are exposed in parse_hook - " print(ec['sources'])", + print(ec['sources']) # try appending to postinstallcmd to see whether the modification is actually picked up # (required templating to be disabled before parse_hook is called) - " ec['postinstallcmds'].append('echo toy')", - " print(ec['postinstallcmds'][-1])", - '', - "def pre_configure_hook(self):", - " print('pre-configure: toy.source: %s' % os.path.exists('toy.source'))", - '', - "def post_configure_hook(self):", - " print('post-configure: toy.source: %s' % os.path.exists('toy.source'))", - '', - "def post_install_hook(self):", - " print('in post-install hook for %s v%s' % (self.name, self.version))", - " print(', '.join(sorted(os.listdir(self.installdir))))", - '', - "def end_hook():", - " print('end hook triggered, all done!')", - ]) + ec['postinstallcmds'].append('echo toy') + print(ec['postinstallcmds'][-1]) + + def pre_configure_hook(self): + print('pre-configure: toy.source: %s' % os.path.exists('toy.source')) + + def post_configure_hook(self): + print('post-configure: toy.source: %s' % os.path.exists('toy.source')) + + def post_install_hook(self): + print('in post-install hook for %s v%s' % (self.name, self.version)) + print(', '.join(sorted(os.listdir(self.installdir)))) + + def module_write_hook(self, module_path, module_txt): + print('in module-write hook hook for %s' % os.path.basename(module_path)) + return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore') + + def end_hook(): + print('end hook triggered, all done!') + """) write_file(hooks_file, hooks_file_txt) self.mock_stderr(True) @@ -2676,26 +2681,44 @@ def test_toy_build_hooks(self): self.mock_stderr(False) self.mock_stdout(False) + test_mod_path = os.path.join(self.test_installpath, 'modules', 'all') + toy_mod_file = os.path.join(test_mod_path, 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_mod_file += '.lua' + self.assertEqual(stderr, '') - expected_output = '\n'.join([ - "== Running start hook...", - "start hook triggered", - "== Running parse hook for toy-0.0.eb...", - "toy 0.0", - "['%(name)s-%(version)s.tar.gz']", - "echo toy", - "== Running pre-configure hook...", - "pre-configure: toy.source: True", - "== Running post-configure hook...", - "post-configure: toy.source: False", - "== Running post-install hook...", - "in post-install hook for toy v0.0", - "bin, lib", - "== Running end hook...", - "end hook triggered, all done!", - ]) + # There are 4 modules written: + # Sanitycheck for extensions and main easyblock (1 each), main and devel module + expected_output = textwrap.dedent(""" + == Running start hook... + start hook triggered + == Running parse hook for toy-0.0.eb... + toy 0.0 + ['%(name)s-%(version)s.tar.gz'] + echo toy + == Running pre-configure hook... + pre-configure: toy.source: True + == Running post-configure hook... + post-configure: toy.source: False + == Running post-install hook... + in post-install hook for toy v0.0 + bin, lib + == Running module_write hook... + in module-write hook hook for {mod_name} + == Running module_write hook... + in module-write hook hook for {mod_name} + == Running module_write hook... + in module-write hook hook for {mod_name} + == Running module_write hook... + in module-write hook hook for {mod_name} + == Running end hook... + end hook triggered, all done! + """).strip().format(mod_name=os.path.basename(toy_mod_file)) self.assertEqual(stdout.strip(), expected_output) + toy_mod = read_file(toy_mod_file) + self.assertIn('Not a toy anymore', toy_mod) + def test_toy_multi_deps(self): """Test installation of toy easyconfig that uses multi_deps.""" test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')