Skip to content

add pre/post extension hook that is triggered before/after individual extension installations #4193

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lexming merged 2 commits into from
Feb 26, 2023
Merged

add pre/post extension hook that is triggered before/after individual extension installations #4193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7c33ba3
add pre/post extension hook (triggered before/after individual extens…
boegel Jan 21, 2023
6a46414
rename 'extension_hook' to 'single_extension_hook'
boegel Feb 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions easybuild/framework/easyblock.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,9 @@
from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink
from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook
from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP
from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP
from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook
from easybuild.tools.run import check_async_cmd, run_cmd
from easybuild.tools.jenkins import write_to_xml
from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
Expand Down Expand Up @@ -1851,6 +1851,8 @@ def install_extensions_sequential(self, install=True):

self.log.info("Starting extension %s", ext.name)

run_hook(SINGLE_EXTENSION, self.hooks, pre_step_hook=True, args=[ext])

# always go back to original work dir to avoid running stuff from a dir that no longer exists
change_dir(self.orig_workdir)

Expand Down Expand Up @@ -1897,6 +1899,8 @@ def install_extensions_sequential(self, install=True):

self.update_exts_progress_bar(progress_info, progress_size=1)

run_hook(SINGLE_EXTENSION, self.hooks, post_step_hook=True, args=[ext])

def install_extensions_parallel(self, install=True):
"""
Install extensions in parallel.
Expand Down
27 changes: 26 additions & 1 deletion easybuild/tools/hooks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@

START = 'start'
PARSE = 'parse'
SINGLE_EXTENSION = 'single_extension'
MODULE_WRITE = 'module_write'
END = 'end'

Expand All @@ -73,7 +74,31 @@
INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP,
PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP]

HOOK_NAMES = [START, PARSE, MODULE_WRITE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]
# hook names (in order of being triggered)
HOOK_NAMES = [
START,
PARSE,
] + [p + x for x in STEP_NAMES[:STEP_NAMES.index(EXTENSIONS_STEP)]
for p in [PRE_PREF, POST_PREF]] + [
# pre-extensions hook is triggered before starting installation of extensions,
# pre/post extension (singular) hook is triggered when installing individual extensions,
# post-extensions hook is triggered after installation of extensions
PRE_PREF + EXTENSIONS_STEP,
PRE_PREF + SINGLE_EXTENSION,
POST_PREF + SINGLE_EXTENSION,
POST_PREF + EXTENSIONS_STEP,
] + [p + x for x in STEP_NAMES[STEP_NAMES.index(EXTENSIONS_STEP)+1:STEP_NAMES.index(MODULE_STEP)]
for p in [PRE_PREF, POST_PREF]] + [
# pre-module hook hook is triggered before starting module step which creates module file,
# module_write hook is triggered when module file has been written,
# post-module hook hook is triggered before after running module step
PRE_PREF + MODULE_STEP,
MODULE_WRITE,
POST_PREF + MODULE_STEP,
] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:]
for p in [PRE_PREF, POST_PREF]] + [
END,
]
KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES]


Expand Down
34 changes: 32 additions & 2 deletions test/framework/hooks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ def setUp(self):
'',
'def pre_install_hook(self):',
' print("this is run before install step")',
'',
'def pre_single_extension_hook(ext):',
' print("this is run before installing an extension")',
])
write_file(self.test_hooks_pymod, test_hooks_pymod_txt)

Expand All @@ -71,8 +74,15 @@ def test_load_hooks(self):

hooks = load_hooks(self.test_hooks_pymod)

self.assertEqual(len(hooks), 4)
self.assertEqual(sorted(hooks.keys()), ['parse_hook', 'post_configure_hook', 'pre_install_hook', 'start_hook'])
self.assertEqual(len(hooks), 5)
expected = [
'parse_hook',
'post_configure_hook',
'pre_install_hook',
'pre_single_extension_hook',
'start_hook',
]
self.assertEqual(sorted(hooks.keys()), expected)
self.assertTrue(all(callable(h) for h in hooks.values()))

# test caching of hooks
Expand Down Expand Up @@ -101,6 +111,7 @@ def test_find_hook(self):

post_configure_hook = [hooks[k] for k in hooks if k == 'post_configure_hook'][0]
pre_install_hook = [hooks[k] for k in hooks if k == 'pre_install_hook'][0]
pre_single_extension_hook = [hooks[k] for k in hooks if k == 'pre_single_extension_hook'][0]
start_hook = [hooks[k] for k in hooks if k == 'start_hook'][0]

self.assertEqual(find_hook('configure', hooks), None)
Expand All @@ -111,6 +122,14 @@ def test_find_hook(self):
self.assertEqual(find_hook('install', hooks, pre_step_hook=True), pre_install_hook)
self.assertEqual(find_hook('install', hooks, post_step_hook=True), None)

self.assertEqual(find_hook('single_extension', hooks), None)
self.assertEqual(find_hook('single_extension', hooks, pre_step_hook=True), pre_single_extension_hook)
self.assertEqual(find_hook('single_extension', hooks, post_step_hook=True), None)

self.assertEqual(find_hook('extensions', hooks), None)
self.assertEqual(find_hook('extensions', hooks, pre_step_hook=True), None)
self.assertEqual(find_hook('extensions', hooks, post_step_hook=True), None)

self.assertEqual(find_hook('build', hooks), None)
self.assertEqual(find_hook('build', hooks, pre_step_hook=True), None)
self.assertEqual(find_hook('build', hooks, post_step_hook=True), None)
Expand All @@ -136,6 +155,11 @@ def test_run_hook(self):
run_hook('build', hooks, post_step_hook=True, args=[None])
run_hook('install', hooks, pre_step_hook=True, args=[None])
run_hook('install', hooks, post_step_hook=True, args=[None])
run_hook('extensions', hooks, pre_step_hook=True, args=[None])
for _ in range(3):
run_hook('single_extension', hooks, pre_step_hook=True, args=[None])
run_hook('single_extension', hooks, post_step_hook=True, args=[None])
run_hook('extensions', hooks, post_step_hook=True, args=[None])
stdout = self.get_stdout()
stderr = self.get_stderr()
self.mock_stdout(False)
Expand All @@ -151,6 +175,12 @@ def test_run_hook(self):
"running foo helper method",
"== Running pre-install hook...",
"this is run before install step",
"== Running pre-single_extension hook...",
"this is run before installing an extension",
"== Running pre-single_extension hook...",
"this is run before installing an extension",
"== Running pre-single_extension hook...",
"this is run before installing an extension",
])

self.assertEqual(stdout.strip(), expected_stdout)
Expand Down
60 changes: 60 additions & 0 deletions test/framework/options.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -659,6 +659,66 @@ def run_test(custom=None, extra_params=[], fmt=None):
run_test(custom='bar', extra_params=['bar_extra1', 'bar_extra2'], fmt=fmt)
run_test(custom='EB_foofoo', extra_params=['foofoo_extra1', 'foofoo_extra2'], fmt=fmt)

def test_avail_hooks(self):
"""
Test listing available hooks via --avail-hooks
"""

self.mock_stderr(True)
self.mock_stdout(True)
self.eb_main(['--avail-hooks'], verbose=True, raise_error=True)
stderr, stdout = self.get_stderr(), self.get_stdout()
self.mock_stderr(False)
self.mock_stdout(False)

self.assertFalse(stderr)

expected = '\n'.join([
"List of supported hooks (in order of execution):",
" start_hook",
" parse_hook",
" pre_fetch_hook",
" post_fetch_hook",
" pre_ready_hook",
" post_ready_hook",
" pre_source_hook",
" post_source_hook",
" pre_patch_hook",
" post_patch_hook",
" pre_prepare_hook",
" post_prepare_hook",
" pre_configure_hook",
" post_configure_hook",
" pre_build_hook",
" post_build_hook",
" pre_test_hook",
" post_test_hook",
" pre_install_hook",
" post_install_hook",
" pre_extensions_hook",
" pre_single_extension_hook",
" post_single_extension_hook",
" post_extensions_hook",
" pre_postproc_hook",
" post_postproc_hook",
" pre_sanitycheck_hook",
" post_sanitycheck_hook",
" pre_cleanup_hook",
" post_cleanup_hook",
" pre_module_hook",
" module_write_hook",
" post_module_hook",
" pre_permissions_hook",
" post_permissions_hook",
" pre_package_hook",
" post_package_hook",
" pre_testcases_hook",
" post_testcases_hook",
" end_hook",
'',
])
self.assertEqual(stdout, expected)

# double underscore to make sure it runs first, which is required to detect certain types of bugs,
# e.g. running with non-initialized EasyBuild config (truly mimicing 'eb --list-toolchains')
def test__list_toolchains(self):
Expand Down
39 changes: 35 additions & 4 deletions test/framework/toy_build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2878,6 +2878,11 @@ def test_toy_build_trace(self):

def test_toy_build_hooks(self):
"""Test use of --hooks."""
toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
test_ec = os.path.join(self.test_prefix, 'test.eb')
test_ec_txt = read_file(toy_ec) + "\nexts_list = [('bar', '0.0'), ('toy', '0.0')]"
write_file(test_ec, test_ec_txt)

hooks_file = os.path.join(self.test_prefix, 'my_hooks.py')
hooks_file_txt = textwrap.dedent("""
import os
Expand Down Expand Up @@ -2908,14 +2913,20 @@ def module_write_hook(self, module_path, module_txt):
print('in module-write hook hook for %s' % os.path.basename(module_path))
return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore')

def post_single_extension_hook(ext):
print('installing of extension %s is done!' % ext.name)

def pre_sanitycheck_hook(self):
print('pre_sanity_check_hook')

def end_hook():
print('end hook triggered, all done!')
""")
write_file(hooks_file, hooks_file_txt)

self.mock_stderr(True)
self.mock_stdout(True)
self.test_toy_build(extra_args=['--hooks=%s' % hooks_file], raise_error=True)
self.test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True)
stderr = self.get_stderr()
stdout = self.get_stdout()
self.mock_stderr(False)
Expand All @@ -2927,12 +2938,16 @@ def end_hook():
toy_mod_file += '.lua'

self.assertEqual(stderr, '')
# There are 4 modules written:
# Sanitycheck for extensions and main easyblock (1 each), main and devel module
# parse hook is triggered 3 times: once for main install, and then again for each extension;
# module write hook is triggered 5 times:
# - before installing extensions
# - for fake module file being created during sanity check (triggered twice, for main + toy install)
# - for final module file
# - for devel module file
expected_output = textwrap.dedent("""
== Running start hook...
start hook triggered
== Running parse hook for toy-0.0.eb...
== Running parse hook for test.eb...
toy 0.0
['%(name)s-%(version)s.tar.gz']
echo toy
Expand All @@ -2945,6 +2960,22 @@ def end_hook():
bin, lib
== Running module_write hook...
in module-write hook hook for {mod_name}
== Running parse hook...
toy 0.0
['%(name)s-%(version)s.tar.gz']
echo toy
== Running parse hook...
toy 0.0
['%(name)s-%(version)s.tar.gz']
echo toy
== Running post-single_extension hook...
installing of extension bar is done!
== Running post-single_extension hook...
installing of extension toy is done!
== Running pre-sanitycheck hook...
pre_sanity_check_hook
== Running module_write hook...
in module-write hook hook for {mod_name}
== Running module_write hook...
in module-write hook hook for {mod_name}
== Running module_write hook...
Expand Down