Skip to content

Commit 9b480c8

Browse files
lexminglexming
authored
Merge pull request #4193 from boegel/extension_hook
add pre/post extension hook (triggered before/after individual extension installations)
2 parents 426c9c6 + 6a46414 commit 9b480c8

5 files changed

Lines changed: 160 additions & 10 deletions

File tree

‎easybuild/framework/easyblock.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,9 @@
8484
from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
8585
from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink
8686
from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
87-
from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
88-
from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
89-
from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook
87+
from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP
88+
from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP
89+
from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook
9090
from easybuild.tools.run import check_async_cmd, run_cmd
9191
from easybuild.tools.jenkins import write_to_xml
9292
from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
@@ -1851,6 +1851,8 @@ def install_extensions_sequential(self, install=True):
18511851

18521852
self.log.info("Starting extension %s", ext.name)
18531853

1854+
run_hook(SINGLE_EXTENSION, self.hooks, pre_step_hook=True, args=[ext])
1855+
18541856
# always go back to original work dir to avoid running stuff from a dir that no longer exists
18551857
change_dir(self.orig_workdir)
18561858

@@ -1897,6 +1899,8 @@ def install_extensions_sequential(self, install=True):
18971899

18981900
self.update_exts_progress_bar(progress_info, progress_size=1)
18991901

1902+
run_hook(SINGLE_EXTENSION, self.hooks, post_step_hook=True, args=[ext])
1903+
19001904
def install_extensions_parallel(self, install=True):
19011905
"""
19021906
Install extensions in parallel.

‎easybuild/tools/hooks.py‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161

6262
START = 'start'
6363
PARSE = 'parse'
64+
SINGLE_EXTENSION = 'single_extension'
6465
MODULE_WRITE = 'module_write'
6566
END = 'end'
6667

@@ -73,7 +74,31 @@
7374
INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP,
7475
PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP]
7576

76-
HOOK_NAMES = [START, PARSE, MODULE_WRITE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]
77+
# hook names (in order of being triggered)
78+
HOOK_NAMES = [
79+
START,
80+
PARSE,
81+
] + [p + x for x in STEP_NAMES[:STEP_NAMES.index(EXTENSIONS_STEP)]
82+
for p in [PRE_PREF, POST_PREF]] + [
83+
# pre-extensions hook is triggered before starting installation of extensions,
84+
# pre/post extension (singular) hook is triggered when installing individual extensions,
85+
# post-extensions hook is triggered after installation of extensions
86+
PRE_PREF + EXTENSIONS_STEP,
87+
PRE_PREF + SINGLE_EXTENSION,
88+
POST_PREF + SINGLE_EXTENSION,
89+
POST_PREF + EXTENSIONS_STEP,
90+
] + [p + x for x in STEP_NAMES[STEP_NAMES.index(EXTENSIONS_STEP)+1:STEP_NAMES.index(MODULE_STEP)]
91+
for p in [PRE_PREF, POST_PREF]] + [
92+
# pre-module hook hook is triggered before starting module step which creates module file,
93+
# module_write hook is triggered when module file has been written,
94+
# post-module hook hook is triggered before after running module step
95+
PRE_PREF + MODULE_STEP,
96+
MODULE_WRITE,
97+
POST_PREF + MODULE_STEP,
98+
] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:]
99+
for p in [PRE_PREF, POST_PREF]] + [
100+
END,
101+
]
77102
KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES]
78103

79104

‎test/framework/hooks.py‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ def setUp(self):
6161
'',
6262
'def pre_install_hook(self):',
6363
' print("this is run before install step")',
64+
'',
65+
'def pre_single_extension_hook(ext):',
66+
' print("this is run before installing an extension")',
6467
])
6568
write_file(self.test_hooks_pymod, test_hooks_pymod_txt)
6669

@@ -71,8 +74,15 @@ def test_load_hooks(self):
7174

7275
hooks = load_hooks(self.test_hooks_pymod)
7376

74-
self.assertEqual(len(hooks), 4)
75-
self.assertEqual(sorted(hooks.keys()), ['parse_hook', 'post_configure_hook', 'pre_install_hook', 'start_hook'])
77+
self.assertEqual(len(hooks), 5)
78+
expected = [
79+
'parse_hook',
80+
'post_configure_hook',
81+
'pre_install_hook',
82+
'pre_single_extension_hook',
83+
'start_hook',
84+
]
85+
self.assertEqual(sorted(hooks.keys()), expected)
7686
self.assertTrue(all(callable(h) for h in hooks.values()))
7787

7888
# test caching of hooks
@@ -101,6 +111,7 @@ def test_find_hook(self):
101111

102112
post_configure_hook = [hooks[k] for k in hooks if k == 'post_configure_hook'][0]
103113
pre_install_hook = [hooks[k] for k in hooks if k == 'pre_install_hook'][0]
114+
pre_single_extension_hook = [hooks[k] for k in hooks if k == 'pre_single_extension_hook'][0]
104115
start_hook = [hooks[k] for k in hooks if k == 'start_hook'][0]
105116

106117
self.assertEqual(find_hook('configure', hooks), None)
@@ -111,6 +122,14 @@ def test_find_hook(self):
111122
self.assertEqual(find_hook('install', hooks, pre_step_hook=True), pre_install_hook)
112123
self.assertEqual(find_hook('install', hooks, post_step_hook=True), None)
113124

125+
self.assertEqual(find_hook('single_extension', hooks), None)
126+
self.assertEqual(find_hook('single_extension', hooks, pre_step_hook=True), pre_single_extension_hook)
127+
self.assertEqual(find_hook('single_extension', hooks, post_step_hook=True), None)
128+
129+
self.assertEqual(find_hook('extensions', hooks), None)
130+
self.assertEqual(find_hook('extensions', hooks, pre_step_hook=True), None)
131+
self.assertEqual(find_hook('extensions', hooks, post_step_hook=True), None)
132+
114133
self.assertEqual(find_hook('build', hooks), None)
115134
self.assertEqual(find_hook('build', hooks, pre_step_hook=True), None)
116135
self.assertEqual(find_hook('build', hooks, post_step_hook=True), None)
@@ -136,6 +155,11 @@ def test_run_hook(self):
136155
run_hook('build', hooks, post_step_hook=True, args=[None])
137156
run_hook('install', hooks, pre_step_hook=True, args=[None])
138157
run_hook('install', hooks, post_step_hook=True, args=[None])
158+
run_hook('extensions', hooks, pre_step_hook=True, args=[None])
159+
for _ in range(3):
160+
run_hook('single_extension', hooks, pre_step_hook=True, args=[None])
161+
run_hook('single_extension', hooks, post_step_hook=True, args=[None])
162+
run_hook('extensions', hooks, post_step_hook=True, args=[None])
139163
stdout = self.get_stdout()
140164
stderr = self.get_stderr()
141165
self.mock_stdout(False)
@@ -151,6 +175,12 @@ def test_run_hook(self):
151175
"running foo helper method",
152176
"== Running pre-install hook...",
153177
"this is run before install step",
178+
"== Running pre-single_extension hook...",
179+
"this is run before installing an extension",
180+
"== Running pre-single_extension hook...",
181+
"this is run before installing an extension",
182+
"== Running pre-single_extension hook...",
183+
"this is run before installing an extension",
154184
])
155185

156186
self.assertEqual(stdout.strip(), expected_stdout)

‎test/framework/options.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -661,6 +661,66 @@ def run_test(custom=None, extra_params=[], fmt=None):
661661
run_test(custom='bar', extra_params=['bar_extra1', 'bar_extra2'], fmt=fmt)
662662
run_test(custom='EB_foofoo', extra_params=['foofoo_extra1', 'foofoo_extra2'], fmt=fmt)
663663

664+
def test_avail_hooks(self):
665+
"""
666+
Test listing available hooks via --avail-hooks
667+
"""
668+
669+
self.mock_stderr(True)
670+
self.mock_stdout(True)
671+
self.eb_main(['--avail-hooks'], verbose=True, raise_error=True)
672+
stderr, stdout = self.get_stderr(), self.get_stdout()
673+
self.mock_stderr(False)
674+
self.mock_stdout(False)
675+
676+
self.assertFalse(stderr)
677+
678+
expected = '\n'.join([
679+
"List of supported hooks (in order of execution):",
680+
" start_hook",
681+
" parse_hook",
682+
" pre_fetch_hook",
683+
" post_fetch_hook",
684+
" pre_ready_hook",
685+
" post_ready_hook",
686+
" pre_source_hook",
687+
" post_source_hook",
688+
" pre_patch_hook",
689+
" post_patch_hook",
690+
" pre_prepare_hook",
691+
" post_prepare_hook",
692+
" pre_configure_hook",
693+
" post_configure_hook",
694+
" pre_build_hook",
695+
" post_build_hook",
696+
" pre_test_hook",
697+
" post_test_hook",
698+
" pre_install_hook",
699+
" post_install_hook",
700+
" pre_extensions_hook",
701+
" pre_single_extension_hook",
702+
" post_single_extension_hook",
703+
" post_extensions_hook",
704+
" pre_postproc_hook",
705+
" post_postproc_hook",
706+
" pre_sanitycheck_hook",
707+
" post_sanitycheck_hook",
708+
" pre_cleanup_hook",
709+
" post_cleanup_hook",
710+
" pre_module_hook",
711+
" module_write_hook",
712+
" post_module_hook",
713+
" pre_permissions_hook",
714+
" post_permissions_hook",
715+
" pre_package_hook",
716+
" post_package_hook",
717+
" pre_testcases_hook",
718+
" post_testcases_hook",
719+
" end_hook",
720+
'',
721+
])
722+
self.assertEqual(stdout, expected)
723+
664724
# double underscore to make sure it runs first, which is required to detect certain types of bugs,
665725
# e.g. running with non-initialized EasyBuild config (truly mimicing 'eb --list-toolchains')
666726
def test__list_toolchains(self):

‎test/framework/toy_build.py‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2877,6 +2877,11 @@ def test_toy_build_trace(self):
28772877

28782878
def test_toy_build_hooks(self):
28792879
"""Test use of --hooks."""
2880+
toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
2881+
test_ec = os.path.join(self.test_prefix, 'test.eb')
2882+
test_ec_txt = read_file(toy_ec) + "\nexts_list = [('bar', '0.0'), ('toy', '0.0')]"
2883+
write_file(test_ec, test_ec_txt)
2884+
28802885
hooks_file = os.path.join(self.test_prefix, 'my_hooks.py')
28812886
hooks_file_txt = textwrap.dedent("""
28822887
import os
@@ -2907,14 +2912,20 @@ def module_write_hook(self, module_path, module_txt):
29072912
print('in module-write hook hook for %s' % os.path.basename(module_path))
29082913
return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore')
29092914
2915+
def post_single_extension_hook(ext):
2916+
print('installing of extension %s is done!' % ext.name)
2917+
2918+
def pre_sanitycheck_hook(self):
2919+
print('pre_sanity_check_hook')
2920+
29102921
def end_hook():
29112922
print('end hook triggered, all done!')
29122923
""")
29132924
write_file(hooks_file, hooks_file_txt)
29142925

29152926
self.mock_stderr(True)
29162927
self.mock_stdout(True)
2917-
self.test_toy_build(extra_args=['--hooks=%s' % hooks_file], raise_error=True)
2928+
self.test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True)
29182929
stderr = self.get_stderr()
29192930
stdout = self.get_stdout()
29202931
self.mock_stderr(False)
@@ -2926,12 +2937,16 @@ def end_hook():
29262937
toy_mod_file += '.lua'
29272938

29282939
self.assertEqual(stderr, '')
2929-
# There are 4 modules written:
2930-
# Sanitycheck for extensions and main easyblock (1 each), main and devel module
2940+
# parse hook is triggered 3 times: once for main install, and then again for each extension;
2941+
# module write hook is triggered 5 times:
2942+
# - before installing extensions
2943+
# - for fake module file being created during sanity check (triggered twice, for main + toy install)
2944+
# - for final module file
2945+
# - for devel module file
29312946
expected_output = textwrap.dedent("""
29322947
== Running start hook...
29332948
start hook triggered
2934-
== Running parse hook for toy-0.0.eb...
2949+
== Running parse hook for test.eb...
29352950
toy 0.0
29362951
['%(name)s-%(version)s.tar.gz']
29372952
echo toy
@@ -2944,6 +2959,22 @@ def end_hook():
29442959
bin, lib
29452960
== Running module_write hook...
29462961
in module-write hook hook for {mod_name}
2962+
== Running parse hook...
2963+
toy 0.0
2964+
['%(name)s-%(version)s.tar.gz']
2965+
echo toy
2966+
== Running parse hook...
2967+
toy 0.0
2968+
['%(name)s-%(version)s.tar.gz']
2969+
echo toy
2970+
== Running post-single_extension hook...
2971+
installing of extension bar is done!
2972+
== Running post-single_extension hook...
2973+
installing of extension toy is done!
2974+
== Running pre-sanitycheck hook...
2975+
pre_sanity_check_hook
2976+
== Running module_write hook...
2977+
in module-write hook hook for {mod_name}
29472978
== Running module_write hook...
29482979
in module-write hook hook for {mod_name}
29492980
== Running module_write hook...

0 commit comments

Comments
 (0)