From 4929b7b3031b40c4a3389e7dc0fbe55e731aa406 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 16 Mar 2021 19:25:39 +0000 Subject: [PATCH 1/4] added configuration option to define the env command to use for shebangs --- easybuild/framework/easyblock.py | 2 +- easybuild/tools/config.py | 4 +++ easybuild/tools/options.py | 1 + test/framework/toy_build.py | 43 +++++++++++++++++++++++++++++++- 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 5fb593aee0..5b0b780029 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2367,7 +2367,7 @@ def fix_shebang(self): if isinstance(fix_shebang_for, string_type): fix_shebang_for = [fix_shebang_for] - shebang = '#!/usr/bin/env %s' % lang + shebang = '#!%s %s' % (build_option('env_for_shebang'), lang) for glob_pattern in fix_shebang_for: paths = glob.glob(os.path.join(self.installdir, glob_pattern)) self.log.info("Fixing '%s' shebang to '%s' for files that match '%s': %s", diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 1a7ff08c88..f49aefa4ef 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -79,6 +79,7 @@ DEFAULT_CONT_TYPE = CONT_TYPE_SINGULARITY DEFAULT_BRANCH = 'develop' +DEFAULT_ENV_FOR_SHEBANG = '/usr/bin/env' DEFAULT_ENVVAR_USERS_MODULES = 'HOME' DEFAULT_INDEX_MAX_AGE = 7 * 24 * 60 * 60 # 1 week (in seconds) DEFAULT_JOB_BACKEND = 'GC3Pie' @@ -295,6 +296,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_BRANCH: [ 'pr_target_branch', ], + DEFAULT_ENV_FOR_SHEBANG: [ + 'env_for_shebang', + ], DEFAULT_INDEX_MAX_AGE: [ 'index_max_age', ], diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 605ed46ab9..76df62643a 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -375,6 +375,7 @@ def override_options(self): None, 'store', None, 'e', {'metavar': 'CLASS'}), 'enforce-checksums': ("Enforce availability of checksums for all sources/patches, so they can be verified", None, 'store_true', False), + 'env-for-shebang': ("Define the env command to use when fixing shebangs", None, 'store', '/usr/bin/env'), 'experimental': ("Allow experimental code (with behaviour that can be changed/removed at any given time).", None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9809e5b999..a9bfac92c6 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -40,7 +40,7 @@ import sys import tempfile from distutils.version import LooseVersion -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from test.framework.package import mock_fpm from unittest import TextTestRunner @@ -2854,6 +2854,47 @@ def test_fix_shebang(self): self.assertTrue(bash_shebang_regex.match(bashbin_txt), "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) + # now test with a custom env command + init_config(build_options={'env_for_shebang': "/usr/bin/env -S"}) + + self.test_toy_build(ec_file=test_ec, raise_error=True) + + toy_bindir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin') + + # bin/toy and bin/toy2 should *not* be patched, since they're binary files + toy_txt = read_file(os.path.join(toy_bindir, 'toy'), mode='rb') + for fn in ['toy.perl', 'toy.python']: + fn_txt = read_file(os.path.join(toy_bindir, fn), mode='rb') + # no shebang added + self.assertFalse(fn_txt.startswith(b"#!/")) + # exact same file as original binary (untouched) + self.assertEqual(toy_txt, fn_txt) + + # no re.M, this should match at start of file! + py_shebang_regex = re.compile(r'^#!/usr/bin/env -S python\n# test$') + for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py']: + pybin_path = os.path.join(toy_bindir, pybin) + pybin_txt = read_file(pybin_path) + self.assertTrue(py_shebang_regex.match(pybin_txt), + "Pattern '%s' found in %s: %s" % (py_shebang_regex.pattern, pybin_path, pybin_txt)) + + # no re.M, this should match at start of file! + perl_shebang_regex = re.compile(r'^#!/usr/bin/env -S perl\n# test$') + for perlbin in ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl']: + perlbin_path = os.path.join(toy_bindir, perlbin) + perlbin_txt = read_file(perlbin_path) + self.assertTrue(perl_shebang_regex.match(perlbin_txt), + "Pattern '%s' found in %s: %s" % (perl_shebang_regex.pattern, perlbin_path, perlbin_txt)) + + # There are 2 bash files which shouldn't be influenced by fix_shebang + bash_shebang_regex = re.compile(r'^#!/usr/bin/env bash\n# test$') + for bashbin in ['b1.sh', 'b2.sh']: + bashbin_path = os.path.join(toy_bindir, bashbin) + bashbin_txt = read_file(bashbin_path) + self.assertTrue(bash_shebang_regex.match(bashbin_txt), + "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) + + def test_toy_system_toolchain_alias(self): """Test use of 'system' toolchain alias.""" toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') From 00f8e7921f7f49d68604d25715d1c188190c49da Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 16 Mar 2021 19:30:28 +0000 Subject: [PATCH 2/4] removed extra blank line --- test/framework/toy_build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index a9bfac92c6..0cf73cae29 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2894,7 +2894,6 @@ def test_fix_shebang(self): self.assertTrue(bash_shebang_regex.match(bashbin_txt), "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) - def test_toy_system_toolchain_alias(self): """Test use of 'system' toolchain alias.""" toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') From ccde372d16c81df7af38c4ab78b460e7ab112c9a Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 16 Mar 2021 19:50:29 +0000 Subject: [PATCH 3/4] reuse DEFAULT_ENV_FOR_SHEBANG --- easybuild/tools/options.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 76df62643a..860f97fb48 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -60,8 +60,8 @@ from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES -from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_ENVVAR_USERS_MODULES, DEFAULT_FORCE_DOWNLOAD -from easybuild.tools.config import DEFAULT_INDEX_MAX_AGE +from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_ENV_FOR_SHEBANG, DEFAULT_ENVVAR_USERS_MODULES +from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL @@ -375,7 +375,8 @@ def override_options(self): None, 'store', None, 'e', {'metavar': 'CLASS'}), 'enforce-checksums': ("Enforce availability of checksums for all sources/patches, so they can be verified", None, 'store_true', False), - 'env-for-shebang': ("Define the env command to use when fixing shebangs", None, 'store', '/usr/bin/env'), + 'env-for-shebang': ("Define the env command to use when fixing shebangs", None, 'store', + DEFAULT_ENV_FOR_SHEBANG), 'experimental': ("Allow experimental code (with behaviour that can be changed/removed at any given time).", None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", From 4a498d97a183b0b41c48ee00437110cb883ea820 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 Mar 2021 21:51:35 +0100 Subject: [PATCH 4/4] pass custom env command in test_fix_shebang via --env-for-shebang command line option --- test/framework/toy_build.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 0cf73cae29..296ab8b05b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -40,7 +40,7 @@ import sys import tempfile from distutils.version import LooseVersion -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from test.framework.package import mock_fpm from unittest import TextTestRunner @@ -2855,9 +2855,8 @@ def test_fix_shebang(self): "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) # now test with a custom env command - init_config(build_options={'env_for_shebang': "/usr/bin/env -S"}) - - self.test_toy_build(ec_file=test_ec, raise_error=True) + extra_args = ['--env-for-shebang=/usr/bin/env -S'] + self.test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True) toy_bindir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin')