From a3a427bf76b06c6ab6ff88fcc8a500d2eda38449 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 09:54:36 +0000 Subject: [PATCH 1/8] detect Fortran .mod files in GCCcore installations --- easybuild/framework/easyblock.py | 21 +++++++++++++++++++++ easybuild/tools/config.py | 1 + easybuild/tools/options.py | 1 + 3 files changed, 23 insertions(+) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 52a700bdfe..7d31953520 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3303,6 +3303,19 @@ def regex_for_lib(lib): return fail_msg + def sanity_check_mod_files(self): + """ + Check installation for Fortran .mod files + """ + self.log.debug("Check for .mod files in install directory") + mod_files = glob.glob(os.path.join(self.installdir, '**', '*.mod'), recursive=True) + + fail_msg = None + if mod_files: + fail_msg = ".mod files (%s) found in the installation." % ', '.join(mod_files) + + return fail_msg + def _sanity_check_step_common(self, custom_paths, custom_commands): """ Determine sanity check paths and commands to use. @@ -3625,6 +3638,14 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) + if self.toolchain.name in ['GCCcore']: + mod_files_found = self.sanity_check_mod_files() + if mod_files_found: + if build_option('fail_on_mod_files'): + self.sanity_check_fail_msgs.append(mod_files_found) + else: + print_warning(mod_files_found) + # cleanup if self.fake_mod_data: self.clean_up_fake_module(self.fake_mod_data) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index b18bae480c..c5db28b05f 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -277,6 +277,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'enforce_checksums', 'experimental', 'extended_dry_run', + 'fail_on_mod_files', 'force', 'generate_devel_module', 'group_writable_installdir', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 999b62bb5c..faa1f48e80 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -400,6 +400,7 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), + 'fail-on-mod-files': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', False), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " From f148ffbf6e67fa49e13cc9f317d21e7576678264 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 10:19:40 +0000 Subject: [PATCH 2/8] easyconfig param to skip check --- easybuild/framework/easyconfig/default.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index dd91229d1e..74b96cffde 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -130,6 +130,7 @@ 'sanity_check_paths': [{}, ("List of files and directories to check " "(format: {'files':, 'dirs':})"), BUILD], 'skip': [False, "Skip existing software", BUILD], + 'skip_mod_files_check': [False, "Skip the check for .mod files in a GCCcore level install", BUILD], 'skipsteps': [[], "Skip these steps", BUILD], 'source_urls': [[], "List of URLs for source files", BUILD], 'sources': [[], "List of source files", BUILD], From ec6ece644c7b87d43515c3317d1e0e1688088e60 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 10:46:55 +0000 Subject: [PATCH 3/8] skip check when not needed --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 7d31953520..f789ffea2b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3638,7 +3638,7 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) - if self.toolchain.name in ['GCCcore']: + if self.toolchain.name in ['GCCcore'] and not self.cfg._config['skip_mod_files_check']: mod_files_found = self.sanity_check_mod_files() if mod_files_found: if build_option('fail_on_mod_files'): From d887426f83df97dbd9c5d26bd78747cba7efb0eb Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 11:00:12 +0000 Subject: [PATCH 4/8] correctly check for if the parameter is set --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f789ffea2b..a57db2591a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3638,7 +3638,7 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) - if self.toolchain.name in ['GCCcore'] and not self.cfg._config['skip_mod_files_check']: + if self.toolchain.name in ['GCCcore'] and not self.cfg['skip_mod_files_check']: mod_files_found = self.sanity_check_mod_files() if mod_files_found: if build_option('fail_on_mod_files'): From 809b6948d5f024fe0e9e137eb5c972834a09dcf6 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 12:52:53 +0000 Subject: [PATCH 5/8] add testing --- test/framework/toy_build.py | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 40384d7582..54582ec48e 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3955,6 +3955,44 @@ def test_toy_build_sanity_check_linked_libs(self): self._test_toy_build(ec_file=test_ec, extra_args=args, force=False, raise_error=True, verbose=False, verify=False) + def test_toy_mod_files(self): + """Check detection of .mod files""" + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + test_ec_txt = read_file(toy_ec) + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec) + + test_ec_txt += "\npostinstallcmds += ['touch %(installdir)s/lib/file.mod']" + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec) + + args = ['--try-toolchain=GCCcore,6.2.0', '--disable-map-toolchains'] + self.mock_stdout(True) + self.mock_stderr(True) + self._test_toy_build(ec_file=test_ec, extra_args=args) + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + pattern = r"WARNING: .mod files (.*) found in the installation." + self.assertRegex(stderr.strip(), pattern) + + args += ['--fail-on-mod-files'] + pattern = r"Sanity check failed: .mod files (.*) found in the installation." + self.assertErrorRegex(EasyBuildError, pattern, self.run_test_toy_build_with_output, ec_file=test_ec, + extra_args=args, verify=False, fails=True, verbose=False, raise_error=True) + + test_ec_txt += "\nskip_mod_files_check = True" + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec, extra_args=args) + def test_toy_ignore_test_failure(self): """Check whether use of --ignore-test-failure is mentioned in build output.""" args = ['--ignore-test-failure'] From 0b3daf25fcb1237c39d867028d9045ebeff82aab Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 13:03:42 +0000 Subject: [PATCH 6/8] address review comments --- easybuild/framework/easyblock.py | 14 +++++++------- easybuild/framework/easyconfig/default.py | 2 +- easybuild/tools/config.py | 2 +- easybuild/tools/options.py | 2 +- test/framework/toy_build.py | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a57db2591a..954bb3212e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3307,7 +3307,7 @@ def sanity_check_mod_files(self): """ Check installation for Fortran .mod files """ - self.log.debug("Check for .mod files in install directory") + self.log.debug(f"Checking for .mod files in install directory {self.installdir}...") mod_files = glob.glob(os.path.join(self.installdir, '**', '*.mod'), recursive=True) fail_msg = None @@ -3638,13 +3638,13 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) - if self.toolchain.name in ['GCCcore'] and not self.cfg['skip_mod_files_check']: - mod_files_found = self.sanity_check_mod_files() - if mod_files_found: - if build_option('fail_on_mod_files'): - self.sanity_check_fail_msgs.append(mod_files_found) + if self.toolchain.name in ['GCCcore'] and not self.cfg['skip_mod_files_sanity_check']: + mod_files_found_msg = self.sanity_check_mod_files() + if mod_files_found_msg: + if build_option('fail_on_mod_files_gcccore'): + self.sanity_check_fail_msgs.append(mod_files_found_msg) else: - print_warning(mod_files_found) + print_warning(mod_files_found_msg) # cleanup if self.fake_mod_data: diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 74b96cffde..5de9799e54 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -130,7 +130,7 @@ 'sanity_check_paths': [{}, ("List of files and directories to check " "(format: {'files':, 'dirs':})"), BUILD], 'skip': [False, "Skip existing software", BUILD], - 'skip_mod_files_check': [False, "Skip the check for .mod files in a GCCcore level install", BUILD], + 'skip_mod_files_sanity_check': [False, "Skip the check for .mod files in a GCCcore level install", BUILD], 'skipsteps': [[], "Skip these steps", BUILD], 'source_urls': [[], "List of URLs for source files", BUILD], 'sources': [[], "List of source files", BUILD], diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index c5db28b05f..d8e37ac0da 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -277,7 +277,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'enforce_checksums', 'experimental', 'extended_dry_run', - 'fail_on_mod_files', + 'fail_on_mod_files_gcccore', 'force', 'generate_devel_module', 'group_writable_installdir', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index faa1f48e80..766b891bda 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -400,7 +400,7 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), - 'fail-on-mod-files': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', False), + 'fail-on-mod-files-gcccore': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', False), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 54582ec48e..9bf85607fa 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3982,12 +3982,12 @@ def test_toy_mod_files(self): pattern = r"WARNING: .mod files (.*) found in the installation." self.assertRegex(stderr.strip(), pattern) - args += ['--fail-on-mod-files'] + args += ['--fail-on-mod-files-gcccore'] pattern = r"Sanity check failed: .mod files (.*) found in the installation." self.assertErrorRegex(EasyBuildError, pattern, self.run_test_toy_build_with_output, ec_file=test_ec, extra_args=args, verify=False, fails=True, verbose=False, raise_error=True) - test_ec_txt += "\nskip_mod_files_check = True" + test_ec_txt += "\nskip_mod_files_sanity_check = True" write_file(test_ec, test_ec_txt) with self.mocked_stdout_stderr(): From 1644c930c5126edd73311395d313d4b0ddb4b109 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 13:04:48 +0000 Subject: [PATCH 7/8] appease the hound --- easybuild/tools/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 766b891bda..1d0330ee30 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -400,7 +400,8 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), - 'fail-on-mod-files-gcccore': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', False), + 'fail-on-mod-files-gcccore': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', + False), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " From d7f2fafcfca7df863b58a1ef1ad4421138d5a557 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 16:03:53 +0100 Subject: [PATCH 8/8] tweak warning/error message when .mod files are found --- easybuild/framework/easyblock.py | 6 ++++-- test/framework/toy_build.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 954bb3212e..4fba08ccab 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3312,7 +3312,7 @@ def sanity_check_mod_files(self): fail_msg = None if mod_files: - fail_msg = ".mod files (%s) found in the installation." % ', '.join(mod_files) + fail_msg = f"One or more .mod files found in {self.installdir}: " + ', '.join(mod_files) return fail_msg @@ -3638,7 +3638,9 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) - if self.toolchain.name in ['GCCcore'] and not self.cfg['skip_mod_files_sanity_check']: + # software installed with GCCcore toolchain should not have Fortran module files (.mod), + # unless that's explicitly allowed + if self.toolchain.name in ('GCCcore',) and not self.cfg['skip_mod_files_sanity_check']: mod_files_found_msg = self.sanity_check_mod_files() if mod_files_found_msg: if build_option('fail_on_mod_files_gcccore'): diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9bf85607fa..909c3b6664 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3979,11 +3979,11 @@ def test_toy_mod_files(self): stderr = self.get_stderr() self.mock_stdout(False) self.mock_stderr(False) - pattern = r"WARNING: .mod files (.*) found in the installation." + pattern = r"WARNING: One or more \.mod files found in .*/software/toy/0.0-GCCcore-6.2.0: .*/lib64/file.mod" self.assertRegex(stderr.strip(), pattern) args += ['--fail-on-mod-files-gcccore'] - pattern = r"Sanity check failed: .mod files (.*) found in the installation." + pattern = r"Sanity check failed: One or more \.mod files found in .*/toy/0.0-GCCcore-6.2.0: .*/lib/file.mod" self.assertErrorRegex(EasyBuildError, pattern, self.run_test_toy_build_with_output, ec_file=test_ec, extra_args=args, verify=False, fails=True, verbose=False, raise_error=True)