cleanup, changes + more tests for mpi_cmd_{for,prefix} and %(mpi_cmd_prefix)s

easybuild/framework/easyconfig/templates.py

@@ -266,10 +266,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
                 template_values['mpi_cmd_prefix'] = mpi_cmd_prefix
         except EasyBuildError as err:
             # don't fail just because we couldn't resolve this template
-            if "get_software_version software version for" in str(err):
-                _log.warning("Failed to create mpi_cmd_prefix template, error was:\n%s", str(err))
-            else:
-                raise err
+            _log.warning("Failed to create mpi_cmd_prefix template, error was:\n%s", err)
 
     return template_values
 

easybuild/tools/toolchain/mpi.py

@@ -170,11 +170,14 @@ def mpi_cmd_prefix(self, nr_ranks=1):
         """Construct an MPI command prefix to precede an executable"""
 
         # Verify that the command appears at the end of mpi_cmd_for
-        if self.mpi_cmd_for('xcommandx', '1').rstrip().endswith('xcommandx'):
-            result = self.mpi_cmd_for('', nr_ranks)
+        test_cmd = 'xxx_command_xxx'
+        mpi_cmd = self.mpi_cmd_for(test_cmd, nr_ranks)
+        if mpi_cmd.rstrip().endswith(test_cmd):
+            result = mpi_cmd.replace(test_cmd, '').rstrip()
         else:
-            self.log.warning("mpi_cmd_for cannot be used to construct mpi_cmd_prefix, requires that cmd template "
-                             "appears last in result")
+            warning_msg = "mpi_cmd_for cannot be used by mpi_cmd_prefix, "
+            warning_msg += "requires that %(cmd)s template appears at the end"
+            self.log.warning(warning_msg)
             result = None
 
         return result
@@ -253,16 +256,19 @@ def mpi_cmd_for(self, cmd, nr_ranks):
             else:
                 raise EasyBuildError("Don't know which template MPI command to use for MPI family '%s'", mpi_family)
 
+        missing = []
+        for key in sorted(params.keys()):
+            tmpl = '%(' + key + ')s'
+            if tmpl not in mpi_cmd_template:
+                missing.append(tmpl)
+        if missing:
+            raise EasyBuildError("Missing templates in mpi-cmd-template value '%s': %s",
+                                 mpi_cmd_template, ', '.join(missing))
+
         try:
             res = mpi_cmd_template % params
         except KeyError as err:
-            missing = []
-            for key in params:
-                tmpl = '%(' + key + ')s'
-                if tmpl not in mpi_cmd_template:
-                    missing.append(tmpl)
-            if missing:
-                raise EasyBuildError("Missing templates in mpi-cmd-template value '%s': %s", mpi_cmd_template,
-                                     ', '.join(missing))
+            raise EasyBuildError("Failed to complete MPI cmd template '%s' with %s: KeyError %s",
+                                 mpi_cmd_template, params, err)
 
         return res

test/framework/easyconfig.py

@@ -1008,6 +1008,19 @@ def test_templating(self):
         eb['description'] = "test easyconfig % %% %s% %%% %(name)s %%(name)s %%%(name)s %%%%(name)s"
         self.assertEqual(eb['description'], "test easyconfig % %% %s% %%% PI %(name)s %PI %%(name)s")
 
+        # test use of %(mpi_cmd_prefix)s template
+        test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
+        gompi_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a.eb')
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        write_file(test_ec, read_file(gompi_ec) + "\nsanity_check_commands = ['%(mpi_cmd_prefix)s toy']")
+
+        ec = EasyConfig(test_ec)
+        self.assertEqual(ec['sanity_check_commands'], ['mpirun -n 1 toy'])
+
+        init_config(build_options={'mpi_cmd_template': "mpiexec -np %(nr_ranks)s -- %(cmd)s  "})
+        ec = EasyConfig(test_ec)
+        self.assertEqual(ec['sanity_check_commands'], ['mpiexec -np 1 -- toy'])
+
     def test_templating_doc(self):
         """test templating documentation"""
         doc = avail_easyconfig_templates()

test/framework/toolchain.py

@@ -954,38 +954,42 @@ def test_mpi_cmd_prefix(self):
 
         tc = self.get_toolchain('gompi', version='2018a')
         tc.prepare()
-        self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2 ")
-        self.assertEqual(tc.mpi_cmd_prefix(nr_ranks='2'), "mpirun -n 2 ")
-        self.assertEqual(tc.mpi_cmd_prefix(), "mpirun -n 1 ")
+        self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2")
+        self.assertEqual(tc.mpi_cmd_prefix(nr_ranks='2'), "mpirun -n 2")
+        self.assertEqual(tc.mpi_cmd_prefix(), "mpirun -n 1")
         self.modtool.purge()
 
         self.setup_sandbox_for_intel_fftw(self.test_prefix)
         tc = self.get_toolchain('intel', version='2018a')
         tc.prepare()
-        self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2 ")
-        self.assertEqual(tc.mpi_cmd_prefix(nr_ranks='2'), "mpirun -n 2 ")
-        self.assertEqual(tc.mpi_cmd_prefix(), "mpirun -n 1 ")
+        self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2")
+        self.assertEqual(tc.mpi_cmd_prefix(nr_ranks='2'), "mpirun -n 2")
+        self.assertEqual(tc.mpi_cmd_prefix(), "mpirun -n 1")
         self.modtool.purge()
 
         self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='10.2.6.038')
         tc = self.get_toolchain('intel', version='2012a')
         tc.prepare()
 
-        mpi_exec_nranks_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 4 ")
+        mpi_exec_nranks_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 4")
         self.assertTrue(mpi_exec_nranks_re.match(tc.mpi_cmd_prefix(nr_ranks=4)))
-        mpi_exec_nranks_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 1 ")
+        mpi_exec_nranks_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 1")
         self.assertTrue(mpi_exec_nranks_re.match(tc.mpi_cmd_prefix()))
 
         # test specifying custom template for MPI commands
         init_config(build_options={'mpi_cmd_template': "mpiexec -np %(nr_ranks)s -- %(cmd)s", 'silent': True})
-        self.assertEqual(tc.mpi_cmd_prefix(nr_ranks="7"), "mpiexec -np 7 -- ")
-        self.assertEqual(tc.mpi_cmd_prefix(), "mpiexec -np 1 -- ")
+        self.assertEqual(tc.mpi_cmd_prefix(nr_ranks="7"), "mpiexec -np 7 --")
+        self.assertEqual(tc.mpi_cmd_prefix(), "mpiexec -np 1 --")
 
-        # check that we return none when command does not appear at the end of the template
+        # check that we return None when command does not appear at the end of the template
         init_config(build_options={'mpi_cmd_template': "mpiexec -np %(nr_ranks)s -- %(cmd)s option", 'silent': True})
         self.assertEqual(tc.mpi_cmd_prefix(nr_ranks="7"), None)
         self.assertEqual(tc.mpi_cmd_prefix(), None)
 
+        # template with extra spaces at the end if fine though
+        init_config(build_options={'mpi_cmd_template': "mpirun -np %(nr_ranks)s %(cmd)s  ", 'silent': True})
+        self.assertEqual(tc.mpi_cmd_prefix(), "mpirun -np 1")
+
     def test_mpi_cmd_for(self):
         """Test mpi_cmd_for function."""
         self.modtool.prepend_module_path(self.test_prefix)
@@ -1012,6 +1016,16 @@ def test_mpi_cmd_for(self):
         init_config(build_options={'mpi_cmd_template': "mpiexec -np %(nr_ranks)s -- %(cmd)s", 'silent': True})
         self.assertEqual(tc.mpi_cmd_for('test123', '7'), "mpiexec -np 7 -- test123")
 
+        # check whether expected error is raised when a template with missing keys is used;
+        # %(ranks)s should be %(nr_ranks)s
+        init_config(build_options={'mpi_cmd_template': "mpiexec -np %(ranks)s -- %(cmd)s", 'silent': True})
+        error_pattern = r"Missing templates in mpi-cmd-template value 'mpiexec -np %\(ranks\)s -- %\(cmd\)s': %\(nr_ranks\)s"
+        self.assertErrorRegex(EasyBuildError, error_pattern, tc.mpi_cmd_for, 'test', 1)
+
+        init_config(build_options={'mpi_cmd_template': "mpirun %(foo)s -np %(nr_ranks)s %(cmd)s", 'silent': True})
+        error_pattern = "Failed to complete MPI cmd template .* with .*: KeyError 'foo'"
+        self.assertErrorRegex(EasyBuildError, error_pattern, tc.mpi_cmd_for, 'test', 1)
+
     def test_prepare_deps(self):
         """Test preparing for a toolchain when dependencies are involved."""
         tc = self.get_toolchain('GCC', version='6.4.0-2.28')