From 475d0afef819f97544454f25a6747243585df867 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 8 Aug 2024 08:42:38 +0100 Subject: [PATCH 1/4] limit `maxparallel` to 16 by default --- easybuild/framework/easyconfig/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index aedb9edfa1..438bcea4e1 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -108,7 +108,7 @@ BUILD], 'hidden': [False, "Install module file as 'hidden' by prefixing its version with '.'", BUILD], 'installopts': ['', 'Extra options for installation', BUILD], - 'maxparallel': [None, 'Max degree of parallelism', BUILD], + 'maxparallel': [16, 'Max degree of parallelism', BUILD], 'module_only': [False, 'Only generate module file', BUILD], 'parallel': [None, ('Degree of parallelism for e.g. make (default: based on the number of ' 'cores, active cpuset and restrictions in ulimit)'), BUILD], From dcbcbeb1e2f9ef7404283c5a1b45e3fe3e1a4168 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 8 Aug 2024 10:51:20 +0100 Subject: [PATCH 2/4] alter tests --- test/framework/easyblock.py | 4 ++-- test/framework/easyconfig.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 30b95c18db..730b0ba180 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2184,7 +2184,7 @@ def test_parallel(self): handle, toy_ec1 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec1, toytxt + "\nparallel = 123") + write_file(toy_ec1, toytxt + "\nparallel = 13") handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) @@ -2202,7 +2202,7 @@ def test_parallel(self): # only 'parallel' easyconfig parameter specified (no 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec1)) test_eb.check_readiness_step() - self.assertEqual(test_eb.cfg['parallel'], 123) + self.assertEqual(test_eb.cfg['parallel'], 13) # both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec2)) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 67edcd271e..b3acedd1f2 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3476,7 +3476,7 @@ def test_template_constant_dict(self): except AttributeError: pass # Ignore if not present orig_get_avail_core_count = st.get_avail_core_count - st.get_avail_core_count = lambda: 42 + st.get_avail_core_count = lambda: 12 # also check template values after running check_readiness_step (which runs set_parallel) eb = EasyBlock(ec) @@ -3487,7 +3487,7 @@ def test_template_constant_dict(self): res = template_constant_dict(ec) res.pop('arch') - expected['parallel'] = 42 + expected['parallel'] = 12 self.assertEqual(res, expected) toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-deps.eb') From 013e891db8de46b49fe64e171c4c86c055c3ca56 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:42:43 +0100 Subject: [PATCH 3/4] fix tests --- test/framework/easyblock.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 730b0ba180..7668413f8d 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2188,7 +2188,7 @@ def test_parallel(self): handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec2, toytxt + "\nparallel = 123\nmaxparallel = 67") + write_file(toy_ec2, toytxt + "\nparallel = 14\nmaxparallel = 7") handle, toy_ec3 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) @@ -2207,7 +2207,7 @@ def test_parallel(self): # both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec2)) test_eb.check_readiness_step() - self.assertEqual(test_eb.cfg['parallel'], 67) + self.assertEqual(test_eb.cfg['parallel'], 7) # make sure 'parallel = False' is not overriden (no 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec3)) @@ -2215,20 +2215,20 @@ def test_parallel(self): self.assertEqual(test_eb.cfg['parallel'], False) # only 'parallel' build option specified - init_config(build_options={'parallel': '97', 'validate': False}) + init_config(build_options={'parallel': '9', 'validate': False}) test_eb = EasyBlock(EasyConfig(toy_ec)) test_eb.check_readiness_step() - self.assertEqual(test_eb.cfg['parallel'], 97) + self.assertEqual(test_eb.cfg['parallel'], 9) # both 'parallel' build option and easyconfig parameter specified (no 'maxparallel') test_eb = EasyBlock(EasyConfig(toy_ec1)) test_eb.check_readiness_step() - self.assertEqual(test_eb.cfg['parallel'], 97) + self.assertEqual(test_eb.cfg['parallel'], 9) # both 'parallel' and 'maxparallel' easyconfig parameters specified + 'parallel' build option test_eb = EasyBlock(EasyConfig(toy_ec2)) test_eb.check_readiness_step() - self.assertEqual(test_eb.cfg['parallel'], 67) + self.assertEqual(test_eb.cfg['parallel'], 7) # make sure 'parallel = False' is not overriden (with 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec3)) From db97a36b19797fe157902251512f2c3f1fd41dad Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 8 Aug 2024 13:21:20 +0100 Subject: [PATCH 4/4] revert unneeded changes --- test/framework/easyblock.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 7668413f8d..01014e3ee1 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2188,7 +2188,7 @@ def test_parallel(self): handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) - write_file(toy_ec2, toytxt + "\nparallel = 14\nmaxparallel = 7") + write_file(toy_ec2, toytxt + "\nparallel = 123\nmaxparallel = 67") handle, toy_ec3 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) @@ -2207,7 +2207,7 @@ def test_parallel(self): # both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec2)) test_eb.check_readiness_step() - self.assertEqual(test_eb.cfg['parallel'], 7) + self.assertEqual(test_eb.cfg['parallel'], 67) # make sure 'parallel = False' is not overriden (no 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec3)) @@ -2228,7 +2228,7 @@ def test_parallel(self): # both 'parallel' and 'maxparallel' easyconfig parameters specified + 'parallel' build option test_eb = EasyBlock(EasyConfig(toy_ec2)) test_eb.check_readiness_step() - self.assertEqual(test_eb.cfg['parallel'], 7) + self.assertEqual(test_eb.cfg['parallel'], 9) # make sure 'parallel = False' is not overriden (with 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec3))