From 830754978a345bf77b66ca5daa67cdf1fb955d64 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 2 Mar 2020 09:21:10 +0100 Subject: [PATCH 1/9] add get_cpu_arch_name function to systemtools, using archspec to determine CPU arch name --- easybuild/tools/systemtools.py | 23 +++++++++++++++++++++++ test/framework/systemtools.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 0e114cb0dc..5f8853500c 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -59,6 +59,14 @@ _log.debug("Failed to import 'distro' Python module: %s", err) HAVE_DISTRO = False +try: + from archspec.cpu import host as archspec_cpu_host + HAVE_ARCHSPEC = True +except ImportError as err: + _log.debug("Failed to import 'archspec' Python module: %s", err) + HAVE_ARCHSPEC = False + + # Architecture constants AARCH32 = 'AArch32' @@ -344,6 +352,20 @@ def get_cpu_family(): return family +def get_cpu_arch_name(): + """ + Determine CPU architecture name via archspec (if available). + """ + cpu_arch_name = None + if HAVE_ARCHSPEC: + cpu_arch_name = archspec_cpu_host() + + if cpu_arch_name is None: + cpu_arch_name = UNKNOWN + + return cpu_arch_name + + def get_cpu_model(): """ Determine CPU model, e.g., Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz @@ -746,6 +768,7 @@ def get_system_info(): return { 'core_count': get_avail_core_count(), 'total_memory': get_total_memory(), + 'cpu_arch_name': get_cpu_arch_name(), 'cpu_model': get_cpu_model(), 'cpu_speed': get_cpu_speed(), 'cpu_vendor': get_cpu_vendor(), diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index fa1f1331cb..c33d271a66 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -44,8 +44,8 @@ from easybuild.tools.systemtools import CPU_VENDORS, AMD, APM, ARM, CAVIUM, IBM, INTEL from easybuild.tools.systemtools import MAX_FREQ_FP, PROC_CPUINFO_FP, PROC_MEMINFO_FP from easybuild.tools.systemtools import check_python_version, pick_dep_version -from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_architecture, get_cpu_family -from easybuild.tools.systemtools import get_cpu_features, get_cpu_model, get_cpu_speed, get_cpu_vendor +from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_arch_name, get_cpu_architecture +from easybuild.tools.systemtools import get_cpu_family, get_cpu_features, get_cpu_model, get_cpu_speed, get_cpu_vendor from easybuild.tools.systemtools import get_gcc_version, get_glibc_version, get_os_type, get_os_name, get_os_version from easybuild.tools.systemtools import get_platform_name, get_shared_lib_ext, get_system_info, get_total_memory @@ -338,6 +338,11 @@ def setUp(self): self.orig_platform_uname = st.platform.uname self.orig_get_tool_version = st.get_tool_version self.orig_sys_version_info = st.sys.version_info + self.orig_HAVE_ARCHSPEC = st.HAVE_ARCHSPEC + if hasattr(st, 'archspec_cpu_host'): + self.orig_archspec_cpu_host = st.archspec_cpu_host + else: + self.orig_archspec_cpu_host = None def tearDown(self): """Cleanup after systemtools test.""" @@ -349,6 +354,9 @@ def tearDown(self): st.platform.uname = self.orig_platform_uname st.get_tool_version = self.orig_get_tool_version st.sys.version_info = self.orig_sys_version_info + st.HAVE_ARCHSPEC = self.orig_HAVE_ARCHSPEC + if self.orig_archspec_cpu_host is not None: + st.archspec_cpu_host = self.orig_archspec_cpu_host super(SystemToolsTest, self).tearDown() def test_avail_core_count_native(self): @@ -529,6 +537,22 @@ def test_cpu_architecture(self): MACHINE_NAME = name self.assertEqual(get_cpu_architecture(), machine_names[name]) + def test_cpu_arch_name_native(self): + """Test getting CPU arch name.""" + arch_name = get_cpu_arch_name() + self.assertTrue(isinstance(arch_name, string_type)) + + def test_cpu_arch_name(self): + """Test getting CPU arch name.""" + st.HAVE_ARCHSPEC = True + st.archspec_cpu_host = lambda: 'haswell' + arch_name = get_cpu_arch_name() + self.assertEqual(arch_name, 'haswell') + + st.archspec_cpu_host = lambda: None + arch_name = get_cpu_arch_name() + self.assertEqual(arch_name, 'UNKNOWN') + def test_cpu_vendor_native(self): """Test getting CPU vendor.""" cpu_vendor = get_cpu_vendor() From 707a4b71075fa337599623635a69cbb0454dc50f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Mar 2020 16:34:28 +0100 Subject: [PATCH 2/9] mention CPU arch name in comment for uploaded test report, if it's known --- easybuild/tools/testing.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 456e7c0db2..d114992ec7 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -50,7 +50,7 @@ from easybuild.tools.jenkins import aggregate_xml_in_dirs from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel from easybuild.tools.robot import resolve_dependencies -from easybuild.tools.systemtools import get_system_info +from easybuild.tools.systemtools import UNKNOWN, get_system_info from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION @@ -264,6 +264,11 @@ def post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, 'os_version': system_info['os_version'], 'pyver': system_info['python_version'].split(' ')[0], } + + # also mention CPU architecture name, but only if it's known + if system_info['cpu_arch_name'] != UNKNOWN: + short_system_info['cpu_model'] += " (%s)" % system_info['cpu_arch_name'] + comment_lines = [ "Test report by @%s" % user, ('**FAILED**', '**SUCCESS**')[success], From 959f99758efbd20241d0d120b402dcfb96392bd7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Mar 2020 20:21:29 +0100 Subject: [PATCH 3/9] install archspec as optional dependency when testing with Python >= 2.7 --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 3a93ac826f..0a8591c50f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,3 +50,5 @@ GC3Pie python-graph-dot python-hglib requests + +archspec; python_version >= '2.7' From e456a677e32dfe615db7f2bc5cd3a9c19abd7408 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Mar 2020 14:31:51 +0100 Subject: [PATCH 4/9] only get cpu arch name in get_cpu_arch_name, not the whole archspec.MicroArchitecture instance --- easybuild/tools/systemtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 5f8853500c..c155d47853 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -358,7 +358,7 @@ def get_cpu_arch_name(): """ cpu_arch_name = None if HAVE_ARCHSPEC: - cpu_arch_name = archspec_cpu_host() + cpu_arch_name = archspec_cpu_host().name if cpu_arch_name is None: cpu_arch_name = UNKNOWN From 0fc2a4b00845358c7e97deef40ef7064bd45657f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Mar 2020 17:11:07 +0100 Subject: [PATCH 5/9] take into account that result of archspec.cpu.host() may be None --- easybuild/tools/systemtools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index c155d47853..13e28cc6f3 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -358,7 +358,9 @@ def get_cpu_arch_name(): """ cpu_arch_name = None if HAVE_ARCHSPEC: - cpu_arch_name = archspec_cpu_host().name + res = archspec_cpu_host() + if res: + cpu_arch_name = res.name if cpu_arch_name is None: cpu_arch_name = UNKNOWN From a6cb6b505091f5d345f1b543c506de06f067b6b0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Mar 2020 17:11:22 +0100 Subject: [PATCH 6/9] fix test_cpu_arch_name --- test/framework/systemtools.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index c33d271a66..bda6813014 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -544,8 +544,13 @@ def test_cpu_arch_name_native(self): def test_cpu_arch_name(self): """Test getting CPU arch name.""" + + class MicroArch(object): + def __init__(self, name): + self.name = name + st.HAVE_ARCHSPEC = True - st.archspec_cpu_host = lambda: 'haswell' + st.archspec_cpu_host = lambda: MicroArch('haswell') arch_name = get_cpu_arch_name() self.assertEqual(arch_name, 'haswell') From 8a1115f16790afae1b7e1be59e25fb34daa133c6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 29 Mar 2020 16:34:34 +0200 Subject: [PATCH 7/9] fix adding of CPU arch name in PR comment for test report --- easybuild/tools/testing.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index d114992ec7..d44080dd3c 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -256,6 +256,11 @@ def post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, # post comment to report test result system_info = init_session_state['system_info'] + + # also mention CPU architecture name, but only if it's known + if system_info['cpu_arch_name'] != UNKNOWN: + system_info['cpu_model'] += " (%s)" % system_info['cpu_arch_name'] + short_system_info = "%(hostname)s - %(os_type)s %(os_name)s %(os_version)s, %(cpu_model)s, Python %(pyver)s" % { 'cpu_model': system_info['cpu_model'], 'hostname': system_info['hostname'], @@ -265,10 +270,6 @@ def post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, 'pyver': system_info['python_version'].split(' ')[0], } - # also mention CPU architecture name, but only if it's known - if system_info['cpu_arch_name'] != UNKNOWN: - short_system_info['cpu_model'] += " (%s)" % system_info['cpu_arch_name'] - comment_lines = [ "Test report by @%s" % user, ('**FAILED**', '**SUCCESS**')[success], From 1d571f0b7c1e5639651e3715fb15879b2d995c94 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 29 Mar 2020 16:38:22 +0200 Subject: [PATCH 8/9] mention CPU arch name in --show-system-info output, if archspec is available --- easybuild/tools/options.py | 9 ++++++++- test/framework/options.py | 5 +++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 6a23c2d82e..987f707be6 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1153,6 +1153,7 @@ def show_system_info(self): """Show system information.""" system_info = get_system_info() cpu_features = get_cpu_features() + cpu_arch_name = system_info['cpu_arch_name'] lines = [ "System information (%s):" % system_info['hostname'], '', @@ -1166,6 +1167,11 @@ def show_system_info(self): " -> vendor: %s" % system_info['cpu_vendor'], " -> architecture: %s" % get_cpu_architecture(), " -> family: %s" % get_cpu_family(), + ] + if cpu_arch_name: + lines.append(" -> arch name: %s" % cpu_arch_name) + + lines.extend([ " -> model: %s" % system_info['cpu_model'], " -> speed: %s" % system_info['cpu_speed'], " -> cores: %s" % system_info['core_count'], @@ -1175,7 +1181,8 @@ def show_system_info(self): " -> glibc version: %s" % system_info['glibc_version'], " -> Python binary: %s" % sys.executable, " -> Python version: %s" % sys.version.split(' ')[0], - ] + ]) + return '\n'.join(lines) def show_config(self): diff --git a/test/framework/options.py b/test/framework/options.py index d09bfdc17a..a6c544d1b2 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -59,6 +59,7 @@ from easybuild.tools.py2vs3 import URLError, reload, sort_looseversions from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import HAVE_ARCHSPEC from easybuild.tools.version import VERSION from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config @@ -4562,6 +4563,10 @@ def test_show_system_info(self): "^ -> Python binary: .*/[pP]ython[0-9]?", "^ -> Python version: [0-9.]+", ] + + if HAVE_ARCHSPEC: + patterns.append(r"^ -> arch name: \w+") + for pattern in patterns: regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) From 983fcafaf40e88bf5a16ce0dc9779ff69d2da120 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Mar 2020 09:45:35 +0200 Subject: [PATCH 9/9] handle unknown CPU arch name in --show-system-info --- easybuild/tools/options.py | 8 +++++--- test/framework/options.py | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 987f707be6..dd92f897f8 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -95,8 +95,8 @@ from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME from easybuild.tools.repository.repository import avail_repositories -from easybuild.tools.systemtools import check_python_version, get_cpu_architecture, get_cpu_family, get_cpu_features -from easybuild.tools.systemtools import get_system_info +from easybuild.tools.systemtools import UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family +from easybuild.tools.systemtools import get_cpu_features, get_system_info from easybuild.tools.version import this_is_easybuild @@ -1168,7 +1168,9 @@ def show_system_info(self): " -> architecture: %s" % get_cpu_architecture(), " -> family: %s" % get_cpu_family(), ] - if cpu_arch_name: + if cpu_arch_name == UNKNOWN: + lines.append(" -> arch name: UNKNOWN (archspec is not installed?)") + else: lines.append(" -> arch name: %s" % cpu_arch_name) lines.extend([ diff --git a/test/framework/options.py b/test/framework/options.py index a6c544d1b2..17f390d1f5 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4565,7 +4565,9 @@ def test_show_system_info(self): ] if HAVE_ARCHSPEC: - patterns.append(r"^ -> arch name: \w+") + patterns.append(r"^ -> arch name: \w+$") + else: + patterns.append(r"^ -> arch name: UNKNOWN \(archspec is not installed\?\)$") for pattern in patterns: regex = re.compile(pattern, re.M)