add a CUDA device code sanity check

easybuild/framework/easyconfig/default.py

@@ -124,6 +124,11 @@
                        'after make (for e.g.,"test" for make test)'), BUILD],
     'bin_lib_subdirs': [[], "List of subdirectories for binaries and libraries, which is used during sanity check "
                             "to check RPATH linking and banned/required libraries", BUILD],
+    'cuda_sanity_ignore_files': [[], "List of files (relative to the installation prefix) for which failures in "
+                                     "the CUDA sanity check step are ignored. Typically used for files where you "
+                                     "know the CUDA architectures in those files don't match the "
+                                     "--cuda-compute-capabitilities configured for EasyBuild AND where you know "
+                                     "that this is ok / reasonable (e.g. binary installations)", BUILD],
     'sanity_check_commands': [[], ("format: [(name, options)] e.g. [('gzip','-h')]. "
                                    "Using a non-tuple is equivalent to (name, '-h')"), BUILD],
     'sanity_check_paths': [{}, ("List of files and directories to check "

easybuild/tools/config.py

@@ -297,6 +297,10 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'backup_patched_files',
         'consider_archived_easyconfigs',
         'container_build_image',
+        'cuda_sanity_check_accept_ptx_as_devcode',
+        'cuda_sanity_check_accept_missing_ptx',
+        'cuda_sanity_check_error_on_fail',
+        'cuda_sanity_check_strict',
         'debug',
         'debug_lmod',
         'dump_autopep8',

easybuild/tools/options.py

@@ -398,7 +398,41 @@ def override_options(self):
                                    int, 'store_or_None', None),
             'cuda-compute-capabilities': ("List of CUDA compute capabilities to use when building GPU software; "
                                           "values should be specified as digits separated by a dot, "
-                                          "for example: 3.5,5.0,7.2", 'strlist', 'extend', None),
+                                          "for example: 3.5,5.0,7.2. EasyBuild will (where possible) compile fat "
+                                          "binaries with support for (at least) all requested CUDA compute "
+                                          "capabilities, and PTX code for the highest CUDA compute capability (for "
+                                          "forwards compatibility). The check on this behavior may be relaxed using "
+                                          "--cuda-sanity-check-accept-missing-ptx, "
+                                          "--cuda-sanity-check-accept-ptx-as-devcode, "
+                                          "or made more stringent using --cuda-sanity-check-strict.",
+                                          'strlist', 'extend', None),
+            'cuda-sanity-check-accept-missing-ptx': ("CUDA sanity check also passes if PTX code for the highest "
+                                                     "requested CUDA compute capability is not present (but will "
+                                                     "print a warning)",
+                                                     None, 'store_true', False),
+            'cuda-sanity-check-accept-ptx-as-devcode': ("CUDA sanity check also passes if requested device code is "
+                                                        "not present, as long as PTX code is present that can be "
+                                                        "JIT-compiled for each target in --cuda-compute-capabilities "
+                                                        "E.g. if --cuda-compute-capabilities=8.0 and a binary is "
+                                                        "found in the installation that does not have device code for "
+                                                        "8.0, but it does have PTX code for 7.0, the sanity check "
+                                                        "will pass if, and only if, this option is True. "
+                                                        "Note that JIT-compiling means the binary will work on the "
+                                                        "requested architecture, but is it not necessarily as well "
+                                                        "optimized as when actual device code is present for the "
+                                                        "requested architecture ",
+                                                        None, 'store_true', False),
+            'cuda-sanity-check-error-on-fail': ("If True, failures in the CUDA sanity check will produce an error. "
+                                                "If False, the CUDA sanity check will be performed, and failures will "
+                                                "be reported, but they will not result in an error",
+                                                None, 'store_true', False),
+            'cuda-sanity-check-strict': ("Perform strict CUDA sanity check. Without this option, the CUDA sanity "
+                                         "check will fail if the CUDA binaries don't contain code for (at least) "
+                                         "all compute capabilities defined in --cude-compute-capabilities, but will "
+                                         "accept if code for additional compute capabilities is present. "
+                                         "With this setting, the sanity check will also fail if code is present for "
+                                         "more compute capabilities than defined in --cuda-compute-capabilities.",
+                                         None, 'store_true', False),
             'debug-lmod': ("Run Lmod modules tool commands in debug module", None, 'store_true', False),
             'default-opt-level': ("Specify default optimisation level", 'choice', 'store', DEFAULT_OPT_LEVEL,
                                   Compiler.COMPILER_OPT_OPTIONS),
@@ -544,7 +578,7 @@ def override_options(self):
                 "Git commit to use for the target software build (robot capabilities are automatically disabled)",
                 None, 'store', None),
             'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False),
-            'strict-rpath-sanity-check': ("Perform strict RPATH sanity check, which involces unsetting "
+            'strict-rpath-sanity-check': ("Perform strict RPATH sanity check, which involves unsetting "
                                           "$LD_LIBRARY_PATH before checking whether all required libraries are found",
                                           None, 'store_true', False),
             'sysroot': ("Location root directory of system, prefix for standard paths like /usr/lib and /usr/include",
@@ -944,7 +978,7 @@ def validate(self):
         # values passed to --cuda-compute-capabilities must be of form X.Y (with both X and Y integers),
         # see https://developer.nvidia.com/cuda-gpus
         if self.options.cuda_compute_capabilities:
-            cuda_cc_regex = re.compile(r'^[0-9]+\.[0-9]+$')
+            cuda_cc_regex = re.compile(r'^[0-9]+\.[0-9]+a?$')
             faulty_cuda_ccs = [x for x in self.options.cuda_compute_capabilities if not cuda_cc_regex.match(x)]
             if faulty_cuda_ccs:
                 error_msg = "Incorrect values in --cuda-compute-capabilities (expected pattern: '%s'): %s"

easybuild/tools/systemtools.py

@@ -29,6 +29,7 @@
 
 * Jens Timmerman (Ghent University)
 * Ward Poelmans (Ghent University)
+* Jasper Grimm (UoY)
 * Jan Andre Reuter (Forschungszentrum Juelich GmbH)
 """
 import csv
@@ -41,6 +42,7 @@
 import platform
 import pwd
 import re
+import shutil
 import struct
 import sys
 import termios
@@ -64,6 +66,7 @@
     pass
 
 from easybuild.base import fancylogger
+from easybuild.tools import LooseVersion
 from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning
 from easybuild.tools.config import IGNORE
 from easybuild.tools.filetools import is_readable, read_file, which
@@ -998,6 +1001,106 @@ def get_glibc_version():
     return glibc_ver
 
 
+def get_cuda_object_dump_raw(path):
+    """
+    Get raw ouput from command which extracts information from CUDA binary files in a human-readable format,
+    or None for files containing no CUDA device code.
+    See https://docs.nvidia.com/cuda/cuda-binary-utilities/index.html#cuobjdump
+    """
+
+    res = run_shell_cmd("file %s" % path, fail_on_error=False, hidden=True, output_file=False, stream_output=False)
+    if res.exit_code != EasyBuildExit.SUCCESS:
+        fail_msg = "Failed to run 'file %s': %s" % (path, res.output)
+        _log.warning(fail_msg)
+
+    # check that the file is an executable or object (shared library) or archive (static library)
+    result = None
+    if any(x in res.output for x in ['executable', 'object', 'archive']):
+        # Make sure we have a cuobjdump command
+        if not shutil.which('cuobjdump'):
+            raise EasyBuildError("Failed to get object dump from CUDA file: cuobjdump command not found")
+        cuda_cmd = f"cuobjdump {path}"
+        res = run_shell_cmd(cuda_cmd, fail_on_error=False, hidden=True, output_file=False, stream_output=False)
+        if res.exit_code == EasyBuildExit.SUCCESS:
+            result = res.output
+        else:
+            # Check and report for the common case that this is simply not a CUDA binary, i.e. does not
+            # contain CUDA device code
+            no_device_code_match = re.search(r'does not contain device code', res.output)
+            if no_device_code_match is not None:
+                # File is a regular executable, object or library, but not a CUDA file
+                msg = "'%s' does not appear to be a CUDA binary: cuobjdump failed to find device code in this file"
+                _log.debug(msg, path)
+            else:
+                # This should not happen: there was no string saying this was NOT a CUDA file, yet no device code
+                # was found at all
+                msg = "Dumping CUDA binary file information for '%s' via '%s' failed! Output: '%s'"
+                raise EasyBuildError(msg, path, cuda_cmd, res.output)
+
+    return result
+
+
+def get_cuda_architectures(path, section_type):
+    """
+    Get a sorted list of CUDA architectures supported in the file in 'path'.
+    path: full path to a CUDA file
+    section_type: the type of section in the cuobjdump output to check for architectures ('elf' or 'ptx')
+    Returns None if no CUDA device code is present in the file
+    """
+
+    # Note that typical output for a cuobjdump call will look like this for device code:
+    #
+    # Fatbin elf code:
+    # ================
+    # arch = sm_90
+    # code version = [1,7]
+    # host = linux
+    # compile_size = 64bit
+    #
+    # And for ptx code, it will look like this:
+    #
+    # Fatbin ptx code:
+    # ================
+    # arch = sm_90
+    # code version = [8,1]
+    # host = linux
+    # compile_size = 64bit
+
+    # Pattern to extract elf code architectures and ptx code architectures respectively
+    code_regex = re.compile(f'Fatbin {section_type} code:\n=+\narch = sm_([0-9]+)([0-9]a?)')
+
+    # resolve symlinks
+    if os.path.islink(path) and os.path.exists(path):
+        path = os.path.realpath(path)
+
+    cc_archs = None
+    cuda_raw = get_cuda_object_dump_raw(path)
+    if cuda_raw is not None:
+        # extract unique device code architectures from raw dump
+        code_matches = re.findall(code_regex, cuda_raw)
+        if code_matches:
+            # convert match tuples into unique list of cuda compute capabilities
+            # e.g. [('8', '6'), ('8', '6'), ('9', '0')] -> ['8.6', '9.0']
+            cc_archs = sorted(['.'.join(m) for m in set(code_matches)], key=LooseVersion)
+        else:
+            # Try to be clear in the warning... did we not find elf/ptx code sections at all? or was the arch missing?
+            section_regex = re.compile(f'Fatbin {section_type} code')
+            section_matches = re.findall(section_regex, cuda_raw)
+            if section_matches:
+                fail_msg = f"Found Fatbin {section_type} code section(s) in cuobjdump output for {path}, "
+                fail_msg += "but failed to extract CUDA architecture"
+            else:
+                # In this case, the "Fatbin {section_type} code" section is simply missing from the binary
+                # It is entirely possible for a CUDA binary to have only device code or only ptx code (and thus the
+                # other section could be missing). However, considering --cuda-compute-capabilities is supposed to
+                # generate both PTX and device code (at least for the highest CC in that list), it is unexpected
+                # in an EasyBuild context and thus we print a warning
+                fail_msg = f"Failed to find Fatbin {section_type} code section(s) in cuobjdump output for {path}."
+            _log.warning(fail_msg)
+
+    return cc_archs
+
+
 def get_linked_libs_raw(path):
     """
     Get raw output from command that reports linked libraries for dynamically linked executables/libraries,