add support for data installations

easybuild/framework/easyblock.py

@@ -73,7 +73,8 @@
 from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES
 from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
 from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
-from easybuild.tools.config import install_path, log_path, package_path, source_paths
+from easybuild.tools.config import install_path, log_path, package_path, source_paths, source_paths_data
+from easybuild.tools.config import DATA, SOFTWARE
 from easybuild.tools.environment import restore_env, sanitize_env
 from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256
 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock
@@ -124,6 +125,10 @@
 class EasyBlock(object):
     """Generic support for building and installing software, base class for actual easyblocks."""
 
+    # indicates whether or not this represents an EasyBlock for data or software
+    # set default value as class attribute, allowing custom easyblocks to override it
+    easyblock_type = SOFTWARE
+
     # static class method for extra easyconfig parameter definitions
     # this makes it easy to access the information without needing an instance
     # subclasses of EasyBlock should call this method with a dictionary
@@ -163,7 +168,7 @@ def __init__(self, ec):
 
         # build/install directories
         self.builddir = None
-        self.installdir = None  # software
+        self.installdir = None  # software or data
         self.installdir_mod = None  # module file
 
         # extensions
@@ -177,6 +182,11 @@ def __init__(self, ec):
         # may be set to True by ExtensionEasyBlock
         self.is_extension = False
 
+        known_easyblock_types = [DATA, SOFTWARE]
+        if self.easyblock_type not in known_easyblock_types:
+            raise EasyBuildError(
+                "EasyBlock type %s is not in list of known types %s", self.easyblock_type, known_easyblock_types)
+
         # easyconfig for this application
         if isinstance(ec, EasyConfig):
             self.cfg = ec
@@ -276,6 +286,17 @@ def __init__(self, ec):
         if group_name is not None:
             self.group = use_group(group_name)
 
+        if self.easyblock_type == SOFTWARE and self.cfg['data_sources']:
+            raise EasyBuildError(
+                "Easyconfig parameter 'data_sources' not supported for software installations. Use 'sources' instead.")
+        if self.easyblock_type == DATA and self.cfg['sources']:
+            raise EasyBuildError(
+                "Easyconfig parameter 'sources' not supported for data installations. Use 'data_sources' instead.")
+
+        # use 'data_sources' as alias for 'sources'
+        if self.cfg['data_sources']:
+            self.cfg['sources'] = self.cfg['data_sources']
+
         # generate build/install directories
         self.gen_builddir()
         self.gen_installdir()
@@ -752,7 +773,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
         :param download_instructions: instructions to manually add source (used for complex cases)
         :param alt_location: alternative location to use instead of self.name
         """
-        srcpaths = source_paths()
+        srcpaths_map = {SOFTWARE: source_paths, DATA: source_paths_data}
+        srcpaths = srcpaths_map[self.easyblock_type]()
 
         update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
 
@@ -964,7 +986,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                         msg = "\nDownload instructions:\n\n" + download_instructions + '\n'
                         print_msg(msg, prefix=False, stderr=True)
                         error_msg += "please follow the download instructions above, and make the file available "
-                        error_msg += "in the active source path (%s)" % ':'.join(source_paths())
+                        error_msg += "in the active source path (%s)" % ':'.join(srcpaths_map[self.easyblock_type]())
                     else:
                         # flatten list to string with '%' characters escaped (literal '%' desired in 'sprintf')
                         failedpaths_msg = ', '.join(failedpaths).replace('%', '%%')
@@ -1106,7 +1128,7 @@ def gen_installdir(self):
         """
         Generate the name of the installation directory.
         """
-        basepath = install_path()
+        basepath = install_path(self.easyblock_type)
         if basepath:
             self.install_subdir = ActiveMNS().det_install_subdir(self.cfg)
             self.installdir = os.path.join(os.path.abspath(basepath), self.install_subdir)
@@ -2607,7 +2629,7 @@ def patch_step(self, beginpath=None, patches=None):
             copy_patch = 'copy' in patch and 'sourcepath' not in patch
 
             self.log.debug("Source index: %s; patch level: %s; source path suffix: %s; copy patch: %s",
-                           srcind, level, srcpathsuffix, copy)
+                           srcind, level, srcpathsuffix, copy_patch)
 
             if beginpath is None:
                 try:

easybuild/framework/easyconfig/default.py

@@ -92,6 +92,7 @@
     'checksums': [[], "Checksums for sources and patches", BUILD],
     'configopts': ['', 'Extra options passed to configure (default already has --prefix)', BUILD],
     'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with (if supported)", BUILD],
+    'data_sources': [[], "List of source files for data", BUILD],
     'download_instructions': ['', "Specify steps to acquire necessary file, if obtaining it is difficult", BUILD],
     'easyblock': [None, "EasyBlock to use for building; if set to None, an easyblock is selected "
                         "based on the software name", BUILD],
@@ -132,7 +133,7 @@
     'skip': [False, "Skip existing software", BUILD],
     'skipsteps': [[], "Skip these steps", BUILD],
     'source_urls': [[], "List of URLs for source files", BUILD],
-    'sources': [[], "List of source files", BUILD],
+    'sources': [[], "List of source files for software", BUILD],
     'stop': [None, 'Keyword to halt the build process after a certain step.', BUILD],
     'testopts': ['', 'Extra options for test.', BUILD],
     'tests': [[], ("List of test-scripts to run after install. A test script should return a "

easybuild/framework/easyconfig/format/format.py

@@ -63,7 +63,7 @@
     ['name', 'version', 'versionprefix', 'versionsuffix'],
     ['homepage', 'description'],
     ['toolchain', 'toolchainopts'],
-    ['source_urls', 'sources', 'patches', 'checksums'],
+    ['source_urls', 'sources', 'data_sources', 'patches', 'checksums'],
     DEPENDENCY_PARAMETERS + ['multi_deps'],
     ['osdependencies'],
     ['preconfigopts', 'configopts'],

easybuild/tools/config.py

@@ -70,6 +70,10 @@
 
 EMPTY_LIST = 'empty_list'
 
+DATA = 'data'
+MODULES = 'modules'
+SOFTWARE = 'software'
+
 PKG_TOOL_FPM = 'fpm'
 PKG_TYPE_RPM = 'rpm'
 
@@ -110,8 +114,10 @@
     'packagepath': 'packages',
     'repositorypath': 'ebfiles_repo',
     'sourcepath': 'sources',
-    'subdir_modules': 'modules',
-    'subdir_software': 'software',
+    'sourcepath_data': 'data_sources',
+    'subdir_data': DATA,
+    'subdir_modules': MODULES,
+    'subdir_software': SOFTWARE,
 }
 DEFAULT_PKG_RELEASE = '1'
 DEFAULT_PKG_TOOL = PKG_TOOL_FPM
@@ -470,6 +476,7 @@ class ConfigurationVariables(BaseConfigurationVariables):
         'installpath',
         'installpath_modules',
         'installpath_software',
+        'installpath_data',
         'job_backend',
         'logfile_format',
         'moduleclasses',
@@ -482,8 +489,10 @@ class ConfigurationVariables(BaseConfigurationVariables):
         'repository',
         'repositorypath',
         'sourcepath',
+        'sourcepath_data',
         'subdir_modules',
         'subdir_software',
+        'subdir_data',
         'tmp_logdir',
     ]
     KNOWN_KEYS = REQUIRED  # KNOWN_KEYS must be defined for FrozenDictKnownKeys functionality
@@ -522,13 +531,14 @@ def init(options, config_options_dict):
     """
     tmpdict = copy.deepcopy(config_options_dict)
 
-    # make sure source path is a list
-    sourcepath = tmpdict['sourcepath']
-    if isinstance(sourcepath, string_type):
-        tmpdict['sourcepath'] = sourcepath.split(':')
-        _log.debug("Converted source path ('%s') to a list of paths: %s" % (sourcepath, tmpdict['sourcepath']))
-    elif not isinstance(sourcepath, (tuple, list)):
-        raise EasyBuildError("Value for sourcepath has invalid type (%s): %s", type(sourcepath), sourcepath)
+    for srcpath in ['sourcepath', 'sourcepath_data']:
+        # make sure source path is a list
+        sourcepath = tmpdict[srcpath]
+        if isinstance(sourcepath, string_type):
+            tmpdict[srcpath] = sourcepath.split(':')
+            _log.debug("Converted source path ('%s') to a list of paths: %s" % (sourcepath, tmpdict[srcpath]))
+        elif not isinstance(sourcepath, (tuple, list)):
+            raise EasyBuildError("Value for %s has invalid type (%s): %s", srcpath, type(sourcepath), sourcepath)
 
     # initialize configuration variables (any future calls to ConfigurationVariables() will yield the same instance
     variables = ConfigurationVariables(tmpdict, ignore_unknown_keys=True)
@@ -661,11 +671,18 @@ def build_path():
 
 def source_paths():
     """
-    Return the list of source paths
+    Return the list of source paths for software
     """
     return ConfigurationVariables()['sourcepath']
 
 
+def source_paths_data():
+    """
+    Return the list of source paths for data
+    """
+    return ConfigurationVariables()['sourcepath_data']
+
+
 def source_path():
     """NO LONGER SUPPORTED: use source_paths instead"""
     _log.nosupport("source_path() is replaced by source_paths()", '2.0')
@@ -674,15 +691,16 @@ def source_path():
 def install_path(typ=None):
     """
     Returns the install path
-    - subdir 'software' for actual installation (default)
+    - subdir 'software' for actual software installation (default)
     - subdir 'modules' for environment modules (typ='mod')
+    - subdir 'data' for data installation (typ='data')
     """
     if typ is None:
-        typ = 'software'
+        typ = SOFTWARE
     elif typ == 'mod':
-        typ = 'modules'
+        typ = MODULES
 
-    known_types = ['modules', 'software']
+    known_types = [MODULES, SOFTWARE, DATA]
     if typ not in known_types:
         raise EasyBuildError("Unknown type specified in install_path(): %s (known: %s)", typ, ', '.join(known_types))
 

easybuild/tools/options.py

@@ -578,6 +578,8 @@ def config_options(self):
                                     None, 'store', None),
             'installpath-software': ("Install path for software (if None, combine --installpath and --subdir-software)",
                                      None, 'store', None),
+            'installpath-data': ("Install path for data (if None, combine --installpath and --subdir-data)",
+                                 None, 'store', None),
             'job-backend': ("Backend to use for submitting jobs", 'choice', 'store',
                             DEFAULT_JOB_BACKEND, sorted(avail_job_backends().keys())),
             # purposely take a copy for the default logfile format
@@ -604,7 +606,7 @@ def config_options(self):
                             None, 'store', mk_full_default_path('packagepath')),
             'package-naming-scheme': ("Packaging naming scheme choice",
                                       'choice', 'store', DEFAULT_PNS, sorted(avail_package_naming_schemes().keys())),
-            'prefix': (("Change prefix for buildpath, installpath, sourcepath and repositorypath "
+            'prefix': (("Change prefix for buildpath, installpath, sourcepath, sourcepath-data, and repositorypath "
                         "(used prefix for defaults %s)" % DEFAULT_PREFIX),
                        None, 'store', None),
             'recursive-module-unload': ("Enable generating of modules that unload recursively.",
@@ -615,11 +617,15 @@ def config_options(self):
                                 "(is passed as list of arguments to create the repository instance). "
                                 "For more info, use --avail-repositories."),
                                'strlist', 'store', self.default_repositorypath),
-            'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)",
+            'sourcepath': ("Path(s) to where software sources should be downloaded (string, colon-separated)",
                            None, 'store', mk_full_default_path('sourcepath')),
+            'sourcepath-data': ("Path(s) to where data sources should be downloaded (string, colon-separated)",
+                                None, 'store', mk_full_default_path('sourcepath_data')),
             'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']),
             'subdir-software': ("Installpath subdir for software",
                                 None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']),
+            'subdir-data': ("Installpath subdir for data",
+                            None, 'store', DEFAULT_PATH_SUBDIRS['subdir_data']),
             'subdir-user-modules': ("Base path of user-specific modules relative to --envvars-user-modules",
                                     None, 'store', None),
             'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS),
@@ -1134,8 +1140,8 @@ def _postprocess_config(self):
         #   which can be done in variety of formats (git@<url>:<org>/<repo>), https://<url>, etc.)
         #   (see also https://github.com/easybuilders/easybuild-framework/issues/3892);
         path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath',
-                          'installpath_modules', 'installpath_software', 'prefix', 'packagepath',
-                          'robot_paths', 'sourcepath']
+                          'installpath_modules', 'installpath_software', 'installpath_data', 'prefix', 'packagepath',
+                          'robot_paths', 'sourcepath', 'sourcepath_data']
 
         for opt_name in path_opt_names:
             self._ensure_abs_path(opt_name)
@@ -1144,7 +1150,7 @@ def _postprocess_config(self):
             # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account
             # in the legacy-style configuration, repository is initialised in configuration file itself
             path_opts = ['buildpath', 'containerpath', 'installpath', 'packagepath', 'repository', 'repositorypath',
-                         'sourcepath']
+                         'sourcepath', 'sourcepath_data']
             for dest in path_opts:
                 if not self.options._action_taken.get(dest, False):
                     if dest == 'repository':
@@ -1388,7 +1394,8 @@ def show_config(self):
 
         # options that should never/always be printed
         ignore_opts = ['show_config', 'show_full_config']
-        include_opts = ['buildpath', 'containerpath', 'installpath', 'repositorypath', 'robot_paths', 'sourcepath']
+        include_opts = ['buildpath', 'containerpath', 'installpath', 'repositorypath', 'robot_paths', 'sourcepath',
+                        'sourcepath_data']
         cmdline_opts_dict = self.dict_by_prefix()
 
         def reparse_cfg(args=None, withcfg=True):