kickstart driver script + build list for pilot setup (WIP)

eessi-build-list-2020.06.py

@@ -0,0 +1,18 @@
+eessi_2020_06 = {
+    'GROMACS': {
+        '2020.1-foss-2020a-Python-3.8.2': {
+            'except': ['gpu'],
+        },
+    },
+    'OpenFOAM': {
+        '7-foss-2019b': {},
+    },
+    'TensorFlow': {
+        '2.2.0-foss-2019b-Python-3.7.4': {
+            'except': ['gpu'],
+        },
+        '2.2.0-fosscuda-2019b-Python-3.7.4': {
+            'only': ['gpu'],
+        }
+    },
+}

eessi.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+import re, sys, copy, os, tempfile
+
+from easybuild.tools.options import set_up_configuration
+import easybuild.main as ebmain
+from easybuild.tools.robot import resolve_dependencies
+from easybuild.tools.modules import modules_tool
+from easybuild.framework.easyconfig.tools import det_easyconfig_paths
+from easybuild.framework.easyconfig.tools import parse_easyconfigs, skip_available
+from easybuild.tools.environment import modify_env
+
+
+def error(msg):
+    """Print error message and exit."""
+    sys.stderr.write("ERROR: %s\n" % msg)
+    sys.exit(1)
+
+
+def det_host_attrs():
+    """Determine attributes for current host system."""
+    attrs = {
+        # FIXME figure out whether host has a GPU (leverage archspec for this?)
+        'gpu': False,
+    }
+    return attrs
+
+
+def parse_build_list(path):
+    """Parse build list at specified path and return result."""
+    res = None
+
+    orig_locals = sorted(locals().keys())
+    with open(path) as fp:
+        exec(fp.read())
+
+    key_regex = re.compile('^eessi_[0-9]')
+    for key in locals():
+        if key not in orig_locals and key_regex.search(key):
+            res = locals()[key]
+            break
+
+    if res is None:
+        error("No variable matching '%s' found in %s" % (key_regex.pattern, path))
+
+    return res
+
+
+def filter_build_list(build_list, host_attrs):
+    """Filter provided build list based on specified host attributes."""
+    filtered_build_list = {}
+
+    for name in build_list:
+        filtered_build_list[name] = {}
+        for key in build_list[name]:
+            # make sure all 'only' attributes are set for current host;
+            # if not, filter out the key
+            only_attrs = build_list[name][key].get('only', [])
+            if not all(host_attrs.get(x, False) for x in only_attrs):
+                continue
+
+            # make sure no 'except' attributes are set for current host;
+            # if so, filter out the key
+            except_attrs = build_list[name][key].get('except', [])
+            if any(host_attrs.get(x, False) for x in except_attrs):
+                continue
+
+            filtered_build_list[name][key] = build_list[name][key]
+
+    return filtered_build_list
+
+# Get EasyBuilds robot functionality to give us back an ordered list that we
+# can use to build without robot
+def unroll_robot(easyconfig):
+    print("Unrolling %s (WIP)" % easyconfig)
+    # unrolled result will be stored here
+    easyconfiglist = [easyconfig]
+    return easyconfiglist
+
+def main():
+    orig_env = copy.deepcopy(os.environ)
+    eb_go, _ = set_up_configuration(args=sys.argv, silent=True)
+
+    if len(eb_go.args) != 2:
+        error("Usage: %s <path to build list file>" % sys.argv[0])
+
+    host_attrs = det_host_attrs()
+    build_list = parse_build_list(eb_go.args[1])
+    filtered_build_list = filter_build_list(build_list, host_attrs)
+
+    modtool = modules_tool()
+
+    ec_list = [ "{0}-{1}.eb".format(name, version)
+            for name in sorted(filtered_build_list)
+            for version in filtered_build_list[name] ]
+
+    print("EasyConfig names extracted from build list:")
+    print(ec_list)
+
+    ec_paths = det_easyconfig_paths(ec_list)
+    ecs, _ = parse_easyconfigs([(p, False) for p in ec_paths], validate=False)
+    # call skip_available, reduces the nr of ecs for resolve_deps. Good for large buildlists.
+    retained_ecs = skip_available(ecs, modtool)
+    ordered_ecs = resolve_dependencies(retained_ecs, modtool, retain_all_deps=False)
+
+    print("Ordered list of resolved dependencies")
+    for ec in ordered_ecs:
+        print(ec['spec'])
+
+    # TODO: loop still fails after one iteration, because the first build removes the tmpdir after completing succesfully. I guess we need to re-initialize before calling main again or something?
+    print("Start building")
+    for ec in ordered_ecs:
+        # Without this restory of environment and tempfile, each iteration of this
+        # loop would create a new tempdir in the previous tempdir. This fails, because
+        # the tempdir from the previous iteration is cleaned up at the end of ebmain.main()
+        modify_env(os.environ, orig_env, verbose=False)
+        tempfile.tempdir=None
+        # eb_go, _ = set_up_configuration(args=sys.argv, silent=True)
+        print(ec['spec'])
+        # Just to see if we can call EB like this, do -D to not actually build anything for now
+        ebargs=["{}".format(ec['spec']), '--hooks=gentoo_hooks.py', '--sourcepath="~/.local/easybuild/sources:/shared/maintainers/sources"', '--installpath=/shared/maintainers/EESSI-pilot/easybuild-test/haswell/']
+        #print(ebargs)
+        ebmain.main(ebargs)
+    print("Building completed")
+
+main()

gentoo_hooks.py

@@ -0,0 +1,49 @@
+PREPEND = 1
+APPEND = 2
+REPLACE = 3
+APPEND_LIST = 4
+DROP = 5
+
+opts_changes = {
+    'GCCcore': {
+        # build EPREFIX-aware GCCcore
+        'preconfigopts': (
+                    "if [ -f ../gcc/gcc.c ]; then sed -i 's/--sysroot=%R//' ../gcc/gcc.c; " +
+                    "for h in ../gcc/config/*/*linux*.h; do " +
+                    r'sed -i -r "/_DYNAMIC_LINKER/s,([\":])(/lib),\1${EPREFIX}\2,g" $h; done; fi; ',
+                    PREPEND ),
+        'configopts': ("--with-sysroot=$EPREFIX", PREPEND),
+        # remove .la files, as they mess up rpath when libtool is used
+        'postinstallcmds': (["find %(installdir)s -name '*.la' -delete"], REPLACE),
+    },
+}
+
+def modify_all_opts(ec, opts_changes,
+        opts_to_skip=['builddependencies', 'dependencies', 'modluafooter', 'toolchainopts', 'version', 'multi_deps'],
+        opts_to_change='ALL'):
+    if 'modaltsoftname' in ec and ec['modaltsoftname'] in opts_changes:
+        name = ec['modaltsoftname']
+    else:
+        name = ec['name']
+
+    possible_keys = [(name, ec['version']), name]
+
+    for key in possible_keys:
+        if key in opts_changes.keys():
+            for opt, value in opts_changes[key].items():
+                # we don't modify those in this stage
+                if opt in opts_to_skip:
+                    continue
+                if opts_to_change == 'ALL' or opt in opts_to_change:
+                    if isinstance(value, list):
+                        values = value
+                    else:
+                        values = [value]
+
+                    for v in values:
+                        update_opts(ec, v[0], opt, v[1])
+            break
+
+def parse_hook(ec, *args, **kwargs):
+    """Example parse hook to inject a patch file for a fictive software package named 'Example'."""
+    modify_all_opts(ec, opts_changes, opts_to_skip=[], opts_to_change=['dependencies', 'builddependencies', 'license_file', 'version', 'multi_deps'])