diff --git a/eessi-build-list-2020.06.py b/eessi-build-list-2020.06.py new file mode 100644 index 0000000000..b22cf3edce --- /dev/null +++ b/eessi-build-list-2020.06.py @@ -0,0 +1,18 @@ +eessi_2020_06 = { + 'GROMACS': { + '2020.1-foss-2020a-Python-3.8.2': { + 'except': ['gpu'], + }, + }, + 'OpenFOAM': { + '7-foss-2019b': {}, + }, + 'TensorFlow': { + '2.2.0-foss-2019b-Python-3.7.4': { + 'except': ['gpu'], + }, + '2.2.0-fosscuda-2019b-Python-3.7.4': { + 'only': ['gpu'], + } + }, +} \ No newline at end of file diff --git a/eessi.py b/eessi.py new file mode 100644 index 0000000000..2e6cf61add --- /dev/null +++ b/eessi.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +import re, sys, copy, os, tempfile + +from easybuild.tools.options import set_up_configuration +import easybuild.main as ebmain +from easybuild.tools.robot import resolve_dependencies +from easybuild.tools.modules import modules_tool +from easybuild.framework.easyconfig.tools import det_easyconfig_paths +from easybuild.framework.easyconfig.tools import parse_easyconfigs, skip_available +from easybuild.tools.environment import modify_env + + +def error(msg): + """Print error message and exit.""" + sys.stderr.write("ERROR: %s\n" % msg) + sys.exit(1) + + +def det_host_attrs(): + """Determine attributes for current host system.""" + attrs = { + # FIXME figure out whether host has a GPU (leverage archspec for this?) + 'gpu': False, + } + return attrs + + +def parse_build_list(path): + """Parse build list at specified path and return result.""" + res = None + + orig_locals = sorted(locals().keys()) + with open(path) as fp: + exec(fp.read()) + + key_regex = re.compile('^eessi_[0-9]') + for key in locals(): + if key not in orig_locals and key_regex.search(key): + res = locals()[key] + break + + if res is None: + error("No variable matching '%s' found in %s" % (key_regex.pattern, path)) + + return res + + +def filter_build_list(build_list, host_attrs): + """Filter provided build list based on specified host attributes.""" + filtered_build_list = {} + + for name in build_list: + filtered_build_list[name] = {} + for key in build_list[name]: + # make sure all 'only' attributes are set for current host; + # if not, filter out the key + only_attrs = build_list[name][key].get('only', []) + if not all(host_attrs.get(x, False) for x in only_attrs): + continue + + # make sure no 'except' attributes are set for current host; + # if so, filter out the key + except_attrs = build_list[name][key].get('except', []) + if any(host_attrs.get(x, False) for x in except_attrs): + continue + + filtered_build_list[name][key] = build_list[name][key] + + return filtered_build_list + +# Get EasyBuilds robot functionality to give us back an ordered list that we +# can use to build without robot +def unroll_robot(easyconfig): + print("Unrolling %s (WIP)" % easyconfig) + # unrolled result will be stored here + easyconfiglist = [easyconfig] + return easyconfiglist + +def main(): + orig_env = copy.deepcopy(os.environ) + eb_go, _ = set_up_configuration(args=sys.argv, silent=True) + + if len(eb_go.args) != 2: + error("Usage: %s " % sys.argv[0]) + + host_attrs = det_host_attrs() + build_list = parse_build_list(eb_go.args[1]) + filtered_build_list = filter_build_list(build_list, host_attrs) + + modtool = modules_tool() + + ec_list = [ "{0}-{1}.eb".format(name, version) + for name in sorted(filtered_build_list) + for version in filtered_build_list[name] ] + + print("EasyConfig names extracted from build list:") + print(ec_list) + + ec_paths = det_easyconfig_paths(ec_list) + ecs, _ = parse_easyconfigs([(p, False) for p in ec_paths], validate=False) + # call skip_available, reduces the nr of ecs for resolve_deps. Good for large buildlists. + retained_ecs = skip_available(ecs, modtool) + ordered_ecs = resolve_dependencies(retained_ecs, modtool, retain_all_deps=False) + + print("Ordered list of resolved dependencies") + for ec in ordered_ecs: + print(ec['spec']) + + # TODO: loop still fails after one iteration, because the first build removes the tmpdir after completing succesfully. I guess we need to re-initialize before calling main again or something? + print("Start building") + for ec in ordered_ecs: + # Without this restory of environment and tempfile, each iteration of this + # loop would create a new tempdir in the previous tempdir. This fails, because + # the tempdir from the previous iteration is cleaned up at the end of ebmain.main() + modify_env(os.environ, orig_env, verbose=False) + tempfile.tempdir=None + # eb_go, _ = set_up_configuration(args=sys.argv, silent=True) + print(ec['spec']) + # Just to see if we can call EB like this, do -D to not actually build anything for now + ebargs=["{}".format(ec['spec']), '--hooks=gentoo_hooks.py', '--sourcepath="~/.local/easybuild/sources:/shared/maintainers/sources"', '--installpath=/shared/maintainers/EESSI-pilot/easybuild-test/haswell/'] + #print(ebargs) + ebmain.main(ebargs) + print("Building completed") + +main() diff --git a/gentoo_hooks.py b/gentoo_hooks.py new file mode 100644 index 0000000000..2e710b0d61 --- /dev/null +++ b/gentoo_hooks.py @@ -0,0 +1,49 @@ +PREPEND = 1 +APPEND = 2 +REPLACE = 3 +APPEND_LIST = 4 +DROP = 5 + +opts_changes = { + 'GCCcore': { + # build EPREFIX-aware GCCcore + 'preconfigopts': ( + "if [ -f ../gcc/gcc.c ]; then sed -i 's/--sysroot=%R//' ../gcc/gcc.c; " + + "for h in ../gcc/config/*/*linux*.h; do " + + r'sed -i -r "/_DYNAMIC_LINKER/s,([\":])(/lib),\1${EPREFIX}\2,g" $h; done; fi; ', + PREPEND ), + 'configopts': ("--with-sysroot=$EPREFIX", PREPEND), + # remove .la files, as they mess up rpath when libtool is used + 'postinstallcmds': (["find %(installdir)s -name '*.la' -delete"], REPLACE), + }, +} + +def modify_all_opts(ec, opts_changes, + opts_to_skip=['builddependencies', 'dependencies', 'modluafooter', 'toolchainopts', 'version', 'multi_deps'], + opts_to_change='ALL'): + if 'modaltsoftname' in ec and ec['modaltsoftname'] in opts_changes: + name = ec['modaltsoftname'] + else: + name = ec['name'] + + possible_keys = [(name, ec['version']), name] + + for key in possible_keys: + if key in opts_changes.keys(): + for opt, value in opts_changes[key].items(): + # we don't modify those in this stage + if opt in opts_to_skip: + continue + if opts_to_change == 'ALL' or opt in opts_to_change: + if isinstance(value, list): + values = value + else: + values = [value] + + for v in values: + update_opts(ec, v[0], opt, v[1]) + break + +def parse_hook(ec, *args, **kwargs): + """Example parse hook to inject a patch file for a fictive software package named 'Example'.""" + modify_all_opts(ec, opts_changes, opts_to_skip=[], opts_to_change=['dependencies', 'builddependencies', 'license_file', 'version', 'multi_deps'])