diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index ba52da9052..f2e399e026 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -42,6 +42,7 @@ import os import shutil import stat +import tempfile import time import traceback from distutils.version import LooseVersion @@ -57,7 +58,8 @@ from easybuild.framework.easyconfig.tools import get_paths_for from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP from easybuild.tools.build_details import get_build_stats -from easybuild.tools.build_log import EasyBuildError, print_error, print_msg +from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs +from easybuild.tools.build_log import print_error, print_msg from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env @@ -141,7 +143,8 @@ def __init__(self, ec): # build/install directories self.builddir = None - self.installdir = None + self.installdir = None # software + self.installdir_mod = None # module file # extensions self.exts = None @@ -197,12 +200,15 @@ def __init__(self, ec): # keep track of initial environment we start in, so we can restore it if needed self.initial_environ = copy.deepcopy(os.environ) - # initialize logger - self._init_log() - # should we keep quiet? self.silent = build_option('silent') + # are we doing a dry run? + self.dry_run = build_option('extended_dry_run') + + # initialize logger + self._init_log() + # try and use the specified group (if any) group_name = build_option('group') if self.cfg['group'] is not None: @@ -217,6 +223,11 @@ def __init__(self, ec): self.gen_builddir() self.gen_installdir() + self.ignored_errors = False + + if self.dry_run: + self.init_dry_run() + self.log.info("Init completed for application name %s version %s" % (self.name, self.version)) # INIT/CLOSE LOG @@ -235,8 +246,13 @@ def _init_log(self): self.log.info(this_is_easybuild()) this_module = inspect.getmodule(self) - self.log.info("This is easyblock %s from module %s (%s)", - self.__class__.__name__, this_module.__name__, this_module.__file__) + eb_class = self.__class__.__name__ + eb_mod_name = this_module.__name__ + eb_mod_loc = this_module.__file__ + self.log.info("This is easyblock %s from module %s (%s)", eb_class, eb_mod_name, eb_mod_loc) + + if self.dry_run: + self.dry_run_msg("*** DRY RUN using '%s' easyblock (%s @ %s) ***\n", eb_class, eb_mod_name, eb_mod_loc) def close_log(self): """ @@ -245,6 +261,26 @@ def close_log(self): self.log.info("Closing log for application name %s version %s" % (self.name, self.version)) fancylogger.logToFile(self.logfile, enable=False) + # + # DRY RUN UTILITIES + # + def init_dry_run(self): + """Initialise easyblock instance for performing a dry run.""" + # replace build/install dirs with temporary directories in dry run mode + tmp_root_dir = os.path.realpath(os.path.join(tempfile.gettempdir(), '__ROOT__')) + self.builddir = os.path.join(tmp_root_dir, self.builddir.lstrip(os.path.sep)) + self.installdir = os.path.join(tmp_root_dir, self.installdir.lstrip(os.path.sep)) + self.installdir_mod = os.path.join(tmp_root_dir, self.installdir_mod.lstrip(os.path.sep)) + + # register fake build/install dirs so the original values can be printed during dry run + dry_run_set_dirs(tmp_root_dir, self.builddir, self.installdir, self.installdir_mod) + + def dry_run_msg(self, msg, *args): + """Print dry run message.""" + if args: + msg = msg % args + dry_run_msg(msg, silent=self.silent) + # # FETCH UTILITY FUNCTIONS # @@ -371,6 +407,10 @@ def fetch_extension_sources(self): self.cfg.enable_templating = False exts_list = self.cfg['exts_list'] self.cfg.enable_templating = True + + if self.dry_run: + self.dry_run_msg("\nList of sources/patches for extensions:") + for ext in exts_list: if (isinstance(ext, list) or isinstance(ext, tuple)) and ext: @@ -542,6 +582,8 @@ def obtain_file(self, filename, extension=False, urls=None): break # no need to try other source paths if foundfile: + if self.dry_run: + self.dry_run_msg(" * %s found at %s", filename, foundfile) return foundfile else: # try and download source files from specified source URLs @@ -574,16 +616,24 @@ def obtain_file(self, filename, extension=False, urls=None): self.log.warning("Source URL %s is of unknown type, so ignoring it." % url) continue - self.log.debug("Trying to download file %s from %s to %s ..." % (filename, fullurl, targetpath)) - downloaded = False - try: - if download_file(filename, fullurl, targetpath): - downloaded = True + if self.dry_run: + self.dry_run_msg(" * %s will be downloaded to %s", filename, targetpath) + if extension and urls: + # extensions typically have custom source URLs specified, only mention first + self.dry_run_msg(" (from %s, ...)", fullurl) + downloaded = True - except IOError, err: - self.log.debug("Failed to download %s from %s: %s" % (filename, url, err)) - failedpaths.append(fullurl) - continue + else: + self.log.debug("Trying to download file %s from %s to %s ..." % (filename, fullurl, targetpath)) + downloaded = False + try: + if download_file(filename, fullurl, targetpath): + downloaded = True + + except IOError, err: + self.log.debug("Failed to download %s from %s: %s" % (filename, url, err)) + failedpaths.append(fullurl) + continue if downloaded: # if fetching from source URL worked, we're done @@ -592,8 +642,12 @@ def obtain_file(self, filename, extension=False, urls=None): else: failedpaths.append(fullurl) - raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " - "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + if self.dry_run: + self.dry_run_msg(" * %s (MISSING)", filename) + return filename + else: + raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) # # GETTER/SETTER UTILITY FUNCTIONS @@ -697,7 +751,12 @@ def gen_installdir(self): if basepath: self.install_subdir = ActiveMNS().det_install_subdir(self.cfg) self.installdir = os.path.join(os.path.abspath(basepath), self.install_subdir) - self.log.info("Install dir set to %s" % self.installdir) + self.log.info("Software install dir set to %s" % self.installdir) + + mod_basepath = install_path('mod') + mod_path_suffix = build_option('suffix_modules_path') + self.installdir_mod = os.path.join(os.path.abspath(mod_basepath), mod_path_suffix) + self.log.info("Module install dir set to %s" % self.installdir_mod) else: raise EasyBuildError("Can't set installation directory") @@ -837,10 +896,9 @@ def make_module_dep(self): self.log.debug("Full list of dependencies: %s" % deps) # exclude dependencies that extend $MODULEPATH and form the path to the top of the module tree (if any) - mod_install_path = os.path.join(install_path('mod'), build_option('suffix_modules_path')) - full_mod_subdir = os.path.join(mod_install_path, self.cfg.mod_subdir) + full_mod_subdir = os.path.join(self.installdir_mod, self.cfg.mod_subdir) init_modpaths = mns.det_init_modulepaths(self.cfg) - top_paths = [mod_install_path] + [os.path.join(mod_install_path, p) for p in init_modpaths] + top_paths = [self.installdir_mod] + [os.path.join(self.installdir_mod, p) for p in init_modpaths] excluded_deps = self.modules_tool.path_to_top_of_module_tree(top_paths, self.cfg.short_mod_name, full_mod_subdir, deps) @@ -955,11 +1013,9 @@ def make_module_extend_modpath(self): """ txt = '' if self.cfg['include_modpath_extensions']: - top_modpath = install_path('mod') - mod_path_suffix = build_option('suffix_modules_path') modpath_exts = ActiveMNS().det_modpath_extensions(self.cfg) self.log.debug("Including module path extensions returned by module naming scheme: %s" % modpath_exts) - full_path_modpath_extensions = [os.path.join(top_modpath, mod_path_suffix, ext) for ext in modpath_exts] + full_path_modpath_extensions = [os.path.join(self.installdir_mod, ext) for ext in modpath_exts] # module path extensions must exist, otherwise loading this module file will fail for modpath_extension in full_path_modpath_extensions: mkdir(modpath_extension, parents=True) @@ -1149,6 +1205,13 @@ def guess_start_dir(self): else: topdir = self.builddir + # during dry run, use subdirectory that would likely result from unpacking + if self.dry_run and os.path.samefile(topdir, self.builddir): + topdir = os.path.join(self.builddir, '%s-%s' % (self.name, self.version)) + self.log.info("Modified parent directory of start dir in dry run mode to likely path %s", topdir) + # make sure start_dir subdir exists (cfr. check below) + mkdir(os.path.join(topdir, start_dir), parents=True) + abs_start_dir = os.path.join(topdir, start_dir) if topdir.endswith(start_dir) and not os.path.exists(abs_start_dir): self.cfg['start_dir'] = topdir @@ -1219,6 +1282,7 @@ def check_readiness_step(self): self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) else: self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) + self.cfg['parallel'] = det_parallelism(par=par, maxpar=self.cfg['maxparallel']) self.log.info("Setting parallelism: %s" % self.cfg['parallel']) @@ -1251,9 +1315,8 @@ def check_readiness_step(self): self.log.info("No module %s found. Not skipping anything." % self.full_mod_name) def fetch_step(self, skip_checksums=False): - """ - prepare for building - """ + """Fetch source files and patches (incl. extensions).""" + # check EasyBuild version easybuild_version = self.cfg['easybuild_version'] if not easybuild_version: @@ -1267,15 +1330,27 @@ def fetch_step(self, skip_checksums=False): raise EasyBuildError("EasyBuild-version %s is newer than the currently running one. Aborting!", easybuild_version) + if self.dry_run: + + self.dry_run_msg("Available download URLs for sources/patches:") + if self.cfg['source_urls']: + for source_url in self.cfg['source_urls']: + self.dry_run_msg(" * %s/$source", source_url) + else: + self.dry_run_msg('(none)') + + # actual list of sources is printed via _obtain_file_dry_run method + self.dry_run_msg("\nList of sources:") + # fetch sources if self.cfg['sources']: self.fetch_sources(self.cfg['sources'], checksums=self.cfg['checksums']) else: self.log.info('no sources provided') - # fetch extensions - if len(self.cfg['exts_list']) > 0: - self.exts = self.fetch_extension_sources() + if self.dry_run: + # actual list of patches is printed via _obtain_file_dry_run method + self.dry_run_msg("\nList of patches:") # fetch patches if self.cfg['patches']: @@ -1287,22 +1362,27 @@ def fetch_step(self, skip_checksums=False): self.fetch_patches(checksums=patches_checksums) else: self.log.info('no patches provided') + if self.dry_run: + self.dry_run_msg('(none)') # compute checksums for all source and patch files - if not skip_checksums: + if not (skip_checksums or self.dry_run): for fil in self.src + self.patches: check_sum = compute_checksum(fil['path'], checksum_type=DEFAULT_CHECKSUM) fil[DEFAULT_CHECKSUM] = check_sum self.log.info("%s checksum for %s: %s" % (DEFAULT_CHECKSUM, fil['path'], fil[DEFAULT_CHECKSUM])) + # fetch extensions + if self.cfg['exts_list']: + self.exts = self.fetch_extension_sources() + # create parent dirs in install and modules path already # this is required when building in parallel - mod_path_suffix = build_option('suffix_modules_path') mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.cfg) parent_subdir = os.path.dirname(self.install_subdir) pardirs = [ - os.path.join(install_path(), parent_subdir), - os.path.join(install_path('mod'), mod_path_suffix, parent_subdir), + self.installdir, + os.path.join(self.installdir_mod, parent_subdir), ] for mod_symlink_path in mod_symlink_paths: pardirs.append(os.path.join(install_path('mod'), mod_symlink_path, parent_subdir)) @@ -1314,11 +1394,16 @@ def fetch_step(self, skip_checksums=False): def checksum_step(self): """Verify checksum of sources and patches, if a checksum is available.""" for fil in self.src + self.patches: - ok = verify_checksum(fil['path'], fil['checksum']) - if not ok: - raise EasyBuildError("Checksum verification for %s using %s failed.", fil['path'], fil['checksum']) + if self.dry_run: + # dry run mode: only report checksums, don't actually verify them + filename = os.path.basename(fil['path']) + expected_checksum = fil['checksum'] or '(none)' + self.dry_run_msg("* expected checksum for %s: %s", filename, expected_checksum) else: - self.log.info("Checksum verification for %s using %s passed." % (fil['path'], fil['checksum'])) + if not verify_checksum(fil['path'], fil['checksum']): + raise EasyBuildError("Checksum verification for %s using %s failed.", fil['path'], fil['checksum']) + else: + self.log.info("Checksum verification for %s using %s passed." % (fil['path'], fil['checksum'])) def extract_step(self): """ @@ -1372,11 +1457,14 @@ def prepare_step(self): """ Pre-configure step. Set's up the builddir just before starting configure """ + if self.dry_run: + self.dry_run_msg("Defining build environment, based on toolchain (options) and specified dependencies...\n") + # clean environment, undefine any unwanted environment variables that may be harmful self.cfg['unwanted_env_vars'] = env.unset_env_vars(self.cfg['unwanted_env_vars']) # prepare toolchain: load toolchain module and dependencies, set up build environment - self.toolchain.prepare(self.cfg['onlytcmod']) + self.toolchain.prepare(self.cfg['onlytcmod'], silent=self.silent) # guess directory to start configure/build/install process in, and move there self.guess_start_dir() @@ -1421,7 +1509,9 @@ def extensions_step(self, fetch=False): return # load fake module - fake_mod_data = self.load_fake_module(purge=True) + fake_mod_data = None + if not self.dry_run: + fake_mod_data = self.load_fake_module(purge=True) self.prepare_for_extensions() @@ -1439,7 +1529,7 @@ def extensions_step(self, fetch=False): exts_classmap = self.cfg['exts_classmap'] # we really need a default class - if not exts_defaultclass: + if not exts_defaultclass and fake_mod_data: self.clean_up_fake_module(fake_mod_data) raise EasyBuildError("ERROR: No default extension class set for %s", self.name) @@ -1503,6 +1593,11 @@ def extensions_step(self, fetch=False): else: self.log.debug("Installing extension %s with class %s (from %s)" % (ext['name'], class_name, mod_path)) + if self.dry_run: + eb_class = cls.__name__ + msg = "\n* installing extension %s %s using '%s' easyblock\n" % (ext['name'], ext['version'], eb_class) + self.dry_run_msg(msg) + # real work inst.prerun() txt = inst.run() @@ -1514,7 +1609,8 @@ def extensions_step(self, fetch=False): self.ext_instances.append(inst) # cleanup (unload fake module, remove fake module dir) - self.clean_up_fake_module(fake_mod_data) + if fake_mod_data: + self.clean_up_fake_module(fake_mod_data) def package_step(self): """Package installed software (e.g., into an RPM), if requested, using selected package tool.""" @@ -1555,17 +1651,28 @@ def post_install_step(self): raise EasyBuildError("Invalid element in 'postinstallcmds', not a string: %s", cmd) run_cmd(cmd, simple=True, log_ok=True, log_all=True) - def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=False): + def sanity_check_step(self, *args, **kwargs): """ Do a sanity check on the installation - if *any* of the files/subdirectories in the installation directory listed in sanity_check_paths are non-existent (or empty), the sanity check fails """ + if self.dry_run: + self._sanity_check_step_dry_run(*args, **kwargs) + else: + self._sanity_check_step(*args, **kwargs) + + def _sanity_check_step_common(self, custom_paths, custom_commands): + """Determine sanity check paths and commands to use.""" + # supported/required keys in for sanity check paths, along with function used to check the paths path_keys_and_check = { - 'files': lambda fp: os.path.exists(fp) and not os.path.isdir(fp), # files must exist and not be a directory - 'dirs': lambda dp: os.path.isdir(dp) and os.listdir(dp), # directories must exist and be non-empty + # files must exist and not be a directory + 'files': ('file', lambda fp: os.path.exists(fp) and not os.path.isdir(fp)), + # directories must exist and be non-empty + 'dirs': ("(non-empty) directory", lambda dp: os.path.isdir(dp) and os.listdir(dp)), } + # prepare sanity check paths paths = self.cfg['sanity_check_paths'] if not paths: @@ -1581,7 +1688,6 @@ def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=F else: self.log.info("Using specified sanity check paths: %s" % paths) - # check sanity check paths ks = sorted(paths.keys()) valnottypes = [not isinstance(x, list) for x in paths.values()] lenvals = [len(x) for x in paths.values()] @@ -1590,7 +1696,65 @@ def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=F raise EasyBuildError("Incorrect format for sanity_check_paths (should (only) have %s keys, " "values should be lists (at least one non-empty)).", ','.join(req_keys)) - for key, check_fn in path_keys_and_check.items(): + commands = self.cfg['sanity_check_commands'] + if not commands: + if custom_commands: + commands = custom_commands + self.log.info("Using customised sanity check commands: %s" % commands) + else: + commands = [] + self.log.info("Using specified sanity check commands: %s" % commands) + + for i, command in enumerate(commands): + # set command to default. This allows for config files with + # non-tuple commands + if isinstance(command, basestring): + self.log.debug("Using %s as sanity check command" % command) + command = (command, None) + elif not isinstance(command, tuple): + self.log.debug("Setting sanity check command to default") + command = (None, None) + + # Build substition dictionary + check_cmd = { + 'name': self.name.lower(), + 'options': '-h', + } + if command[0] is not None: + check_cmd['name'] = command[0] + if command[1] is not None: + check_cmd['options'] = command[1] + + commands[i] = "%(name)s %(options)s" % check_cmd + + return paths, path_keys_and_check, commands + + def _sanity_check_step_dry_run(self, custom_paths=None, custom_commands=None, **_): + """Dry run version of sanity_check_step method.""" + paths, path_keys_and_check, commands = self._sanity_check_step_common(custom_paths, custom_commands) + + for key, (typ, _) in path_keys_and_check.items(): + self.dry_run_msg("Sanity check paths - %s ['%s']", typ, key) + if paths[key]: + for path in sorted(paths[key]): + self.dry_run_msg(" * %s", str(path)) + else: + self.dry_run_msg(" (none)") + + self.dry_run_msg("Sanity check commands") + if commands: + for command in sorted(commands): + self.dry_run_msg(" * %s", str(command)) + else: + self.dry_run_msg(" (none)") + + def _sanity_check_step(self, custom_paths=None, custom_commands=None, extension=False): + """Real version of sanity_check_step method.""" + paths, path_keys_and_check, commands = self._sanity_check_step_common(custom_paths, custom_commands) + + # check sanity check paths + for key, (typ, check_fn) in path_keys_and_check.items(): + for xs in paths[key]: if isinstance(xs, basestring): xs = (xs,) @@ -1601,13 +1765,13 @@ def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=F for name in xs: path = os.path.join(self.installdir, name) if check_fn(path): - self.log.debug("Sanity check: found %s %s in %s" % (key[:-1], name, self.installdir)) + self.log.debug("Sanity check: found %s %s in %s" % (typ, name, self.installdir)) found = True break else: - self.log.debug("Could not find %s %s in %s" % (key[:-1], name, self.installdir)) + self.log.debug("Could not find %s %s in %s" % (typ, name, self.installdir)) if not found: - self.sanity_check_fail_msgs.append("no %s of %s in %s" % (key[:-1], xs, self.installdir)) + self.sanity_check_fail_msgs.append("no %s of %s in %s" % (typ, xs, self.installdir)) self.log.warning("Sanity check: %s" % self.sanity_check_fail_msgs[-1]) fake_mod_data = None @@ -1628,39 +1792,15 @@ def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=F raise EasyBuildError("Failed to move to installdir %s: %s", self.installdir, err) # run sanity check commands - commands = self.cfg['sanity_check_commands'] - if not commands: - if custom_commands: - commands = custom_commands - self.log.info("Using customised sanity check commands: %s" % commands) - else: - commands = [] - self.log.info("Using specified sanity check commands: %s" % commands) - for command in commands: - # set command to default. This allows for config files with - # non-tuple commands - if not isinstance(command, tuple): - self.log.debug("Setting sanity check command to default") - command = (None, None) - - # Build substition dictionary - check_cmd = {'name': self.name.lower(), 'options': '-h'} - if command[0] is not None: - check_cmd['name'] = command[0] - - if command[1] is not None: - check_cmd['options'] = command[1] - - cmd = "%(name)s %(options)s" % check_cmd - - out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False) + out, ec = run_cmd(command, simple=False, log_ok=False, log_all=False) if ec != 0: - self.sanity_check_fail_msgs.append("sanity check command %s exited with code %s (output: %s)" % (cmd, ec, out)) + fail_msg = "sanity check command %s exited with code %s (output: %s)" % (command, ec, out) + self.sanity_check_fail_msgs.append(fail_msg) self.log.warning("Sanity check: %s" % self.sanity_check_fail_msgs[-1]) else: - self.log.debug("sanity check command %s ran successfully! (output: %s)" % (cmd, out)) + self.log.debug("sanity check command %s ran successfully! (output: %s)" % (command, out)) if not extension: failed_exts = [ext.name for ext in self.ext_instances if not ext.sanity_check_step()] @@ -1711,7 +1851,9 @@ def cleanup_step(self): def make_module_step(self, fake=False): """ - Generate a module file. + Generate module file + + @param fake: generate 'fake' module in temporary location, rather than actual module file """ modpath = self.module_generator.prepare(fake=fake) @@ -1723,19 +1865,28 @@ def make_module_step(self, fake=False): txt += self.make_module_footer() mod_filepath = self.module_generator.get_module_filepath(fake=fake) - write_file(mod_filepath, txt) - self.log.info("Module file %s written: %s", mod_filepath, txt) + if self.dry_run: + # only report generating actual module file during dry run, don't mention temporary module files + if not fake: + self.dry_run_msg("Generating module file %s, with contents:\n", mod_filepath) + for line in txt.split('\n'): + self.dry_run_msg(' ' * 4 + line) + + else: + write_file(mod_filepath, txt) - # only update after generating final module file - if not fake: - self.modules_tool.update() + self.log.info("Module file %s written: %s", mod_filepath, txt) - mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.cfg) - self.module_generator.create_symlinks(mod_symlink_paths, fake=fake) + # only update after generating final module file + if not fake: + self.modules_tool.update() - if not fake: - self.make_devel_module() + mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.cfg) + self.module_generator.create_symlinks(mod_symlink_paths, fake=fake) + + if not fake: + self.make_devel_module() return modpath @@ -1830,15 +1981,34 @@ def _skip_step(self, step, skippable): return skip - def run_step(self, step, methods): + def run_step(self, step, step_methods): """ Run step, returns false when execution should be stopped """ self.log.info("Starting %s step", step) self.update_config_template_run_step() - for m in methods: - self.log.info("Running method %s part of step %s" % ('_'.join(m.func_code.co_names), step)) - m(self) + for step_method in step_methods: + self.log.info("Running method %s part of step %s" % ('_'.join(step_method.func_code.co_names), step)) + + if self.dry_run: + self.dry_run_msg("[%s method]", step_method(self).__name__) + + # if an known possible error occurs, just report it and continue + try: + # step_method is a lambda function that takes an EasyBlock instance as an argument, + # and returns the actual method, so use () to execute it + step_method(self)() + except Exception as err: + if build_option('extended_dry_run_ignore_errors'): + dry_run_warning("ignoring error %s" % err, silent=self.silent) + self.ignored_errors = True + else: + raise + self.dry_run_msg('') + else: + # step_method is a lambda function that takes an EasyBlock instance as an argument, + # and returns the actual method, so use () to execute it + step_method(self)() if self.cfg['stop'] == step: self.log.info("Stopping after %s step.", step) @@ -1855,46 +2025,46 @@ def get_step(tag, descr, substeps, skippable, initial=True): # list of substeps for steps that are slightly different from 2nd iteration onwards ready_substeps = [ - (False, lambda x: x.check_readiness_step()), - (True, lambda x: x.make_builddir()), - (True, lambda x: env.reset_changes()), - (True, lambda x: x.handle_iterate_opts()), + (False, lambda x: x.check_readiness_step), + (True, lambda x: x.make_builddir), + (True, lambda x: env.reset_changes), + (True, lambda x: x.handle_iterate_opts), ] ready_step_spec = lambda initial: get_step(READY_STEP, "creating build dir, resetting environment", ready_substeps, False, initial=initial) source_substeps = [ - (False, lambda x: x.checksum_step()), - (True, lambda x: x.extract_step()), + (False, lambda x: x.checksum_step), + (True, lambda x: x.extract_step), ] source_step_spec = lambda initial: get_step(SOURCE_STEP, "unpacking", source_substeps, True, initial=initial) def prepare_step_spec(initial): """Return prepare step specification.""" if initial: - substeps = [lambda x: x.prepare_step()] + substeps = [lambda x: x.prepare_step] else: - substeps = [lambda x: x.guess_start_dir()] + substeps = [lambda x: x.guess_start_dir] return (PREPARE_STEP, 'preparing', substeps, False) install_substeps = [ - (False, lambda x: x.stage_install_step()), - (False, lambda x: x.make_installdir()), - (True, lambda x: x.install_step()), + (False, lambda x: x.stage_install_step), + (False, lambda x: x.make_installdir), + (True, lambda x: x.install_step), ] install_step_spec = lambda initial: get_step('install', "installing", install_substeps, True, initial=initial) # format for step specifications: (stop_name: (description, list of functions, skippable)) # core steps that are part of the iterated loop - patch_step_spec = (PATCH_STEP, 'patching', [lambda x: x.patch_step()], True) - configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step()], True) - build_step_spec = (BUILD_STEP, 'building', [lambda x: x.build_step()], True) - test_step_spec = (TEST_STEP, 'testing', [lambda x: x.test_step()], True) + patch_step_spec = (PATCH_STEP, 'patching', [lambda x: x.patch_step], True) + configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step], True) + build_step_spec = (BUILD_STEP, 'building', [lambda x: x.build_step], True) + test_step_spec = (TEST_STEP, 'testing', [lambda x: x.test_step], True) # part 1: pre-iteration + first iteration steps_part1 = [ - (FETCH_STEP, 'fetching files', [lambda x: x.fetch_step()], False), + (FETCH_STEP, 'fetching files', [lambda x: x.fetch_step], False), ready_step_spec(True), source_step_spec(True), patch_step_spec, @@ -1919,13 +2089,13 @@ def prepare_step_spec(initial): ] * (iteration_count - 1) # part 3: post-iteration part steps_part3 = [ - (EXTENSIONS_STEP, 'taking care of extensions', [lambda x: x.extensions_step()], False), - (POSTPROC_STEP, 'postprocessing', [lambda x: x.post_install_step()], True), - (SANITYCHECK_STEP, 'sanity checking', [lambda x: x.sanity_check_step()], False), - (CLEANUP_STEP, 'cleaning up', [lambda x: x.cleanup_step()], False), - (MODULE_STEP, 'creating module', [lambda x: x.make_module_step()], False), - (PERMISSIONS_STEP, 'permissions', [lambda x: x.permissions_step()], False), - (PACKAGE_STEP, 'packaging', [lambda x: x.package_step()], False), + (EXTENSIONS_STEP, 'taking care of extensions', [lambda x: x.extensions_step], False), + (POSTPROC_STEP, 'postprocessing', [lambda x: x.post_install_step], True), + (SANITYCHECK_STEP, 'sanity checking', [lambda x: x.sanity_check_step], False), + (CLEANUP_STEP, 'cleaning up', [lambda x: x.cleanup_step], False), + (MODULE_STEP, 'creating module', [lambda x: x.make_module_step], False), + (PERMISSIONS_STEP, 'permissions', [lambda x: x.permissions_step], False), + (PACKAGE_STEP, 'packaging', [lambda x: x.package_step], False), ] # full list of steps, included iterated steps @@ -1933,8 +2103,8 @@ def prepare_step_spec(initial): if run_test_cases: steps.append((TESTCASES_STEP, 'running test cases', [ - lambda x: x.load_module(), - lambda x: x.test_cases_step(), + lambda x: x.load_module, + lambda x: x.test_cases_step, ], False)) return steps @@ -1949,13 +2119,16 @@ def run_all_steps(self, run_test_cases): steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt()) - print_msg("building and installing %s..." % self.full_mod_name, self.log, silent=self.silent) + print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) try: for (step_name, descr, step_methods, skippable) in steps: if self._skip_step(step_name, skippable): - print_msg("%s [skipped]" % descr, self.log, silent=self.silent) + print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) else: - print_msg("%s..." % descr, self.log, silent=self.silent) + if self.dry_run: + self.dry_run_msg("%s... [DRY RUN]\n", descr) + else: + print_msg("%s..." % descr, log=self.log, silent=self.silent) self.run_step(step_name, step_methods) except StopException: @@ -1965,6 +2138,20 @@ def run_all_steps(self, run_test_cases): return True +def print_dry_run_note(loc, silent=True): + """Print note on interpreting dry run output.""" + msg = '\n'.join([ + '', + "Important note: the actual build & install procedure that will be performed may diverge", + "(slightly) from what is outlined %s, due to conditions in the easyblock which are" % loc, + "incorrectly handled in a dry run.", + "Any errors that may occur are ignored and reported as warnings, on a per-step basis.", + "Please be aware of this, and only use the information %s for quick debugging purposes." % loc, + '', + ]) + dry_run_msg(msg, silent=silent) + + def build_and_install_one(ecdict, init_env): """ Build the software @@ -1977,6 +2164,10 @@ def build_and_install_one(ecdict, init_env): rawtxt = ecdict['ec'].rawtxt name = ecdict['ec']['name'] + dry_run = build_option('extended_dry_run') + + if dry_run: + dry_run_msg('', silent=silent) print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent) # restore original environment @@ -1993,6 +2184,11 @@ def build_and_install_one(ecdict, init_env): try: app_class = get_easyblock_class(easyblock, name=name) + + if dry_run: + # print note on interpreting dry run output (argument is reference to location of dry run messages) + print_dry_run_note('below', silent=silent) + app = app_class(ecdict['ec']) _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock)) except EasyBuildError, err: @@ -2022,17 +2218,20 @@ def build_and_install_one(ecdict, init_env): errormsg = "build failed (first %d chars): %s" % (first_n, err.msg[:first_n]) _log.warning(errormsg) result = False + app.close_log() - ended = "ended" + ended = 'ended' # make sure we're back in original directory before we finish up os.chdir(cwd) - # successful build - if result: + application_log = None + + # successful (non-dry-run) build + if result and not dry_run: if app.cfg['stop']: - ended = "STOPPED" + ended = 'STOPPED' if app.builddir is not None: new_log_dir = os.path.join(app.builddir, config.log_path()) else: @@ -2062,12 +2261,7 @@ def build_and_install_one(ecdict, init_env): except EasyBuildError, err: _log.warn("Unable to commit easyconfig to repository: %s", err) - success = True - succ = "successfully" - summary = "COMPLETED" - # cleanup logs - app.close_log() log_fn = os.path.basename(get_log_filename(app.name, app.version)) application_log = os.path.join(new_log_dir, log_fn) move_logs(app.logfile, application_log) @@ -2087,10 +2281,14 @@ def build_and_install_one(ecdict, init_env): # take away user write permissions (again) adjust_permissions(new_log_dir, stat.S_IWUSR, add=False, recursive=False) - # build failed + if result: + success = True + summary = 'COMPLETED' + succ = 'successfully' else: + # build failed success = False - summary = "FAILED" + summary = 'FAILED' build_dir = '' if app.builddir: @@ -2110,9 +2308,19 @@ def build_and_install_one(ecdict, init_env): _log, silent=silent) if app.postmsg: - print_msg("\nWARNING: %s\n" % app.postmsg, _log, silent=silent) + print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent) + + if dry_run: + # print note on interpreting dry run output (argument is reference to location of dry run messages) + print_dry_run_note('above', silent=silent) + + if app.ignored_errors: + dry_run_warning("One or more errors were ignored, see warnings above", silent=silent) + else: + dry_run_msg("(no ignored errors during dry run)\n", silent=silent) - print_msg("Results of the build can be found in the log file %s" % application_log, _log, silent=silent) + if application_log: + print_msg("Results of the build can be found in the log file %s" % application_log, log=_log, silent=silent) del app diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 227b0b1f6e..f31b0e51c0 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -484,7 +484,21 @@ def toolchain(self): returns the Toolchain used """ if self._toolchain is None: - self._toolchain = get_toolchain(self['toolchain'], self['toolchainopts'], mns=ActiveMNS()) + # provide list of (direct) toolchain dependencies (name & version), if easyconfig can be found for toolchain + tcdeps = None + tcname, tcversion = self['toolchain']['name'], self['toolchain']['version'] + if tcname != DUMMY_TOOLCHAIN_NAME: + tc_ecfile = robot_find_easyconfig(tcname, tcversion) + if tc_ecfile is None: + self.log.debug("No easyconfig found for toolchain %s version %s, can't determine dependencies", + tcname, tcversion) + else: + self.log.debug("Found easyconfig for toolchain %s version %s: %s", tcname, tcversion, tc_ecfile) + tc_ec = process_easyconfig(tc_ecfile)[0] + tcdeps = tc_ec['dependencies'] + self.log.debug("Toolchain dependencies based on easyconfig: %s", tcdeps) + + self._toolchain = get_toolchain(self['toolchain'], self['toolchainopts'], mns=ActiveMNS(), tcdeps=tcdeps) tc_dict = self._toolchain.as_dict() self.log.debug("Initialized toolchain: %s (opts: %s)" % (tc_dict, self['toolchainopts'])) return self._toolchain @@ -1014,18 +1028,20 @@ def create_paths(path, name, version): def robot_find_easyconfig(name, version): """ - Find an easyconfig for module in path + Find an easyconfig for module in path, returns (absolute) path to easyconfig file (or None, if none is found). """ key = (name, version) if key in _easyconfig_files_cache: _log.debug("Obtained easyconfig path from cache for %s: %s" % (key, _easyconfig_files_cache[key])) return _easyconfig_files_cache[key] + paths = build_option('robot_path') - if not paths: - raise EasyBuildError("No robot path specified, which is required when looking for easyconfigs (use --robot)") - if not isinstance(paths, (list, tuple)): + if paths is None: + paths = [] + elif not isinstance(paths, (list, tuple)): paths = [paths] - # candidate easyconfig paths + + res = None for path in paths: easyconfigs_paths = create_paths(path, name, version) for easyconfig_path in easyconfigs_paths: @@ -1033,9 +1049,12 @@ def robot_find_easyconfig(name, version): if os.path.isfile(easyconfig_path): _log.debug("Found easyconfig file for name %s, version %s at %s" % (name, version, easyconfig_path)) _easyconfig_files_cache[key] = os.path.abspath(easyconfig_path) - return _easyconfig_files_cache[key] + res = _easyconfig_files_cache[key] + break + if res: + break - return None + return res class ActiveMNS(object): diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 1713130057..5723c1aa22 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -37,7 +37,7 @@ import os from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import build_path +from easybuild.tools.config import build_option, build_path from easybuild.tools.run import run_cmd @@ -62,7 +62,9 @@ def __init__(self, mself, ext): self.patches = self.ext.get('patches', []) self.options = copy.deepcopy(self.ext.get('options', {})) - self.toolchain.prepare(self.cfg['onlytcmod']) + # don't re-prepare the build environment when doing a dry run, since it'll be the same as for the parent + if not build_option('extended_dry_run'): + self.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True) self.sanity_check_fail_msgs = [] diff --git a/easybuild/main.py b/easybuild/main.py index 68ccea97a4..c51e80a7f4 100755 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -129,7 +129,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): # dump test report next to log file test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state) - if 'log_file' in ec_res: + if 'log_file' in ec_res and ec_res['log_file']: test_report_fp = "%s_test_report.md" % '.'.join(ec_res['log_file'].split('.')[:-1]) parent_dir = os.path.dirname(test_report_fp) # parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used @@ -292,7 +292,7 @@ def main(args=None, logfile=None, do_build=None, testing=False): sys.exit(0) # skip modules that are already installed unless forced - if not options.force: + if not (options.force or options.extended_dry_run): retained_ecs = skip_available(easyconfigs) if not testing: for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index 94b53c7372..32fb162eee 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -31,7 +31,8 @@ import os from distutils.version import LooseVersion -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, dry_run_warning +from easybuild.tools.config import build_option from easybuild.toolchains.fft.fftw import Fftw from easybuild.tools.modules import get_software_root, get_software_version @@ -90,5 +91,9 @@ def _set_fftw_variables(self): if all([fftw_lib_exists(lib) for lib in check_fftw_libs]): self.FFT_LIB = fftw_libs else: - raise EasyBuildError("Not all FFTW interface libraries %s are found in %s, can't set FFT_LIB.", - check_fftw_libs, fft_lib_dirs) + msg = "Not all FFTW interface libraries %s are found in %s" % (check_fftw_libs, fft_lib_dirs) + msg += ", can't set $FFT_LIB." + if build_option('extended_dry_run'): + dry_run_warning(msg, silent=build_option('silent')) + else: + raise EasyBuildError(msg) diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index bd9f50853b..feacd97f2a 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -32,6 +32,7 @@ @author: Jens Timmerman (Ghent University) """ import os +import re import sys import tempfile from copy import copy @@ -52,6 +53,10 @@ DEPRECATED_DOC_URL = 'http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html' +DRY_RUN_BUILD_DIR = None +DRY_RUN_SOFTWARE_INSTALL_DIR = None +DRY_RUN_MODULES_INSTALL_DIR = None + class EasyBuildError(LoggedException): """ @@ -225,6 +230,42 @@ def print_msg(msg, log=None, silent=False, prefix=True): print msg +def dry_run_set_dirs(prefix, builddir, software_installdir, module_installdir): + """ + Initialize for printing dry run messages. + + Define DRY_RUN_*DIR constants, so they can be used in dry_run_msg to replace fake build/install dirs. + + @param prefix: prefix of fake build/install dirs, that can be stripped off when printing + @param builddir: fake build dir + @param software_installdir: fake software install directory + @param module_installdir: fake module install directory + """ + global DRY_RUN_BUILD_DIR + DRY_RUN_BUILD_DIR = (re.compile(builddir), builddir[len(prefix):]) + + global DRY_RUN_MODULES_INSTALL_DIR + DRY_RUN_MODULES_INSTALL_DIR = (re.compile(module_installdir), module_installdir[len(prefix):]) + + global DRY_RUN_SOFTWARE_INSTALL_DIR + DRY_RUN_SOFTWARE_INSTALL_DIR = (re.compile(software_installdir), software_installdir[len(prefix):]) + + +def dry_run_msg(msg, silent=False): + """Print dry run message.""" + # replace fake build/install dir in dry run message with original value + for dry_run_var in [DRY_RUN_BUILD_DIR, DRY_RUN_MODULES_INSTALL_DIR, DRY_RUN_SOFTWARE_INSTALL_DIR]: + if dry_run_var is not None: + msg = dry_run_var[0].sub(dry_run_var[1], msg) + + print_msg(msg, silent=silent, prefix=False) + + +def dry_run_warning(msg, silent=False): + """Print dry run message.""" + dry_run_msg("\n!!!\n!!! WARNING: %s\n!!!\n" % msg, silent=silent) + + def print_error(message, log=None, exitCode=1, opt_parser=None, exit_on_error=True, silent=False): """ Print error message and exit EasyBuild diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 309268f618..e0226d6e4c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -43,11 +43,8 @@ from vsc.utils.missing import FrozenDictKnownKeys from vsc.utils.patterns import Singleton -import easybuild.tools.environment as env -from easybuild.tools import run -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.module_naming_scheme import GENERAL_CLASS -from easybuild.tools.run import run_cmd _log = fancylogger.getLogger('config', fname=False) @@ -77,7 +74,6 @@ DEFAULT_PNS = 'EasyBuildPNS' DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' -DEFAULT_STRICT = run.WARN # utility function for obtaining default paths @@ -124,6 +120,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'allow_modules_tool_mismatch', 'debug', 'dump_autopep8', + 'extended_dry_run', 'experimental', 'force', 'group_writable_installdir', @@ -142,8 +139,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): True: [ 'cleanup_builddir', 'cleanup_tmpdir', + 'extended_dry_run_ignore_errors', ], - DEFAULT_STRICT: [ + 'warn': [ 'strict', ], DEFAULT_PKG_RELEASE: [ @@ -303,7 +301,9 @@ def init_build_options(build_options=None, cmdline_options=None): cmdline_options.force = True retain_all_deps = True - if cmdline_options.dep_graph or cmdline_options.dry_run or cmdline_options.dry_run_short: + auto_ignore_osdeps_options = [cmdline_options.dep_graph, cmdline_options.dry_run, cmdline_options.dry_run_short, + cmdline_options.extended_dry_run] + if any(auto_ignore_osdeps_options): _log.info("Ignoring OS dependencies for --dep-graph/--dry-run") cmdline_options.ignore_osdeps = True @@ -333,9 +333,15 @@ def init_build_options(build_options=None, cmdline_options=None): return BuildOptions(bo) -def build_option(key): +def build_option(key, **kwargs): """Obtain value specified build option.""" - return BuildOptions()[key] + build_options = BuildOptions() + if key in build_options: + return build_options[key] + elif 'default' in kwargs: + return kwargs['default'] + else: + raise EasyBuildError("Undefined build option: %s", key) def build_path(): @@ -517,46 +523,3 @@ def module_classes(): def read_environment(env_vars, strict=False): """NO LONGER SUPPORTED: use read_environment from easybuild.tools.environment instead""" _log.nosupport("read_environment has moved to easybuild.tools.environment", '2.0') - - -def set_tmpdir(tmpdir=None, raise_error=False): - """Set temporary directory to be used by tempfile and others.""" - try: - if tmpdir is not None: - if not os.path.exists(tmpdir): - os.makedirs(tmpdir) - current_tmpdir = tempfile.mkdtemp(prefix='eb-', dir=tmpdir) - else: - # use tempfile default parent dir - current_tmpdir = tempfile.mkdtemp(prefix='eb-') - except OSError, err: - raise EasyBuildError("Failed to create temporary directory (tmpdir: %s): %s", tmpdir, err) - - _log.info("Temporary directory used in this EasyBuild run: %s" % current_tmpdir) - - for var in ['TMPDIR', 'TEMP', 'TMP']: - env.setvar(var, current_tmpdir) - - # reset to make sure tempfile picks up new temporary directory to use - tempfile.tempdir = None - - # test if temporary directory allows to execute files, warn if it doesn't - try: - fd, tmptest_file = tempfile.mkstemp() - os.close(fd) - os.chmod(tmptest_file, 0700) - if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False): - msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() - msg += "This can cause problems in the build process, consider using --tmpdir." - if raise_error: - raise EasyBuildError(msg) - else: - _log.warning(msg) - else: - _log.debug("Temporary directory %s allows to execute files, good!" % tempfile.gettempdir()) - os.remove(tmptest_file) - - except OSError, err: - raise EasyBuildError("Failed to test whether temporary directory allows to execute files: %s", err) - - return current_tmpdir diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index cd6b290d77..0ba413653f 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -33,7 +33,8 @@ from vsc.utils import fancylogger from vsc.utils.missing import shell_quote -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, dry_run_msg +from easybuild.tools.config import build_option # take copy of original environemt, so we can restore (parts of) it later @@ -79,10 +80,12 @@ def get_changes(): return _changes -def setvar(key, value): +def setvar(key, value, verbose=True): """ put key in the environment with value tracks added keys until write_changes has been called + + @param verbose: include message in dry run output for defining this environment variable """ if key in os.environ: oldval_info = "previous value: '%s'" % os.environ[key] @@ -93,6 +96,12 @@ def setvar(key, value): _changes[key] = value _log.info("Environment variable %s set to %s (%s)", key, value, oldval_info) + if verbose and build_option('extended_dry_run'): + quoted_value = shell_quote(value) + if quoted_value[0] not in ['"', "'"]: + quoted_value = '"%s"' % quoted_value + dry_run_msg(" export %s=%s" % (key, quoted_value), silent=build_option('silent')) + def unset_env_vars(keys): """ @@ -139,7 +148,7 @@ def read_environment(env_vars, strict=False): return result -def modify_env(old, new): +def modify_env(old, new, verbose=True): """ Compares 2 os.environ dumps. Adapts final environment. """ @@ -151,10 +160,10 @@ def modify_env(old, new): ## hmm, smart checking with debug logging if not new[key] == old[key]: _log.debug("Key in new environment found that is different from old one: %s (%s)" % (key, new[key])) - setvar(key, new[key]) + setvar(key, new[key], verbose=verbose) else: _log.debug("Key in new environment found that is not in old one: %s (%s)" % (key, new[key])) - setvar(key, new[key]) + setvar(key, new[key], verbose=verbose) for key in oldKeys: if not key in newKeys: @@ -167,4 +176,4 @@ def restore_env(env): """ Restore active environment based on specified dictionary. """ - modify_env(os.environ, env) + modify_env(os.environ, env, verbose=False) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 324834d75a..a7095a4ecd 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -35,19 +35,22 @@ @author: Fotis Georgatos (Uni.Lu, NTUA) @author: Sotiris Fragkiskos (NTUA, CERN) """ +import fileinput import glob import hashlib import os import re import shutil import stat +import sys import time import urllib2 import zlib from vsc.utils import fancylogger from vsc.utils.missing import nub -from easybuild.tools.build_log import EasyBuildError, print_msg # import build_log must stay, to use of EasyBuildLog +# import build_log must stay, to use of EasyBuildLog +from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg from easybuild.tools.config import build_option from easybuild.tools import run @@ -146,22 +149,20 @@ def read_file(path, log_error=True): return None -def write_file(path, txt, append=False): +def write_file(path, txt, append=False, forced=False): """Write given contents to file at given path (overwrites current file contents!).""" - f = None + + # early exit in 'dry run' mode + if not forced and build_option('extended_dry_run'): + dry_run_msg("file written: %s" % path, silent=build_option('silent')) + return + # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block try: mkdir(os.path.dirname(path), parents=True) - if append: - f = open(path, 'a') - else: - f = open(path, 'w') - f.write(txt) - f.close() + with open(path, 'a' if append else 'w') as handle: + handle.write(txt) except IOError, err: - # make sure file handle is always closed - if f is not None: - f.close() raise EasyBuildError("Failed to write to %s: %s", path, err) @@ -179,7 +180,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False): Given filename fn, try to extract in directory dest - returns the directory name in case of success """ - if not os.path.isfile(fn): + if not os.path.isfile(fn) and not build_option('extended_dry_run'): raise EasyBuildError("Can't extract file %s: no such file", fn) mkdir(dest, parents=True) @@ -244,7 +245,7 @@ def det_common_path_prefix(paths): return None -def download_file(filename, url, path): +def download_file(filename, url, path, forced=False): """Download a file from the given URL, to the specified path.""" _log.debug("Trying to download %s from %s to %s", filename, url, path) @@ -269,7 +270,7 @@ def download_file(filename, url, path): # urllib2 does the right thing for http proxy setups, urllib does not! url_fd = urllib2.urlopen(url, timeout=timeout) _log.debug('response code for given url %s: %s' % (url, url_fd.getcode())) - write_file(path, url_fd.read()) + write_file(path, url_fd.read(), forced=forced) _log.info("Downloaded file %s from url %s to %s" % (filename, url, path)) downloaded = True url_fd.close() @@ -597,33 +598,41 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None): - assume unified diff created with "diff -ru old new" """ - if not os.path.isfile(patch_file): + if build_option('extended_dry_run'): + # skip checking of files in dry run mode + patch_filename = os.path.basename(patch_file) + dry_run_msg("* applying patch file %s" % patch_filename, silent=build_option('silent')) + + elif not os.path.isfile(patch_file): raise EasyBuildError("Can't find patch %s: no such file", patch_file) - return - if fn and not os.path.isfile(fn): + elif fn and not os.path.isfile(fn): raise EasyBuildError("Can't patch file %s: no such file", fn) - return - if not os.path.isdir(dest): + elif not os.path.isdir(dest): raise EasyBuildError("Can't patch directory %s: no such directory", dest) - return # copy missing files if copy: - try: - shutil.copy2(patch_file, dest) - _log.debug("Copied patch %s to dir %s" % (patch_file, dest)) - return 'ok' - except IOError, err: - raise EasyBuildError("Failed to copy %s to dir %s: %s", patch_file, dest, err) - return + if build_option('extended_dry_run'): + dry_run_msg(" %s copied to %s" % (patch_file, dest), silent=build_option('silent')) + else: + try: + shutil.copy2(patch_file, dest) + _log.debug("Copied patch %s to dir %s" % (patch_file, dest)) + # early exit, work is done after copying + return True + except IOError, err: + raise EasyBuildError("Failed to copy %s to dir %s: %s", patch_file, dest, err) # use absolute paths apatch = os.path.abspath(patch_file) adest = os.path.abspath(dest) - if not level: + if level is None and build_option('extended_dry_run'): + level = '' + + elif level is None: # guess value for -p (patch level) # - based on +++ lines # - first +++ line that matches an existing file determines guessed level @@ -634,34 +643,48 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None): raise EasyBuildError("Can't guess patchlevel from patch %s: no testfile line found in patch", apatch) return - patch_level = guess_patch_level(patched_files, adest) + level = guess_patch_level(patched_files, adest) - if patch_level is None: # patch_level can also be 0 (zero), so don't use "not patch_level" + if level is None: # level can also be 0 (zero), so don't use "not level" # no match raise EasyBuildError("Can't determine patch level for patch %s from directory %s", patch_file, adest) else: - _log.debug("Guessed patch level %d for patch %s" % (patch_level, patch_file)) + _log.debug("Guessed patch level %d for patch %s" % (level, patch_file)) else: - patch_level = level - _log.debug("Using specified patch level %d for patch %s" % (patch_level, patch_file)) + _log.debug("Using specified patch level %d for patch %s" % (level, patch_file)) - try: - os.chdir(adest) - _log.debug("Changing to directory %s" % adest) - except OSError, err: - raise EasyBuildError("Can't change to directory %s: %s", adest, err) - return - - patch_cmd = "patch -b -p%d -i %s" % (patch_level, apatch) - result = run.run_cmd(patch_cmd, simple=True) + patch_cmd = "patch -b -p%s -i %s" % (level, apatch) + result = run.run_cmd(patch_cmd, simple=True, path=adest) if not result: raise EasyBuildError("Patching with patch %s failed", patch_file) - return return result +def apply_regex_substitutions(path, regex_subs): + """ + Apply specified list of regex substitutions. + + @param path: path to file to patch + @param regex_subs: list of substitutions to apply, specified as (, ) + """ + # only report when in 'dry run' mode + if build_option('extended_dry_run'): + dry_run_msg("applying regex substitutions to file %s" % path, silent=build_option('silent')) + for regex, subtxt in regex_subs: + dry_run_msg(" * regex pattern '%s', replacement string '%s'" % (regex, subtxt)) + + else: + for i, (regex, subtxt) in enumerate(regex_subs): + regex_subs[i] = (re.compile(regex), subtxt) + + for line in fileinput.input(path, inplace=1, backup='.orig.eb'): + for regex, subtxt in regex_subs: + line = regex.sub(subtxt, line) + sys.stdout.write(line) + + def modify_env(old, new): """NO LONGER SUPPORTED: use modify_env from easybuild.tools.environment instead""" _log.nosupport("moved modify_env to easybuild.tools.environment", "2.0") @@ -772,20 +795,25 @@ def patch_perl_script_autoflush(path): # patch Perl script to enable autoflush, # so that e.g. run_cmd_qa receives all output to answer questions - txt = read_file(path) - origpath = "%s.eb.orig" % path - write_file(origpath, txt) - _log.debug("Patching Perl script %s for autoflush, original script copied to %s" % (path, origpath)) + # only report when in 'dry run' mode + if build_option('extended_dry_run'): + dry_run_msg("Perl script patched: %s" % path, silent=build_option('silent')) - # force autoflush for Perl print buffer - lines = txt.split('\n') - newtxt = '\n'.join([ - lines[0], # shebang line - "\nuse IO::Handle qw();", - "STDOUT->autoflush(1);\n", # extra newline to separate from actual script - ] + lines[1:]) - - write_file(path, newtxt) + else: + txt = read_file(path) + origpath = "%s.eb.orig" % path + write_file(origpath, txt) + _log.debug("Patching Perl script %s for autoflush, original script copied to %s" % (path, origpath)) + + # force autoflush for Perl print buffer + lines = txt.split('\n') + newtxt = '\n'.join([ + lines[0], # shebang line + "\nuse IO::Handle qw();", + "STDOUT->autoflush(1);\n", # extra newline to separate from actual script + ] + lines[1:]) + + write_file(path, newtxt) def mkdir(path, parents=False, set_gid=None, sticky=None): diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index f6d0705b42..c95ef1b917 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -320,7 +320,7 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): # determine list of changed files via diff diff_fn = os.path.basename(pr_data['diff_url']) diff_filepath = os.path.join(path, diff_fn) - download_file(diff_fn, pr_data['diff_url'], diff_filepath) + download_file(diff_fn, pr_data['diff_url'], diff_filepath, forced=True) diff_txt = read_file(diff_filepath) os.remove(diff_filepath) @@ -342,7 +342,7 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): sha = last_commit['sha'] full_url = URL_SEPARATOR.join([GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, sha, patched_file]) _log.info("Downloading %s from %s" % (fn, full_url)) - download_file(fn, full_url, path=os.path.join(path, fn)) + download_file(fn, full_url, path=os.path.join(path, fn), forced=True) all_files = [os.path.basename(x) for x in patched_files] tmp_files = os.listdir(path) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 5958508430..1e9bc7bbd3 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -97,7 +97,7 @@ def prepare(self, fake=False): mkdir(os.path.dirname(mod_filepath), parents=True) # remove module file if it's there (it'll be recreated), see EasyBlock.make_module - if os.path.exists(mod_filepath): + if os.path.exists(mod_filepath) and not build_option('extended_dry_run'): self.log.debug("Removing existing module file %s", mod_filepath) os.remove(mod_filepath) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index b670a4078f..2aab6fcd33 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -42,7 +42,6 @@ from subprocess import PIPE from vsc.utils import fancylogger from vsc.utils.missing import get_subclasses -from vsc.utils.patterns import Singleton from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_modules_tool, install_path @@ -246,7 +245,7 @@ def check_module_function(self, allow_mismatch=False, regex=None): else: out, ec = None, 1 else: - out, ec = run_cmd("type module", simple=False, log_ok=False, log_all=False) + out, ec = run_cmd("type module", simple=False, log_ok=False, log_all=False, force_in_dry_run=True) if regex is None: regex = r".*%s" % os.path.basename(self.cmd) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index b4b729ab0c..90c3308408 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -38,9 +38,11 @@ import re import shutil import sys +import tempfile from distutils.version import LooseVersion from vsc.utils.missing import nub +import easybuild.tools.environment as env from easybuild.framework.easyblock import MODULE_ONLY_STEPS, SOURCE_STEP, EasyBlock from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.constants import constant_documentation @@ -55,8 +57,8 @@ from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX from easybuild.tools.config import DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS from easybuild.tools.config import DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL, DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX -from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_STRICT -from easybuild.tools.config import get_pretend_installpath, mk_full_default_path, set_tmpdir +from easybuild.tools.config import DEFAULT_REPOSITORY +from easybuild.tools.config import get_pretend_installpath, mk_full_default_path from easybuild.tools.configobj import ConfigObj, ConfigObjError from easybuild.tools.docs import FORMAT_RST, FORMAT_TXT, avail_easyconfig_params from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, fetch_github_token @@ -68,6 +70,7 @@ from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes from easybuild.tools.modules import Lmod from easybuild.tools.ordereddict import OrderedDict +from easybuild.tools.run import run_cmd from easybuild.tools.package.utilities import avail_package_naming_schemes from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.repository.repository import avail_repositories @@ -84,6 +87,9 @@ DEFAULT_USER_CFGFILE = os.path.join(XDG_CONFIG_HOME, 'easybuild', 'config.cfg') +_log = fancylogger.getLogger('options', fname=False) + + class EasyBuildOptions(GeneralOption): """Easybuild generaloption class""" VERSION = this_is_easybuild() @@ -129,6 +135,9 @@ def basic_options(self): opts = OrderedDict({ 'dry-run': ("Print build overview incl. dependencies (full paths)", None, 'store_true', False), 'dry-run-short': ("Print build overview incl. dependencies (short paths)", None, 'store_true', False, 'D'), + 'extended-dry-run': ("Print build environment and (expected) build procedure that will be performed", + None, 'store_true', False, 'x'), + 'extended-dry-run-ignore-errors': ("Ignore errors that occur during dry run", None, 'store_true', True), 'force': ("Force to rebuild software even if it's already installed (i.e. if it can be found as module)", None, 'store_true', False, 'f'), 'job': ("Submit the build as a job", None, 'store_true', False), @@ -142,7 +151,7 @@ def basic_options(self): None, 'store_true', False, 'k'), 'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', SOURCE_STEP, 's', all_stops), - 'strict': ("Set strictness level", 'choice', 'store', DEFAULT_STRICT, strictness_options), + 'strict': ("Set strictness level", 'choice', 'store', run.WARN, strictness_options), }) self.log.debug("basic_options: descr %s opts %s" % (descr, opts)) @@ -940,3 +949,46 @@ def process_software_build_specs(options): build_specs.update({param: value}) return (try_to_generate, build_specs) + + +def set_tmpdir(tmpdir=None, raise_error=False): + """Set temporary directory to be used by tempfile and others.""" + try: + if tmpdir is not None: + if not os.path.exists(tmpdir): + os.makedirs(tmpdir) + current_tmpdir = tempfile.mkdtemp(prefix='eb-', dir=tmpdir) + else: + # use tempfile default parent dir + current_tmpdir = tempfile.mkdtemp(prefix='eb-') + except OSError, err: + raise EasyBuildError("Failed to create temporary directory (tmpdir: %s): %s", tmpdir, err) + + _log.info("Temporary directory used in this EasyBuild run: %s" % current_tmpdir) + + for var in ['TMPDIR', 'TEMP', 'TMP']: + env.setvar(var, current_tmpdir, verbose=False) + + # reset to make sure tempfile picks up new temporary directory to use + tempfile.tempdir = None + + # test if temporary directory allows to execute files, warn if it doesn't + try: + fd, tmptest_file = tempfile.mkstemp() + os.close(fd) + os.chmod(tmptest_file, 0700) + if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True): + msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() + msg += "This can cause problems in the build process, consider using --tmpdir." + if raise_error: + raise EasyBuildError(msg) + else: + _log.warning(msg) + else: + _log.debug("Temporary directory %s allows to execute files, good!" % tempfile.gettempdir()) + os.remove(tmptest_file) + + except OSError, err: + raise EasyBuildError("Failed to test whether temporary directory allows to execute files: %s", err) + + return current_tmpdir diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index fe55d9d6a4..a6d29bd46a 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -43,7 +43,8 @@ from vsc.utils import fancylogger from easybuild.tools.asyncprocess import PIPE, STDOUT, Popen, recv_some, send_all -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option +from easybuild.tools.build_log import EasyBuildError, dry_run_msg _log = fancylogger.getLogger('run', fname=False) @@ -59,20 +60,38 @@ # default strictness level strictness = WARN - -def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None): +def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, + force_in_dry_run=False, verbose=True): """ - Executes a command cmd - - returns exitcode and stdout+stderr (mixed) - - no input though stdin - - if log_ok or log_all are set -> will raise EasyBuildError if non-zero exit-code - - if simple is True -> instead of returning a tuple (output, ec) it will just return True or False signifying succes - - inp is the input given to the command - - regexp -> Regex used to check the output for errors. If True will use default (see parselogForError) - - if log_output is True -> all output of command will be logged to a tempfile - - path is the path run_cmd should chdir to before doing anything + Run specified command (in a subshell) + @param cmd: command to run + @param log_ok: only run output/exit code for failing commands (exit code non-zero) + @param log_all: always log command output and exit code + @param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) + @param inp: the input given to the command via stdin + @param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) + @param log_output: indicate whether all output of command should be logged to a separate tempoary logfile + @param path: path to execute the command in; current working directory is used if unspecified + @param force_in_dry_run: force running the command during dry run + @param verbose: include message on running the command in dry run output """ cwd = os.getcwd() + + # early exit in 'dry run' mode, after printing the command that would be run (unless running the command is forced) + if not force_in_dry_run and build_option('extended_dry_run'): + if path is None: + path = cwd + if verbose: + dry_run_msg(" running command \"%s\"" % cmd, silent=build_option('silent')) + dry_run_msg(" (in %s)" % path, silent=build_option('silent')) + + # make sure we get the type of the return value right + if simple: + return True + else: + # output, exit code + return ('', 0) + try: if path: os.chdir(path) @@ -132,18 +151,31 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None, maxhits=50): """ - Executes a command cmd - - looks for questions and tries to answer based on qa dictionary - - provided answers can be either strings or lists of strings (which will be used iteratively) - - returns exitcode and stdout+stderr (mixed) - - no input though stdin - - if log_ok or log_all are set -> will log.error if non-zero exit-code - - if simple is True -> instead of returning a tuple (output, ec) it will just return True or False signifying succes - - regexp -> Regex used to check the output for errors. If True will use default (see parselogForError) - - if log_output is True -> all output of command will be logged to a tempfile - - path is the path run_cmd should chdir to before doing anything + Run specified interactive command (in a subshell) + @param cmd: command to run + @param qa: dictionary which maps question to answers + @param no_qa: list of patters that are not questions + @param log_ok: only run output/exit code for failing commands (exit code non-zero) + @param log_all: always log command output and exit code + @param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) + @param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) + @param std_qa: dictionary which maps question regex patterns to answers + @param path: path to execute the command is; current working directory is used if unspecified """ cwd = os.getcwd() + + # early exit in 'dry run' mode, after printing the command that would be run + if build_option('extended_dry_run'): + if path is None: + path = cwd + dry_run_msg(" running interactive command \"%s\"" % cmd, silent=build_option('silent')) + dry_run_msg(" (in %s)" % path, silent=build_option('silent')) + if simple: + return True + else: + # output, exit code + return ('', 0) + try: if path: os.chdir(path) @@ -332,7 +364,14 @@ def check_answers_list(answers): def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): """ - will parse and perform error checks based on strictness setting + Parse command output and construct return value. + @param cmd: executed command + @param stdouterr: combined stdout/stderr of executed command + @param ec: exit code of executed command + @param simple: if True, just return True/False to indicate success, else return a tuple: (output, exit_code) + @param log_all: always log command output and exit code + @param log_ok: only run output/exit code for failing commands (exit code non-zero) + @param regex: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) """ if strictness == IGNORE: check_ec = False diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 8f3c5891d0..569a9536cb 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -88,7 +88,7 @@ def get_avail_core_count(): core_cnt = int(sum(sched_getaffinity().cpus)) else: # BSD-type systems - out, _ = run_cmd('sysctl -n hw.ncpu') + out, _ = run_cmd('sysctl -n hw.ncpu', force_in_dry_run=True) try: if int(out) > 0: core_cnt = int(out) @@ -130,7 +130,7 @@ def get_cpu_vendor(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" - out, ec = run_cmd(cmd) + out, ec = run_cmd(cmd, force_in_dry_run=True) out = out.strip() if ec == 0 and out in VENDORS: vendor = VENDORS[out] @@ -191,7 +191,7 @@ def get_cpu_model(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.brand_string" - out, ec = run_cmd(cmd) + out, ec = run_cmd(cmd, force_in_dry_run=True) if ec == 0: model = out.strip() _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) @@ -236,7 +236,7 @@ def get_cpu_speed(): elif os_type == DARWIN: cmd = "sysctl -n hw.cpufrequency_max" _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd) - out, ec = run_cmd(cmd) + out, ec = run_cmd(cmd, force_in_dry_run=True) if ec == 0: # returns clock frequency in cycles/sec, but we want MHz cpu_freq = float(out.strip())/(1000**2) @@ -370,11 +370,11 @@ def check_os_dependency(dep): cmd = None if which('rpm'): cmd = "rpm -q %s" % dep - found = run_cmd(cmd, simple=True, log_all=False, log_ok=False) + found = run_cmd(cmd, simple=True, log_all=False, log_ok=False, force_in_dry_run=True) if not found and which('dpkg'): cmd = "dpkg -s %s" % dep - found = run_cmd(cmd, simple=True, log_all=False, log_ok=False) + found = run_cmd(cmd, simple=True, log_all=False, log_ok=False, force_in_dry_run=True) if cmd is None: # fallback for when os-dependency is a binary/library @@ -383,7 +383,7 @@ def check_os_dependency(dep): # try locate if it's available if not found and which('locate'): cmd = 'locate --regexp "/%s$"' % dep - found = run_cmd(cmd, simple=True, log_all=False, log_ok=False) + found = run_cmd(cmd, simple=True, log_all=False, log_ok=False, force_in_dry_run=True) return found @@ -393,7 +393,7 @@ def get_tool_version(tool, version_option='--version'): Get output of running version option for specific command line tool. Output is returned as a single-line string (newlines are replaced by '; '). """ - out, ec = run_cmd(' '.join([tool, version_option]), simple=False, log_ok=False) + out, ec = run_cmd(' '.join([tool, version_option]), simple=False, log_ok=False, force_in_dry_run=True) if ec: _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, out)) return UNKNOWN @@ -484,7 +484,7 @@ def det_parallelism(par=None, maxpar=None): else: par = get_avail_core_count() # check ulimit -u - out, ec = run_cmd('ulimit -u') + out, ec = run_cmd('ulimit -u', force_in_dry_run=True) try: if out.startswith("unlimited"): out = 2 ** 32 - 1 diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 907bb65734..44c942a3e3 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -32,9 +32,10 @@ """ import copy import os +import tempfile from vsc.utils import fancylogger -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, dry_run_msg from easybuild.tools.config import build_option, install_path from easybuild.tools.environment import setvar from easybuild.tools.module_generator import dependencies_for @@ -77,7 +78,7 @@ def _is_toolchain_for(cls, name): _is_toolchain_for = classmethod(_is_toolchain_for) - def __init__(self, name=None, version=None, mns=None, class_constants=None): + def __init__(self, name=None, version=None, mns=None, class_constants=None, tcdeps=None): """Toolchain constructor.""" self.base_init() @@ -101,6 +102,11 @@ def __init__(self, name=None, version=None, mns=None, class_constants=None): self._init_class_constants(class_constants) + self.tcdeps = tcdeps + + # toolchain instances are created before initiating build options sometimes, e.g. for --list-toolchains + self.dry_run = build_option('extended_dry_run', default=False) + self.modules_tool = modules_tool() self.mns = mns self.mod_full_name = None @@ -332,14 +338,24 @@ def add_dependencies(self, dependencies): """ Verify if the given dependencies exist and add them """ self.log.debug("add_dependencies: adding toolchain dependencies %s" % dependencies) dep_mod_names = [dep['full_mod_name'] for dep in dependencies] - deps_exist = self.modules_tool.exist(dep_mod_names) + + # check whether modules exist + if self.dry_run: + deps_exist = [True] * len(dep_mod_names) + else: + deps_exist = self.modules_tool.exist(dep_mod_names) + + missing_dep_mods = [] for dep, dep_mod_name, dep_exists in zip(dependencies, dep_mod_names, deps_exist): self.log.debug("add_dependencies: MODULEPATH: %s" % os.environ['MODULEPATH']) - if not dep_exists: - raise EasyBuildError("add_dependencies: no module '%s' found for dependency %s", dep_mod_name, dep) - else: + if dep_exists: self.dependencies.append(dep) self.log.debug('add_dependencies: added toolchain dependency %s' % str(dep)) + else: + missing_dep_mods.append(dep_mod_name) + + if missing_dep_mods: + raise EasyBuildError("Missing modules for one or more dependencies: %s", ', '.join(missing_dep_mods)) def is_required(self, name): """Determine whether this is a required toolchain element.""" @@ -363,79 +379,144 @@ def is_dep_in_toolchain_module(self, name): """Check whether a specific software name is listed as a dependency in the module for this toolchain.""" return any(map(lambda m: self.mns.is_short_modname_for(m, name), self.toolchain_dep_mods)) - def _prepare_dependency_external_module(self, dep): - """Set environment variables picked up by utility functions for dependencies specified as external modules.""" - mod_name = dep['full_mod_name'] - metadata = dep['external_module_metadata'] - self.log.debug("Defining $EB* environment variables for external module %s", mod_name) - - names = metadata.get('name', []) - versions = metadata.get('version', [None]*len(names)) - self.log.debug("Metadata for external module %s: %s", mod_name, metadata) - - for name, version in zip(names, versions): - self.log.debug("Defining $EB* environment variables for external module %s under name %s", mod_name, name) - - # define $EBROOT env var for install prefix, picked up by get_software_root - prefix = metadata.get('prefix') - if prefix is not None: - if prefix in os.environ: - val = os.environ[prefix] - self.log.debug("Using value of $%s as prefix for external module %s: %s", prefix, mod_name, val) - else: - val = prefix - self.log.debug("Using specified prefix for external module %s: %s", mod_name, val) - setvar(get_software_root_env_var_name(name), val) + def _simulated_load_dependency_module(self, name, version, metadata, verbose=False): + """ + Set environment variables picked up by utility functions for dependencies specified as external modules. + + @param name: software name + @param version: software version + @param metadata: dictionary with software metadata ('prefix' for software installation prefix) + """ + + self.log.debug("Defining $EB* environment variables for software named %s", name) + + # define $EBROOT env var for install prefix, picked up by get_software_root + prefix = metadata.get('prefix') + if prefix is not None: + if prefix in os.environ: + val = os.environ[prefix] + self.log.debug("Using value of $%s as prefix for software named %s: %s", prefix, name, val) + else: + val = prefix + self.log.debug("Using specified prefix for software named %s: %s", name, val) + setvar(get_software_root_env_var_name(name), val, verbose=verbose) + + # define $EBVERSION env var for software version, picked up by get_software_version + if version is not None: + setvar(get_software_version_env_var_name(name), version, verbose=verbose) + + def _load_toolchain_module(self, silent=False): + """Load toolchain module.""" + + tc_mod = self.det_short_module_name() + + if self.dry_run: + dry_run_msg("Loading toolchain module...\n", silent=silent) + + # load toolchain module, or simulate load of toolchain components if it is not available + if self.modules_tool.exist([tc_mod])[0]: + self.modules_tool.load([tc_mod]) + dry_run_msg("module load %s" % tc_mod, silent=silent) + else: + # first simulate loads for toolchain dependencies, if required information is available + if self.tcdeps is not None: + for tcdep in self.tcdeps: + modname = tcdep['short_mod_name'] + dry_run_msg("module load %s [SIMULATED]" % modname, silent=silent) + # 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output) + deproot = '$%s' % get_software_root_env_var_name(tcdep['name']) + self._simulated_load_dependency_module(tcdep['name'], tcdep['version'], {'prefix': deproot}) + + dry_run_msg("module load %s [SIMULATED]" % tc_mod, silent=silent) + # use name of $EBROOT* env var as value for $EBROOT* env var (results in sensible dry run output) + tcroot = '$%s' % get_software_root_env_var_name(self.name) + self._simulated_load_dependency_module(self.name, self.version, {'prefix': tcroot}) + else: + # make sure toolchain is available using short module name by running 'module use' on module path subdir + if self.init_modpaths: + mod_path_suffix = build_option('suffix_modules_path') + for modpath in self.init_modpaths: + self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath)) - # define $EBVERSION env var for software version, picked up by get_software_version - if version is not None: - setvar(get_software_version_env_var_name(name), version) + # load modules for all dependencies + self.log.debug("Loading module for toolchain: %s" % tc_mod) + self.modules_tool.load([tc_mod]) - def _prepare_dependencies(self): + def _load_dependencies_modules(self, silent=False): """Load modules for dependencies, and handle special cases like external modules.""" - # load modules for all dependencies - dep_mods = [dep['short_mod_name'] for dep in self.dependencies] - self.log.debug("Loading modules for dependencies: %s" % dep_mods) - self.modules_tool.load(dep_mods) + + if self.dry_run: + dry_run_msg("\nLoading modules for dependencies...\n", silent=silent) + + mod_names = [dep['short_mod_name'] for dep in self.dependencies] + mods_exist = self.modules_tool.exist(mod_names) + + # load available modules for dependencies, simulate load for others + for dep, dep_mod_exists in zip(self.dependencies, mods_exist): + mod_name = dep['short_mod_name'] + if dep_mod_exists: + self.modules_tool.load([mod_name]) + dry_run_msg("module load %s" % mod_name, silent=silent) + else: + dry_run_msg("module load %s [SIMULATED]" % mod_name, silent=silent) + # 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output) + deproot = '$%s' % get_software_root_env_var_name(dep['name']) + self._simulated_load_dependency_module(dep['name'], dep['version'], {'prefix': deproot}) + else: + # load modules for all dependencies + dep_mods = [dep['short_mod_name'] for dep in self.dependencies] + self.log.debug("Loading modules for dependencies: %s" % dep_mods) + self.modules_tool.load(dep_mods) # define $EBROOT* and $EBVERSION* for external modules, if metadata is available for dep in [d for d in self.dependencies if d['external_module']]: - self._prepare_dependency_external_module(dep) + mod_name = dep['full_mod_name'] + metadata = dep['external_module_metadata'] + self.log.debug("Metadata for external module %s: %s", mod_name, metadata) - def prepare(self, onlymod=None): - """ - Prepare a set of environment parameters based on name/version of toolchain - - load modules for toolchain and dependencies - - generate extra variables and set them in the environment + names = metadata.get('name', []) + versions = metadata.get('version', [None] * len(names)) + self.log.debug("Defining $EB* environment variables for external module %s using names %s, versions %s", + mod_name, names, versions) - onlymod: Boolean/string to indicate if the toolchain should only load the environment - with module (True) or also set all other variables (False) like compiler CC etc - (If string: comma separated list of variables that will be ignored). - """ + for name, version in zip(names, versions): + self._simulated_load_dependency_module(name, version, metadata, verbose=True) + + def _load_modules(self, silent=False): + """Load modules for toolchain and dependencies.""" if self.modules_tool is None: raise EasyBuildError("No modules tool defined in Toolchain instance.") - if not self._toolchain_exists(): + if not self._toolchain_exists() and not self.dry_run: raise EasyBuildError("No module found for toolchain: %s", self.mod_short_name) if self.name == DUMMY_TOOLCHAIN_NAME: if self.version == DUMMY_TOOLCHAIN_VERSION: self.log.info('prepare: toolchain dummy mode, dummy version; not loading dependencies') + if self.dry_run: + dry_run_msg("(no modules are loaded for a dummy-dummy toolchain)", silent=silent) else: self.log.info('prepare: toolchain dummy mode and loading dependencies') - self._prepare_dependencies() - return - - # Load the toolchain and dependencies modules - self.log.debug("Loading toolchain module and dependencies...") - # make sure toolchain is available using short module name by running 'module use' on module path subdir - if self.init_modpaths: - mod_path_suffix = build_option('suffix_modules_path') - for modpath in self.init_modpaths: - self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath)) - self.modules_tool.load([self.det_short_module_name()]) - self._prepare_dependencies() + self._load_dependencies_modules(silent=silent) + else: + # load the toolchain and dependencies modules + self.log.debug("Loading toolchain module and dependencies...") + self._load_toolchain_module(silent=silent) + self._load_dependencies_modules(silent=silent) + + # include list of loaded modules in dry run output + if self.dry_run: + loaded_mods = self.modules_tool.list() + dry_run_msg("\nFull list of loaded modules:", silent=silent) + if loaded_mods: + for i, mod_name in enumerate([m['mod_name'] for m in loaded_mods]): + dry_run_msg(" %d) %s" % (i+1, mod_name), silent=silent) + else: + dry_run_msg(" (none)", silent=silent) + dry_run_msg('', silent=silent) + def _verify_toolchain(self): + """Verify toolchain: check toolchain definition against dependencies of toolchain module.""" # determine direct toolchain dependencies mod_name = self.det_short_module_name() self.toolchain_dep_mods = dependencies_for(mod_name, depth=0) @@ -461,21 +542,38 @@ def prepare(self, onlymod=None): raise EasyBuildError("List of toolchain dependency modules and toolchain definition do not match " "(%s vs %s)", self.toolchain_dep_mods, toolchain_definition) - # Generate the variables to be set - self.set_variables() + def prepare(self, onlymod=None, silent=False): + """ + Prepare a set of environment parameters based on name/version of toolchain + - load modules for toolchain and dependencies + - generate extra variables and set them in the environment + + onlymod: Boolean/string to indicate if the toolchain should only load the environment + with module (True) or also set all other variables (False) like compiler CC etc + (If string: comma separated list of variables that will be ignored). + """ + self._load_modules(silent=silent) - # set the variables - # onlymod can be comma-separated string of variables not to be set - if onlymod == True: - self.log.debug("prepare: do not set additional variables onlymod=%s" % onlymod) - self.generate_vars() - else: - self.log.debug("prepare: set additional variables onlymod=%s" % onlymod) + if self.name != DUMMY_TOOLCHAIN_NAME: - # add LDFLAGS and CPPFLAGS from dependencies to self.vars - self._add_dependency_variables() - self.generate_vars() - self._setenv_variables(onlymod) + if not self.dry_run: + self._verify_toolchain() + + # Generate the variables to be set + self.set_variables() + + # set the variables + # onlymod can be comma-separated string of variables not to be set + if onlymod == True: + self.log.debug("prepare: do not set additional variables onlymod=%s" % onlymod) + self.generate_vars() + else: + self.log.debug("prepare: set additional variables onlymod=%s" % onlymod) + + # add LDFLAGS and CPPFLAGS from dependencies to self.vars + self._add_dependency_variables() + self.generate_vars() + self._setenv_variables(onlymod, verbose=not silent) def _add_dependency_variables(self, names=None, cpp=None, ld=None): """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies @@ -514,9 +612,12 @@ def _add_dependency_variables(self, names=None, cpp=None, ld=None): self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths) self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths) - def _setenv_variables(self, donotset=None): + def _setenv_variables(self, donotset=None, verbose=True): """Actually set the environment variables""" + self.log.debug("_setenv_variables: setting variables: donotset=%s" % donotset) + if self.dry_run: + dry_run_msg("Defining build environment...\n", silent=not verbose) donotsetlist = [] if isinstance(donotset, str): @@ -525,19 +626,19 @@ def _setenv_variables(self, donotset=None): elif isinstance(donotset, list): donotsetlist = donotset - for key, val in self.vars.items(): + for key, val in sorted(self.vars.items()): if key in donotsetlist: self.log.debug("_setenv_variables: not setting environment variable %s (value: %s)." % (key, val)) continue self.log.debug("_setenv_variables: setting environment variable %s to %s" % (key, val)) - setvar(key, val) + setvar(key, val, verbose=verbose) # also set unique named variables that can be used in Makefiles # - so you can have 'CFLAGS = $(EBVARCFLAGS)' # -- 'CLFLAGS = $(CFLAGS)' gives '*** Recursive variable `CFLAGS' # references itself (eventually). Stop' error - setvar("EBVAR%s" % key, val) + setvar("EBVAR%s" % key, val, verbose=False) def get_flag(self, name): """Get compiler flag for a certain option.""" diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index 73bf78c7c1..f9032cd7ae 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -112,7 +112,7 @@ def search_toolchain(name): return None, found_tcs -def get_toolchain(tc, tcopts, mns): +def get_toolchain(tc, tcopts, mns=None, tcdeps=None): """ Return an initialized toolchain for the given specifications. If none is available in the toolchain instances cache, a new one is created. @@ -124,9 +124,9 @@ def get_toolchain(tc, tcopts, mns): else: tc_class, all_tcs = search_toolchain(tc['name']) if not tc_class: - all_tcs_names = ",".join([x.NAME for x in all_tcs]) + all_tcs_names = ','.join([x.NAME for x in all_tcs]) raise EasyBuildError("Toolchain %s not found, available toolchains: %s", tc['name'], all_tcs_names) - tc_inst = tc_class(version=tc['version'], mns=mns) + tc_inst = tc_class(version=tc['version'], mns=mns, tcdeps=tcdeps) tc_dict = tc_inst.as_dict() _log.debug("Obtained new toolchain instance for %s: %s" % (key, tc_dict)) diff --git a/test/framework/config.py b/test/framework/config.py index ecbb28fe2c..414bd8b3b5 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -41,7 +41,7 @@ from easybuild.tools import run from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, build_path, source_paths, install_path, get_repositorypath -from easybuild.tools.config import set_tmpdir, BuildOptions, ConfigurationVariables +from easybuild.tools.config import BuildOptions, ConfigurationVariables from easybuild.tools.config import get_build_log_path, DEFAULT_PATH_SUBDIRS, init_build_options from easybuild.tools.environment import modify_env from easybuild.tools.filetools import mkdir, write_file @@ -337,35 +337,6 @@ def test_generaloption_config_file(self): del os.environ['EASYBUILD_CONFIGFILES'] sys.path[:] = orig_sys_path - def test_set_tmpdir(self): - """Test set_tmpdir config function.""" - self.purge_environment() - - for tmpdir in [None, os.path.join(tempfile.gettempdir(), 'foo')]: - parent = tmpdir - if parent is None: - parent = tempfile.gettempdir() - - mytmpdir = set_tmpdir(tmpdir=tmpdir) - - for var in ['TMPDIR', 'TEMP', 'TMP']: - self.assertTrue(os.environ[var].startswith(os.path.join(parent, 'eb-'))) - self.assertEqual(os.environ[var], mytmpdir) - self.assertTrue(tempfile.gettempdir().startswith(os.path.join(parent, 'eb-'))) - tempfile_tmpdir = tempfile.mkdtemp() - self.assertTrue(tempfile_tmpdir.startswith(os.path.join(parent, 'eb-'))) - fd, tempfile_tmpfile = tempfile.mkstemp() - self.assertTrue(tempfile_tmpfile.startswith(os.path.join(parent, 'eb-'))) - - # tmp_logdir follows tmpdir - self.assertEqual(get_build_log_path(), mytmpdir) - - # cleanup - os.close(fd) - shutil.rmtree(mytmpdir) - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None - def test_configuration_variables(self): """Test usage of ConfigurationVariables.""" # delete instance of ConfigurationVariables diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index c8f19c3193..cb34331de3 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -560,7 +560,7 @@ def test_check_readiness(self): try: eb.check_readiness_step() except EasyBuildError, err: - err_regex = re.compile("no module 'nosuchsoftware/1.2.3-GCC-4.6.4' found for dependency .*") + err_regex = re.compile("Missing modules for one or more dependencies: nosuchsoftware/1.2.3-GCC-4.6.4") self.assertTrue(err_regex.search(str(err)), "Pattern '%s' found in '%s'" % (err_regex.pattern, err)) shutil.rmtree(tmpdir) diff --git a/test/framework/environment.py b/test/framework/environment.py new file mode 100644 index 0000000000..dd68506d48 --- /dev/null +++ b/test/framework/environment.py @@ -0,0 +1,79 @@ +# # +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for environment.py + +@author: Kenneth Hoste (Ghent University) +""" +import os +from test.framework.utilities import EnhancedTestCase, init_config +from unittest import TestLoader, main + +import easybuild.tools.environment as env + + +class EnvironmentTest(EnhancedTestCase): + """ Testcase for run module """ + + def test_setvar(self): + """Test setvar function.""" + self.mock_stdout(True) + env.setvar('FOO', 'bar') + txt = self.get_stdout() + self.mock_stdout(False) + self.assertEqual(os.getenv('FOO'), 'bar') + self.assertEqual(os.environ['FOO'], 'bar') + # no printing if dry run is not enabled + self.assertEqual(txt, '') + + build_options = { + 'extended_dry_run': True, + 'silent': False, + } + init_config(build_options=build_options) + self.mock_stdout(True) + env.setvar('FOO', 'foobaz') + txt = self.get_stdout() + self.mock_stdout(False) + self.assertEqual(os.getenv('FOO'), 'foobaz') + self.assertEqual(os.environ['FOO'], 'foobaz') + self.assertEqual(txt, " export FOO=\"foobaz\"\n") + + # disabling verbose + self.mock_stdout(True) + env.setvar('FOO', 'barfoo', verbose=False) + txt = self.get_stdout() + self.mock_stdout(False) + self.assertEqual(os.getenv('FOO'), 'barfoo') + self.assertEqual(os.environ['FOO'], 'barfoo') + self.assertEqual(txt, '') + + +def suite(): + """ returns all the testcases in this module """ + return TestLoader().loadTestsFromTestCase(EnvironmentTest) + +if __name__ == '__main__': + main() diff --git a/test/framework/options.py b/test/framework/options.py index 7f10003976..60b9ddd038 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -44,12 +44,12 @@ from easybuild.framework.easyconfig import BUILD, CUSTOM, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, LICENSE from easybuild.framework.easyconfig import MANDATORY, MODULES, OTHER, TOOLCHAIN from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import DEFAULT_MODULECLASSES, get_module_syntax +from easybuild.tools.config import DEFAULT_MODULECLASSES, get_build_log_path, get_module_syntax from easybuild.tools.environment import modify_env from easybuild.tools.filetools import mkdir, read_file, write_file from easybuild.tools.github import fetch_github_token from easybuild.tools.modules import modules_tool -from easybuild.tools.options import EasyBuildOptions +from easybuild.tools.options import EasyBuildOptions, set_tmpdir from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX from easybuild.tools.version import VERSION from vsc.utils import fancylogger @@ -69,6 +69,12 @@ def setUp(self): super(CommandLineOptionsTest, self).setUp() self.github_token = fetch_github_token(GITHUB_TEST_ACCOUNT) + def purge_environment(self): + """Remove any leftover easybuild variables""" + for var in os.environ.keys(): + if var.startswith('EASYBUILD_'): + del os.environ[var] + def test_help_short(self, txt=None): """Test short help message.""" @@ -416,7 +422,7 @@ def test__list_toolchains(self): info_msg = r"INFO List of known toolchains \(toolchainname: module\[,module\.\.\.\]\):" logtxt = read_file(self.logfile) - self.assertTrue(re.search(info_msg, logtxt), "Info message with list of known compiler toolchains") + self.assertTrue(re.search(info_msg, logtxt), "Info message with list of known toolchains found in: %s" % logtxt) # toolchain elements should be in alphabetical order tcs = { 'dummy': [], @@ -1501,7 +1507,7 @@ def test_robot(self): eb_file, '--robot-paths=%s' % test_ecs_path, ] - error_regex = 'no module .* found for dependency' + error_regex = "Missing modules for one or more dependencies: .*" self.assertErrorRegex(EasyBuildError, error_regex, self.eb_main, args, raise_error=True, do_build=True) # enable robot, but without passing path required to resolve toy dependency => FAIL @@ -1898,6 +1904,76 @@ def test_review_pr(self): self.mock_stdout(False) self.assertTrue(re.search(r"^Comparing zlib-1.2.8\S* with zlib-1.2.8", txt)) + def test_set_tmpdir(self): + """Test set_tmpdir config function.""" + self.purge_environment() + + for tmpdir in [None, os.path.join(tempfile.gettempdir(), 'foo')]: + parent = tmpdir + if parent is None: + parent = tempfile.gettempdir() + + mytmpdir = set_tmpdir(tmpdir=tmpdir) + + for var in ['TMPDIR', 'TEMP', 'TMP']: + self.assertTrue(os.environ[var].startswith(os.path.join(parent, 'eb-'))) + self.assertEqual(os.environ[var], mytmpdir) + self.assertTrue(tempfile.gettempdir().startswith(os.path.join(parent, 'eb-'))) + tempfile_tmpdir = tempfile.mkdtemp() + self.assertTrue(tempfile_tmpdir.startswith(os.path.join(parent, 'eb-'))) + fd, tempfile_tmpfile = tempfile.mkstemp() + self.assertTrue(tempfile_tmpfile.startswith(os.path.join(parent, 'eb-'))) + + # tmp_logdir follows tmpdir + self.assertEqual(get_build_log_path(), mytmpdir) + + # cleanup + os.close(fd) + shutil.rmtree(mytmpdir) + modify_env(os.environ, self.orig_environ) + tempfile.tempdir = None + + def test_extended_dry_run(self): + """Test use of --extended-dry-run/-x.""" + ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') + args = [ + ec_file, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, + '--debug', + ] + # *no* output in testing mode (honor 'silent') + self.mock_stdout(True) + self.eb_main(args + ['--extended-dry-run'], do_build=True, raise_error=True, testing=True) + stdout = self.get_stdout() + self.mock_stdout(False) + self.assertEqual(len(stdout), 0) + + msg_regexs = [ + re.compile(r"the actual build \& install procedure that will be performed may diverge", re.M), + re.compile(r"^\*\*\* DRY RUN using 'EB_toy' easyblock", re.M), + re.compile(r"^== COMPLETED: Installation ended successfully", re.M), + re.compile(r"^\(no ignored errors during dry run\)", re.M), + ] + ignoring_error_regex = re.compile(r"WARNING: ignoring error", re.M) + ignored_error_regex = re.compile(r"WARNING: One or more errors were ignored, see warnings above", re.M) + + for opt in ['--extended-dry-run', '-x']: + # check for expected patterns in output of --extended-dry-run/-x + self.mock_stdout(True) + self.eb_main(args + [opt], do_build=True, raise_error=True, testing=False) + stdout = self.get_stdout() + self.mock_stdout(False) + + for msg_regex in msg_regexs: + self.assertTrue(msg_regex.search(stdout), "Pattern '%s' found in: %s" % (msg_regex.pattern, stdout)) + + # no ignored errors should occur + for notthere_regex in [ignoring_error_regex, ignored_error_regex]: + msg = "Pattern '%s' NOT found in: %s" % (notthere_regex.pattern, stdout) + self.assertFalse(notthere_regex.search(stdout), msg) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/run.py b/test/framework/run.py index 4d46392ddf..0bd0261e1c 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -30,11 +30,13 @@ @author: Stijn De Weirdt (Ghent University) """ import os -from test.framework.utilities import EnhancedTestCase +import re +from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader, main from vsc.utils.fancylogger import setLogLevelDebug, logToScreen from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import read_file from easybuild.tools.run import run_cmd, run_cmd_qa, parse_log_for_error from easybuild.tools.run import _log as run_log @@ -95,6 +97,56 @@ def test_parse_log_error(self): errors = parse_log_for_error("error failed", True) self.assertEqual(len(errors), 1) + def test_dry_run(self): + """Test use of functions under (extended) dry run.""" + build_options = { + 'extended_dry_run': True, + 'silent': False, + } + init_config(build_options=build_options) + + self.mock_stdout(True) + run_cmd("somecommand foo 123 bar") + txt = self.get_stdout() + self.mock_stdout(False) + + expected_regex = re.compile('\n'.join([ + r" running command \"somecommand foo 123 bar\"", + r" \(in .*\)", + ])) + self.assertTrue(expected_regex.match(txt), "Pattern %s matches with: %s" % (expected_regex.pattern, txt)) + + # check disabling 'verbose' + self.mock_stdout(True) + run_cmd("somecommand foo 123 bar", verbose=False) + txt = self.get_stdout() + self.mock_stdout(False) + self.assertEqual(txt, '') + + # check forced run + outfile = os.path.join(self.test_prefix, 'cmd.out') + self.assertFalse(os.path.exists(outfile)) + self.mock_stdout(True) + run_cmd("echo 'This is always echoed' > %s" % outfile, force_in_dry_run=True) + txt = self.get_stdout() + self.mock_stdout(False) + # nothing printed to stdout, but command was run + self.assertEqual(txt, '') + self.assertTrue(os.path.exists(outfile)) + self.assertEqual(read_file(outfile), "This is always echoed\n") + + # Q&A commands + self.mock_stdout(True) + run_cmd_qa("some_qa_cmd", {'question1': 'answer1'}) + txt = self.get_stdout() + self.mock_stdout(False) + + expected_regex = re.compile('\n'.join([ + r" running interactive command \"some_qa_cmd\"", + r" \(in .*\)", + ])) + self.assertTrue(expected_regex.match(txt), "Pattern %s matches with: %s" % (expected_regex.pattern, txt)) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index f0c02aa99f..b09a0bfb71 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -58,7 +58,9 @@ def configure_step(self, name=None): if self.cfg['allow_system_deps']: if get_software_root('Python') != 'Python' or get_software_version('Python') != platform.python_version(): raise EasyBuildError("Sanity check on allowed Python system dep failed.") - os.rename('%s.source' % name, '%s.c' % name) + + if os.path.exists("%s.source" % name): + os.rename('%s.source' % name, '%s.c' % name) def build_step(self, name=None): """Build toy.""" @@ -75,7 +77,8 @@ def install_step(self, name=None): name = self.name bindir = os.path.join(self.installdir, 'bin') mkdir(bindir, parents=True) - shutil.copy2(name, bindir) + if os.path.exists(name): + shutil.copy2(name, bindir) # also install a dummy libtoy.a, to make the default sanity check happy libdir = os.path.join(self.installdir, 'lib') mkdir(libdir, parents=True) diff --git a/test/framework/suite.py b/test/framework/suite.py index 7207a56447..8b38f3d0f9 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -39,7 +39,7 @@ # initialize EasyBuild logging, so we disable it from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import set_tmpdir +from easybuild.tools.options import set_tmpdir # set plain text key ring to be used, so a GitHub token stored in it can be obtained without having to provide a password try: @@ -62,6 +62,7 @@ import test.framework.easyconfigformat as ef import test.framework.ebconfigobj as ebco import test.framework.easyconfigversion as ev +import test.framework.environment as env import test.framework.docs as d import test.framework.filetools as f import test.framework.format_convert as f_c @@ -105,7 +106,7 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, tw, - p, i, pkg, d, et] + p, i, pkg, d, env, et] SUITE = unittest.TestSuite([x.suite() for x in tests]) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 1210d08d0b..757d805d93 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -46,13 +46,13 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.main import main from easybuild.tools import config -from easybuild.tools.config import module_classes, set_tmpdir +from easybuild.tools.config import module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.environment import modify_env from easybuild.tools.filetools import mkdir, read_file from easybuild.tools.module_naming_scheme import GENERAL_CLASS from easybuild.tools.modules import modules_tool -from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX, EasyBuildOptions +from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX, EasyBuildOptions, set_tmpdir # make sure tests are robust against any non-default configuration settings; @@ -155,7 +155,7 @@ def tearDown(self): os.chdir(self.cwd) # restore original environment - modify_env(os.environ, self.orig_environ) + modify_env(os.environ, self.orig_environ, verbose=False) # restore original Python search path sys.path = self.orig_sys_path @@ -333,6 +333,7 @@ def init_config(args=None, build_options=None): # initialize build options if build_options is None: build_options = { + 'extended_dry_run': False, 'external_modules_metadata': ConfigObj(), 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()],