Merge pull request #5 from boegel/sources_from_git

minor cleanup in get_source_tarball_from_git + add & use remove_dir/remove function + add tests

Author: mboisson

.travis.yml

@@ -3,8 +3,8 @@ python: 2.6
 env:
   matrix:
     # purposely specifying slowest builds first, to gain time overall
-    - LMOD_VERSION=5.8
-    - LMOD_VERSION=5.8 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
+    - LMOD_VERSION=6.6.3
+    - LMOD_VERSION=6.6.3 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
     - LMOD_VERSION=7.7.16
     - LMOD_VERSION=7.7.16 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
     - ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl
@@ -19,7 +19,7 @@ matrix:
   include:
     # also test default configuration with Python 2.7
     - python: 2.7
-      env: LMOD_VERSION=5.8
+      env: LMOD_VERSION=6.6.3
 addons:
   apt:
     packages:
@@ -51,7 +51,7 @@ before_install:
     # autopep8 1.3.4 is last one to support Python 2.6
     - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install 'autopep8<1.3.5'; else pip install autopep8; fi
     # optional Python packages for EasyBuild
-    - pip install GC3Pie pycodestyle python-graph-dot python-hglib PyYAML
+    - pip install GC3Pie pycodestyle python-graph-dot python-hglib PyYAML requests
     # git config is required to make actual git commits (cfr. tests for GitRepository)
     - git config --global user.name "Travis CI"
     - git config --global user.email "travis@travis-ci.org"
@@ -100,7 +100,7 @@ script:
     - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
     - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
     - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-    - EB_BOOTSTRAP_EXPECTED="20180531.01 21d9704055c4fcf2cd42fe9825dd5925fa508268e4c7c423cb5958a0903d7c2e"
+    - EB_BOOTSTRAP_EXPECTED="20180916.01 7e7563787a8bab8c30efdbdf95df6ec8ed63230ebcd361fee7b9574cbe9e74ed"
     - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
     # test bootstrap script
     - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap

easybuild/framework/easyblock.py

@@ -72,12 +72,13 @@
 from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256
 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name
 from easybuild.tools.filetools import compute_checksum, copy_file, derive_alt_pypi_url, diff_files, download_file
-from easybuild.tools.filetools import encode_class_name, extract_file, is_alt_pypi_url, is_sha256_checksum
-from easybuild.tools.filetools import get_source_tarball_from_git, mkdir, move_logs, read_file, remove_file, rmtree2
+from easybuild.tools.filetools import encode_class_name, extract_file, get_source_tarball_from_git, is_alt_pypi_url
+from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_logs, read_file, remove_file, rmtree2
 from easybuild.tools.filetools import verify_checksum, weld_paths, write_file
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
 from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTPROC_STEP, PREPARE_STEP
-from easybuild.tools.hooks import READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP, run_hook
+from easybuild.tools.hooks import READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
+from easybuild.tools.hooks import load_hooks, run_hook
 from easybuild.tools.run import run_cmd
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
@@ -124,7 +125,7 @@ def extra_options(extra=None):
     #
     # INIT
     #
-    def __init__(self, ec, hooks=None):
+    def __init__(self, ec):
         """
         Initialize the EasyBlock instance.
         :param ec: a parsed easyconfig file (EasyConfig instance)
@@ -134,7 +135,7 @@ def __init__(self, ec, hooks=None):
         self.orig_workdir = os.getcwd()
 
         # list of pre- and post-step hooks
-        self.hooks = hooks or []
+        self.hooks = load_hooks(build_option('hooks'))
 
         # list of patch/source files, along with checksums
         self.patches = []
@@ -492,7 +493,7 @@ def fetch_extension_sources(self, skip_checksums=False):
                         'options': ext_options,
                     }
 
-                    checksums = ext_options.get('checksums', None)
+                    checksums = ext_options.get('checksums', [])
 
                     if ext_options.get('source_tmpl', None):
                         fn = resolve_template(ext_options['source_tmpl'], ext_src)
@@ -515,14 +516,15 @@ def fetch_extension_sources(self, skip_checksums=False):
                                     src_checksum = compute_checksum(src_fn, checksum_type=checksum_type)
                                     self.log.info("%s checksum for %s: %s", checksum_type, src_fn, src_checksum)
 
-                                if checksums:
-                                    fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0)
-                                    if verify_checksum(src_fn, fn_checksum):
-                                        self.log.info('Checksum for extension source %s verified', fn)
-                                    elif build_option('ignore_checksums'):
-                                        print_warning("Ignoring failing checksum verification for %s" % fn)
-                                    else:
-                                        raise EasyBuildError('Checksum verification for extension source %s failed', fn)
+                                # verify checksum (if provided)
+                                self.log.debug('Verifying checksums for extension source...')
+                                fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0)
+                                if verify_checksum(src_fn, fn_checksum):
+                                    self.log.info('Checksum for extension source %s verified', fn)
+                                elif build_option('ignore_checksums'):
+                                    print_warning("Ignoring failing checksum verification for %s" % fn)
+                                else:
+                                    raise EasyBuildError('Checksum verification for extension source %s failed', fn)
 
                             ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True)
                             if ext_patches:
@@ -537,16 +539,16 @@ def fetch_extension_sources(self, skip_checksums=False):
                                             checksum = compute_checksum(patch, checksum_type=checksum_type)
                                             self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
 
-                                    if checksums:
-                                        self.log.debug('Verifying checksums for extension patches...')
-                                        for idx, patch in enumerate(ext_patches):
-                                            checksum = self.get_checksum_for(checksums[1:], filename=patch, index=idx)
-                                            if verify_checksum(patch, checksum):
-                                                self.log.info('Checksum for extension patch %s verified', patch)
-                                            elif build_option('ignore_checksums'):
-                                                print_warning("Ignoring failing checksum verification for %s" % patch)
-                                            else:
-                                                raise EasyBuildError('Checksum for extension patch %s failed', patch)
+                                    # verify checksum (if provided)
+                                    self.log.debug('Verifying checksums for extension patches...')
+                                    for idx, patch in enumerate(ext_patches):
+                                        checksum = self.get_checksum_for(checksums[1:], filename=patch, index=idx)
+                                        if verify_checksum(patch, checksum):
+                                            self.log.info('Checksum for extension patch %s verified', patch)
+                                        elif build_option('ignore_checksums'):
+                                            print_warning("Ignoring failing checksum verification for %s" % patch)
+                                        else:
+                                            raise EasyBuildError('Checksum for extension patch %s failed', patch)
                             else:
                                 self.log.debug('No patches found for extension %s.' % ext_name)
 
@@ -1687,6 +1689,63 @@ def checksum_step(self):
                 else:
                     raise EasyBuildError("Checksum verification for %s using %s failed.", fil['path'], fil['checksum'])
 
+    def check_checksums_for(self, ent, sub='', source_cnt=None):
+        """
+        Utility method: check whether checksums for all sources/patches are available, for given entity
+        """
+        ec_fn = os.path.basename(self.cfg.path)
+        checksum_issues = []
+
+        sources = ent.get('sources', [])
+        patches = ent.get('patches', [])
+        checksums = ent.get('checksums', [])
+
+        if source_cnt is None:
+            source_cnt = len(sources)
+        patch_cnt, checksum_cnt = len(patches), len(checksums)
+
+        if (source_cnt + patch_cnt) != checksum_cnt:
+            if sub:
+                sub = "%s in %s" % (sub, ec_fn)
+            else:
+                sub = "in %s" % ec_fn
+            msg = "Checksums missing for one or more sources/patches %s: " % sub
+            msg += "found %d sources + %d patches " % (source_cnt, patch_cnt)
+            msg += "vs %d checksums" % checksum_cnt
+            checksum_issues.append(msg)
+
+        for fn, checksum in zip(sources + patches, checksums):
+            if not is_sha256_checksum(checksum):
+                msg = "Non-SHA256 checksum found for %s: %s" % (fn, checksum)
+                checksum_issues.append(msg)
+
+        return checksum_issues
+
+    def check_checksums(self):
+        """
+        Check whether a SHA256 checksum is available for all sources & patches (incl. extensions).
+
+        :return: list of strings describing checksum issues (missing checksums, wrong checksum type, etc.)
+        """
+        checksum_issues = []
+
+        # check whether a checksum if available for every source + patch
+        checksum_issues.extend(self.check_checksums_for(self.cfg))
+
+        # also check checksums for extensions
+        for ext in self.cfg['exts_list']:
+            # just skip extensions for which only a name is specified
+            # those are just there to check for things that are in the "standard library"
+            if not isinstance(ext, basestring):
+                ext_name = ext[0]
+                # take into account that extension may be a 2-tuple with just name/version
+                ext_opts = ext[2] if len(ext) == 3 else {}
+                # only a single source per extension is supported (see source_tmpl)
+                res = self.check_checksums_for(ext_opts, sub="of extension %s" % ext_name, source_cnt=1)
+                checksum_issues.extend(res)
+
+        return checksum_issues
+
     def extract_step(self):
         """
         Unpack the source files.
@@ -2346,6 +2405,17 @@ def cleanup_step(self):
 
         self.restore_iterate_opts()
 
+    def invalidate_module_caches(self, modpath):
+        """Helper method to invalidate module caches for specified module path."""
+        # invalidate relevant 'module avail'/'module show' cache entries
+        # consider both paths: for short module name, and subdir indicated by long module name
+        paths = [modpath]
+        if self.mod_subdir:
+            paths.append(os.path.join(modpath, self.mod_subdir))
+
+        for path in paths:
+            invalidate_module_caches_for(path)
+
     def make_module_step(self, fake=False):
         """
         Generate module file
@@ -2395,14 +2465,7 @@ def make_module_step(self, fake=False):
                 self.log.info(diff_msg)
                 print_msg(diff_msg, log=self.log)
 
-            # invalidate relevant 'module avail'/'module show' cache entries
-            # consider both paths: for short module name, and subdir indicated by long module name
-            paths = [modpath]
-            if self.mod_subdir:
-                paths.append(os.path.join(modpath, self.mod_subdir))
-
-            for path in paths:
-                invalidate_module_caches_for(path)
+            self.invalidate_module_caches(modpath)
 
             # only update after generating final module file
             if not fake:
@@ -2507,8 +2570,11 @@ def _skip_step(self, step, skippable):
         force = build_option('force') or build_option('rebuild')
         skip = False
 
-        # skip step if specified as individual (skippable) step
-        if skippable and (self.skip or step in self.cfg['skipsteps']):
+        # under --skip, sanity check is not skipped
+        cli_skip = self.skip and step != SANITYCHECK_STEP
+
+        # skip step if specified as individual (skippable) step, or if --skip is used
+        if skippable and (cli_skip or step in self.cfg['skipsteps']):
             self.log.info("Skipping %s step (skip: %s, skipsteps: %s)", step, self.skip, self.cfg['skipsteps'])
             skip = True
 
@@ -2654,7 +2720,7 @@ def install_step_spec(initial):
         steps_part3 = [
             (EXTENSIONS_STEP, 'taking care of extensions', [lambda x: x.extensions_step], False),
             (POSTPROC_STEP, 'postprocessing', [lambda x: x.post_install_step], True),
-            (SANITYCHECK_STEP, 'sanity checking', [lambda x: x.sanity_check_step], False),
+            (SANITYCHECK_STEP, 'sanity checking', [lambda x: x.sanity_check_step], True),
             (CLEANUP_STEP, 'cleaning up', [lambda x: x.cleanup_step], False),
             (MODULE_STEP, 'creating module', [lambda x: x.make_module_step], False),
             (PERMISSIONS_STEP, 'permissions', [lambda x: x.permissions_step], False),
@@ -2717,7 +2783,7 @@ def print_dry_run_note(loc, silent=True):
     dry_run_msg(msg, silent=silent)
 
 
-def build_and_install_one(ecdict, init_env, hooks=None):
+def build_and_install_one(ecdict, init_env):
     """
     Build the software
     :param ecdict: dictionary contaning parsed easyconfig + metadata
@@ -2755,7 +2821,7 @@ def build_and_install_one(ecdict, init_env, hooks=None):
     try:
         app_class = get_easyblock_class(easyblock, name=name)
 
-        app = app_class(ecdict['ec'], hooks=hooks)
+        app = app_class(ecdict['ec'])
         _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock))
     except EasyBuildError, err:
         print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg),
@@ -2816,13 +2882,14 @@ def build_and_install_one(ecdict, init_env, hooks=None):
             buildstats = get_build_stats(app, start_time, build_option('command_line'))
             _log.info("Build stats: %s" % buildstats)
 
-            if build_option("minimal_toolchains"):
-                # for reproducability we dump out the parsed easyconfig since the contents are affected when
-                # --minimal-toolchains (and --use-existing-modules) is used
-                # TODO --try-toolchain needs to be fixed so this doesn't play havoc with it's usability
-                reprod_spec = os.path.join(new_log_dir, 'reprod', ec_filename)
+            # for reproducability we dump out the fully processed easyconfig since the contents can be affected
+            # by subtoolchain resolution (and related options) and/or hooks
+            reprod_spec = os.path.join(new_log_dir, 'reprod', ec_filename)
+            try:
                 app.cfg.dump(reprod_spec)
-                _log.debug("Dumped easyconfig tweaked via --minimal-toolchains to %s", reprod_spec)
+                _log.info("Dumped fully processed easyconfig to %s", reprod_spec)
+            except NotImplementedError as err:
+                _log.warn("Unable to dumped fully processed easyconfig to %s: %s", reprod_spec, err)
 
             try:
                 # upload easyconfig (and patch files) to central repository

easybuild/framework/easyconfig/default.py

@@ -90,7 +90,8 @@
     'easyblock': [None, "EasyBlock to use for building; if set to None, an easyblock is selected "
                         "based on the software name", BUILD],
     'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD],
-    'github_account': [None, "GitHub account name to be used to resolve template values in source URLs", BUILD],
+    'github_account': ['%(namelower)s', "GitHub account name to be used to resolve template values in source URLs",
+                       BUILD],
     'hidden': [False, "Install module file as 'hidden' by prefixing its version with '.'", BUILD],
     'installopts': ['', 'Extra options for installation', BUILD],
     'maxparallel': [None, 'Max degree of parallelism', BUILD],