Update config.guess for all R packages

easybuild/easyblocks/generic/configuremake.py

@@ -40,13 +40,14 @@
 import stat
 from datetime import datetime
 
+from easybuild.base import fancylogger
 from easybuild.easyblocks import VERSION as EASYBLOCKS_VERSION
 from easybuild.framework.easyblock import EasyBlock
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools.build_log import print_warning
-from easybuild.tools.config import source_paths
+from easybuild.tools.config import source_paths, build_option
 from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file
-from easybuild.tools.filetools import read_file, remove_file, verify_checksum
+from easybuild.tools.filetools import read_file, remove_file
 from easybuild.tools.run import run_cmd
 
 # string that indicates that a configure script was generated by Autoconf
@@ -55,7 +56,6 @@
 AUTOCONF_GENERATED_MSG = b"Generated by GNU Autoconf"
 
 # download location & SHA256 for config.guess script
-# note: if this is updated, don't forget to trash the cached download from generic/Configure/<eb_version>/!
 CONFIG_GUESS_VERSION = '2018-08-29'
 CONFIG_GUESS_URL_STUB = "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb="
 CONFIG_GUESS_COMMIT_ID = "59e2ce0e6b46bb47ef81b68b600ed087e14fdaad"
@@ -67,6 +67,103 @@
 DEFAULT_INSTALL_CMD = 'make install'
 
 
+def check_config_guess(config_guess):
+    """Check timestamp & SHA256 checksum of config.guess script.
+
+    :param config_guess: Path to config.guess script to check
+    :return: Whether the script is valid (matches the version and checksum)
+    """
+    log = fancylogger.getLogger('check_config_guess')
+
+    # config.guess includes a "timestamp='...'" indicating the version
+    config_guess_version = None
+    version_regex = re.compile("^timestamp='(.*)'", re.M)
+    res = version_regex.search(read_file(config_guess))
+    if res:
+        config_guess_version = res.group(1)
+
+    config_guess_checksum = compute_checksum(config_guess, checksum_type=CHECKSUM_TYPE_SHA256)
+    try:
+        config_guess_timestamp = datetime.fromtimestamp(os.stat(config_guess).st_mtime).isoformat()
+    except OSError as err:
+        log.warning("Failed to determine timestamp of %s: %s", config_guess, err)
+        config_guess_timestamp = None
+
+    log.info("config.guess version: %s (last updated: %s, SHA256 checksum: %s)",
+             config_guess_version, config_guess_timestamp, config_guess_checksum)
+
+    result = True
+
+    # check version & SHA256 checksum before declaring victory
+    if config_guess_version != CONFIG_GUESS_VERSION:
+        result = False
+        log.warning("config.guess version at %s does not match expected version: %s vs %s",
+                    config_guess, config_guess_version, CONFIG_GUESS_VERSION)
+
+    elif config_guess_checksum != CONFIG_GUESS_SHA256:
+        result = False
+        log.warning("SHA256 checksum of config.guess at %s does not match expected checksum: %s vs %s",
+                    config_guess, config_guess_checksum, CONFIG_GUESS_SHA256)
+
+    return result
+
+
+def obtain_config_guess(download_source_path=None, search_source_paths=None):
+    """
+    Locate or download an up-to-date config.guess
+
+    :param download_source_path: Path to download config.guess to
+    :param search_source_paths: Paths to search for config.guess
+    :return: Path to config.guess or None
+    """
+    log = fancylogger.getLogger('obtain_config_guess')
+
+    eb_source_paths = source_paths()
+
+    if download_source_path is None:
+        download_source_path = eb_source_paths[0]
+    else:
+        log.deprecated("Specifying custom source path to download config.guess via 'download_source_path'", '5.0')
+
+    if search_source_paths is None:
+        search_source_paths = eb_source_paths
+    else:
+        log.deprecated("Specifying custom location to search for updated config.guess via 'search_source_paths'", '5.0')
+
+    config_guess = 'config.guess'
+    sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake')
+
+    config_guess_path = None
+
+    # check if config.guess has already been downloaded to source path
+    for path in search_source_paths:
+        cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess)
+        if os.path.isfile(cand_config_guess_path) and check_config_guess(cand_config_guess_path):
+            force_download = build_option('force_download')
+            if force_download:
+                print_warning("Found file %s at %s, but re-downloading it anyway..."
+                              % (config_guess, cand_config_guess_path))
+            else:
+                config_guess_path = cand_config_guess_path
+                log.info("Found %s at %s", config_guess, config_guess_path)
+            break
+
+    if not config_guess_path:
+        cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess)
+        config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID
+        if not download_file(config_guess, config_guess_url, cand_config_guess_path):
+            print_warning("Failed to download recent %s to %s", config_guess, cand_config_guess_path, log=log)
+        elif not check_config_guess(cand_config_guess_path):
+            print_warning("Verification failed for file %s, not using it!", cand_config_guess_path, log=log)
+            remove_file(cand_config_guess_path)
+        else:
+            config_guess_path = cand_config_guess_path
+            adjust_permissions(config_guess_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True)
+            log.info("Verified %s at %s, using it if required", config_guess, config_guess_path)
+
+    return config_guess_path
+
+
 class ConfigureMake(EasyBlock):
     """
     Support for building and installing applications with configure/make/make install
@@ -106,74 +203,13 @@ def obtain_config_guess(self, download_source_path=None, search_source_paths=Non
         :param search_source_paths: Paths to search for config.guess
         :return: Path to config.guess or None
         """
-        eb_source_paths = source_paths()
-        if download_source_path is None:
-            download_source_path = eb_source_paths[0]
-        if search_source_paths is None:
-            search_source_paths = eb_source_paths
-
-        config_guess = 'config.guess'
-        sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake')
-
-        config_guess_path = None
-
-        # check if config.guess has already been downloaded to source path
-        for path in eb_source_paths:
-            cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess)
-            if os.path.isfile(cand_config_guess_path):
-                config_guess_path = cand_config_guess_path
-                self.log.info("Found recent %s at %s, using it if required", config_guess, config_guess_path)
-                break
-
-        # if not found, try to download it
-        if config_guess_path is None:
-            cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess)
-            config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID
-            downloaded_path = download_file(config_guess, config_guess_url, cand_config_guess_path)
-            if downloaded_path is not None:
-                # verify SHA256 checksum of download to avoid using a corrupted download
-                if verify_checksum(downloaded_path, CONFIG_GUESS_SHA256):
-                    config_guess_path = downloaded_path
-                    # add execute permissions
-                    adjust_permissions(downloaded_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True)
-                    self.log.info("Downloaded recent %s to %s, using it if required", config_guess, config_guess_path)
-                else:
-                    self.log.warning("Checksum failed for downloaded file %s, not using it!", downloaded_path)
-                    remove_file(downloaded_path)
-            else:
-                self.log.warning("Failed to download recent %s to %s for use with ConfigureMake easyblock (if needed)",
-                                 config_guess, cand_config_guess_path)
-
-        return config_guess_path
+        return obtain_config_guess(download_source_path=download_source_path, search_source_paths=search_source_paths)
 
     def check_config_guess(self):
         """Check timestamp & SHA256 checksum of config.guess script."""
         # log version, timestamp & SHA256 checksum of config.guess that was found (if any)
         if self.config_guess:
-            # config.guess includes a "timestamp='...'" indicating the version
-            config_guess_version = None
-            version_regex = re.compile("^timestamp='(.*)'", re.M)
-            res = version_regex.search(read_file(self.config_guess))
-            if res:
-                config_guess_version = res.group(1)
-
-            config_guess_checksum = compute_checksum(self.config_guess, checksum_type=CHECKSUM_TYPE_SHA256)
-            try:
-                config_guess_timestamp = datetime.fromtimestamp(os.stat(self.config_guess).st_mtime).isoformat()
-            except OSError as err:
-                self.log.warning("Failed to determine timestamp of %s: %s", self.config_guess, err)
-                config_guess_timestamp = None
-
-            self.log.info("config.guess version: %s (last updated: %s, SHA256 checksum: %s)",
-                          config_guess_version, config_guess_timestamp, config_guess_checksum)
-
-            if config_guess_version != CONFIG_GUESS_VERSION:
-                tup = (self.config_guess, config_guess_version, CONFIG_GUESS_VERSION)
-                print_warning("config.guess version at %s does not match expected version: %s vs %s" % tup)
-
-            if config_guess_checksum != CONFIG_GUESS_SHA256:
-                tup = (self.config_guess, config_guess_checksum, CONFIG_GUESS_SHA256)
-                print_warning("SHA256 checksum of config.guess at %s does not match expected checksum: %s vs %s" % tup)
+            check_config_guess(self.config_guess)
 
     def fetch_step(self, *args, **kwargs):
         """Custom fetch step for ConfigureMake so we use an updated config.guess."""

easybuild/easyblocks/generic/rpackage.py

@@ -33,13 +33,13 @@
 @author: Balazs Hajgato (Vrije Universiteit Brussel)
 """
 import os
-import shutil
 
 from easybuild.easyblocks.r import EXTS_FILTER_R_PACKAGES, EB_R
+from easybuild.easyblocks.generic.configuremake import check_config_guess, obtain_config_guess
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.filetools import mkdir
+from easybuild.tools.filetools import mkdir, copy_file
 from easybuild.tools.run import run_cmd, parse_log_for_error
 
 
@@ -132,12 +132,8 @@ def make_cmdline_cmd(self, prefix=None):
         else:
             prefix = ''
 
-        if self.cfg['unpack_sources']:
-            loc = self.start_dir
-        elif self.patches:
-            loc = self.ext_dir
-        else:
-            loc = self.ext_src
+        loc = self.start_dir or self.ext_dir or self.ext_src
+
         cmd = ' '.join([
             self.cfg['preinstallopts'],
             "R CMD INSTALL",
@@ -152,23 +148,6 @@ def make_cmdline_cmd(self, prefix=None):
         self.log.debug("make_cmdline_cmd returns %s" % cmd)
         return cmd, None
 
-    def extract_step(self):
-        """Source should not be extracted."""
-
-        if self.cfg['unpack_sources']:
-            super(RPackage, self).extract_step()
-        elif len(self.src) > 1:
-            raise EasyBuildError("Don't know how to handle R packages with multiple sources.'")
-        else:
-            try:
-                shutil.copy2(self.src[0]['path'], self.builddir)
-            except OSError as err:
-                raise EasyBuildError("Failed to copy source to build dir: %s", err)
-            self.ext_src = self.src[0]['name']
-
-            # set final path since it can't be determined from unpacked sources (used for guessing start_dir)
-            self.src[0]['finalpath'] = self.builddir
-
     def configure_step(self):
         """No configuration for installing R packages."""
         pass
@@ -195,9 +174,23 @@ def install_R_package(self, cmd, inp=None):
         else:
             self.log.debug("R package %s installed succesfully" % self.name)
 
+    def update_config_guess(self, path):
+        """Update any config.guess found in specified directory"""
+        for config_guess_dir in (root for root, _, files in os.walk(path) if 'config.guess' in files):
+            config_guess = os.path.join(config_guess_dir, 'config.guess')
+            if not check_config_guess(config_guess):
+                updated_config_guess = obtain_config_guess()
+                if updated_config_guess:
+                    self.log.debug("Replacing outdated %s with more recent %s", config_guess, updated_config_guess)
+                    copy_file(updated_config_guess, config_guess)
+                else:
+                    raise EasyBuildError("Failed to obtain updated config.guess")
+
     def install_step(self):
         """Install procedure for R packages."""
-
+        # Update config.guess if the package was extracted
+        if self.start_dir:
+            self.update_config_guess(self.start_dir)
         cmd, stdin = self.make_cmdline_cmd(prefix=os.path.join(self.installdir, self.cfg['exts_subdir']))
         self.install_R_package(cmd, inp=stdin)
 
@@ -215,16 +208,15 @@ def run(self):
             lib_install_prefix = os.path.join(self.installdir, self.cfg['exts_subdir'])
             mkdir(lib_install_prefix, parents=True)
 
-        if self.patches:
-            super(RPackage, self).run(unpack_src=True)
-        else:
-            super(RPackage, self).run()
-
         if self.src:
+            super(RPackage, self).run(unpack_src=True)
             self.ext_src = self.src
+            self.update_config_guess(self.ext_dir)
             self.log.debug("Installing R package %s version %s." % (self.name, self.version))
             cmd, stdin = self.make_cmdline_cmd(prefix=lib_install_prefix)
         else:
+            if self.patches:
+                raise EasyBuildError("Cannot patch R package %s as no explicit source is given!", self.name)
             self.log.debug("Installing most recent version of R package %s (source not found)." % self.name)
             cmd, stdin = self.make_r_cmd(prefix=lib_install_prefix)