support for eb --check-versions (WIP)

easybuild/framework/easyblock.py

@@ -58,7 +58,7 @@
 from easybuild.framework.easyconfig.easyconfig import get_module_path, letter_dir_for, resolve_template
 from easybuild.framework.easyconfig.format.format import INDENT_4SPACES
 from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig
-from easybuild.framework.easyconfig.tools import get_paths_for
+from easybuild.framework.easyconfig.tools import PYPI_PKG_URL_PATTERN, get_paths_for
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP
 from easybuild.tools.build_details import get_build_stats
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs
@@ -92,9 +92,6 @@
 
 MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP]
 
-# string part of URL for Python packages on PyPI that indicates needs to be rewritten (see derive_alt_pypi_url)
-PYPI_PKG_URL_PATTERN = 'pypi.python.org/packages/source/'
-
 
 _log = fancylogger.getLogger('easyblock')
 
@@ -1386,6 +1383,10 @@ def skip_extensions(self):
     # MISCELLANEOUS UTILITY FUNCTIONS
     #
 
+    def check_versions(self):
+        """Check which versions are available for this software"""
+        raise NotImplementedError
+
     @property
     def start_dir(self):
         """Start directory in build directory"""

easybuild/framework/easyconfig/tools.py

@@ -43,7 +43,9 @@
 import sys
 import tempfile
 from distutils.version import LooseVersion
+from pkg_resources import parse_version
 from vsc.utils import fancylogger
+from vsc.utils.missing import nub
 
 from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
 from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, EasyConfig
@@ -52,7 +54,8 @@
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import build_option
 from easybuild.tools.environment import restore_env
-from easybuild.tools.filetools import find_easyconfigs, is_patch_file, resolve_path, which, write_file
+from easybuild.tools.filetools import KNOWN_EXTS, download_file, find_easyconfigs, find_extension, is_patch_file
+from easybuild.tools.filetools import pypi_source_urls, read_file, resolve_path, which, write_file
 from easybuild.tools.github import fetch_easyconfigs_from_pr, download_repo
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.multidiff import multidiff
@@ -83,6 +86,9 @@
 except ImportError:
     pass
 
+# string part of URL for Python packages on PyPI that indicates needs to be rewritten (see derive_alt_pypi_url)
+PYPI_PKG_URL_PATTERN = 'pypi.python.org/packages/source/'
+
 _log = fancylogger.getLogger('easyconfig.tools', fname=False)
 
 
@@ -594,3 +600,121 @@ def categorize_files_by_type(paths):
             res['easyconfigs'].append(path)
 
     return res
+
+
+def mk_src_regex(src, version):
+    """Create regex pattern based on given source filename."""
+    known_exts = sorted(KNOWN_EXTS, key=len, reverse=True)
+
+    # replace file extension with regex pattern for known extensions for source files
+    ext_regex = re.compile(find_extension(src).replace('.', '\\.') + '$')
+    src_pattern = ext_regex.sub('(' + '|'.join(known_exts) + ')', src) # '(?![a-zA-Z.]))', src)
+
+    # replace version with placeholder (which doesn't include dots)
+    version_placeholder = '@version@'
+    src_pattern = src_pattern.replace(version, version_placeholder)
+    # escape all dots to avoid that they match any character
+    src_pattern = src_pattern.replace('.', '\\.')
+    # replace version placeholder with regex pattern for versions;
+    # versions are assumed not to include a dot followed by a letter,
+    # to avoid including parts of an extension in the matched version
+    src_pattern = src_pattern.replace(version_placeholder, '(?P<version>([\w-]|\.(?![a-zA-Z]))*)')
+
+    return re.compile(src_pattern)
+
+
+def check_software_versions_via_url(name, src_pattern, url):
+    """Check available software versions via provided URL."""
+    versions = None
+    known_url_types = [
+        (PYPI_PKG_URL_PATTERN, check_software_versions_pypi),
+    ]
+    for url_pattern, url_handler in known_url_types:
+        if re.search(url_pattern, url):
+            versions = url_handler(name, src_pattern, url)
+            break
+
+    if not versions:
+        versions = check_software_versions_via_dir_listing(name, src_pattern, url)
+
+    return versions
+
+
+def check_software_versions_pypi(name, src_regex, _):
+    """Determine available software versions for specified PyPI URL."""
+    versions = []
+    pkg_urls = pypi_source_urls(name)
+    for pkg_url in pkg_urls:
+        res = src_regex.search(pkg_url)
+        if res:
+            versions.append(res.group('version'))
+
+    return versions
+
+
+def check_software_versions_via_dir_listing(name, src_regex, url):
+    """Try to determine available software versions by scraping URL as directory listing."""
+    versions = []
+
+    target = os.path.join(tempfile.mkdtemp(), 'dir.list')
+    if download_file(os.path.basename(target), url, target):
+        dir_list = read_file(target)
+        _log.debug("Directory listing @ %s:\n%s", url, dir_list)
+        for res in src_regex.finditer(dir_list):
+            _log.debug("Found match for source regex '%s': %s", src_regex.pattern, res.groups())
+            version = res.group('version')
+            versions.append(res.group('version'))
+
+    return versions
+
+
+def check_software_versions(easyconfigs):
+    """Check available software versions for each of the specified easyconfigs"""
+    lines = ['']
+    for ec in easyconfigs:
+        ec = ec['ec']
+        lines.extend([
+            "* available software versions for %s:" % ec['name'],
+            "  (based on %s)" % ec.path,
+        ])
+
+        versions = []
+
+        # check if easyblock has a custom way of determining available versions
+        app_class = get_easyblock_class(ec['easyblock'], name=ec['name'])
+        app = app_class(ec)
+        try:
+            versions = app.check_versions()
+        except NotImplementedError:
+            _log.debug("No custom method for determining versions implemented in %s", app_class)
+
+            # try to fall back to method based on type of source URL
+            for src_spec in ec['sources']:
+                if isinstance(src_spec, basestring):
+                    src = src_spec
+                elif isinstance(src_spec, (list, tuple)):
+                    src = src_spec[0]
+                elif isinstance(src_spec, dict):
+                    src = src_spec.get('download_filename') or src_spec.get('filename')
+                    if src is None:
+                        raise EasyBuildError("Failed to determine source filename from %s", src_spec)
+                else:
+                    raise EasyBuildError("Unknown type of source spec: %s", src_spec)
+
+                # FIXME version will be incorrect here for components...
+                src_regex = mk_src_regex(src, ec['version'])
+                for url in ec['source_urls']:
+                    versions.extend(check_software_versions_via_url(ec['name'], src_regex, url))
+
+        if versions:
+            # replace pre-release tags like 'rc1' with 'b9991' to get correct version ordering
+            # the '999' is added to try and ensure correct ordering against existing 'b<number>' versions
+            rc_regex = re.compile('rc([0-9])')
+            ordered_versions = nub(sorted(versions, key=lambda v: parse_version(v)))
+            lines.extend('\t* %s' % v for v in nub(ordered_versions))
+        else:
+            lines.append("\tNo versions found for %s! :(" % ec['name'])
+
+        # FIXME: also handle extensions
+
+    print_msg('\n'.join(lines) + '\n', prefix=False)

easybuild/main.py

@@ -52,9 +52,10 @@
 from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
 from easybuild.framework.easyconfig.easyconfig import verify_easyconfig_filename
 from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check
-from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, categorize_files_by_type, dep_graph
-from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for
-from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, skip_available
+from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, categorize_files_by_type
+from easybuild.framework.easyconfig.tools import check_software_versions, dep_graph, det_easyconfig_paths
+from easybuild.framework.easyconfig.tools import dump_env_script, get_paths_for, parse_easyconfigs, review_pr
+from easybuild.framework.easyconfig.tools import skip_available
 from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
 from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option
 from easybuild.tools.docs import list_software
@@ -390,9 +391,10 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
 
     forced = options.force or options.rebuild
     dry_run_mode = options.dry_run or options.dry_run_short
+    no_skip = options.extended_dry_run or options.inject_checksums or options.check_versions
 
     # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
-    if not (forced or dry_run_mode or options.extended_dry_run or new_update_preview_pr or options.inject_checksums):
+    if not (forced or dry_run_mode or new_update_preview_pr or no_skip):
         retained_ecs = skip_available(easyconfigs, modtool)
         if not testing:
             for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
@@ -436,6 +438,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
         else:
             print_msg("\nNo conflicts detected!\n", prefix=False)
 
+    elif options.check_versions:
+        check_software_versions(easyconfigs)
+
     # dump source script to set up build environment
     elif options.dump_env_script:
         dump_env_script(easyconfigs)
@@ -444,7 +449,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
         inject_checksums(ordered_ecs, options.inject_checksums)
 
     # cleanup and exit after dry run, searching easyconfigs or submitting regression test
-    stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums]
+    stop_options = [options.check_conflicts, options.check_versions, dry_run_mode, options.dump_env_script,
+                    options.inject_checksums]
     if any(no_ec_opts) or any(stop_options):
         cleanup(logfile, eb_tmpdir, testing)
         sys.exit(0)

easybuild/tools/filetools.py

@@ -145,6 +145,7 @@
     # tar.Z: using compress (LZW)
     '.tar.z':   "tar xZf %(filepath)s",
 }
+KNOWN_EXTS = EXTRACT_CMDS.keys()
 
 
 class ZlibChecksum(object):
@@ -744,7 +745,7 @@ def get_local_dirs_purged():
 def find_extension(filename):
     """Find best match for filename extension."""
     # sort by length, so longest file extensions get preference
-    suffixes = sorted(EXTRACT_CMDS.keys(), key=len, reverse=True)
+    suffixes = sorted(KNOWN_EXTS, key=len, reverse=True)
     pat = r'(?P<ext>%s)$' % '|'.join([s.replace('.', '\\.') for s in suffixes])
     res = re.search(pat, filename, flags=re.IGNORECASE)
     if res:

easybuild/tools/options.py

@@ -518,6 +518,8 @@ def informative_options(self):
             'avail-hooks': ("Show list of known hooks", None, 'store_true', False),
             'avail-toolchain-opts': ("Show options for toolchain", 'str', 'store', None),
             'check-conflicts': ("Check for version conflicts in dependency graphs", None, 'store_true', False),
+            'check-versions': ("Check which software versions are available for specified easyconfigs",
+                               None, 'store_true', False),
             'dep-graph': ("Create dependency graph", None, 'store', None, {'metavar': 'depgraph.<ext>'}),
             'dump-env-script': ("Dump source script to set up build environment based on toolchain/dependencies",
                                 None, 'store_true', False),