diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 806061fe84..44ec153019 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -58,7 +58,7 @@ from easybuild.framework.easyconfig.easyconfig import get_module_path, letter_dir_for, resolve_template from easybuild.framework.easyconfig.format.format import INDENT_4SPACES from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig -from easybuild.framework.easyconfig.tools import get_paths_for +from easybuild.framework.easyconfig.tools import PYPI_PKG_URL_PATTERN, get_paths_for from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs @@ -92,9 +92,6 @@ MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP] -# string part of URL for Python packages on PyPI that indicates needs to be rewritten (see derive_alt_pypi_url) -PYPI_PKG_URL_PATTERN = 'pypi.python.org/packages/source/' - _log = fancylogger.getLogger('easyblock') @@ -1386,6 +1383,10 @@ def skip_extensions(self): # MISCELLANEOUS UTILITY FUNCTIONS # + def check_versions(self): + """Check which versions are available for this software""" + raise NotImplementedError + @property def start_dir(self): """Start directory in build directory""" diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index c0460c3c71..bdcebae02c 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -43,7 +43,9 @@ import sys import tempfile from distutils.version import LooseVersion +from pkg_resources import parse_version from vsc.utils import fancylogger +from vsc.utils.missing import nub from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, EasyConfig @@ -52,7 +54,8 @@ from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env -from easybuild.tools.filetools import find_easyconfigs, is_patch_file, resolve_path, which, write_file +from easybuild.tools.filetools import KNOWN_EXTS, download_file, find_easyconfigs, find_extension, is_patch_file +from easybuild.tools.filetools import pypi_source_urls, read_file, resolve_path, which, write_file from easybuild.tools.github import fetch_easyconfigs_from_pr, download_repo from easybuild.tools.modules import modules_tool from easybuild.tools.multidiff import multidiff @@ -83,6 +86,9 @@ except ImportError: pass +# string part of URL for Python packages on PyPI that indicates needs to be rewritten (see derive_alt_pypi_url) +PYPI_PKG_URL_PATTERN = 'pypi.python.org/packages/source/' + _log = fancylogger.getLogger('easyconfig.tools', fname=False) @@ -594,3 +600,121 @@ def categorize_files_by_type(paths): res['easyconfigs'].append(path) return res + + +def mk_src_regex(src, version): + """Create regex pattern based on given source filename.""" + known_exts = sorted(KNOWN_EXTS, key=len, reverse=True) + + # replace file extension with regex pattern for known extensions for source files + ext_regex = re.compile(find_extension(src).replace('.', '\\.') + '$') + src_pattern = ext_regex.sub('(' + '|'.join(known_exts) + ')', src) # '(?![a-zA-Z.]))', src) + + # replace version with placeholder (which doesn't include dots) + version_placeholder = '@version@' + src_pattern = src_pattern.replace(version, version_placeholder) + # escape all dots to avoid that they match any character + src_pattern = src_pattern.replace('.', '\\.') + # replace version placeholder with regex pattern for versions; + # versions are assumed not to include a dot followed by a letter, + # to avoid including parts of an extension in the matched version + src_pattern = src_pattern.replace(version_placeholder, '(?P([\w-]|\.(?![a-zA-Z]))*)') + + return re.compile(src_pattern) + + +def check_software_versions_via_url(name, src_pattern, url): + """Check available software versions via provided URL.""" + versions = None + known_url_types = [ + (PYPI_PKG_URL_PATTERN, check_software_versions_pypi), + ] + for url_pattern, url_handler in known_url_types: + if re.search(url_pattern, url): + versions = url_handler(name, src_pattern, url) + break + + if not versions: + versions = check_software_versions_via_dir_listing(name, src_pattern, url) + + return versions + + +def check_software_versions_pypi(name, src_regex, _): + """Determine available software versions for specified PyPI URL.""" + versions = [] + pkg_urls = pypi_source_urls(name) + for pkg_url in pkg_urls: + res = src_regex.search(pkg_url) + if res: + versions.append(res.group('version')) + + return versions + + +def check_software_versions_via_dir_listing(name, src_regex, url): + """Try to determine available software versions by scraping URL as directory listing.""" + versions = [] + + target = os.path.join(tempfile.mkdtemp(), 'dir.list') + if download_file(os.path.basename(target), url, target): + dir_list = read_file(target) + _log.debug("Directory listing @ %s:\n%s", url, dir_list) + for res in src_regex.finditer(dir_list): + _log.debug("Found match for source regex '%s': %s", src_regex.pattern, res.groups()) + version = res.group('version') + versions.append(res.group('version')) + + return versions + + +def check_software_versions(easyconfigs): + """Check available software versions for each of the specified easyconfigs""" + lines = [''] + for ec in easyconfigs: + ec = ec['ec'] + lines.extend([ + "* available software versions for %s:" % ec['name'], + " (based on %s)" % ec.path, + ]) + + versions = [] + + # check if easyblock has a custom way of determining available versions + app_class = get_easyblock_class(ec['easyblock'], name=ec['name']) + app = app_class(ec) + try: + versions = app.check_versions() + except NotImplementedError: + _log.debug("No custom method for determining versions implemented in %s", app_class) + + # try to fall back to method based on type of source URL + for src_spec in ec['sources']: + if isinstance(src_spec, basestring): + src = src_spec + elif isinstance(src_spec, (list, tuple)): + src = src_spec[0] + elif isinstance(src_spec, dict): + src = src_spec.get('download_filename') or src_spec.get('filename') + if src is None: + raise EasyBuildError("Failed to determine source filename from %s", src_spec) + else: + raise EasyBuildError("Unknown type of source spec: %s", src_spec) + + # FIXME version will be incorrect here for components... + src_regex = mk_src_regex(src, ec['version']) + for url in ec['source_urls']: + versions.extend(check_software_versions_via_url(ec['name'], src_regex, url)) + + if versions: + # replace pre-release tags like 'rc1' with 'b9991' to get correct version ordering + # the '999' is added to try and ensure correct ordering against existing 'b' versions + rc_regex = re.compile('rc([0-9])') + ordered_versions = nub(sorted(versions, key=lambda v: parse_version(v))) + lines.extend('\t* %s' % v for v in nub(ordered_versions)) + else: + lines.append("\tNo versions found for %s! :(" % ec['name']) + + # FIXME: also handle extensions + + print_msg('\n'.join(lines) + '\n', prefix=False) diff --git a/easybuild/main.py b/easybuild/main.py index 02345fb867..e609608efa 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -52,9 +52,10 @@ from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.easyconfig import verify_easyconfig_filename from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check -from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, categorize_files_by_type, dep_graph -from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for -from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, skip_available +from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, categorize_files_by_type +from easybuild.framework.easyconfig.tools import check_software_versions, dep_graph, det_easyconfig_paths +from easybuild.framework.easyconfig.tools import dump_env_script, get_paths_for, parse_easyconfigs, review_pr +from easybuild.framework.easyconfig.tools import skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.docs import list_software @@ -390,9 +391,10 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short + no_skip = options.extended_dry_run or options.inject_checksums or options.check_versions # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping - if not (forced or dry_run_mode or options.extended_dry_run or new_update_preview_pr or options.inject_checksums): + if not (forced or dry_run_mode or new_update_preview_pr or no_skip): retained_ecs = skip_available(easyconfigs, modtool) if not testing: for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: @@ -436,6 +438,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): else: print_msg("\nNo conflicts detected!\n", prefix=False) + elif options.check_versions: + check_software_versions(easyconfigs) + # dump source script to set up build environment elif options.dump_env_script: dump_env_script(easyconfigs) @@ -444,7 +449,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): inject_checksums(ordered_ecs, options.inject_checksums) # cleanup and exit after dry run, searching easyconfigs or submitting regression test - stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums] + stop_options = [options.check_conflicts, options.check_versions, dry_run_mode, options.dump_env_script, + options.inject_checksums] if any(no_ec_opts) or any(stop_options): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 5176ec7180..2f83053bcd 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -145,6 +145,7 @@ # tar.Z: using compress (LZW) '.tar.z': "tar xZf %(filepath)s", } +KNOWN_EXTS = EXTRACT_CMDS.keys() class ZlibChecksum(object): @@ -744,7 +745,7 @@ def get_local_dirs_purged(): def find_extension(filename): """Find best match for filename extension.""" # sort by length, so longest file extensions get preference - suffixes = sorted(EXTRACT_CMDS.keys(), key=len, reverse=True) + suffixes = sorted(KNOWN_EXTS, key=len, reverse=True) pat = r'(?P%s)$' % '|'.join([s.replace('.', '\\.') for s in suffixes]) res = re.search(pat, filename, flags=re.IGNORECASE) if res: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 85d0620314..cd645f0b15 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -518,6 +518,8 @@ def informative_options(self): 'avail-hooks': ("Show list of known hooks", None, 'store_true', False), 'avail-toolchain-opts': ("Show options for toolchain", 'str', 'store', None), 'check-conflicts': ("Check for version conflicts in dependency graphs", None, 'store_true', False), + 'check-versions': ("Check which software versions are available for specified easyconfigs", + None, 'store_true', False), 'dep-graph': ("Create dependency graph", None, 'store', None, {'metavar': 'depgraph.'}), 'dump-env-script': ("Dump source script to set up build environment based on toolchain/dependencies", None, 'store_true', False),