diff --git a/easybuild/build.py b/easybuild/build.py index d4d26d17da..3a73243af5 100644 --- a/easybuild/build.py +++ b/easybuild/build.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2012 Stijn De Weirdt, Dries Verdegem, Kenneth Hoste, Pieter De Baets, Jens Timmerman +# Copyright 2009-2012 Stijn De Weirdt, Dries Verdegem, Kenneth Hoste, Pieter De Baets, Jens Timmerman, Toon Willems # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). @@ -85,6 +85,7 @@ def add_build_options(parser): parser.add_option("-d", "--debug" , action="store_true", help="log debug messages") parser.add_option("-v", "--version", action="store_true", help="show version") parser.add_option("--regtest", action="store_true", help="enable regression test mode") + parser.add_option("--regtest-online", action="store_true", help="enable online regression test mode") def main(): """ @@ -182,7 +183,7 @@ def main(): error("Can't find path %s" % path) try: - packages.extend(findEasyconfigs(path, log, blocks)) + packages.extend(findEasyconfigs(path, log, blocks, options.regtest_online)) except IOError, err: log.error("Processing easyconfigs in path %s failed: %s" % (path, err)) @@ -247,12 +248,12 @@ def error(message, exitCode=1, optparser=None): optparser.print_help() sys.exit(exitCode) -def findEasyconfigs(path, log, onlyBlocks=None): +def findEasyconfigs(path, log, onlyBlocks=None, regtest_online=False): """ Find .eb easyconfig files in path and process them """ if os.path.isfile(path): - return processEasyconfig(path, log, onlyBlocks) + return processEasyconfig(path, log, onlyBlocks, regtest_online) ## Walk through the start directory, retain all files that end in .eb files = [] @@ -268,10 +269,10 @@ def findEasyconfigs(path, log, onlyBlocks=None): packages = [] for filename in files: - packages.extend(processEasyconfig(filename, log, onlyBlocks)) + packages.extend(processEasyconfig(filename, log, onlyBlocks, regtest_online)) return packages -def processEasyconfig(path, log, onlyBlocks=None): +def processEasyconfig(path, log, onlyBlocks=None, regtest_online=False): """ Process easyconfig, returning some information for each block """ @@ -285,7 +286,7 @@ def processEasyconfig(path, log, onlyBlocks=None): try: app = Application(debug=LOGDEBUG) - app.process_ebfile(spec) + app.process_ebfile(spec, regtest_online) except EasyBuildError, err: msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg) log.exception(msg) @@ -576,7 +577,7 @@ def build(module, options, log, origEnviron, exitOnFailure=True): # timing info starttime = time.time() try: - result = app.autobuild(spec, runTests=not options.skip_tests) + result = app.autobuild(spec, runTests=not options.skip_tests, regtest_online=options.regtest_online) except EasyBuildError, err: lastn = 300 errormsg = "autoBuild Failed (last %d chars): %s" % (lastn, err.msg[-lastn:]) @@ -671,7 +672,7 @@ def build(module, options, log, origEnviron, exitOnFailure=True): ## Check for errors if exitCode > 0 or filetools.errorsFoundInLog > 0: print_msg("\nWARNING: Build exited with exit code %d. %d possible error(s) were detected in the " \ - "build logs, please verify the build.\n" % (exitCode, filetools.errorsFoundInLog), + "build logs, please verify the build.\n" % (exitCode, filetools.errorsFoundInLog), log) if app.postmsg: diff --git a/easybuild/easybuild_config.py b/easybuild/easybuild_config.py index 7874354f99..b2f584987a 100644 --- a/easybuild/easybuild_config.py +++ b/easybuild/easybuild_config.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2012 Stijn De Weirdt, Dries Verdegem, Kenneth Hoste, Pieter De Baets, Jens Timmerman +# Copyright 2009-2012 Stijn De Weirdt, Dries Verdegem, Kenneth Hoste, Pieter De Baets, Jens Timmerman, Toon Willems # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). diff --git a/easybuild/framework/application.py b/easybuild/framework/application.py index 8011953bae..15a5071c43 100644 --- a/easybuild/framework/application.py +++ b/easybuild/framework/application.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2012 Stijn De Weirdt, Dries Verdegem, Kenneth Hoste, Pieter De Baets, Jens Timmerman +# Copyright 2009-2012 Stijn De Weirdt, Dries Verdegem, Kenneth Hoste, Pieter De Baets, Jens Timmerman, Toon Willems # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). @@ -23,6 +23,7 @@ import glob import grp #@UnresolvedImport import os +import re import shutil import time import urllib @@ -153,11 +154,11 @@ def __init__(self, name=None, version=None, newBuild=True, debug=False): # mandatory config entries self.mandatory = ['name', 'version', 'homepage', 'description', 'toolkit'] - def autobuild(self, ebfile, runTests): + def autobuild(self, ebfile, runTests, regtest_online): """ Build the software package described by cfg. """ - self.process_ebfile(ebfile) + self.process_ebfile(ebfile, regtest_online) if self.getcfg('stop') and self.getcfg('stop') == 'cfg': return True @@ -349,9 +350,9 @@ def parse_dependency(self, dep): return result - ## process EasyBuild spec file + ## process EasyBuild spec file - def process_ebfile(self, fn): + def process_ebfile(self, fn, regtest_online=False): """ Read file fn, eval and add info - assume certain predefined variable names @@ -397,16 +398,7 @@ def process_ebfile(self, fn): self.setcfg(k, locs[k]) self.log.info("Using cfg option %s: value %s" % (k, self.getcfg(k))) - for k in self.mandatory: - if not k in locs: - self.log.error("No cfg option %s provided" % k) - - if self.getcfg('stop') and not (self.getcfg('stop') in self.validstops): - self.log.error("Stop provided %s is not valid: %s" % (self.cfg['stop'], self.validstops)) - - if not (self.getcfg('moduleclass') in self.validmoduleclasses): - self.log.error("Moduleclass provided %s is not valid: %s" % (self.cfg['moduleclass'], self.validmoduleclasses)) - + # NOTE: this option ('cfg') cannot be provided on the commandline because it will not yet be set by Easybuild if self.getcfg('stop') == 'cfg': self.log.info("Stopping in parsing cfg") return @@ -428,8 +420,6 @@ def process_ebfile(self, fn): self.log.debug("toolkit: %s" % self.getcfg('toolkit')) tk = self.getcfg('toolkit') self.settoolkit(tk['name'], tk['version']) - else: - self.log.error('no toolkit defined') if self.getcfg('toolkitopts'): self.tk.setOptions(self.getcfg('toolkitopts')) @@ -447,6 +437,29 @@ def process_ebfile(self, fn): self.setparallelism() self.make_installversion() + self.verify_config(regtest_online) + + def verify_config(self, regtest_online=False): + """ + verify the config settings + """ + for k in self.mandatory: + if not self.getcfg(k): + self.log.error("No cfg option %s provided" % k) + + if self.getcfg('stop') and not (self.getcfg('stop') in self.validstops): + self.log.error("Stop provided %s is not valid: %s" % (self.cfg['stop'], self.validstops)) + + if not (self.getcfg('moduleclass') in self.validmoduleclasses): + self.log.error("Moduleclass provided %s is not valid: %s" % (self.cfg['moduleclass'], self.validmoduleclasses)) + + if not self.getcfg('toolkit'): + self.log.error('no toolkit defined') + + if regtest_online and not self.verify_homepage(): + self.log.error("Homepage (%s) does not seem to contain anything relevant to %s" % (self.getcfg("homepage"), + self.name())) + def getcfg(self, key): """ @@ -499,7 +512,7 @@ def check_osdeps(self, osdeps): else: self.log.error("One or more OS dependencies were not found: %s" % not_found) - ## BUILD + ## BUILD def ready2build(self): """ @@ -674,7 +687,7 @@ def download(filename, url, path): if foundfile: return foundfile else: - # try and download source files from specified source URLs + # try and download source files from specified source URLs sourceURLs = self.getcfg('sourceURLs') targetdir = candidate_filepaths[0] if not os.path.isdir(targetdir): @@ -720,6 +733,36 @@ def download(filename, url, path): self.log.error("Couldn't find file %s anywhere, and downloading it didn't work either...\nPaths attempted (in order): %s " % (filename, ', '.join(failedpaths))) + + def verify_homepage(self): + """ + Download homepage, verify if the name of the software is mentioned + """ + homepage = self.getcfg("homepage") + + try: + page = urllib.urlopen(homepage) + except IOError: + self.log.error("Homepage (%s) is unavailable." % homepage) + return False + + regex = re.compile(self.name(), re.I) + + # if url contains software name and is available we are satisfied + if regex.search(homepage): + return True + + # Perform a lowercase compare against the entire contents of the html page + # (does not care about html) + for line in page: + if regex.search(line): + return True + + return False + + + + def apply_patch(self, beginpath=None): """ Apply the patches @@ -877,8 +920,8 @@ def postproc(self): def cleanup(self): """ Cleanup leftover mess: remove/clean build directory - - except when we're building in the installation directory, + + except when we're building in the installation directory, otherwise we remove the installation """ if not self.build_in_installdir: @@ -891,7 +934,7 @@ def cleanup(self): def sanitycheck(self): """ Do a sanity check on the installation - - if *any* of the files/subdirectories in the installation directory listed + - if *any* of the files/subdirectories in the installation directory listed in sanityCheckPaths are non-existent (or empty), the sanity check fails """ # prepare sanity check paths @@ -924,7 +967,7 @@ def sanitycheck(self): self.log.debug("Sanity check: found file %s in %s" % (f, self.installdir)) if self.sanityCheckOK: - # check if directories exist, and whether they are non-empty + # check if directories exist, and whether they are non-empty for d in self.sanityCheckPaths['dirs']: p = os.path.join(self.installdir, d) if not os.path.isdir(p) or not os.listdir(p): @@ -1010,7 +1053,7 @@ def make_builddir(self): """ if not self.build_in_installdir: # make a unique build dir - ## if a tookitversion starts with a -, remove the - so prevent a -- in the path name + ## if a tookitversion starts with a -, remove the - so prevent a -- in the path name tkversion = self.tk.version if tkversion.startswith('-'): tkversion = tkversion[1:] @@ -1228,7 +1271,7 @@ def make_module_extra_packages(self): def packages(self): """ After make install, run this. - - only if variable len(pkglist) > 0 + - only if variable len(pkglist) > 0 - optionally: load module that was just created using temp module file - find source for packages, in pkgs - run extraPackages @@ -1281,7 +1324,7 @@ def find_package_patches(self, pkgName): def find_package_sources(self): """ - Find source file for packages. + Find source file for packages. """ pkgSources = [] for pkg in self.getcfg('pkglist'): @@ -1384,7 +1427,7 @@ def filter_packages(self): """ Called when self.skip is True - use this to detect existing packages and to remove them from self.pkgs - - based on initial R version + - based on initial R version """ cmdtmpl = self.getcfg('pkgfilter')[0] cmdinputtmpl = self.getcfg('pkgfilter')[1] @@ -1517,7 +1560,7 @@ def get_instance(easyblock, log, name=None): """ Get instance for a particular application class (or Application) """ - #TODO: create proper factory for this, as explained here + #TODO: create proper factory for this, as explained here #http://stackoverflow.com/questions/456672/class-factory-in-python try: if not easyblock: