diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index cf689d441e..6fc41daf0f 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -525,6 +525,7 @@ def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None): path = os.path.join(path, "easybuild", subdir) _log.debug("Checking for easybuild/%s at %s" % (subdir, path)) try: + # FIXME: will not work on paths in zipped egg if os.path.exists(path): paths.append(os.path.abspath(path)) _log.debug("Added %s to list of paths for easybuild/%s" % (path, subdir)) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index b826839271..0ff7b6edca 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -29,11 +29,11 @@ """ import glob import os +import pkgutil import string import sys from vsc.utils import fancylogger -import easybuild.tools.environment as env from easybuild.tools.build_log import EasyBuildError @@ -109,18 +109,26 @@ def import_available_modules(namespace): @param namespace: The namespace to import modules from. """ - modules = [] + # determine list of paths where top-level package name is available + top_pkg = namespace.split('.')[0] + paths = [] for path in sys.path: - for module in sorted(glob.glob(os.path.sep.join([path] + namespace.split('.') + ['*.py']))): - if not module.endswith('__init__.py'): - mod_name = module.split(os.path.sep)[-1].split('.')[0] - modpath = '.'.join([namespace, mod_name]) - _log.debug("importing module %s" % modpath) - try: - mod = __import__(modpath, globals(), locals(), ['']) - except ImportError as err: - raise EasyBuildError("import_available_modules: Failed to import %s: %s", modpath, err) - modules.append(mod) + mod_names = [name for (_, name, _) in pkgutil.iter_modules(path=[path])] + if any([name.split('.')[0] == top_pkg for name in mod_names]): + paths.append(os.path.join(os.path.abspath(path), *namespace.split('.'))) + + # determine all (unique) module names + modnames = [mod for (_, mod, ispkg) in pkgutil.walk_packages(path=paths, prefix='%s.' % namespace) if not ispkg] + + # import all modules found + modules = [] + for modname in modnames: + try: + mod = __import__(modname, globals(), locals(), ['']) + except ImportError as err: + raise EasyBuildError("import_available_modules: Failed to import %s: %s", modname, err) + modules.append(mod) + return modules diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 80e6032d44..4f708fe9a0 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -53,16 +53,21 @@ def get_git_revision(): relies on GitPython (see http://gitorious.org/git-python) """ + res = UNKNOWN try: import git + # __file__ may contain path in a zipped egg + if os.path.exists(__file__): + path = os.path.dirname(__file__) + try: + gitrepo = git.Git(path) + res = gitrepo.rev_list("HEAD").splitlines()[0] + except git.GitCommandError: + pass except ImportError: - return UNKNOWN - try: - path = os.path.dirname(__file__) - gitrepo = git.Git(path) - return gitrepo.rev_list("HEAD").splitlines()[0] - except git.GitCommandError: - return UNKNOWN + pass + + return res git_rev = get_git_revision() if git_rev == UNKNOWN: diff --git a/setup.cfg b/setup.cfg index baf390bd17..65eff4cc9e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ [bdist_rpm] requires = environment-modules, bash, python >= 2.6, python < 3 + +[easy_install] + diff --git a/setup.py b/setup.py index 6e5cd1e109..1163241fba 100644 --- a/setup.py +++ b/setup.py @@ -106,8 +106,8 @@ def find_rel_test(): platforms="Linux", provides=["eb"] + easybuild_packages, test_suite="test.framework.suite", - zip_safe=False, - install_requires=["vsc-base >= 2.2.4"], + zip_safe=True, + install_requires=["vsc-base >= 2.3.0"], extras_require = { 'yeb': ["PyYAML >= 3.11"], }, diff --git a/test/framework/options.py b/test/framework/options.py index 96c3d01e80..55479abe9c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -177,7 +177,7 @@ def test_force(self): """Test forcing installation even if the module is already available.""" # use GCC-4.6.3.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'GCC-4.6.3.eb') + eb_file = os.path.join(self.test_easyconfigs, 'GCC-4.6.3.eb') # check log message without --force args = [ @@ -203,7 +203,7 @@ def test_skip(self): """Test skipping installation of module (--skip, -k).""" # use toy-0.0.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb') + eb_file = os.path.join(self.test_easyconfigs, 'toy-0.0.eb') # check log message with --skip for existing module args = [ @@ -263,7 +263,7 @@ def test_job(self): """Test submitting build as a job.""" # use gzip-1.4.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.4.eb') + eb_file = os.path.join(self.test_easyconfigs, 'gzip-1.4.eb') def check_args(job_args, passed_args=None): """Check whether specified args yield expected result.""" @@ -330,7 +330,7 @@ def test_zzz_logtostdout(self): stdoutorig = sys.stdout sys.stdout = open("/dev/null", 'w') - toy_ecfile = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'toy-0.0.eb') + toy_ecfile = os.path.join(self.test_easyconfigs, 'toy-0.0.eb') self.logfile = None out = self.eb_main([toy_ecfile, '--debug', '-l', '--force'], raise_error=True) @@ -477,8 +477,7 @@ def test_avail_cfgfile_constants(self): tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') - shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + shutil.copytree(self.test_easyconfigs, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) orig_sys_path = sys.path[:] sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs @@ -560,7 +559,7 @@ def test_search(self): args = [ '--search=gzip', - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--robot=%s' % self.test_easyconfigs, '--unittest-file=%s' % self.logfile, ] self.eb_main(args, logfile=dummylogfn) @@ -578,7 +577,7 @@ def test_search(self): args = [ '--search=^gcc.*2.eb', - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--robot=%s' % self.test_easyconfigs, '--unittest-file=%s' % self.logfile, ] self.eb_main(args, logfile=dummylogfn) @@ -600,7 +599,7 @@ def test_search(self): search_arg, 'toy-0.0', '-r', - os.path.join(os.path.dirname(__file__), 'easyconfigs'), + self.test_easyconfigs, '--unittest-file=%s' % self.logfile, ] self.eb_main(args, logfile=dummylogfn, raise_error=True, verbose=True) @@ -621,10 +620,10 @@ def test_dry_run(self): os.close(fd) args = [ - os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb'), + os.path.join(self.test_easyconfigs, 'gzip-1.4-GCC-4.6.3.eb'), '--dry-run', # implies enabling dependency resolution '--unittest-file=%s' % self.logfile, - '--robot-paths=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--robot-paths=%s' % self.test_easyconfigs, ] self.eb_main(args, logfile=dummylogfn) logtxt = read_file(self.logfile) @@ -652,8 +651,7 @@ def test_dry_run_short(self): tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') - shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + shutil.copytree(self.test_easyconfigs, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) orig_sys_path = sys.path[:] sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs @@ -696,9 +694,8 @@ def test_try_robot_force(self): os.close(fd) # use toy-0.0.eb easyconfig file that comes with the tests - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') - eb_file1 = os.path.join(test_ecs_dir, 'FFTW-3.3.3-gompi-1.4.10.eb') - eb_file2 = os.path.join(test_ecs_dir, 'ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb') + eb_file1 = os.path.join(self.test_easyconfigs, 'FFTW-3.3.3-gompi-1.4.10.eb') + eb_file2 = os.path.join(self.test_easyconfigs, 'ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb') # check log message with --skip for existing module args = [ @@ -709,7 +706,7 @@ def test_try_robot_force(self): '--installpath=%s' % self.test_installpath, '--debug', '--force', - '--robot=%s' % test_ecs_dir, + '--robot=%s' % self.test_easyconfigs, '--try-toolchain=gompi,1.3.12', '--dry-run', '--unittest-file=%s' % self.logfile, @@ -736,17 +733,16 @@ def test_dry_run_hierarchical(self): fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') args = [ - os.path.join(test_ecs, 'gzip-1.5-goolf-1.4.10.eb'), - os.path.join(test_ecs, 'OpenMPI-1.6.4-GCC-4.7.2.eb'), + os.path.join(self.test_easyconfigs, 'gzip-1.5-goolf-1.4.10.eb'), + os.path.join(self.test_easyconfigs, 'OpenMPI-1.6.4-GCC-4.7.2.eb'), '--dry-run', '--unittest-file=%s' % self.logfile, '--module-naming-scheme=HierarchicalMNS', '--ignore-osdeps', '--force', '--debug', - '--robot-paths=%s' % os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs'), + '--robot-paths=%s' % self.test_easyconfigs, ] outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True, raise_error=True) @@ -777,17 +773,16 @@ def test_dry_run_categorized(self): os.close(fd) self.setup_categorized_hmns_modules() - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') args = [ - os.path.join(test_ecs, 'gzip-1.5-goolf-1.4.10.eb'), - os.path.join(test_ecs, 'OpenMPI-1.6.4-GCC-4.7.2.eb'), + os.path.join(self.test_easyconfigs, 'gzip-1.5-goolf-1.4.10.eb'), + os.path.join(self.test_easyconfigs, 'OpenMPI-1.6.4-GCC-4.7.2.eb'), '--dry-run', '--unittest-file=%s' % self.logfile, '--module-naming-scheme=CategorizedHMNS', '--ignore-osdeps', '--force', '--debug', - '--robot-paths=%s' % os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs'), + '--robot-paths=%s' % self.test_easyconfigs, ] outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True, raise_error=True) @@ -827,7 +822,7 @@ def test_from_pr(self): '--from-pr=1239', '--dry-run', # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--robot=%s' % self.test_easyconfigs, '--unittest-file=%s' % self.logfile, '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user '--tmpdir=%s' % tmpdir, @@ -873,10 +868,9 @@ def test_from_pr_listed_ecs(self): os.close(fd) # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') ecstmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(ecstmpdir, 'easybuild'), parents=True) - shutil.copytree(test_ecs_path, os.path.join(ecstmpdir, 'easybuild', 'easyconfigs')) + shutil.copytree(self.test_easyconfigs, os.path.join(ecstmpdir, 'easybuild', 'easyconfigs')) # inject path to test easyconfigs into head of Python search path sys.path.insert(0, ecstmpdir) @@ -890,7 +884,7 @@ def test_from_pr_listed_ecs(self): '--from-pr=1239', '--dry-run', # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed - '--robot=%s' % test_ecs_path, + '--robot=%s' % self.test_easyconfigs, '--unittest-file=%s' % self.logfile, '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user '--tmpdir=%s' % tmpdir, @@ -898,13 +892,13 @@ def test_from_pr_listed_ecs(self): try: outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) modules = [ - (test_ecs_path, 'toy/0.0'), # not included in PR - (test_ecs_path, 'GCC/4.9.2'), # not included in PR + (self.test_easyconfigs, 'toy/0.0'), # not included in PR + (self.test_easyconfigs, 'GCC/4.9.2'), # not included in PR (tmpdir, 'hwloc/1.10.0-GCC-4.9.2'), (tmpdir, 'numactl/2.0.10-GCC-4.9.2'), (tmpdir, 'OpenMPI/1.8.4-GCC-4.9.2'), (tmpdir, 'gompi/2015a'), - (test_ecs_path, 'GCC/4.6.3'), # not included in PR + (self.test_easyconfigs, 'GCC/4.6.3'), # not included in PR ] for path_prefix, module in modules: ec_fn = "%s.eb" % '-'.join(module.split('/')) @@ -959,7 +953,7 @@ def test_footer(self): write_file(modules_footer, module_footer_txt) # use toy-0.0.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') + eb_file = os.path.join(self.test_easyconfigs, 'toy-0.0.eb') # check log message with --skip for existing module args = [ @@ -988,7 +982,7 @@ def test_recursive_module_unload(self): """Test generating recursively unloading modules.""" # use toy-0.0.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0-deps.eb') + eb_file = os.path.join(self.test_easyconfigs, 'toy-0.0-deps.eb') # check log message with --skip for existing module args = [ @@ -1016,7 +1010,7 @@ def test_tmpdir(self): tmpdir = tempfile.mkdtemp() # use toy-0.0.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') + eb_file = os.path.join(self.test_easyconfigs, 'toy-0.0.eb') # check log message with --skip for existing module args = [ @@ -1179,7 +1173,7 @@ def test_allow_modules_tool_mismatch(self): # make sure MockModulesTool is available from test.framework.modulestool import MockModulesTool - ec_file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb') + ec_file = os.path.join(self.test_easyconfigs, 'toy-0.0.eb') # keep track of original module definition so we can restore it orig_module = os.environ.get('module', None) @@ -1230,9 +1224,8 @@ def test_allow_modules_tool_mismatch(self): def test_try(self): """Test whether --try options are taken into account.""" - ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') - shutil.copy2(os.path.join(ecs_path, 'toy-0.0.eb'), tweaked_toy_ec) + shutil.copy2(os.path.join(self.test_easyconfigs, 'toy-0.0.eb'), tweaked_toy_ec) f = open(tweaked_toy_ec, 'a') f.write("easyblock = 'ConfigureMake'") f.close() @@ -1243,7 +1236,7 @@ def test_try(self): '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, '--dry-run', - '--robot=%s' % ecs_path, + '--robot=%s' % self.test_easyconfigs, ] test_cases = [ @@ -1288,17 +1281,16 @@ def test_try(self): def test_recursive_try(self): """Test whether recursive --try-X works.""" - ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + ecs_path = os.path.join(self.test_easyconfigs) tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') shutil.copy2(os.path.join(ecs_path, 'toy-0.0.eb'), tweaked_toy_ec) f = open(tweaked_toy_ec, 'a') f.write("dependencies = [('gzip', '1.4')]\n") # add fictious dependency f.close() - sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') args = [ tweaked_toy_ec, - '--sourcepath=%s' % sourcepath, + '--sourcepath=%s' % self.test_sourcepath, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, '--try-toolchain=gompi,1.4.10', @@ -1347,7 +1339,7 @@ def test_recursive_try(self): def test_cleanup_builddir(self): """Test cleaning up of build dir and --disable-cleanup-builddir.""" - toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') + toy_ec = os.path.join(self.test_easyconfigs, 'toy-0.0.eb') toy_buildpath = os.path.join(self.test_buildpath, 'toy', '0.0', 'dummy-dummy') args = [ @@ -1375,15 +1367,14 @@ def test_cleanup_builddir(self): def test_filter_deps(self): """Test use of --filter-deps.""" - test_dir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(test_dir, 'easyconfigs', 'goolf-1.4.10.eb') - os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') + ec_file = os.path.join(self.test_easyconfigs, 'goolf-1.4.10.eb') args = [ ec_file, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, - '--robot=%s' % os.path.join(test_dir, 'easyconfigs'), + '--robot=%s' % self.test_easyconfigs, '--dry-run', + '--force', ] outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) self.assertTrue(re.search('module: FFTW/3.3.3-gompi', outtxt)) @@ -1402,15 +1393,14 @@ def test_filter_deps(self): def test_hide_deps(self): """Test use of --hide-deps.""" - test_dir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(test_dir, 'easyconfigs', 'goolf-1.4.10.eb') - os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') + ec_file = os.path.join(self.test_easyconfigs, 'goolf-1.4.10.eb') args = [ ec_file, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, - '--robot=%s' % os.path.join(test_dir, 'easyconfigs'), + '--robot=%s' % self.test_easyconfigs, '--dry-run', + '--force', ] outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) self.assertTrue(re.search('module: GCC/4.7.2', outtxt)) @@ -1442,7 +1432,7 @@ def test_test_report_env_filter(self): def toy(extra_args=None): """Build & install toy, return contents of test report.""" - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') + eb_file = os.path.join(self.test_easyconfigs, 'toy-0.0.eb') args = [ eb_file, '--sourcepath=%s' % self.test_sourcepath, @@ -1491,8 +1481,7 @@ def test_robot(self): # unset $EASYBUILD_ROBOT_PATHS that was defined in setUp os.environ['EASYBUILD_ROBOT_PATHS'] = self.test_prefix - test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') - eb_file = os.path.join(test_ecs_path, 'gzip-1.4-GCC-4.6.3.eb') # includes 'toy/.0.0-deps' as a dependency + eb_file = os.path.join(self.test_easyconfigs, 'gzip-1.4-GCC-4.6.3.eb') # includes 'toy/.0.0-deps' as a dependency # hide test modules self.reset_modulepath([]) @@ -1500,7 +1489,7 @@ def test_robot(self): # dependency resolution is disabled by default, even if required paths are available args = [ eb_file, - '--robot-paths=%s' % test_ecs_path, + '--robot-paths=%s' % self.test_easyconfigs, ] error_regex = "Missing modules for one or more dependencies: .*" self.assertErrorRegex(EasyBuildError, error_regex, self.eb_main, args, raise_error=True, do_build=True) @@ -1514,13 +1503,13 @@ def test_robot(self): self.assertErrorRegex(EasyBuildError, 'Irresolvable dependencies', self.eb_main, args, raise_error=True) # add path to test easyconfigs to robot paths, so dependencies can be resolved - self.eb_main(args + ['--robot-paths=%s' % test_ecs_path], raise_error=True) + self.eb_main(args + ['--robot-paths=%s' % self.test_easyconfigs], raise_error=True) # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory # to check whether easyconfigs install path is auto-included in robot path tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) - shutil.copytree(test_ecs_path, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + shutil.copytree(self.test_easyconfigs, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) # prepend path to test easyconfigs into Python search path, so it gets picked up as --robot-paths default del os.environ['EASYBUILD_ROBOT_PATHS'] @@ -1534,14 +1523,14 @@ def test_robot(self): # make sure that paths specified to --robot get preference over --robot-paths args = [ eb_file, - '--robot=%s' % test_ecs_path, + '--robot=%s' % self.test_easyconfigs, '--robot-paths=%s' % os.path.join(tmpdir, 'easybuild', 'easyconfigs'), '--dry-run', ] outtxt = self.eb_main(args, raise_error=True) for ecfile in ['GCC-4.6.3.eb', 'ictce-4.1.13.eb', 'toy-0.0-deps.eb', 'gzip-1.4-GCC-4.6.3.eb']: - ec_regex = re.compile(r'^\s\*\s\[[xF ]\]\s%s' % os.path.join(test_ecs_path, ecfile), re.M) + ec_regex = re.compile(r'^\s\*\s\[[xF ]\]\s%s' % os.path.join(self.test_easyconfigs, ecfile), re.M) self.assertTrue(ec_regex.search(outtxt), "Pattern %s found in %s" % (ec_regex.pattern, outtxt)) def test_missing_cfgfile(self): @@ -1695,8 +1684,7 @@ def test_include_easyblocks(self): self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) - test_easyblocks = os.path.dirname(os.path.abspath(__file__)) - path_pattern = os.path.join(test_easyblocks, 'sandbox', 'easybuild', 'easyblocks', 'f', 'foo.py') + path_pattern = os.path.join(self.test_easyblocks, 'f', 'foo.py') foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) @@ -1788,7 +1776,7 @@ def test_use_included_module_naming_scheme(self): ]) write_file(os.path.join(self.test_prefix, 'test_mns.py'), mns_txt) - eb_file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb') + eb_file = os.path.join(self.test_easyconfigs, 'toy-0.0.eb') args = [ '--unittest-file=%s' % self.logfile, '--module-naming-scheme=AnotherTestIncludedMNS', @@ -1862,7 +1850,7 @@ def test_include_toolchains(self): def test_cleanup_tmpdir(self): """Test --cleanup-tmpdir.""" args = [ - os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'toy-0.0.eb'), + os.path.join(self.test_easyconfigs, 'toy-0.0.eb'), '--dry-run', '--try-software-version=1.0', # so we get a tweaked easyconfig ] diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 757d805d93..401c1d267f 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -30,10 +30,13 @@ import copy import fileinput import os +import pkgutil import re import shutil import sys import tempfile +from zipfile import ZipFile + from vsc.utils import fancylogger from vsc.utils.patterns import Singleton from vsc.utils.testing import EnhancedTestCase as _EnhancedTestCase @@ -77,6 +80,14 @@ newkey = '%s_%s' % (CONFIG_ENV_VAR_PREFIX, key[len(test_env_var_prefix):]) os.environ[newkey] = val +TESTDIR = os.path.dirname(os.path.abspath(__file__)) +if not os.path.exists(TESTDIR): + tmpdir = tempfile.mkdtemp() + eggpath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + ZipFile(eggpath).extractall(tmpdir) + TESTDIR = os.path.join(tmpdir, 'test', 'framework') + + class EnhancedTestCase(_EnhancedTestCase): """Enhanced test case, provides extra functionality (e.g. an assertErrorRegex method).""" @@ -105,9 +116,7 @@ def setUp(self): # keep track of original environment/Python search path to restore self.orig_sys_path = sys.path[:] - testdir = os.path.dirname(os.path.abspath(__file__)) - - self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources') + self.test_sourcepath = os.path.join(TESTDIR, 'sandbox', 'sources') os.environ['EASYBUILD_SOURCEPATH'] = self.test_sourcepath os.environ['EASYBUILD_PREFIX'] = self.test_prefix self.test_buildpath = tempfile.mkdtemp() @@ -116,7 +125,11 @@ def setUp(self): os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath # make sure that the tests only pick up easyconfigs provided with the tests - os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs') + self.test_easyconfigs = os.path.realpath(os.path.join(TESTDIR, 'easyconfigs')) + os.environ['EASYBUILD_ROBOT_PATHS'] = self.test_easyconfigs + + self.test_easyblocks = os.path.join(TESTDIR, 'sandbox', 'easybuild', 'easyblocks') + self.test_modules = os.path.realpath(os.path.join(TESTDIR, 'modules')) # make sure no deprecated behaviour is being triggered (unless intended by the test) # trip *all* log.deprecated statements by setting deprecation version ridiculously high @@ -127,12 +140,13 @@ def setUp(self): # remove any entries in Python search path that seem to provide easyblocks for path in sys.path[:]: - if os.path.exists(os.path.join(path, 'easybuild', 'easyblocks', '__init__.py')): + mods = [mod for (_, mod, _) in pkgutil.iter_modules(path=[os.path.join(path, 'easybuild')])] + if 'easyblocks' in mods: sys.path.remove(path) # add test easyblocks to Python search path and (re)import and reload easybuild modules import easybuild - sys.path.append(os.path.join(testdir, 'sandbox')) + sys.path.append(os.path.join(TESTDIR, 'sandbox')) reload(easybuild) import easybuild.easyblocks reload(easybuild.easyblocks) @@ -143,7 +157,7 @@ def setUp(self): modtool = modules_tool() # purge out any loaded modules with original $MODULEPATH before running each test modtool.purge() - self.reset_modulepath([os.path.join(testdir, 'modules')]) + self.reset_modulepath([os.path.join(TESTDIR, 'modules')]) def tearDown(self): """Clean up after running testcase.""" @@ -250,7 +264,7 @@ def setup_hierarchical_modules(self): # EasyBuild is responsible for making sure that the toolchain can be loaded using the short module name mkdir(mod_prefix, parents=True) for mod_subdir in ['Core', 'Compiler', 'MPI']: - src_mod_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules', mod_subdir) + src_mod_path = os.path.join(self.test_modules, mod_subdir) shutil.copytree(src_mod_path, os.path.join(mod_prefix, mod_subdir)) # make sure only modules in a hierarchical scheme are available, mixing modules installed with @@ -284,8 +298,7 @@ def setup_categorized_hmns_modules(self): # EasyBuild is responsible for making sure that the toolchain can be loaded using the short module name mkdir(mod_prefix, parents=True) for mod_subdir in ['Core', 'Compiler', 'MPI']: - src_mod_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'modules', 'CategorizedHMNS', mod_subdir) + src_mod_path = os.path.join(self.test_modules, 'CategorizedHMNS', mod_subdir) shutil.copytree(src_mod_path, os.path.join(mod_prefix, mod_subdir)) # create empty module file directory to make C/Tcl modules happy mpi_pref = os.path.join(mod_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4')