From 20caa7b0199ff6d861fa035e9209e6227ece9603 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Jan 2016 17:19:02 +0100 Subject: [PATCH 1/3] add support for --search-filename and --terse --- easybuild/main.py | 32 ++++++++++++++++++-------------- easybuild/tools/filetools.py | 20 ++++++++++++++------ easybuild/tools/options.py | 15 +++++++++++---- easybuild/tools/robot.py | 5 +++-- 4 files changed, 46 insertions(+), 26 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 0e4eb1d915..23a20bfd4a 100755 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -178,7 +178,7 @@ def main(args=None, logfile=None, do_build=None, testing=False): # initialise logging for main global _log - _log, logfile = init_logging(logfile, logtostdout=options.logtostdout, silent=testing or options.last_log) + _log, logfile = init_logging(logfile, logtostdout=options.logtostdout, silent=testing or options.terse) # disallow running EasyBuild as root if os.getuid() == 0: @@ -223,8 +223,6 @@ def main(args=None, logfile=None, do_build=None, testing=False): # print location to last log file, and exit last_log = find_last_log(logfile) or '(none)' print_msg(last_log, log=_log, prefix=False) - cleanup(logfile, eb_tmpdir, testing, silent=True) - sys.exit(0) # check whether packaging is supported when it's being used if options.package: @@ -232,13 +230,6 @@ def main(args=None, logfile=None, do_build=None, testing=False): else: _log.debug("Packaging not enabled, so not checking for packaging support.") - # update session state - eb_config = eb_go.generate_cmd_line(add_default=True) - modlist = session_module_list(testing=testing) # build options must be initialized first before 'module list' works - init_session_state.update({'easybuild_configuration': eb_config}) - init_session_state.update({'module_list': modlist}) - _log.debug("Initial session state: %s" % init_session_state) - # GitHub integration if options.review_pr or options.new_pr or options.update_pr: if options.review_pr: @@ -254,9 +245,22 @@ def main(args=None, logfile=None, do_build=None, testing=False): sys.exit(0) # search for easyconfigs, if a query is specified - query = options.search or options.search_short + query = options.search or options.search_filename or options.search_short if query: - search_easyconfigs(query, short=not options.search) + search_easyconfigs(query, short=options.search_short, filename_only=options.search_filename, + terse=options.terse) + + # non-verbose cleanup and exit after printing terse info + if options.terse: + cleanup(logfile, eb_tmpdir, testing, silent=True) + sys.exit(0) + + # update session state + eb_config = eb_go.generate_cmd_line(add_default=True) + modlist = session_module_list(testing=testing) # build options must be initialized first before 'module list' works + init_session_state.update({'easybuild_configuration': eb_config}) + init_session_state.update({'module_list': modlist}) + _log.debug("Initial session state: %s" % init_session_state) # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) @@ -264,8 +268,8 @@ def main(args=None, logfile=None, do_build=None, testing=False): _log.warning("Failed to determine install path for easybuild-easyconfigs package.") # command line options that do not require any easyconfigs to be specified - no_ec_opts = [options.aggregate_regtest, options.new_pr, options.review_pr, options.search, options.search_short, - options.regtest, options.update_pr] + no_ec_opts = [options.aggregate_regtest, options.new_pr, options.review_pr, options.search, + options.search_filename, options.search_short, options.regtest, options.update_pr] # determine paths to easyconfigs paths = det_easyconfig_paths(orig_paths) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 9fc67412b8..41c64af6fc 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -320,7 +320,7 @@ def find_easyconfigs(path, ignore_dirs=None): return files -def search_file(paths, query, short=False, ignore_dirs=None, silent=False): +def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filename_only=False, terse=False): """ Search for a particular file (only prints) """ @@ -340,7 +340,8 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False): for path in paths: hits = [] hit_in_path = False - print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent) + if not terse: + print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent) for (dirpath, dirnames, filenames) in os.walk(path, topdown=True): for filename in filenames: @@ -349,7 +350,10 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False): var = "CFGS%d" % var_index var_index += 1 hit_in_path = True - hits.append(os.path.join(dirpath, filename)) + if filename_only: + hits.append(filename) + else: + hits.append(os.path.join(dirpath, filename)) # do not consider (certain) hidden directories # note: we still need to consider e.g., .local ! @@ -359,7 +363,7 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False): hits = sorted(hits) - if hits: + if hits and not terse: common_prefix = det_common_path_prefix(hits) if short and common_prefix is not None and len(common_prefix) > len(var) * 2: var_lines.append("%s=%s" % (var, common_prefix)) @@ -367,8 +371,12 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False): else: hit_lines.extend([" * %s" % fn for fn in hits]) - for line in var_lines + hit_lines: - print_msg(line, log=_log, silent=silent, prefix=False) + if terse: + for line in hits: + print line + else: + for line in var_lines + hit_lines: + print_msg(line, log=_log, silent=silent, prefix=False) def compute_checksum(path, checksum_type=DEFAULT_CHECKSUM): diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 4f780a02d9..59f28d99f4 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -362,13 +362,16 @@ def informative_options(self): 'choice', 'store_or_None', 'simple', ['simple', 'detailed']), 'list-toolchains': ("Show list of known toolchains", None, 'store_true', False), - 'search': ("Search for easyconfig files in the robot directory, print full paths", - None, 'store', None, {'metavar': 'STR'}), - 'search-short': ("Search for easyconfig files in the robot directory, print short paths", - None, 'store', None, 'S', {'metavar': 'STR'}), + 'search': ("Search for easyconfig files in the robot search path, print full paths", + None, 'store', None, {'metavar': 'REGEX'}), + 'search-filename': ("Search for easyconfig files in the robot search path, print only filenames", + None, 'store', None, {'metavar': 'REGEX'}), + 'search-short': ("Search for easyconfig files in the robot search path, print short paths", + None, 'store', None, 'S', {'metavar': 'REGEX'}), 'show-default-configfiles': ("Show list of default config files", None, 'store_true', False), 'show-default-moduleclasses': ("Show default module classes with description", None, 'store_true', False), + 'terse': ("Terse output (machine-readable)", None, 'store_true', False), }) self.log.debug("informative_options: descr %s opts %s" % (descr, opts)) @@ -566,6 +569,10 @@ def postprocess(self): if not HAVE_AUTOPEP8: raise EasyBuildError("Python 'autopep8' module required to reformat dumped easyconfigs as requested") + # some options imply enabling --terse + if self.options.last_log: + self.options.terse = True + self._postprocess_external_modules_metadata() self._postprocess_config() diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index f481075fc0..099a685a1c 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -258,7 +258,7 @@ def resolve_dependencies(easyconfigs, retain_all_deps=False, minimal_toolchains= return ordered_ecs -def search_easyconfigs(query, short=False): +def search_easyconfigs(query, short=False, filename_only=False, terse=False): """Search for easyconfigs, if a query is provided.""" robot_path = build_option('robot_path') if robot_path: @@ -267,4 +267,5 @@ def search_easyconfigs(query, short=False): search_path = [os.getcwd()] ignore_dirs = build_option('ignore_dirs') silent = build_option('silent') - search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, silent=silent) + search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, silent=silent, filename_only=filename_only, + terse=terse) From e97bd728e43a56886a6c857855a24404586a41ae Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Jan 2016 13:56:57 +0100 Subject: [PATCH 2/3] add unit test for --search-filename( --terse) --- test/framework/options.py | 98 ++++++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 33 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 7f380935e6..4379194156 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -559,65 +559,97 @@ def test_list_easyblocks(self): def test_search(self): """Test searching for easyconfigs.""" - fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') - os.close(fd) + test_easyconfigs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + # simple search args = [ '--search=gzip', - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), - '--unittest-file=%s' % self.logfile, + '--robot=%s' % test_easyconfigs_dir, ] - self.eb_main(args, logfile=dummylogfn) - logtxt = read_file(self.logfile) + self.mock_stdout(True) + self.eb_main(args, testing=False) + txt = self.get_stdout() + self.mock_stdout(False) info_msg = r"Searching \(case-insensitive\) for 'gzip' in" - self.assertTrue(re.search(info_msg, logtxt), "Info message when searching for easyconfigs in '%s'" % logtxt) + self.assertTrue(re.search(info_msg, txt), "Info message when searching for easyconfigs in '%s'" % txt) for ec in ["gzip-1.4.eb", "gzip-1.4-GCC-4.6.3.eb"]: - self.assertTrue(re.search(r" \* \S*%s$" % ec, logtxt, re.M), "Found easyconfig %s in '%s'" % (ec, logtxt)) - - if os.path.exists(dummylogfn): - os.remove(dummylogfn) - - write_file(self.logfile, '') + regex = re.compile(r" \* \S*%s$" % ec, re.M) + self.assertTrue(regex.search(txt), "Found pattern '%s' in: %s" % (regex.pattern, txt)) + # search w/ regex args = [ '--search=^gcc.*2.eb', - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), - '--unittest-file=%s' % self.logfile, + '--robot=%s' % test_easyconfigs_dir, ] - self.eb_main(args, logfile=dummylogfn) - logtxt = read_file(self.logfile) + self.mock_stdout(True) + self.eb_main(args, testing=False) + txt = self.get_stdout() + self.mock_stdout(False) info_msg = r"Searching \(case-insensitive\) for '\^gcc.\*2.eb' in" - self.assertTrue(re.search(info_msg, logtxt), "Info message when searching for easyconfigs in '%s'" % logtxt) + self.assertTrue(re.search(info_msg, txt), "Info message when searching for easyconfigs in '%s'" % txt) for ec in ['GCC-4.7.2.eb', 'GCC-4.8.2.eb', 'GCC-4.9.2.eb']: - self.assertTrue(re.search(r" \* \S*%s$" % ec, logtxt, re.M), "Found easyconfig %s in '%s'" % (ec, logtxt)) + regex = re.compile(r" \* \S*%s$" % ec, re.M) + self.assertTrue(regex.search(txt), "Found pattern '%s' in: %s" % (regex.pattern, txt)) - if os.path.exists(dummylogfn): - os.remove(dummylogfn) + gcc_ecs = [ + 'GCC-4.6.3.eb', + 'GCC-4.6.4.eb', + 'GCC-4.7.2.eb', + 'GCC-4.8.2.eb', + 'GCC-4.8.3.eb', + 'GCC-4.9.2.eb', + ] - write_file(self.logfile, '') + # test --search-filename + args = [ + '--search-filename=^gcc', + '--robot=%s' % test_easyconfigs_dir, + ] + self.mock_stdout(True) + self.eb_main(args, testing=False) + txt = self.get_stdout() + self.mock_stdout(False) + + for ec in gcc_ecs: + regex = re.compile(r"^ \* %s$" % ec, re.M) + self.assertTrue(regex.search(txt), "Found pattern '%s' in: %s" % (regex.pattern, txt)) + # test --search-filename --terse + args = [ + '--search-filename=^gcc', + '--terse', + '--robot=%s' % test_easyconfigs_dir, + ] + self.mock_stdout(True) + self.eb_main(args, testing=False) + txt = self.get_stdout() + self.mock_stdout(False) + + for ec in gcc_ecs: + regex = re.compile(r"^%s$" % ec, re.M) + self.assertTrue(regex.search(txt), "Found pattern '%s' in: %s" % (regex.pattern, txt)) + + # also test --search-short/-S for search_arg in ['-S', '--search-short']: - open(self.logfile, 'w').write('') args = [ search_arg, 'toy-0.0', '-r', - os.path.join(os.path.dirname(__file__), 'easyconfigs'), - '--unittest-file=%s' % self.logfile, + test_easyconfigs_dir, ] - self.eb_main(args, logfile=dummylogfn, raise_error=True, verbose=True) - logtxt = read_file(self.logfile) + self.mock_stdout(True) + self.eb_main(args, raise_error=True, verbose=True, testing=False) + txt = self.get_stdout() + self.mock_stdout(False) info_msg = r"Searching \(case-insensitive\) for 'toy-0.0' in" - self.assertTrue(re.search(info_msg, logtxt), "Info message when searching for easyconfigs in '%s'" % logtxt) - self.assertTrue(re.search('INFO CFGS\d+=', logtxt), "CFGS line message found in '%s'" % logtxt) + self.assertTrue(re.search(info_msg, txt), "Info message when searching for easyconfigs in '%s'" % txt) + self.assertTrue(re.search('^CFGS\d+=', txt, re.M), "CFGS line message found in '%s'" % txt) for ec in ["toy-0.0.eb", "toy-0.0-multiple.eb"]: - self.assertTrue(re.search(" \* \$CFGS\d+/*%s" % ec, logtxt), "Found easyconfig %s in '%s'" % (ec, logtxt)) - - if os.path.exists(dummylogfn): - os.remove(dummylogfn) + regex = re.compile(r" \* \$CFGS\d+/*%s" % ec, re.M) + self.assertTrue(regex.search(txt), "Found pattern '%s' in: %s" % (regex.pattern, txt)) def test_dry_run(self): """Test dry run (long format).""" From 697c5eb2f4680d1b7418b725c59aefda81288515 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Jan 2016 14:02:51 +0100 Subject: [PATCH 3/3] fix remarks --- easybuild/tools/filetools.py | 2 +- easybuild/tools/options.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 41c64af6fc..beca79959e 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -373,7 +373,7 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen if terse: for line in hits: - print line + print(line) else: for line in var_lines + hit_lines: print_msg(line, log=_log, silent=silent, prefix=False) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 59f28d99f4..316559fe9e 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -569,7 +569,7 @@ def postprocess(self): if not HAVE_AUTOPEP8: raise EasyBuildError("Python 'autopep8' module required to reformat dumped easyconfigs as requested") - # some options imply enabling --terse + # imply --terse for --last-log to avoid extra output that gets in the way if self.options.last_log: self.options.terse = True