Skip to content

add support for --search-filename and --terse (REVIEW) #1577

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
boegel merged 3 commits into from
Feb 5, 2016
Merged

add support for --search-filename and --terse (REVIEW) #1577

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
20caa7b
add support for --search-filename and --terse
boegel Jan 27, 2016
e97bd72
add unit test for --search-filename( --terse)
boegel Jan 29, 2016
697c5eb
fix remarks
boegel Jan 29, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 18 additions & 14 deletions easybuild/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ def main(args=None, logfile=None, do_build=None, testing=False):

# initialise logging for main
global _log
_log, logfile = init_logging(logfile, logtostdout=options.logtostdout, silent=testing or options.last_log)
_log, logfile = init_logging(logfile, logtostdout=options.logtostdout, silent=testing or options.terse)

# disallow running EasyBuild as root
if os.getuid() == 0:
Expand Down Expand Up @@ -223,22 +223,13 @@ def main(args=None, logfile=None, do_build=None, testing=False):
# print location to last log file, and exit
last_log = find_last_log(logfile) or '(none)'
print_msg(last_log, log=_log, prefix=False)
cleanup(logfile, eb_tmpdir, testing, silent=True)
sys.exit(0)

Copy link
Member

@wpoely86 wpoely86 Jan 28, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so, this changes functionally? You can now combine --last-log with --search?

Copy link
Member Author

@boegel boegel Jan 29, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm, that's an unintended side-effect, but does it matter?

you can indeed combine these, but I don't think it makes a lot of sense to do it

doesn't hurt, so fine?

Copy link
Member

@wpoely86 wpoely86 Jan 29, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know. Might bite us in the ass later?

Copy link
Member Author

@boegel boegel Feb 5, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should be fine, there's lots of weird combo's of options you can make now.

Using --last-log and --search together won't break anything (I checked).

# check whether packaging is supported when it's being used
if options.package:
check_pkg_support()
else:
_log.debug("Packaging not enabled, so not checking for packaging support.")

# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = session_module_list(testing=testing) # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)

# GitHub integration
if options.review_pr or options.new_pr or options.update_pr:
if options.review_pr:
Expand All @@ -254,18 +245,31 @@ def main(args=None, logfile=None, do_build=None, testing=False):
sys.exit(0)

# search for easyconfigs, if a query is specified
query = options.search or options.search_short
query = options.search or options.search_filename or options.search_short
if query:
search_easyconfigs(query, short=not options.search)
search_easyconfigs(query, short=options.search_short, filename_only=options.search_filename,
terse=options.terse)

# non-verbose cleanup and exit after printing terse info
if options.terse:
cleanup(logfile, eb_tmpdir, testing, silent=True)
sys.exit(0)

# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = session_module_list(testing=testing) # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)

# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")

# command line options that do not require any easyconfigs to be specified
no_ec_opts = [options.aggregate_regtest, options.new_pr, options.review_pr, options.search, options.search_short,
options.regtest, options.update_pr]
no_ec_opts = [options.aggregate_regtest, options.new_pr, options.review_pr, options.search,
options.search_filename, options.search_short, options.regtest, options.update_pr]

# determine paths to easyconfigs
paths = det_easyconfig_paths(orig_paths)
Expand Down
20 changes: 14 additions & 6 deletions easybuild/tools/filetools.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ def find_easyconfigs(path, ignore_dirs=None):
return files


def search_file(paths, query, short=False, ignore_dirs=None, silent=False):
def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filename_only=False, terse=False):
"""
Search for a particular file (only prints)
"""
Expand All @@ -340,7 +340,8 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False):
for path in paths:
hits = []
hit_in_path = False
print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent)
if not terse:
print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent)

for (dirpath, dirnames, filenames) in os.walk(path, topdown=True):
for filename in filenames:
Expand All @@ -349,7 +350,10 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False):
var = "CFGS%d" % var_index
var_index += 1
hit_in_path = True
hits.append(os.path.join(dirpath, filename))
if filename_only:
hits.append(filename)
else:
hits.append(os.path.join(dirpath, filename))

# do not consider (certain) hidden directories
# note: we still need to consider e.g., .local !
Expand All @@ -359,16 +363,20 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False):

hits = sorted(hits)

if hits:
if hits and not terse:
common_prefix = det_common_path_prefix(hits)
if short and common_prefix is not None and len(common_prefix) > len(var) * 2:
var_lines.append("%s=%s" % (var, common_prefix))
hit_lines.extend([" * %s" % os.path.join('$%s' % var, fn[len(common_prefix) + 1:]) for fn in hits])
else:
hit_lines.extend([" * %s" % fn for fn in hits])

for line in var_lines + hit_lines:
print_msg(line, log=_log, silent=silent, prefix=False)
if terse:
for line in hits:
print(line)
else:
for line in var_lines + hit_lines:
print_msg(line, log=_log, silent=silent, prefix=False)


def compute_checksum(path, checksum_type=DEFAULT_CHECKSUM):
Expand Down
15 changes: 11 additions & 4 deletions easybuild/tools/options.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,13 +362,16 @@ def informative_options(self):
'choice', 'store_or_None', 'simple', ['simple', 'detailed']),
'list-toolchains': ("Show list of known toolchains",
None, 'store_true', False),
'search': ("Search for easyconfig files in the robot directory, print full paths",
None, 'store', None, {'metavar': 'STR'}),
'search-short': ("Search for easyconfig files in the robot directory, print short paths",
None, 'store', None, 'S', {'metavar': 'STR'}),
'search': ("Search for easyconfig files in the robot search path, print full paths",
None, 'store', None, {'metavar': 'REGEX'}),
'search-filename': ("Search for easyconfig files in the robot search path, print only filenames",
None, 'store', None, {'metavar': 'REGEX'}),
'search-short': ("Search for easyconfig files in the robot search path, print short paths",
None, 'store', None, 'S', {'metavar': 'REGEX'}),
'show-default-configfiles': ("Show list of default config files", None, 'store_true', False),
'show-default-moduleclasses': ("Show default module classes with description",
None, 'store_true', False),
'terse': ("Terse output (machine-readable)", None, 'store_true', False),
})

self.log.debug("informative_options: descr %s opts %s" % (descr, opts))
Expand Down Expand Up @@ -566,6 +569,10 @@ def postprocess(self):
if not HAVE_AUTOPEP8:
raise EasyBuildError("Python 'autopep8' module required to reformat dumped easyconfigs as requested")

# imply --terse for --last-log to avoid extra output that gets in the way
if self.options.last_log:
self.options.terse = True

self._postprocess_external_modules_metadata()

self._postprocess_config()
Expand Down
5 changes: 3 additions & 2 deletions easybuild/tools/robot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ def resolve_dependencies(easyconfigs, retain_all_deps=False, minimal_toolchains=
return ordered_ecs


def search_easyconfigs(query, short=False):
def search_easyconfigs(query, short=False, filename_only=False, terse=False):
"""Search for easyconfigs, if a query is provided."""
robot_path = build_option('robot_path')
if robot_path:
Expand All @@ -267,4 +267,5 @@ def search_easyconfigs(query, short=False):
search_path = [os.getcwd()]
ignore_dirs = build_option('ignore_dirs')
silent = build_option('silent')
search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, silent=silent)
search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, silent=silent, filename_only=filename_only,
terse=terse)
98 changes: 65 additions & 33 deletions test/framework/options.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -559,65 +559,97 @@ def test_list_easyblocks(self):
def test_search(self):
"""Test searching for easyconfigs."""

fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
os.close(fd)
test_easyconfigs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')

# simple search
args = [
'--search=gzip',
'--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'),
'--unittest-file=%s' % self.logfile,
'--robot=%s' % test_easyconfigs_dir,
]
self.eb_main(args, logfile=dummylogfn)
logtxt = read_file(self.logfile)
self.mock_stdout(True)
self.eb_main(args, testing=False)
txt = self.get_stdout()
self.mock_stdout(False)

info_msg = r"Searching \(case-insensitive\) for 'gzip' in"
self.assertTrue(re.search(info_msg, logtxt), "Info message when searching for easyconfigs in '%s'" % logtxt)
self.assertTrue(re.search(info_msg, txt), "Info message when searching for easyconfigs in '%s'" % txt)
for ec in ["gzip-1.4.eb", "gzip-1.4-GCC-4.6.3.eb"]:
self.assertTrue(re.search(r" \* \S*%s$" % ec, logtxt, re.M), "Found easyconfig %s in '%s'" % (ec, logtxt))

if os.path.exists(dummylogfn):
os.remove(dummylogfn)

write_file(self.logfile, '')
regex = re.compile(r" \* \S*%s$" % ec, re.M)
self.assertTrue(regex.search(txt), "Found pattern '%s' in: %s" % (regex.pattern, txt))

# search w/ regex
args = [
'--search=^gcc.*2.eb',
'--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'),
'--unittest-file=%s' % self.logfile,
'--robot=%s' % test_easyconfigs_dir,
]
self.eb_main(args, logfile=dummylogfn)
logtxt = read_file(self.logfile)
self.mock_stdout(True)
self.eb_main(args, testing=False)
txt = self.get_stdout()
self.mock_stdout(False)

info_msg = r"Searching \(case-insensitive\) for '\^gcc.\*2.eb' in"
self.assertTrue(re.search(info_msg, logtxt), "Info message when searching for easyconfigs in '%s'" % logtxt)
self.assertTrue(re.search(info_msg, txt), "Info message when searching for easyconfigs in '%s'" % txt)
for ec in ['GCC-4.7.2.eb', 'GCC-4.8.2.eb', 'GCC-4.9.2.eb']:
self.assertTrue(re.search(r" \* \S*%s$" % ec, logtxt, re.M), "Found easyconfig %s in '%s'" % (ec, logtxt))
regex = re.compile(r" \* \S*%s$" % ec, re.M)
self.assertTrue(regex.search(txt), "Found pattern '%s' in: %s" % (regex.pattern, txt))

if os.path.exists(dummylogfn):
os.remove(dummylogfn)
gcc_ecs = [
'GCC-4.6.3.eb',
'GCC-4.6.4.eb',
'GCC-4.7.2.eb',
'GCC-4.8.2.eb',
'GCC-4.8.3.eb',
'GCC-4.9.2.eb',
]

write_file(self.logfile, '')
# test --search-filename
args = [
'--search-filename=^gcc',
'--robot=%s' % test_easyconfigs_dir,
]
self.mock_stdout(True)
self.eb_main(args, testing=False)
txt = self.get_stdout()
self.mock_stdout(False)

for ec in gcc_ecs:
regex = re.compile(r"^ \* %s$" % ec, re.M)
self.assertTrue(regex.search(txt), "Found pattern '%s' in: %s" % (regex.pattern, txt))

# test --search-filename --terse
args = [
'--search-filename=^gcc',
'--terse',
'--robot=%s' % test_easyconfigs_dir,
]
self.mock_stdout(True)
self.eb_main(args, testing=False)
txt = self.get_stdout()
self.mock_stdout(False)

for ec in gcc_ecs:
regex = re.compile(r"^%s$" % ec, re.M)
self.assertTrue(regex.search(txt), "Found pattern '%s' in: %s" % (regex.pattern, txt))

# also test --search-short/-S
for search_arg in ['-S', '--search-short']:
open(self.logfile, 'w').write('')
args = [
search_arg,
'toy-0.0',
'-r',
os.path.join(os.path.dirname(__file__), 'easyconfigs'),
'--unittest-file=%s' % self.logfile,
test_easyconfigs_dir,
]
self.eb_main(args, logfile=dummylogfn, raise_error=True, verbose=True)
logtxt = read_file(self.logfile)
self.mock_stdout(True)
self.eb_main(args, raise_error=True, verbose=True, testing=False)
txt = self.get_stdout()
self.mock_stdout(False)

info_msg = r"Searching \(case-insensitive\) for 'toy-0.0' in"
self.assertTrue(re.search(info_msg, logtxt), "Info message when searching for easyconfigs in '%s'" % logtxt)
self.assertTrue(re.search('INFO CFGS\d+=', logtxt), "CFGS line message found in '%s'" % logtxt)
self.assertTrue(re.search(info_msg, txt), "Info message when searching for easyconfigs in '%s'" % txt)
self.assertTrue(re.search('^CFGS\d+=', txt, re.M), "CFGS line message found in '%s'" % txt)
for ec in ["toy-0.0.eb", "toy-0.0-multiple.eb"]:
self.assertTrue(re.search(" \* \$CFGS\d+/*%s" % ec, logtxt), "Found easyconfig %s in '%s'" % (ec, logtxt))

if os.path.exists(dummylogfn):
os.remove(dummylogfn)
regex = re.compile(r" \* \$CFGS\d+/*%s" % ec, re.M)
self.assertTrue(regex.search(txt), "Found pattern '%s' in: %s" % (regex.pattern, txt))

def test_dry_run(self):
"""Test dry run (long format)."""
Expand Down