diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index c3faf7be00..60e936bac9 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -429,7 +429,8 @@ def change_dir(path): return cwd -def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced=False, change_into_dir=False): +def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced=False, change_into_dir=False, + trace=True): """ Extract file at given path to specified directory :param fn: path to file to extract @@ -439,6 +440,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced :param overwrite: overwrite existing unpacked file :param forced: force extraction in (extended) dry run mode :param change_into_dir: change into resulting directorys + :param trace: produce trace output for extract command being run :return: path to directory (in case of success) """ @@ -468,7 +470,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced if extra_options: cmd = "%s %s" % (cmd, extra_options) - run.run_cmd(cmd, simple=True, force_in_dry_run=forced) + run.run_cmd(cmd, simple=True, force_in_dry_run=forced, trace=trace) # note: find_base_dir also changes into the base dir! base_dir = find_base_dir() @@ -739,7 +741,7 @@ def det_file_size(http_header): return res -def download_file(filename, url, path, forced=False): +def download_file(filename, url, path, forced=False, trace=True): """Download a file from the given URL, to the specified path.""" insecure = build_option('insecure_download') @@ -848,11 +850,13 @@ def download_file(filename, url, path, forced=False): if downloaded: _log.info("Successful download of file %s from url %s to path %s" % (filename, url, path)) - trace_msg("download succeeded: %s" % url) + if trace: + trace_msg("download succeeded: %s" % url) return path else: _log.warning("Download of %s to %s failed, done trying" % (url, path)) - trace_msg("download failed: %s" % url) + if trace: + trace_msg("download failed: %s" % url) return None diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 21e2552264..6fff6c2e53 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -243,7 +243,7 @@ def read(self, path, api=True): if not api: outfile = tempfile.mkstemp()[1] url = '/'.join([GITHUB_RAW, self.githubuser, self.reponame, self.branchname, path]) - download_file(os.path.basename(path), url, outfile) + download_file(os.path.basename(path), url, outfile, trace=False) return outfile else: obj = self.get_path(path).get(ref=self.branchname)[1] @@ -394,10 +394,10 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_M target_path = os.path.join(path, base_name) _log.debug("downloading repo %s/%s as archive from %s to %s" % (account, repo, url, target_path)) - download_file(base_name, url, target_path, forced=True) + download_file(base_name, url, target_path, forced=True, trace=False) _log.debug("%s downloaded to %s, extracting now" % (base_name, path)) - base_dir = extract_file(target_path, path, forced=True, change_into_dir=False) + base_dir = extract_file(target_path, path, forced=True, change_into_dir=False, trace=False) extracted_path = os.path.join(base_dir, extracted_dir_name) # check if extracted_path exists @@ -499,7 +499,7 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi # determine list of changed files via diff diff_fn = os.path.basename(pr_data['diff_url']) diff_filepath = os.path.join(path, diff_fn) - download_file(diff_fn, pr_data['diff_url'], diff_filepath, forced=True) + download_file(diff_fn, pr_data['diff_url'], diff_filepath, forced=True, trace=False) diff_txt = read_file(diff_filepath) _log.debug("Diff for PR #%s:\n%s", pr, diff_txt) @@ -535,7 +535,7 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi sha = pr_data['head']['sha'] full_url = URL_SEPARATOR.join([GITHUB_RAW, github_account, github_repo, sha, patched_file]) _log.info("Downloading %s from %s", fn, full_url) - download_file(fn, full_url, path=os.path.join(path, fn), forced=True) + download_file(fn, full_url, path=os.path.join(path, fn), forced=True, trace=False) final_path = path diff --git a/test/framework/filetools.py b/test/framework/filetools.py index da2ffb3c29..0267b20040 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -480,6 +480,14 @@ def test_download_file(self): except request.URLError: print("Skipping timeout test in test_download_file (working offline)") + # check whether disabling trace output works + target_location = os.path.join(self.test_prefix, 'test.txt') + with self.mocked_stdout_stderr(): + ft.download_file(fn, source_url, target_location, forced=True, trace=False) + stdout = self.get_stdout() + self.assertEqual(stdout, '') + ft.remove_file(target_location) + # also test behaviour of download_file under --dry-run build_options = { 'extended_dry_run': True, @@ -2288,16 +2296,23 @@ def test_extract_file(self): ft.change_dir(cwd) self.assertFalse(os.path.samefile(os.getcwd(), self.test_prefix)) - self.mock_stdout(True) - self.mock_stderr(True) - path = ft.extract_file(toy_tarball, self.test_prefix, change_into_dir=True) - stderr = self.get_stderr().strip() - self.mock_stderr(False) - self.mock_stdout(False) + with self.mocked_stdout_stderr(): + path = ft.extract_file(toy_tarball, self.test_prefix, change_into_dir=True) + stdout = self.get_stdout() + stderr = self.get_stderr() self.assertTrue(os.path.samefile(path, self.test_prefix)) self.assertTrue(os.path.samefile(os.getcwd(), self.test_prefix)) self.assertFalse(stderr) + self.assertTrue("running command" in stdout) + + # check whether disabling trace output works + with self.mocked_stdout_stderr(): + path = ft.extract_file(toy_tarball, self.test_prefix, change_into_dir=True, trace=False) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.assertFalse(stderr) + self.assertFalse(stdout) def test_remove(self): """Test remove_file, remove_dir and join remove functions."""