use separate different progress bars for different aspects of the installations being performed

easybuild/framework/easyblock.py

@@ -89,6 +89,8 @@
 from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
 from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root
 from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name
+from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS
+from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.package.utilities import package
 from easybuild.tools.py2vs3 import extract_method_name, string_type
 from easybuild.tools.repository.repository import init_repository
@@ -304,21 +306,6 @@ def close_log(self):
         self.log.info("Closing log for application name %s version %s" % (self.name, self.version))
         fancylogger.logToFile(self.logfile, enable=False)
 
-    def set_progress_bar(self, progress_bar, task_id):
-        """
-        Set progress bar, the progress bar is needed when writing messages so
-        that the progress counter is always at the bottom
-        """
-        self.progress_bar = progress_bar
-        self.pbar_task = task_id
-
-    def advance_progress(self, tick=1.0):
-        """
-        Advance the progress bar forward with `tick`
-        """
-        if self.progress_bar and self.pbar_task is not None:
-            self.progress_bar.advance(self.pbar_task, tick)
-
     #
     # DRY RUN UTILITIES
     #
@@ -703,6 +690,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
         """
         srcpaths = source_paths()
 
+        update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
+
         # should we download or just try and find it?
         if re.match(r"^(https?|ftp)://", filename):
             # URL detected, so let's try and download it
@@ -1934,6 +1923,27 @@ def fetch_step(self, skip_checksums=False):
                 raise EasyBuildError("EasyBuild-version %s is newer than the currently running one. Aborting!",
                                      easybuild_version)
 
+        # count number of files to download: sources + patches (incl. extensions)
+        # FIXME: to make this count fully correct, we need the Extension instances first,
+        # which requires significant refactoring in fetch_extension_sources
+        # (also needed to fix https://github.com/easybuilders/easybuild-framework/issues/3849)
+        cnt = len(self.cfg['sources']) + len(self.cfg['patches'])
+        if self.cfg['exts_list']:
+            for ext in self.cfg['exts_list']:
+                if isinstance(ext, tuple) and len(ext) >= 3:
+                    ext_opts = ext[2]
+                    if 'source' in ext_opts:
+                        cnt += 1
+                    elif 'sources' in ext_opts:
+                        cnt += len(ext_opts['sources'])
+                    else:
+                        # assume there's always one source file;
+                        # for extensions using PythonPackage, no 'source' or 'sources' may be specified
+                        cnt += 1
+                    cnt += len(ext_opts.get('patches', []))
+
+        start_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, cnt)
+
         if self.dry_run:
 
             self.dry_run_msg("Available download URLs for sources/patches:")
@@ -2016,6 +2026,8 @@ def fetch_step(self, skip_checksums=False):
         else:
             self.log.info("Skipped installation dirs check per user request")
 
+        stop_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL)
+
     def checksum_step(self):
         """Verify checksum of sources and patches, if a checksum is available."""
         for fil in self.src + self.patches:
@@ -2429,15 +2441,22 @@ def extensions_step(self, fetch=False, install=True):
             self.skip_extensions()
 
         exts_cnt = len(self.ext_instances)
+
+        start_progress_bar(PROGRESS_BAR_EXTENSIONS, exts_cnt)
+
         for idx, ext in enumerate(self.ext_instances):
 
             self.log.debug("Starting extension %s" % ext.name)
 
             # always go back to original work dir to avoid running stuff from a dir that no longer exists
             change_dir(self.orig_workdir)
 
+            progress_label = "Installing '%s' extension" % ext.name
+            update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label)
+
             tup = (ext.name, ext.version or '', idx + 1, exts_cnt)
             print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent)
+
             start_time = datetime.now()
 
             if self.dry_run:
@@ -2473,6 +2492,8 @@ def extensions_step(self, fetch=False, install=True):
                         elif self.logdebug or build_option('trace'):
                             print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent)
 
+        stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False)
+
         # cleanup (unload fake module, remove fake module dir)
         if fake_mod_data:
             self.clean_up_fake_module(fake_mod_data)
@@ -3475,6 +3496,7 @@ def run_step(self, step, step_methods):
         run_hook(step, self.hooks, post_step_hook=True, args=[self])
 
         if self.cfg['stop'] == step:
+            update_progress_bar(PROGRESS_BAR_EASYCONFIG)
             self.log.info("Stopping after %s step.", step)
             raise StopException(step)
 
@@ -3587,8 +3609,16 @@ def run_all_steps(self, run_test_cases):
             return True
 
         steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt())
-        # Calculate progress bar tick
-        tick = 1.0 / float(len(steps))
+
+        # figure out how many steps will actually be run (not be skipped)
+        step_cnt = 0
+        for (step_name, _, _, skippable) in steps:
+            if not self.skip_step(step_name, skippable):
+                step_cnt += 1
+            if self.cfg['stop'] == step_name:
+                break
+
+        start_progress_bar(PROGRESS_BAR_EASYCONFIG, step_cnt, label="Installing %s" % self.full_mod_name)
 
         print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent)
         trace_msg("installation prefix: %s" % self.installdir)
@@ -3608,7 +3638,7 @@ def run_all_steps(self, run_test_cases):
             create_lock(lock_name)
 
         try:
-            for (step_name, descr, step_methods, skippable) in steps:
+            for step_name, descr, step_methods, skippable in steps:
                 if self.skip_step(step_name, skippable):
                     print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent)
                 else:
@@ -3627,14 +3657,18 @@ def run_all_steps(self, run_test_cases):
                                 print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent)
                             elif self.logdebug or build_option('trace'):
                                 print_msg("... (took < 1 sec)", log=self.log, silent=self.silent)
-                self.advance_progress(tick)
+
+                    progress_label = "Installing %s: %s" % (self.full_mod_name, descr)
+                    update_progress_bar(PROGRESS_BAR_EASYCONFIG, label=progress_label)
 
         except StopException:
             pass
         finally:
             if not ignore_locks:
                 remove_lock(lock_name)
 
+        stop_progress_bar(PROGRESS_BAR_EASYCONFIG)
+
         # return True for successfull build (or stopped build)
         return True
 
@@ -3653,7 +3687,7 @@ def print_dry_run_note(loc, silent=True):
     dry_run_msg(msg, silent=silent)
 
 
-def build_and_install_one(ecdict, init_env, progress_bar=None, task_id=None):
+def build_and_install_one(ecdict, init_env):
     """
     Build the software
     :param ecdict: dictionary contaning parsed easyconfig + metadata
@@ -3701,11 +3735,6 @@ def build_and_install_one(ecdict, init_env, progress_bar=None, task_id=None):
         print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg),
                     silent=silent)
 
-    # Setup progress bar
-    if progress_bar and task_id is not None:
-        app.set_progress_bar(progress_bar, task_id)
-        _log.info("Updated progress bar instance for easyblock %s", easyblock)
-
     # application settings
     stop = build_option('stop')
     if stop is not None:

easybuild/main.py

@@ -68,7 +68,8 @@
 from easybuild.tools.hooks import START, END, load_hooks, run_hook
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.options import set_up_configuration, use_color
-from easybuild.tools.output import create_progress_bar, print_checks
+from easybuild.tools.output import PROGRESS_BAR_OVERALL, print_checks, rich_live_cm
+from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs
 from easybuild.tools.package.utilities import check_pkg_support
 from easybuild.tools.parallelbuild import submit_jobs
@@ -101,36 +102,27 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=
     return [(ec_file, generated)]
 
 
-def build_and_install_software(ecs, init_session_state, exit_on_failure=True, progress_bar=None):
+def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
     """
     Build and install software for all provided parsed easyconfig files.
 
     :param ecs: easyconfig files to install software with
     :param init_session_state: initial session state, to use in test reports
     :param exit_on_failure: whether or not to exit on installation failure
-    :param progress_bar: progress bar to use to report progress
     """
     # obtain a copy of the starting environment so each build can start afresh
     # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since
     # e.g. via easyconfig.handle_allowed_system_deps
     init_env = copy.deepcopy(os.environ)
 
-    # Initialize progress bar with overall installation task
-    if progress_bar:
-        task_id = progress_bar.add_task("", total=len(ecs))
-    else:
-        task_id = None
+    start_progress_bar(PROGRESS_BAR_OVERALL, size=len(ecs))
 
     res = []
     for ec in ecs:
 
-        if progress_bar:
-            progress_bar.update(task_id, description=ec['short_mod_name'])
-
         ec_res = {}
         try:
-            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progress_bar=progress_bar,
-                                                                      task_id=task_id)
+            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env)
             ec_res['log_file'] = app_log
             if not ec_res['success']:
                 ec_res['err'] = EasyBuildError(err)
@@ -169,6 +161,10 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr
 
         res.append((ec, ec_res))
 
+        update_progress_bar(PROGRESS_BAR_OVERALL)
+
+    stop_progress_bar(PROGRESS_BAR_OVERALL)
+
     return res
 
 
@@ -540,11 +536,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     if not testing or (testing and do_build):
         exit_on_failure = not (options.dump_test_report or options.upload_test_report)
 
-        progress_bar = create_progress_bar()
-        with progress_bar:
+        with rich_live_cm():
             ecs_with_res = build_and_install_software(ordered_ecs, init_session_state,
-                                                      exit_on_failure=exit_on_failure,
-                                                      progress_bar=progress_bar)
+                                                      exit_on_failure=exit_on_failure)
     else:
         ecs_with_res = [(ec, {}) for ec in ordered_ecs]
 

easybuild/tools/filetools.py

@@ -54,13 +54,15 @@
 import tempfile
 import time
 import zlib
+from functools import partial
 
 from easybuild.base import fancylogger
 from easybuild.tools import run
 # import build_log must stay, to use of EasyBuildLog
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning
 from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN
 from easybuild.tools.config import build_option, install_path
+from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type
 from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars
 
@@ -215,7 +217,8 @@ def read_file(path, log_error=True, mode='r'):
     return txt
 
 
-def write_file(path, data, append=False, forced=False, backup=False, always_overwrite=True, verbose=False):
+def write_file(path, data, append=False, forced=False, backup=False, always_overwrite=True, verbose=False,
+               show_progress=False, size=None):
     """
     Write given contents to file at given path;
     overwrites current file contents without backup by default!
@@ -227,6 +230,8 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over
     :param backup: back up existing file before overwriting or modifying it
     :param always_overwrite: don't require --force to overwrite an existing file
     :param verbose: be verbose, i.e. inform where backup file was created
+    :param show_progress: show progress bar while writing file
+    :param size: size (in bytes) of data to write (used for progress bar)
     """
     # early exit in 'dry run' mode
     if not forced and build_option('extended_dry_run'):
@@ -256,15 +261,30 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over
     if sys.version_info[0] >= 3 and (isinstance(data, bytes) or data_is_file_obj):
         mode += 'b'
 
+    # don't bother showing a progress bar for small files (< 10MB)
+    if size and size < 10 * (1024 ** 2):
+        _log.info("Not showing progress bar for downloading small file (size %s)", size)
+        show_progress = False
+
+    if show_progress:
+        start_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE, size, label=os.path.basename(path))
+
     # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block
     try:
         mkdir(os.path.dirname(path), parents=True)
         with open_file(path, mode) as fh:
             if data_is_file_obj:
-                # if a file-like object was provided, use copyfileobj (which reads the file in chunks)
-                shutil.copyfileobj(data, fh)
+                # if a file-like object was provided, read file in 1MB chunks
+                for chunk in iter(partial(data.read, 1024 ** 2), b''):
+                    fh.write(chunk)
+                    if show_progress:
+                        update_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE, progress_size=len(chunk))
             else:
                 fh.write(data)
+
+        if show_progress:
+            stop_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE)
+
     except IOError as err:
         raise EasyBuildError("Failed to write to %s: %s", path, err)
 
@@ -701,6 +721,22 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header
     return urlpat_headers
 
 
+def det_file_size(http_header):
+    """
+    Determine size of file from provided HTTP header info (without downloading it).
+    """
+    res = None
+    len_key = 'Content-Length'
+    if len_key in http_header:
+        size = http_header[len_key]
+        try:
+            res = int(size)
+        except (ValueError, TypeError) as err:
+            _log.warning("Failed to interpret size '%s' as integer value: %s", size, err)
+
+    return res
+
+
 def download_file(filename, url, path, forced=False):
     """Download a file from the given URL, to the specified path."""
 
@@ -757,19 +793,24 @@ def download_file(filename, url, path, forced=False):
                 # urllib does not!
                 url_fd = std_urllib.urlopen(url_req, timeout=timeout)
                 status_code = url_fd.getcode()
+                size = det_file_size(url_fd.info())
             else:
                 response = requests.get(url, headers=headers, stream=True, timeout=timeout)
                 status_code = response.status_code
                 response.raise_for_status()
+                size = det_file_size(response.headers)
                 url_fd = response.raw
                 url_fd.decode_content = True
-            _log.debug('response code for given url %s: %s' % (url, status_code))
+
+            _log.debug("HTTP response code for given url %s: %s", url, status_code)
+            _log.info("File size for %s: %s", url, size)
+
             # note: we pass the file object to write_file rather than reading the file first,
             # to ensure the data is read in chunks (which prevents problems in Python 3.9+);
             # cfr. https://github.com/easybuilders/easybuild-framework/issues/3455
             # and https://bugs.python.org/issue42853
-            write_file(path, url_fd, forced=forced, backup=True)
-            _log.info("Downloaded file %s from url %s to %s" % (filename, url, path))
+            write_file(path, url_fd, forced=forced, backup=True, show_progress=True, size=size)
+            _log.info("Downloaded file %s from url %s to %s", filename, url, path)
             downloaded = True
             url_fd.close()
         except used_urllib.HTTPError as err: