Singularity runtime

.github/actions/run-integration-tests/action.yaml

@@ -23,6 +23,10 @@ runs:
       shell: bash -l -eo pipefail {0}
       run: nextstrain setup conda
 
+    - if: runner.os != 'macOS' && runner.os != 'Windows'
+      shell: bash -l -eo pipefail {0}
+      run: nextstrain setup singularity
+
     - shell: bash -l -eo pipefail {0}
       run: nextstrain check-setup --set-default
 
@@ -43,6 +47,13 @@ runs:
         git -C zika-tutorial clean -dfqx
         nextstrain build --conda --cpus 2 zika-tutorial
 
+    - if: runner.os != 'macOS' && runner.os != 'Windows'
+      name: Build zika-tutorial with --singularity
+      shell: bash -l -eo pipefail {0}
+      run: |
+        git -C zika-tutorial clean -dfqx
+        nextstrain build --singularity --cpus 2 zika-tutorial
+
     - if: runner.os != 'Windows'
       name: Build zika-tutorial with --ambient
       shell: bash -l -eo pipefail {0}

.github/actions/setup-integration-tests/action.yaml

@@ -68,6 +68,34 @@ runs:
         python3 --version | grep -F 'Python ${{ inputs.python-version }}.'
         [[ "$(python --version)" == "$(python3 --version)" ]]
 
+    # Install Singularity on Linux.
+    #
+    # We don't install it with Conda because Conda Forge provides a non-suid
+    # build of Singularity.  We're compatible with Singularity's non-suid mode,
+    # but production usages of Singularity are likely to use its suid mode, so
+    # I'd rather test against that.
+    #   -trs, 6 Jan 2023
+    - if: runner.os == 'Linux'
+      shell: bash -l -eo pipefail {0}
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        # Work in a temp dir to avoid cluttering the caller's working dir
+        pushd "$(mktemp -d)"
+        export "$(grep UBUNTU_CODENAME /etc/os-release)"
+
+        # Download latest SingularityCE .deb for this version of Ubuntu
+        url="$(
+          curl -fsSL --proto '=https' -H "Authorization: Bearer $GITHUB_TOKEN" \
+            https://api.github.com/repos/sylabs/singularity/releases/latest \
+              | jq -r '.assets | map(select(.name | endswith("\(env.UBUNTU_CODENAME)_amd64.deb"))) | .[0].browser_download_url')"
+
+        curl -fsSL --proto '=https' "$url" > singularity.deb
+
+        # Install and check that it runs
+        sudo dpkg -i singularity.deb
+        singularity --version
+
     # Clone the small build we'll use as an integration test case.
     - run: git clone https://github.com/nextstrain/zika-tutorial
       shell: bash -l -eo pipefail {0}

CHANGES.md

@@ -13,6 +13,20 @@ development source code and as such may not be routinely kept up to date.
 
 # __NEXT__
 
+## Improvements
+
+* We've added a new Singularity runtime based on our existing Docker runtime.
+
+  Singularity is a container system freely-available for Linux platforms.  It
+  is commonly available on institutional HPC systems as an alternative to
+  Docker, which is often not supported on such systems.  When you use
+  Singularity with the Nextstrain CLI, you don't need to install any other
+  Nextstrain software dependencies as validated versions are already bundled
+  into a container image by the Nextstrain team.
+
+  Run `nextstrain setup singularity` to get started.
+  ([#248](https://github.com/nextstrain/cli/pull/248))
+
 
 # 6.0.3 (17 January 2023)
 

doc/aws-batch.md

@@ -19,13 +19,13 @@ Batch job, monitors the job status, streams the job logs to your terminal, and
 downloads build results back to the `zika-tutorial/` directory.
 
 The interface aims to be very similar to that of local builds (run in the
-Docker, Conda, or ambient runtimes), so the `nextstrain build` command stays in
-the foreground and result files are written back directly to the local build
-directory.  Alternatively, you can specify the `--detach` option to run AWS
-Batch builds in the background once they're submitted.  The Nextstrain CLI will
-tell you how to reattach to the build later to view the logs and download the
-results.  If you forget to use the `--detach` option, you can press Control-Z
-to detach at any point once the build is submitted.
+Docker, Conda, Singularity, or ambient runtimes), so the `nextstrain build`
+command stays in the foreground and result files are written back directly to
+the local build directory.  Alternatively, you can specify the `--detach`
+option to run AWS Batch builds in the background once they're submitted.  The
+Nextstrain CLI will tell you how to reattach to the build later to view the
+logs and download the results.  If you forget to use the `--detach` option, you
+can press Control-Z to detach at any point once the build is submitted.
 
 [AWS Batch]: https://aws.amazon.com/batch/
 [`zika-tutorial/` directory]: https://github.com/nextstrain/zika-tutorial

doc/installation.rst

@@ -116,7 +116,8 @@ Runtimes
 The Nextstrain CLI provides a consistent interface and computing environment
 for running and visualizing Nextstrain pathogen builds across several different
 computing platforms, such as `Docker <https://docker.com>`__, `Conda
-<https://docs.conda.io/en/latest/miniconda.html>`__, and `AWS Batch
+<https://docs.conda.io/en/latest/miniconda.html>`__,
+:ref:`Singularity <installation/singularity>`, and `AWS Batch
 <https://aws.amazon.com/batch/>`__.
 
 We call the provided computing environments the :term:`Nextstrain runtimes
@@ -131,8 +132,9 @@ At least one of these runtimes must be setup in order for many of
 The default runtime is Docker, using the `nextstrain/base`_ container image.
 Containers provide a tremendous amount of benefit for scientific workflows by
 isolating dependencies and increasing reproducibility. However, they're not
-always appropriate, so a Conda runtime and "ambient" runtime are also supported.
-The installation and setup of supported runtimes is described below.
+always appropriate, so a Conda runtime, Singularity runtime, and "ambient"
+runtime are also supported.  The installation and setup of supported runtimes
+is described below.
 
 .. _nextstrain/base: https://github.com/nextstrain/docker-base
 
@@ -173,6 +175,34 @@ This runtime is not directly supported on Windows, but you can use `WSL2
 <https://docs.microsoft.com/en-us/windows/wsl/wsl2-index>`__ to "switch" to
 Linux and run the above setup command.
 
+.. _installation/singularity:
+
+Singularity
+-----------
+
+Singularity is a container system freely-available for Linux platforms.  It is
+commonly available on institutional HPC systems as an alternative to Docker,
+which is often not supported on such systems.  When you use Singularity with
+the Nextstrain CLI, you don't need to install any other Nextstrain software
+dependencies as validated versions are already bundled into a container image
+by the Nextstrain team.
+
+Run ``nextstrain setup singularity`` to get started.
+
+Note that the Singularity project forked into two separate projects in late
+2021: `SingularityCE`_ under `Sylabs`_ and `Apptainer`_ under the `Linux
+Foundation`_.  Either fork should work with Nextstrain CLI, as both projects
+still provide very similar interfaces and functionality via the ``singularity``
+command.  You can read `Sylab's announcement`_ and `Apptainer's announcement`_
+for more information on the fork.
+
+.. _SingularityCE: https://sylabs.io/singularity/
+.. _Sylabs: https://sylabs.io/
+.. _Apptainer: https://apptainer.org
+.. _Linux Foundation: https://www.linuxfoundation.org/
+.. _Sylab's announcement: https://sylabs.io/2022/06/singularityce-is-singularity/
+.. _Apptainer's announcement: https://apptainer.org/news/community-announcement-20211130
+
 Ambient
 -------
 
@@ -238,6 +268,10 @@ based on what's available. You should see output similar to the following:
    ✔ yes: augur is installed and runnable
    ✔ yes: auspice is installed and runnable
 
+   # singularity is supported
+   ✔ yes: singularity is installed
+   ✔ yes: singularity works
+
    # ambient is not supported
    ✔ yes: snakemake is installed and runnable
    ✘ no: augur is installed and runnable
@@ -248,15 +282,16 @@ based on what's available. You should see output similar to the following:
    ✘ no: job queue "nextstrain-job-queue" exists
    ✘ no: S3 bucket "nextstrain-jobs" exists
 
-   All good!  Supported Nextstrain runtimes: docker, conda
+   All good!  Supported Nextstrain runtimes: docker, conda, singularity
 
    Setting default runtime to docker.
 
 If the output doesn't say "All good!" and list at least one supported
-Nextstrain runtime (typically Docker, Conda, or ambient), then something may be
-wrong with your installation.
+Nextstrain runtime (typically Docker, Conda, Singularity, or ambient), then
+something may be wrong with your installation.
 
 The default is written to the :file:`~/.nextstrain/config` file. If multiple
 runtimes are supported, you can override the default for specific runs
-using command-line options such as ``--docker``, ``--conda``, ``--ambient``,
-and ``--aws-batch``, e.g. ``nextstrain build --ambient …``.
+using command-line options such as ``--docker``, ``--conda``,
+``--singularity``, ``--ambient``, and ``--aws-batch``, e.g. ``nextstrain build
+--ambient …``.

nextstrain/cli/argparse.py

@@ -178,12 +178,12 @@ def runner_module_argument(name: str) -> RunnerModule:
     >>> runner_module_argument("invalid")
     Traceback (most recent call last):
         ...
-    argparse.ArgumentTypeError: invalid runtime name: 'invalid'; valid names are: 'docker', 'conda', 'ambient', 'aws-batch'
+    argparse.ArgumentTypeError: invalid runtime name: 'invalid'; valid names are: 'docker', 'conda', 'singularity', 'ambient', 'aws-batch'
 
     >>> runner_module_argument("Invalid Name")
     Traceback (most recent call last):
         ...
-    argparse.ArgumentTypeError: invalid runtime name: 'Invalid Name' (normalized to 'invalid-name'); valid names are: 'docker', 'conda', 'ambient', 'aws-batch'
+    argparse.ArgumentTypeError: invalid runtime name: 'Invalid Name' (normalized to 'invalid-name'); valid names are: 'docker', 'conda', 'singularity', 'ambient', 'aws-batch'
     """
     try:
         return runner_module(name)

nextstrain/cli/command/check_setup.py

@@ -1,7 +1,7 @@
 """
 Checks for supported runtimes.
 
-Four runtimes are tested by default:
+Five runtimes are tested by default:
 
   • Our Docker image is the preferred runtime.  Docker itself must
     be installed and configured on your computer first, but once it is, the
@@ -12,6 +12,12 @@
     ambient runtime, but is less isolated and robust than the Docker
     runtime.
 
+  • Our Singularity runtime uses the same container image as our Docker
+    runtime.  Singularity must be installed and configured on your computer
+    first, although it is often already present on HPC systems.  This runtime
+    is more isolated and reproducible than the Conda runtime, but potentially
+    less so than the Docker runtime.
+
   • Your ambient setup will be tested for snakemake, augur, and auspice.
     Their presence implies a working runtime, but does not guarantee
     it.

nextstrain/cli/command/shell.py

@@ -9,7 +9,7 @@
 from ..argparse import add_extended_help_flags
 from ..errors import UserError
 from ..paths import SHELL_HISTORY
-from ..runner import docker, conda
+from ..runner import docker, conda, singularity
 from ..util import colored, remove_prefix, runner_name, warn
 from ..volume import store_volume, NamedVolume
 
@@ -37,7 +37,7 @@ def register_parser(subparser):
     runner.register_runners(
         parser,
         exec    = ["bash", ...],
-        runners = [docker, conda])
+        runners = [docker, conda, singularity])
 
     return parser
 
@@ -55,35 +55,35 @@ def run(opts):
 
     overlay_volumes = [v for v in opts.volumes if v is not opts.build]
 
-    if overlay_volumes and opts.__runner__ is not docker:
+    if overlay_volumes and opts.__runner__ not in {docker, singularity}:
         raise UserError(f"""
             The {runner_name(opts.__runner__)} runtime does not support overlays (e.g. of {overlay_volumes[0].name}).
-            Use the Docker runtime (--docker) if overlays are necessary.
+            Use the Docker or Singularity runtimes (via --docker or --singularity) if overlays are necessary.
             """)
 
     print(colored("bold", "Entering the Nextstrain runtime"))
     print()
 
-    if opts.volumes and opts.__runner__ is docker:
+    if opts.volumes and opts.__runner__ in {docker, singularity}:
         print(colored("bold", "Mapped volumes:"))
 
-        # This is more tightly coupled to the Docker runner than I'd like (i.e.
-        # assuming /nextstrain/…), but right now that's the only runner this
-        # command supports (and the only one it makes sense to).
-        #   -trs, 25 Sept 2018
+        # This is more tightly coupled to the Docker/Singularity runners than
+        # I'd like (i.e.  assuming /nextstrain/…), but the number of runtimes
+        # will always be small so some special-casing seems ok.
+        #   -trs, 5 Jan 2023 (updated from 25 Sept 2018)
         for volume in opts.volumes:
-            print("  /nextstrain/%s is from %s" % (volume.name, volume.src.resolve(strict = True)))
+            print("  %s is from %s" % (docker.mount_point(volume), volume.src.resolve(strict = True))) # type: ignore
 
         print()
 
     print(colored("bold", 'Run the command "exit" to leave the runtime.'))
     print()
 
     with resources.as_file("bashrc") as bashrc:
-        # Ensure the history file exists to pass checks the Docker runner
-        # performs for mounted volumes.  This also makes sure that the file is
-        # writable by the Conda runtime too by ensuring the parent directory
-        # exists.
+        # Ensure the history file exists to pass checks the Docker/Singularity
+        # runners perform for mounted volumes.  This also makes sure that the
+        # file is writable by the Conda runtime too by ensuring the parent
+        # directory exists.
         #
         # Don't use strict=True because it's ok if it doesn't exist yet!
         history_file = SHELL_HISTORY.resolve()
@@ -101,7 +101,7 @@ def run(opts):
                 "--rcfile", str(bashrc),
             ]
 
-        elif opts.__runner__ is docker:
+        elif opts.__runner__ in {docker, singularity}:
             opts.volumes.append(NamedVolume("bashrc", bashrc, dir = False, writable = False))
 
             history_volume = NamedVolume("bash_history", history_file, dir = False)

nextstrain/cli/command/update.py

@@ -5,8 +5,8 @@
 without arguments.  Provide a runtime name as an argument to update a specific
 runtime instead.
 
-Only two runtimes currently support updates: Docker and Conda.  Both may take
-several minutes as new software versions are downloaded.
+Three runtimes currently support updates: Docker, Conda, and Singularity.
+Updates may take several minutes as new software versions are downloaded.
 
 This command also checks for newer versions of the Nextstrain CLI (the
 ``nextstrain`` program) itself and will suggest upgrade instructions if an

nextstrain/cli/command/view.py

@@ -55,7 +55,7 @@
 from typing import Iterable, NamedTuple, Tuple, Union
 from .. import runner
 from ..argparse import add_extended_help_flags, SUPPRESS, SKIP_AUTO_DEFAULT_IN_HELP
-from ..runner import docker, ambient, conda
+from ..runner import docker, ambient, conda, singularity
 from ..util import colored, remove_suffix, warn
 from ..volume import NamedVolume
 
@@ -136,7 +136,7 @@ def register_parser(subparser):
     runner.register_runners(
         parser,
         exec    = ["auspice", "view", "--verbose", "--datasetDir=.", "--narrativeDir=."],
-        runners = [docker, ambient, conda])
+        runners = [docker, ambient, conda, singularity])
 
     return parser
 
@@ -176,7 +176,7 @@ def run(opts):
 
     # A volume which will be our working dir.
     working_volume = NamedVolume("auspice/data", data_dir)
-    opts.volumes.append(working_volume) # for Docker
+    opts.volumes.append(working_volume) # for Docker and Singularity
 
     # If auspice/ exists, then use it for datasets.  Otherwise, look for
     # datasets in the given dir.