CI lockfiles

.cirrus.yml

@@ -79,11 +79,10 @@ linux_task_template: &LINUX_TASK_TEMPLATE
       - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${CARTOPY_CACHE_BUILD}"
   nox_cache:
     folder: ${CIRRUS_WORKING_DIR}/.nox
+    reupload_on_changes: true
     fingerprint_script:
       - echo "${CIRRUS_TASK_NAME}"
-      - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${NOX_CACHE_BUILD}"
-      - sha256sum ${CIRRUS_WORKING_DIR}/requirements/ci/py$(echo ${PY_VER} | tr -d ".").yml
-
+      - echo "${NOX_CACHE_BUILD}"
 
 #
 # YAML alias for compute credits.

.github/workflows/refresh-lockfiles.yml

@@ -0,0 +1,73 @@
+# This workflow periodically creates new environment lock files based on the newest
+# available dependency pacakges on conda-forge.
+#
+# For environments that have changed, a pull request will be made and submitted
+# to the master branch
+
+name: Refresh Lockfiles
+
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run once a week on a Saturday night 
+    - cron: 1 0 * * 6
+
+
+jobs:
+
+  gen_lockfiles:
+    # this is a matrix job: it splits to create new lockfiles for each
+    # of the CI test python versions.
+    # this list below should be changed when covering more python versions
+    # TODO: generate this matrix automatically from the list of available py**.yml files
+    #       ref: https://tomasvotruba.com/blog/2020/11/16/how-to-make-dynamic-matrix-in-github-actions/
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python: ['36', '37', '38']
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: install conda-lock
+        run: |
+          source $CONDA/bin/activate base
+          conda install -y -c conda-forge conda-lock
+      - name: generate lockfile
+        run: |
+          $CONDA/bin/conda-lock lock -p linux-64 -f requirements/ci/py${{matrix.python}}.yml
+          mv conda-linux-64.lock py${{matrix.python}}-linux-64.lock
+      - name: output lockfile
+        uses: actions/upload-artifact@v2
+        with:
+          path: py${{matrix.python}}-linux-64.lock
+
+  create_pr:
+    # once the matrix job has completed all the lock files will have been uploaded as artifacts.
+    # Download the artifacts, add them to the repo, and create a PR.
+    runs-on: ubuntu-latest
+    needs: gen_lockfiles
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: get artifacts
+        uses: actions/download-artifact@v2
+        with:
+          path: artifacts
+
+      - name: Update lock files in repo
+        run: | 
+          cp artifacts/artifact/*.lock requirements/ci/nox.lock
+          rm -r artifacts
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@052fc72b4198ba9fbc81b818c6e1859f747d49a8
+        with:
+          commit-message: Updated environment lockfiles
+          delete-branch: true
+          branch: lockfiles
+          branch-suffix: timestamp
+          title: Update CI environment lockfiles
+          body: |
+            Lockfiles updated to the latest resolvable environment.

Makefile

@@ -0,0 +1,2 @@
+lockfiles:
+	python tools/update_lockfiles.py -o requirements/ci/nox.lock requirements/ci/py*.yml

docs/src/developers_guide/contributing_ci_tests.rst

@@ -38,6 +38,36 @@ The above `cirrus-ci`_ tasks are run automatically against all `Iris`_ branches
 on GitHub whenever a pull-request is submitted, updated or merged. See the
 `Cirrus-CI Dashboard`_ for details of recent past and active Iris jobs.
 
+
+.. _cirrus_test_env:
+
+Cirrus CI Test environment
+--------------------------
+
+The test environment on the Cirrus-CI service is determined from the requirement files
+in `requirements/ci/py**.yml`.  These are conda envinroment files that list the entire
+set of build, test and run requirements for iris.
+
+For reproducible test results, these environments are resolved for all their dependencies
+and stored as lock files in `requirements/ci/nox.lock`.  The test environments will not
+resolve the dependencies each time, instead they will use the lock file to reproduce the
+same exact environment each time.
+
+**If you have updated the requirement yaml files with new dependencies, you will need to
+generate new lock files.** To do this, run the command::
+
+   python tools/update_lockfiles.py -o requirements/ci/nox.lock requirements/ci/py*.yml
+
+or simply::
+
+   make lockfiles
+
+and add the changed lockfiles to your pull request.
+
+New lockfiles are generated automatically each week to ensure that iris continues to be
+tested against the latest available version of its dependencies.
+
+
 .. _skipping Cirrus-CI tasks:
 
 Skipping Cirrus-CI Tasks

docs/src/developers_guide/contributing_pull_request_checklist.rst

@@ -29,6 +29,10 @@ is merged.  Before submitting a pull request please consider this list.
 #. **Check all modified and new source files conform to the required**
    :ref:`code_formatting`.
 
+#. **Check all new dependencies added to the requirements/ci/*.yml files.**  If
+   dependencies have been added new nox testing lockfieles should be generated too,
+   see :ref:`cirrus_test_env`.
+
 #. **Check the source documentation been updated to explain all new or changed
    features**.  See :ref:`docstrings`.
 

docs/src/whatsnew/latest.rst

@@ -150,6 +150,8 @@ This document explains the changes made to Iris for this release
 #. `@bjlittle`_ updated the perceptual imagehash graphical test support for
    `matplotlib`_ 3.4.1. (:pull:`4087`)
 
+#. `@jamesp`_ switched `cirrus-ci`_ testing and `nox`_
+   testing to use `conda-lock`_ files for static test environments. (:pull:`4108`)
 
 .. comment
     Whatsnew author names (@github name) in alphabetical order. Note that,
@@ -175,3 +177,4 @@ This document explains the changes made to Iris for this release
 .. _Python 3.8: https://www.python.org/downloads/release/python-380/
 .. _README.md: https://github.com/SciTools/iris#-----
 .. _xxhash: http://cyan4973.github.io/xxHash/
+.. _conda-lock: https://github.com/conda-incubator/conda-lock

noxfile.py

@@ -10,6 +10,7 @@
 from pathlib import Path
 
 import nox
+from nox.logger import logger
 
 
 #: Default to reusing any pre-existing nox environments.
@@ -25,57 +26,64 @@
 CARTOPY_CACHE_DIR = os.environ.get("HOME") / Path(".local/share/cartopy")
 
 
-def venv_cached(session):
-    """
-    Determine whether the nox session environment has been cached.
-
-    Parameters
-    ----------
-    session: object
-        A `nox.sessions.Session` object.
+def session_lockfile(session: nox.sessions.Session) -> Path:
+    """Return the path of the session lockfile."""
+    return Path(
+        f"requirements/ci/nox.lock/py{session.python.replace('.', '')}-linux-64.lock"
+    )
 
-    Returns
-    -------
-    bool
-        Whether the session has been cached.
 
-    """
-    result = False
-    yml = Path(f"requirements/ci/py{session.python.replace('.', '')}.yml")
+def session_cachefile(session: nox.sessions.Session) -> Path:
+    """Returns the path of the session lockfile cache."""
+    lockfile = session_lockfile(session)
     tmp_dir = Path(session.create_tmp())
-    cache = tmp_dir / yml.name
+    cache = tmp_dir / lockfile.name
+    return cache
+
+
+def venv_populated(session: nox.sessions.Session) -> bool:
+    """Returns True if the conda venv has been created
+    and the list of packages in the lockfile installed."""
+    return session_cachefile(session).is_file()
+
+
+def venv_changed(session: nox.sessions.Session) -> bool:
+    """Returns True if the installed session is different to that specified
+    in the lockfile."""
+    cache = session_cachefile(session)
+    lockfile = session_lockfile(session)
     if cache.is_file():
-        with open(yml, "rb") as fi:
+        with open(lockfile, "rb") as fi:
             expected = hashlib.sha256(fi.read()).hexdigest()
         with open(cache, "r") as fi:
             actual = fi.read()
-        result = actual == expected
-    return result
+        return actual != expected
+    else:
+        return False
 
 
-def cache_venv(session):
+def cache_venv(session: nox.sessions.Session):
     """
     Cache the nox session environment.
 
     This consists of saving a hexdigest (sha256) of the associated
-    conda requirements YAML file.
+    conda lock file.
 
     Parameters
     ----------
     session: object
         A `nox.sessions.Session` object.
 
     """
-    yml = Path(f"requirements/ci/py{session.python.replace('.', '')}.yml")
-    with open(yml, "rb") as fi:
+    lockfile = session_lockfile(session)
+    cache = session_cachefile(session)
+    with open(lockfile, "rb") as fi:
         hexdigest = hashlib.sha256(fi.read()).hexdigest()
-    tmp_dir = Path(session.create_tmp())
-    cache = tmp_dir / yml.name
     with open(cache, "w") as fo:
         fo.write(hexdigest)
 
 
-def cache_cartopy(session):
+def cache_cartopy(session: nox.sessions.Session):
     """
     Determine whether to cache the cartopy natural earth shapefiles.
 
@@ -112,41 +120,47 @@ def prepare_venv(session):
       - https://github.com/theacodes/nox/issues/260
 
     """
-    if not venv_cached(session):
-        # Determine the conda requirements yaml file.
-        fname = f"requirements/ci/py{session.python.replace('.', '')}.yml"
-        # Back-door approach to force nox to use "conda env update".
-        command = (
-            "conda",
-            "env",
-            "update",
-            f"--prefix={session.virtualenv.location}",
-            f"--file={fname}",
-            "--prune",
-        )
-        session._run(*command, silent=True, external="error")
+    lockfile = session_lockfile(session)
+    venv_dir = session.virtualenv.location_name
+
+    if not venv_populated(session):
+        # environment has been created but pacakages not yet installed
+        # populate the environment from the lockfile
+        logger.debug(f"Populating conda env at {venv_dir}")
+        session.conda_install("--file", str(lockfile))
         cache_venv(session)
 
+    elif venv_changed(session):
+        # destroy the environment and rebuild it
+        logger.debug(f"Lockfile changed. Re-creating conda env at {venv_dir}")
+        _re_orig = session.virtualenv.reuse_existing
+        session.virtualenv.reuse_existing = False
+        session.virtualenv.create()
+        session.conda_install("--file", str(lockfile))
+        session.virtualenv.reuse_existing = _re_orig
+        cache_venv(session)
+
+    logger.debug(f"Environment {venv_dir} up to date")
+
     cache_cartopy(session)
-    session.install("--no-deps", "--editable", ".")
 
     # Determine whether verbose diagnostics have been requested
     # from the command line.
     verbose = "-v" in session.posargs or "--verbose" in session.posargs
 
     if verbose:
         session.run("conda", "info")
-        session.run("conda", "list", f"--prefix={session.virtualenv.location}")
+        session.run("conda", "list", f"--prefix={venv_dir}")
         session.run(
             "conda",
             "list",
-            f"--prefix={session.virtualenv.location}",
+            f"--prefix={venv_dir}",
             "--explicit",
         )
 
 
 @nox.session
-def flake8(session):
+def flake8(session: nox.sessions.Session):
     """
     Perform flake8 linting of iris.
 
@@ -165,7 +179,7 @@ def flake8(session):
 
 
 @nox.session
-def black(session):
+def black(session: nox.sessions.Session):
     """
     Perform black format checking of iris.
 
@@ -184,7 +198,7 @@ def black(session):
 
 
 @nox.session(python=PY_VER, venv_backend="conda")
-def tests(session):
+def tests(session: nox.sessions.Session):
     """
     Perform iris system, integration and unit tests.
 
@@ -195,6 +209,8 @@ def tests(session):
 
     """
     prepare_venv(session)
+    session.install("--no-deps", "--editable", ".")
+
     session.run(
         "python",
         "-m",
@@ -216,6 +232,7 @@ def gallery(session):
 
     """
     prepare_venv(session)
+    session.install("--no-deps", "--editable", ".")
     session.run(
         "python",
         "-m",
@@ -236,6 +253,7 @@ def doctest(session):
 
     """
     prepare_venv(session)
+    session.install("--no-deps", "--editable", ".")
     session.cd("docs")
     session.run(
         "make",
@@ -262,6 +280,7 @@ def linkcheck(session):
 
     """
     prepare_venv(session)
+    session.install("--no-deps", "--editable", ".")
     session.cd("docs")
     session.run(
         "make",