Requirements re-haul and optional testing against Iris latest

.travis.yml

@@ -3,68 +3,107 @@ dist: xenial
 
 env:
   matrix:
-    - PYTHON_VERSION=3.6
-    - PYTHON_VERSION=3.7
+    - PYTHON_VERSION=36 LIMIT_TO_RELEASED_IRIS=false
+    - PYTHON_VERSION=36 LIMIT_TO_RELEASED_IRIS=true
+    - PYTHON_VERSION=37 LIMIT_TO_RELEASED_IRIS=false
+    - PYTHON_VERSION=37 LIMIT_TO_RELEASED_IRIS=true
 
 install:
+  # Download iris-test-data
+  # -----------------------------------
+  - >
+    export IRIS_TEST_DATA_REF="https://github.com/SciTools/iris-test-data/archive/master.zip";
+    export IRIS_TEST_DATA_LOCATION="${HOME}/iris-test-data";
+    mkdir ${IRIS_TEST_DATA_LOCATION};
+    wget -O ${IRIS_TEST_DATA_LOCATION}/iris-test-data.zip ${IRIS_TEST_DATA_REF};
+    unzip -q ${IRIS_TEST_DATA_LOCATION}/iris-test-data.zip -d ${IRIS_TEST_DATA_LOCATION};
+
   # Install miniconda
   # -----------------
   - >
     echo "Installing miniconda";
     export CONDA_BASE="https://repo.continuum.io/miniconda/Miniconda";
-    if [[ "${PYTHON_VERSION}" == 2* ]]; then
-      wget --quiet ${CONDA_BASE}2-latest-Linux-x86_64.sh -O miniconda.sh;
-    else
-      wget --quiet ${CONDA_BASE}3-latest-Linux-x86_64.sh -O miniconda.sh;
-    fi;
+    wget --quiet ${CONDA_BASE}3-latest-Linux-x86_64.sh -O miniconda.sh;
     bash miniconda.sh -b -p ${HOME}/miniconda;
     export PATH="${HOME}/miniconda/bin:${PATH}";
 
-  # Configure the test environment
-  # ------------------------------
+  # Create the test environment
+  # ---------------------------
   - >
-    echo "Configuring testing environment";
+    echo "Creating testing environment";
     conda config --set always_yes yes --set changeps1 no;
     conda config --set show_channel_urls True;
     conda config --add channels conda-forge;
     conda update --quiet conda;
-    ENV_NAME='test-environment';
-    conda create --quiet -n ${ENV_NAME} python=${PYTHON_VERSION} mock filelock pep8;
+
+  # Get environment spec.
+  - >
+    export ENV_NAME="iris-grib-dev";
+    export ENV_FILE="requirements/ci/py${PYTHON_VERSION}.yml";
+
+  # Optionally download latest Iris from repo,
+  # and replace the Iris dependency with Iris' underlying dependencies.
+  - >
+    if [[ "${LIMIT_TO_RELEASED_IRIS}" == false ]]; then
+      IRIS_REF="https://github.com/SciTools/iris/archive/master.zip";
+      export IRIS_LOCATION="${HOME}/iris";
+      mkdir ${IRIS_LOCATION};
+      wget -O ${IRIS_LOCATION}/iris.zip ${IRIS_REF};
+      unzip -q ${IRIS_LOCATION}/iris.zip -d ${IRIS_LOCATION};
+
+      IRIS_ENV_FILE="${IRIS_LOCATION}/iris-master/requirements/ci/py${PYTHON_VERSION}.yml";
+      sed -i "/- iris/d" ${ENV_FILE};
+      echo "# IRIS DEPENDENCIES." >> ${ENV_FILE};
+      sed -ne "/dependencies:/,$ p" ${IRIS_ENV_FILE} | sed "1d" >> ${ENV_FILE};
+    fi;
+
+  # Create environment.
+  - >
+    conda env create --quiet --file=${ENV_FILE};
     source activate ${ENV_NAME};
 
-  # Customise the test environment
-  # ------------------------------
+  # Check installation of eccodes.
+  - python -m eccodes selfcheck;
+
+  # Install pip, then coveralls.
+  - conda install --quiet pip;
+  - pip install coveralls;
+
+  # Output environment debug info.
   - >
-    conda env update -n ${ENV_NAME} --quiet --file environment.yml;
-    # Check installation of eccodes.
-    python -m eccodes selfcheck;
+    conda list -n ${ENV_NAME};
+    conda info -a;
 
-  # Download and install iris-test-data
-  # -----------------------------------
+  # Configure Iris
+  # --------------
+  - echo "Configuring Iris";
+
+  # Optionally install latest Iris from downloaded source.
   - >
-    export IRIS_TEST_DATA_REF="https://github.com/SciTools/iris-test-data/archive/master.zip";
-    export IRIS_TEST_DATA_LOCATION="${HOME}/iris-test-data";
-    mkdir ${IRIS_TEST_DATA_LOCATION};
-    wget -O ${IRIS_TEST_DATA_LOCATION}/iris-test-data.zip ${IRIS_TEST_DATA_REF};
-    unzip -q ${IRIS_TEST_DATA_LOCATION}/iris-test-data.zip -d ${IRIS_TEST_DATA_LOCATION};
+    if [[ "${LIMIT_TO_RELEASED_IRIS}" == false ]]; then
+      cd ${IRIS_LOCATION}/iris-master;
+      python setup.py --quiet install;
+    fi;
 
-  # Locate iris installation
-  # ------------------------
+  # Locate Iris installation.
   - export IRIS_DIR=$(python -c "import iris; import os.path; print(os.path.dirname(iris.__file__))")
 
-  # Poke site.cfg to reference iris-test-data
+  # Set iris config paths.
   - >
+    export PREFIX="${HOME}/miniconda/envs/${ENV_NAME}";
     SITE_CFG="${IRIS_DIR}/etc/site.cfg";
     echo "[Resources]" > ${SITE_CFG};
-    echo "test_data_dir = ${IRIS_TEST_DATA_LOCATION}/iris-test-data-master/test_data" >> ${SITE_CFG};
+    echo "test_data_dir = $(pwd)/iris-test-data/test_data" >> ${SITE_CFG};
+    echo "doc_dir = $(pwd)/docs/iris" >> ${SITE_CFG};
+    echo "[System]" >> ${SITE_CFG};
+    echo "udunits2_path = ${PREFIX}/lib/libudunits2.so" >> ${SITE_CFG};
 
   # Install iris-grib itself
   # ------------------------
-  - pip install . --no-deps
-
-  # Install coveralls for test coverage
-  # -----------------------------------
-  - pip install coveralls
+  - >
+    echo "Installing Iris-grib.";
+    cd ${HOME}/SciTools/iris-grib;
+    pip install . --no-deps;
 
   # Summarise the environment
   # -------------------------

readthedocs.yml

@@ -1,2 +1,2 @@
 conda:
-    file: environment.yml
+    file: requirements/ci/readthedocs.yml

requirements/all.txt

@@ -0,0 +1,3 @@
+# Optional dependencies.
+
+mo_pack

requirements/ci/iris-grib.yml

@@ -0,0 +1 @@
+py37.yml

requirements/ci/py36.yml

@@ -0,0 +1,22 @@
+name: iris-grib-dev
+
+channels:
+  - conda-forge
+
+dependencies:
+  - python=3.6
+
+# Setup dependencies.
+  - setuptools
+
+# Core dependencies.
+  - iris>=2.4
+  - python-eccodes
+
+# Optional dependencies.
+  - mo_pack
+
+# Test dependencies.
+  - mock
+  - filelock
+  - pep8

requirements/ci/py37.yml

@@ -0,0 +1,22 @@
+name: iris-grib-dev
+
+channels:
+  - conda-forge
+
+dependencies:
+  - python=3.7
+
+# Setup dependencies.
+  - setuptools
+
+# Core dependencies.
+  - iris>=2.4
+  - python-eccodes
+
+# Optional dependencies.
+  - mo_pack
+
+# Test dependencies.
+  - mock
+  - filelock
+  - pep8

requirements/ci/readthedocs.yml

@@ -0,0 +1 @@
+iris-grib.yml

requirements/core.txt

@@ -0,0 +1,4 @@
+# Core dependencies.
+
+scitools-iris>=2.4
+eccodes-python

requirements/setup.txt

@@ -0,0 +1,4 @@
+# Dependencies necessary to run setup.py of iris-grib
+# ---------------------------------------------------
+
+setuptools

requirements/test.txt

@@ -0,0 +1,5 @@
+# Test dependencies.
+
+mock
+filelock
+pep8

setup.py

@@ -2,48 +2,59 @@
 
 import os
 import os.path
-import sys
 from setuptools import setup
-import textwrap
 
 
-name = 'iris_grib'
-
-
-LONG_DESCRIPTION = textwrap.dedent("""
-    Iris loading of GRIB files
-    ==========================
-
-    With this package, iris is able to load GRIB files:
-
-    ```
-    my_data = iris.load(path_to_grib_file)
-    ```
-    """)
-
-
-here = os.path.abspath(os.path.dirname(__file__))
-pkg_root = os.path.join(here, name)
+NAME = 'iris_grib'
+PYPI_NAME = 'iris-grib'
+PACKAGE_DIR = os.path.abspath(os.path.dirname(__file__))
+PACKAGE_ROOT = os.path.join(PACKAGE_DIR, NAME)
 
 packages = []
-for d, _, _ in os.walk(os.path.join(here, name)):
+for d, _, _ in os.walk(os.path.join(PACKAGE_DIR, NAME)):
     if os.path.exists(os.path.join(d, '__init__.py')):
-        packages.append(d[len(here)+1:].replace(os.path.sep, '.'))
+        packages.append(d[len(PACKAGE_DIR) + 1:].replace(os.path.sep, '.'))
+
+
+def pip_requirements(*args):
+    requirements = []
+    for name in args:
+        fname = os.path.join(
+            PACKAGE_DIR, "requirements", "{}.txt".format(name)
+        )
+        if not os.path.exists(fname):
+            emsg = (
+                f"Unable to find the {name!r} requirements file at {fname!r}"
+            )
+            raise RuntimeError(emsg)
+        with open(fname, "r") as fh:
+            for line in fh:
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                requirements.append(line)
+    return requirements
 
 
 def extract_version():
     version = None
-    fdir = os.path.dirname(__file__)
-    fnme = os.path.join(fdir, 'iris_grib', '__init__.py')
-    with open(fnme) as fd:
-        for line in fd:
+    fname = os.path.join(PACKAGE_DIR, 'iris_grib', '__init__.py')
+    with open(fname) as fi:
+        for line in fi:
             if (line.startswith('__version__')):
                 _, version = line.split('=')
-                version = version.strip()[1:-1]  # Remove quotation characters
+                version = version.strip()[1:-1]  # Remove quotations
                 break
     return version
 
 
+def long_description():
+    fname = os.path.join(PACKAGE_DIR, "README.rst")
+    with open(fname, "rb") as fi:
+        result = fi.read().decode("utf-8")
+    return result
+
+
 def file_walk_relative(top, remove=''):
     """
     Returns a generator of files from the top of the tree, removing
@@ -58,13 +69,14 @@ def file_walk_relative(top, remove=''):
 
 
 setup_args = dict(
-    name             = name,
+    name             = PYPI_NAME,
     version          = extract_version(),
     packages         = packages,
     package_data     = {'iris_grib': list(file_walk_relative('iris_grib/tests/results',
                                           remove='iris_grib/'))},
     description      = "GRIB loading for Iris",
-    long_description = LONG_DESCRIPTION,
+    long_description = long_description(),
+    long_description_content_type = "text/x-rst",
     url              = 'https://github.com/SciTools/iris-grib',
     author           = 'UK Met Office',
     author_email     = '[email protected]',
@@ -78,10 +90,14 @@ def file_walk_relative(top, remove=''):
         'Programming Language :: Python :: 3 :: Only',
     ],
     # NOTE: The Python 3 bindings to eccodes (eccodes-python) is available on
-    # PyPI, but the user is required to install eccodes itself manually. See 
+    # PyPI, but the user is required to install eccodes itself manually. See
     # ECMWF ecCodes installation documentation for more information.
-    install_requires=['scitools-iris>=2.4.*'] + ['eccodes-python'],
-    test_suite = 'iris_grib.tests',
+    install_requires=pip_requirements("setup", "core"),
+    test_suite = f'{PYPI_NAME}.tests',
+    extras_require={
+        "all": pip_requirements("all"),
+        "test": pip_requirements("test"),
+    },
 )