Added testing capability using Github Actions

.github/workflows/meta.yaml

@@ -0,0 +1,35 @@
+{% set name = "rat" %}
+{% set version = "0.0" %}
+
+package:
+  name: {{ name|lower }}
+  version: {{ version }}
+
+source:
+  path: ../../
+
+build:
+  noarch: python
+  script: {{ PYTHON }} -m pip install . -vv
+  number: 0
+  entry_points:
+    - rat = rat.cli.rat_cli:main
+
+requirements:
+  host:
+    - python >=3.9
+    - setuptools >=61.0
+    - pip
+  run:
+    - python >=3.9
+
+about:
+  home: https://github.com/UW-SASWE/RAT
+  summary: Resevoir Monitoring using Satellite Remote Sensing
+  license: GPL-3.0-or-later
+  license_file: LICENSE
+
+extra:
+  recipe-maintainers:
+    - pritamd47
+    - SanchitMinocha

.github/workflows/python-package-pypi.yaml

@@ -22,7 +22,7 @@ jobs:
       run: |
         # from refs/tags/v1.2.3 get 1.2.3
         VERSION=$(echo $GITHUB_REF | sed 's#.*/##')
-        PLACEHOLDER='version = \"develop\"'
+        PLACEHOLDER='version = \"0.0\"'
         VERSION_FILE1='setup.py'
         VERSION_FILE2='pyproject.toml'
 
@@ -42,20 +42,20 @@ jobs:
       run: >-
         python -m build --sdist --wheel --outdir dist/ .
     - name: Upload artifact to GA
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: dist
         path: dist/
         retention-days: 5
-        if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn`    
+        if-no-files-found: warn # 'warn' or 'ignore' are also available, defaults to `warn`
 
   publish:
     name: Publish distribution to PyPI
     needs: [build]
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags')
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: dist
           path: dist

.github/workflows/test.yaml

@@ -0,0 +1,72 @@
+name: build-and-test
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: Build package
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+    - name: Install pypa/build
+      run: >-
+        python -m pip install build --user
+    - name: Build a binary wheel and a source tarball
+      run: >-
+        python -m build --sdist --wheel --outdir dist/ .
+    - name: Upload artifact to GA
+      uses: actions/upload-artifact@v4
+      with:
+        name: dist
+        path: dist/
+        retention-days: 5
+        if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn`
+
+  test:
+    name: Test package
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Setup environment with conda
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: test_env.yml
+          environment-name: rat_test_env
+          init-shell: bash
+          create-args: >-
+            python=${{ matrix.python-version }}
+      - name: Test conda
+        shell: bash -l {0}
+        run: |
+          which conda
+          conda info
+          conda env list
+          conda env export --from-history
+      - name: Build RAT conda package
+        shell: bash -l {0}
+        run: |
+          mkdir conda_dist
+          conda-build .github/workflows/ --output-folder conda_dist
+      - name: Install RAT
+        shell: bash -l {0}
+        run: |
+          conda install rat -c ./conda_dist/ 
+      - name: Run tests
+        shell: bash -l {0}
+        env:
+          IMERG_SECRETS: ${{ secrets.IMERG_SECRETS }}
+          KEY_FILE: ${{ secrets.KEY_FILE }}
+        run: |
+          pytest -s tests/

.gitignore

@@ -15,6 +15,7 @@ notebooks/metsim_sample
 .git*/
 _send_files.py
 _send_files_adpc.py
+conda_dist/
 
 # Created by https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks,visualstudiocode,data
 # Edit at https://www.toptal.com/developers/gitignore?templates=python,jupyternotebooks,visualstudiocode,data

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rat"
-version = "develop"
+version = "0.0"
 authors = [
   { name="Pritam Das", email="[email protected]" },
   { name="Sanchit Minocha", email="[email protected]" },

setup.py

@@ -3,7 +3,7 @@
 
 setup(
     name = "rat",
-    version = "develop",
+    version = "0.0",
     license = "GPL-3.0",
     package_dir = {"": "src"}
 )

src/rat/cli/rat_cli.py

@@ -76,7 +76,9 @@ def init_func(args):
             global_data_dir = None
 
     secrets_fp = None
-    if args.secrets is not None:
+    if args.secrets == 'GA':
+        secrets_fp = args.secrets
+    elif args.secrets is not None:
         secrets_fp = Path(args.secrets).resolve()
         assert secrets_fp.exists(), f"Secrets file {secrets_fp} does not exist"
 
@@ -239,8 +241,11 @@ def test_func(args):
 
     test_param_fp = project_dir / 'params' / 'test_config.yml'
 
-    secrets_fp = Path(args.secrets).resolve()
-    assert secrets_fp.exists(), f"{secrets_fp} does not exist - rat requires secrets.ini file to be passed. Please refer to documentation for more details."
+    if args.secrets == 'GA':
+        secrets_fp = args.secrets
+    elif args.secrets is not None:
+        secrets_fp = Path(args.secrets).resolve()
+        assert secrets_fp.exists(), f"{secrets_fp} does not exist - rat requires secrets.ini file to be passed. Please refer to documentation for more details."
 
     if args.drive is not None:
         drive = str(args.drive)
@@ -400,7 +405,7 @@ def run_func(args):
     from rat.run_rat import run_rat
     run_rat(args.param, args.operational_latency)
 
-def main():
+def main(args_param=None):
     ## CLI interface
     p = argparse.ArgumentParser(description='Reservoir Assessment Tool')
 
@@ -548,6 +553,10 @@ def main():
 
     test_parser.set_defaults(func=test_func)
 
+    if args_param is None:
+        args = p.parse_args()
+    else:
+        args = p.parse_args(args_param)
     args = p.parse_args()
     args.func(args)
 

src/rat/core/run_altimetry.py

@@ -55,7 +55,7 @@ def altimeter_routine(reservoir_df, reservoir_column_dict, j3tracks, custom_rese
 
     return (resname,latest_cycle)
 
-def run_altimetry(config, section, res_shpfile, res_shpfile_column_dict, basin_name, basin_data_dir, save_dir):
+def run_altimetry(config, section, res_shpfile, res_shpfile_column_dict, basin_name, basin_data_dir, save_dir, secrets_file=None):
     reservoirs_gdf = gpd.read_file(res_shpfile) 
 
     ## Declaring variables to see if only certain reservoirs needs to be processed or certain range of a reservoir is available
@@ -84,7 +84,10 @@ def run_altimetry(config, section, res_shpfile, res_shpfile_column_dict, basin_n
                                                                                                         tuple,axis=1).to_dict()
 
     secrets = configparser.ConfigParser()
-    secrets.read(config['CONFIDENTIAL']['secrets'])
+    if secret_file == 'GA':
+        secrets.read_string(os.environ['GA'])
+    else:
+        secrets.read(config['CONFIDENTIAL']['secrets'])
     username = secrets["aviso"]["username"]
     pwd = secrets["aviso"]["pwd"]
 

src/rat/data_processing/newdata.py

@@ -677,7 +677,11 @@ def get_newdata(basin_name,basin_bounds,data_dir, basin_data_dir,startdate, endd
                 create_directory(temp_dir_path_var)
 
     secrets = configparser.ConfigParser()
-    secrets.read(secrets_file)  # assuming there's a secret ini file with user/pwd
+    if secrets_file == 'GA':
+        secrets.read_string(os.environ['IMERG_SECRETS'])
+        log.info("Secrets read from IMERG_SECRETS")
+    else:
+        secrets.read(secrets_file)  # assuming there's a secret ini file with user/pwd
 
     enddate = enddate
 

src/rat/rat_basin.py

@@ -646,7 +646,7 @@ def rat_basin(config, rat_logger, forecast_mode=False):
             ##----------- Altimeter height extraction begins -----------##
             # Altimeter
             latest_altimetry_cycle = run_altimetry(config, 'ALTIMETER', basin_reservoir_shpfile_path, reservoirs_gdf_column_dict, 
-                                                                                    basin_name, basin_data_dir, altimetry_savepath)
+                                                                                    basin_name, basin_data_dir, altimetry_savepath, secrets_file=config['CONFIDENTIAL']['secrets'])
             ALTIMETER_STATUS = 1
         except:
             no_errors = no_errors+1

src/rat/run_rat.py

@@ -10,6 +10,7 @@
 import datetime
 import copy
 from dask.distributed import Client, LocalCluster
+from tempfile import NamedTemporaryFile
 
 from rat.utils.logging import init_logger,close_logger,LOG_LEVEL1_NAME
 import rat.ee_utils.ee_config as ee_configuration
@@ -49,13 +50,26 @@ def run_rat(config_fn, operational_latency=None):
 
     log.debug(f"Started client with {config['GLOBAL']['multiprocessing']} workers. Dashboard link: {client.dashboard_link}")
 
+    temp_key_file = None
+
     # Trying the ee credentials given by user
     try:
         log.info("Checking earth engine credentials")
         secrets = configparser.ConfigParser()
-        secrets.read(config['CONFIDENTIAL']['secrets'])
+        if config['CONFIDENTIAL']['secrets'] == 'GA':
+            log.info('USING GITHUB ACTIONS SECRETS')
+            secrets.read_string(os.environ['IMERG_SECRETS'])
+            log.info(os.environ['IMERG_SECRETS'])
+            log.info('checking if secret is present: ', 'dummy' in os.environ['IMERG_SECRETS'])
+            temp_key_file = NamedTemporaryFile(delete=True)
+            temp_key_file.write(os.environ['KEY_FILE'].encode())
+            secrets['ee']['key_file'] = temp_key_file.name
+            log.info(secrets['ee']['key_file'])
+        else:
+            secrets.read(config['CONFIDENTIAL']['secrets'])
         ee_configuration.service_account = secrets["ee"]["service_account"]
         ee_configuration.key_file = secrets["ee"]["key_file"]
+        print(f"{secrets['ee']['service_account']}")
         ee_credentials = ee.ServiceAccountCredentials(ee_configuration.service_account,ee_configuration.key_file)
         ee.Initialize(ee_credentials)
         log.info("Connected to earth engine succesfully.")

test_env.yml

@@ -3,5 +3,19 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - conda-build
-  - mamba
+  - rasterio
+  - xarray
+  - geopandas
+  - rioxarray
+  - earthengine-api
+  - pandas
+  - numpy
+  - netcdf4
+  - dask
+  - scipy
+  - scikit-learn
+  - ruamel_yaml
+  - yaml
+  - gdown
+  - pytest
+  - conda-build

tests/test_rat_gunnison.py

@@ -0,0 +1,49 @@
+import rat
+from rat.cli.rat_cli import init_func
+from rat.cli.rat_cli import test_func as rat_t_func
+from argparse import Namespace
+from pathlib import Path
+import pytest
+
+def assert_directory_structure(project_dir):
+    assert project_dir.exists()
+    assert (project_dir / "data").exists()
+    assert (project_dir / "models").exists()
+
+def test_rat_gunnison():
+    args = Namespace()
+    args.project_dir = Path.home() / "rat_test"
+    args.global_data = False
+    args.global_data_dir = None
+    args.drive = None
+    args.secrets = "GA"
+
+    init_func(args)
+
+    # Test that the project directory was created
+    assert_directory_structure(args.project_dir)
+
+    # run rat for gunnision
+    args = Namespace()
+    args.project_dir = Path.home() / "rat_test"
+    args.test_basin = "GUNNISON"
+    args.secrets = "GA"
+    args.drive = None
+
+    rat_t_func(args)
+
+    run_log_dir = args.project_dir / "data" / "test_output" / "runs" / "logs"
+    run_log_files = list(run_log_dir.glob("RAT_run*.log"))
+    assert run_log_dir.exists()
+    assert len(run_log_files) == 1 # there should only be one log file
+    print("run_log_files", run_log_files)
+    if len(run_log_files) == 1:
+        print(run_log_files[0].read_text())
+
+    basin_log_dir = args.project_dir / "data" / "test_output" / "colorado" / "logs" / "gunnison"
+    basin_log_files = list(basin_log_dir.glob("RAT-gunnison*.log"))
+    assert basin_log_dir.exists()
+    assert len(basin_log_files) == 1 # there should only be one log file
+    print("basin_log_files", basin_log_files)
+    if len(basin_log_files) == 1:
+        print(basin_log_files[0].read_text())