diff --git a/.github/workflows/meta.yaml b/.github/workflows/meta.yaml new file mode 100644 index 00000000..130e42b8 --- /dev/null +++ b/.github/workflows/meta.yaml @@ -0,0 +1,35 @@ +{% set name = "rat" %} +{% set version = "0.0" %} + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + path: ../../ + +build: + noarch: python + script: {{ PYTHON }} -m pip install . -vv + number: 0 + entry_points: + - rat = rat.cli.rat_cli:main + +requirements: + host: + - python >=3.9 + - setuptools >=61.0 + - pip + run: + - python >=3.9 + +about: + home: https://github.com/UW-SASWE/RAT + summary: Resevoir Monitoring using Satellite Remote Sensing + license: GPL-3.0-or-later + license_file: LICENSE + +extra: + recipe-maintainers: + - pritamd47 + - SanchitMinocha \ No newline at end of file diff --git a/.github/workflows/python-package-pypi.yaml b/.github/workflows/python-package-pypi.yaml index b70cda57..bbfb710f 100644 --- a/.github/workflows/python-package-pypi.yaml +++ b/.github/workflows/python-package-pypi.yaml @@ -22,7 +22,7 @@ jobs: run: | # from refs/tags/v1.2.3 get 1.2.3 VERSION=$(echo $GITHUB_REF | sed 's#.*/##') - PLACEHOLDER='version = \"develop\"' + PLACEHOLDER='version = \"0.0\"' VERSION_FILE1='setup.py' VERSION_FILE2='pyproject.toml' @@ -42,12 +42,12 @@ jobs: run: >- python -m build --sdist --wheel --outdir dist/ . - name: Upload artifact to GA - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: dist path: dist/ retention-days: 5 - if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` + if-no-files-found: warn # 'warn' or 'ignore' are also available, defaults to `warn` publish: name: Publish distribution to PyPI @@ -55,7 +55,7 @@ jobs: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags') steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: dist path: dist diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 00000000..b5ca5aaf --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,72 @@ +name: build-and-test + +on: [push, pull_request] + +jobs: + build: + name: Build package + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Install pypa/build + run: >- + python -m pip install build --user + - name: Build a binary wheel and a source tarball + run: >- + python -m build --sdist --wheel --outdir dist/ . + - name: Upload artifact to GA + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + retention-days: 5 + if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` + + test: + name: Test package + needs: build + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Setup environment with conda + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: test_env.yml + environment-name: rat_test_env + init-shell: bash + create-args: >- + python=${{ matrix.python-version }} + - name: Test conda + shell: bash -l {0} + run: | + which conda + conda info + conda env list + conda env export --from-history + - name: Build RAT conda package + shell: bash -l {0} + run: | + mkdir conda_dist + conda-build .github/workflows/ --output-folder conda_dist + - name: Install RAT + shell: bash -l {0} + run: | + conda install rat -c ./conda_dist/ + - name: Run tests + shell: bash -l {0} + env: + IMERG_SECRETS: ${{ secrets.IMERG_SECRETS }} + KEY_FILE: ${{ secrets.KEY_FILE }} + run: | + pytest -s tests/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 22daac28..b5c2f759 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ notebooks/metsim_sample .git*/ _send_files.py _send_files_adpc.py +conda_dist/ # Created by https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks,visualstudiocode,data # Edit at https://www.toptal.com/developers/gitignore?templates=python,jupyternotebooks,visualstudiocode,data diff --git a/pyproject.toml b/pyproject.toml index e3b21a51..6cf7eb82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "rat" -version = "develop" +version = "0.0" authors = [ { name="Pritam Das", email="pdas47@uw.edu" }, { name="Sanchit Minocha", email="msanchit@uw.edu" }, diff --git a/setup.py b/setup.py index 90ab3c5f..854e1b6f 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name = "rat", - version = "develop", + version = "0.0", license = "GPL-3.0", package_dir = {"": "src"} ) diff --git a/src/rat/cli/rat_cli.py b/src/rat/cli/rat_cli.py index a04865db..dda5e479 100644 --- a/src/rat/cli/rat_cli.py +++ b/src/rat/cli/rat_cli.py @@ -76,7 +76,9 @@ def init_func(args): global_data_dir = None secrets_fp = None - if args.secrets is not None: + if args.secrets == 'GA': + secrets_fp = args.secrets + elif args.secrets is not None: secrets_fp = Path(args.secrets).resolve() assert secrets_fp.exists(), f"Secrets file {secrets_fp} does not exist" @@ -239,8 +241,11 @@ def test_func(args): test_param_fp = project_dir / 'params' / 'test_config.yml' - secrets_fp = Path(args.secrets).resolve() - assert secrets_fp.exists(), f"{secrets_fp} does not exist - rat requires secrets.ini file to be passed. Please refer to documentation for more details." + if args.secrets == 'GA': + secrets_fp = args.secrets + elif args.secrets is not None: + secrets_fp = Path(args.secrets).resolve() + assert secrets_fp.exists(), f"{secrets_fp} does not exist - rat requires secrets.ini file to be passed. Please refer to documentation for more details." if args.drive is not None: drive = str(args.drive) @@ -400,7 +405,7 @@ def run_func(args): from rat.run_rat import run_rat run_rat(args.param, args.operational_latency) -def main(): +def main(args_param=None): ## CLI interface p = argparse.ArgumentParser(description='Reservoir Assessment Tool') @@ -548,6 +553,10 @@ def main(): test_parser.set_defaults(func=test_func) + if args_param is None: + args = p.parse_args() + else: + args = p.parse_args(args_param) args = p.parse_args() args.func(args) diff --git a/src/rat/core/run_altimetry.py b/src/rat/core/run_altimetry.py index f2e5d5e1..75dda872 100644 --- a/src/rat/core/run_altimetry.py +++ b/src/rat/core/run_altimetry.py @@ -55,7 +55,7 @@ def altimeter_routine(reservoir_df, reservoir_column_dict, j3tracks, custom_rese return (resname,latest_cycle) -def run_altimetry(config, section, res_shpfile, res_shpfile_column_dict, basin_name, basin_data_dir, save_dir): +def run_altimetry(config, section, res_shpfile, res_shpfile_column_dict, basin_name, basin_data_dir, save_dir, secrets_file=None): reservoirs_gdf = gpd.read_file(res_shpfile) ## Declaring variables to see if only certain reservoirs needs to be processed or certain range of a reservoir is available @@ -84,7 +84,10 @@ def run_altimetry(config, section, res_shpfile, res_shpfile_column_dict, basin_n tuple,axis=1).to_dict() secrets = configparser.ConfigParser() - secrets.read(config['CONFIDENTIAL']['secrets']) + if secret_file == 'GA': + secrets.read_string(os.environ['GA']) + else: + secrets.read(config['CONFIDENTIAL']['secrets']) username = secrets["aviso"]["username"] pwd = secrets["aviso"]["pwd"] diff --git a/src/rat/data_processing/newdata.py b/src/rat/data_processing/newdata.py index cd870f71..a70a80cf 100644 --- a/src/rat/data_processing/newdata.py +++ b/src/rat/data_processing/newdata.py @@ -677,7 +677,11 @@ def get_newdata(basin_name,basin_bounds,data_dir, basin_data_dir,startdate, endd create_directory(temp_dir_path_var) secrets = configparser.ConfigParser() - secrets.read(secrets_file) # assuming there's a secret ini file with user/pwd + if secrets_file == 'GA': + secrets.read_string(os.environ['IMERG_SECRETS']) + log.info("Secrets read from IMERG_SECRETS") + else: + secrets.read(secrets_file) # assuming there's a secret ini file with user/pwd enddate = enddate diff --git a/src/rat/rat_basin.py b/src/rat/rat_basin.py index 2558af26..9b9dc79c 100644 --- a/src/rat/rat_basin.py +++ b/src/rat/rat_basin.py @@ -646,7 +646,7 @@ def rat_basin(config, rat_logger, forecast_mode=False): ##----------- Altimeter height extraction begins -----------## # Altimeter latest_altimetry_cycle = run_altimetry(config, 'ALTIMETER', basin_reservoir_shpfile_path, reservoirs_gdf_column_dict, - basin_name, basin_data_dir, altimetry_savepath) + basin_name, basin_data_dir, altimetry_savepath, secrets_file=config['CONFIDENTIAL']['secrets']) ALTIMETER_STATUS = 1 except: no_errors = no_errors+1 diff --git a/src/rat/run_rat.py b/src/rat/run_rat.py index bdf6acc7..201e7a22 100644 --- a/src/rat/run_rat.py +++ b/src/rat/run_rat.py @@ -10,6 +10,7 @@ import datetime import copy from dask.distributed import Client, LocalCluster +from tempfile import NamedTemporaryFile from rat.utils.logging import init_logger,close_logger,LOG_LEVEL1_NAME import rat.ee_utils.ee_config as ee_configuration @@ -49,13 +50,26 @@ def run_rat(config_fn, operational_latency=None): log.debug(f"Started client with {config['GLOBAL']['multiprocessing']} workers. Dashboard link: {client.dashboard_link}") + temp_key_file = None + # Trying the ee credentials given by user try: log.info("Checking earth engine credentials") secrets = configparser.ConfigParser() - secrets.read(config['CONFIDENTIAL']['secrets']) + if config['CONFIDENTIAL']['secrets'] == 'GA': + log.info('USING GITHUB ACTIONS SECRETS') + secrets.read_string(os.environ['IMERG_SECRETS']) + log.info(os.environ['IMERG_SECRETS']) + log.info('checking if secret is present: ', 'dummy' in os.environ['IMERG_SECRETS']) + temp_key_file = NamedTemporaryFile(delete=True) + temp_key_file.write(os.environ['KEY_FILE'].encode()) + secrets['ee']['key_file'] = temp_key_file.name + log.info(secrets['ee']['key_file']) + else: + secrets.read(config['CONFIDENTIAL']['secrets']) ee_configuration.service_account = secrets["ee"]["service_account"] ee_configuration.key_file = secrets["ee"]["key_file"] + print(f"{secrets['ee']['service_account']}") ee_credentials = ee.ServiceAccountCredentials(ee_configuration.service_account,ee_configuration.key_file) ee.Initialize(ee_credentials) log.info("Connected to earth engine succesfully.") diff --git a/test_env.yml b/test_env.yml index 07e38b8f..d9205796 100644 --- a/test_env.yml +++ b/test_env.yml @@ -3,5 +3,19 @@ channels: - conda-forge - defaults dependencies: - - conda-build - - mamba \ No newline at end of file + - rasterio + - xarray + - geopandas + - rioxarray + - earthengine-api + - pandas + - numpy + - netcdf4 + - dask + - scipy + - scikit-learn + - ruamel_yaml + - yaml + - gdown + - pytest + - conda-build \ No newline at end of file diff --git a/tests/test_rat_gunnison.py b/tests/test_rat_gunnison.py new file mode 100644 index 00000000..060f1028 --- /dev/null +++ b/tests/test_rat_gunnison.py @@ -0,0 +1,49 @@ +import rat +from rat.cli.rat_cli import init_func +from rat.cli.rat_cli import test_func as rat_t_func +from argparse import Namespace +from pathlib import Path +import pytest + +def assert_directory_structure(project_dir): + assert project_dir.exists() + assert (project_dir / "data").exists() + assert (project_dir / "models").exists() + +def test_rat_gunnison(): + args = Namespace() + args.project_dir = Path.home() / "rat_test" + args.global_data = False + args.global_data_dir = None + args.drive = None + args.secrets = "GA" + + init_func(args) + + # Test that the project directory was created + assert_directory_structure(args.project_dir) + + # run rat for gunnision + args = Namespace() + args.project_dir = Path.home() / "rat_test" + args.test_basin = "GUNNISON" + args.secrets = "GA" + args.drive = None + + rat_t_func(args) + + run_log_dir = args.project_dir / "data" / "test_output" / "runs" / "logs" + run_log_files = list(run_log_dir.glob("RAT_run*.log")) + assert run_log_dir.exists() + assert len(run_log_files) == 1 # there should only be one log file + print("run_log_files", run_log_files) + if len(run_log_files) == 1: + print(run_log_files[0].read_text()) + + basin_log_dir = args.project_dir / "data" / "test_output" / "colorado" / "logs" / "gunnison" + basin_log_files = list(basin_log_dir.glob("RAT-gunnison*.log")) + assert basin_log_dir.exists() + assert len(basin_log_files) == 1 # there should only be one log file + print("basin_log_files", basin_log_files) + if len(basin_log_files) == 1: + print(basin_log_files[0].read_text()) \ No newline at end of file