refactor(tests): refactor test framework, scripts and cases

* use modeling utilities from flopy
* use testing utilities/fixtures from modflow-devtools
* add pytest.ini with markers, file patterns & default opts
* use keepable temp dirs instead of manually managed folder
* replace common_regression.py with pytest/modflow-devtools
* refactor get_exes.py with modflow-devtools and get-modflow
* remove binary_file_writer.py, disu_util.py, budget_file_compare.py
* remove build_exes.py (require meson setup/install before testing)
* remove mfpymake dependency from autotests (replaced by devtools)
* test example models separately, rename large.yml -> examples.yml
* strip z0* prefix from examples/testmodel/largetestmodel scripts
* rename Simulation to TestConfig (disambiguate from flopy)
* rename Framework to TestHarness (avoid import errors)
* add optional path argument to update_flopy.py CLI
* partially refactor gwf test with pytest-cases

Author: wpbonelli

.github/workflows/ci.yml

@@ -149,18 +149,32 @@ jobs:
 
       - name: Get executables
         working-directory: modflow6/autotest
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
         run: pytest -v --durations 0 get_exes.py
 
-      - name: Test programs
+      - name: Test
         working-directory: modflow6/autotest
         run: |
           if [ "${{ github.ref_name }}" == "master" ]; then
-            pytest -v -n auto --durations 0 -m "not developmode"
+            pytest -v -n auto --durations 0 -m "not repo and not developmode"
           else
-            pytest -v -n auto --durations 0
+            pytest -v -n auto --durations 0 -m "not repo"
           fi
 
-      - name: Test scripts
+      - name: Test mf5to6 models
+        working-directory: modflow6/autotest
+        run: pytest -v -n auto --durations 0 test_testmodels_mf5to6.py
+        env:
+          REPOS_PATH: ${{ github.workspace }}
+
+      - name: Test mf6 models
+        working-directory: modflow6/autotest
+        run: pytest -v -n auto --durations 0 test_testmodels_mf6.py
+        env:
+          REPOS_PATH: ${{ github.workspace }}
+
+      - name: Test dist scripts
         working-directory: modflow6/distribution
         env:
           GITHUB_TOKEN: ${{ github.token }}
@@ -221,17 +235,31 @@ jobs:
       
       - name: Get executables
         working-directory: modflow6/autotest
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
         run: pytest -v --durations 0 get_exes.py
 
-      - name: Test modflow6
+      - name: Test
         working-directory: modflow6/autotest
         run: |
           if [ "${{ github.ref_name }}" == "master" ]; then
-            pytest -v -n auto --durations 0 -m "not developmode"
+            pytest -v -n auto --durations 0 -m "not repo and not developmode"
           else
-            pytest -v -n auto --durations 0
+            pytest -v -n auto --durations 0 -m "not repo"
           fi
 
+      - name: Test mf5to6 models
+        working-directory: modflow6/autotest
+        run: pytest -v -n auto --durations 0 test_testmodels_mf5to6.py
+        env:
+          REPOS_PATH: ${{ github.workspace }}
+
+      - name: Test mf6 models
+        working-directory: modflow6/autotest
+        run: pytest -v -n auto --durations 0 test_testmodels_mf6.py
+        env:
+          REPOS_PATH: ${{ github.workspace }}
+
   test_ifort:
     name: Test (ifort)
     needs:
@@ -258,12 +286,6 @@ jobs:
           repository: MODFLOW-USGS/modflow6-testmodels
           path: modflow6-testmodels
 
-      - name: Checkout modflow6-examples
-        uses: actions/checkout@v3
-        with:
-          repository: MODFLOW-USGS/modflow6-examples
-          path: modflow6-examples
-
       - name: Setup Micromamba
         uses: mamba-org/provision-with-micromamba@main
         with:
@@ -282,23 +304,6 @@ jobs:
           $mamba_bin = "C:\Users\runneradmin\micromamba-root\envs\modflow6\Scripts"
           echo $mamba_bin | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
 
-      - name: Cache modflow6 examples
-        id: cache-examples
-        uses: actions/cache@v3
-        with:
-          path: modflow6-examples/examples
-          key: modflow6-examples-${{ hashFiles('modflow6-examples/scripts/**') }}
-
-      - name: Install extra Python packages
-        if: steps.cache-examples.outputs.cache-hit != 'true'
-        working-directory: modflow6-examples/etc
-        run: pip install -r requirements.pip.txt
-
-      - name: Build example models
-        if: steps.cache-examples.outputs.cache-hit != 'true'
-        working-directory: modflow6-examples/etc
-        run: python ci_build_files.py
-
       - name: Update version files
         working-directory: modflow6/distribution
         run: python update_version.py
@@ -327,46 +332,84 @@ jobs:
       - name: Get executables
         if: runner.os != 'Windows'
         working-directory: modflow6/autotest
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
         run: pytest -v --durations 0 get_exes.py
 
       - name: Get executables (Windows)
         if: runner.os == 'Windows'
         working-directory: modflow6/autotest
         shell: pwsh
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
         run: pytest -v --durations 0 get_exes.py
 
-      - name: Test programs
+      - name: Test
         if: runner.os != 'Windows'
         working-directory: modflow6/autotest
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
         run: |
           if [ "${{ github.ref_name }}" == "master" ]; then
-            pytest -v -n auto --durations 0 -m "not developmode"
+            pytest -v -n auto --durations 0 -m "not repo and not developmode"
           else
-            pytest -v -n auto --durations 0
+            pytest -v -n auto --durations 0 -m "not repo"
           fi
 
-      - name: Test programs (Windows)
+      - name: Test (Windows)
         if: runner.os == 'Windows'
         working-directory: modflow6/autotest
         shell: pwsh
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
         run: |
           if ( "${{ github.ref_name }}" -eq "master" ) {
-            pytest -v -n auto --durations 0 -m "not developmode"
+            pytest -v -n auto --durations 0 -m "not repo and not developmode"
           } else {
-            pytest -v -n auto --durations 0
+            pytest -v -n auto --durations 0 -m "not repo"
           }
 
-      - name: Test scripts
+      - name: Test mf5to6 models
+        if: runner.os != 'Windows'
+        working-directory: modflow6/autotest
+        run: pytest -v -n auto --durations 0 test_testmodels_mf5to6.py
+        env:
+          REPOS_PATH: ${{ github.workspace }}
+
+      - name: Test mf5to6 models (Windows)
+        if: runner.os == 'Windows'
+        working-directory: modflow6/autotest
+        shell: pwsh
+        run: pytest -v -n auto --durations 0 test_testmodels_mf5to6.py
+        env:
+          REPOS_PATH: ${{ github.workspace }}
+
+      - name: Test mf6 models
+        if: runner.os != 'Windows'
+        working-directory: modflow6/autotest
+        run: pytest -v -n auto --durations 0 test_testmodels_mf6.py
+        env:
+          REPOS_PATH: ${{ github.workspace }}
+
+      - name: Test mf6 models (Windows)
+        if: runner.os == 'Windows'
+        working-directory: modflow6/autotest
+        shell: pwsh
+        run: pytest -v -n auto --durations 0 test_testmodels_mf6.py
+        env:
+          REPOS_PATH: ${{ github.workspace }}
+
+      - name: Test dist scripts
         if: runner.os != 'Windows'
         working-directory: modflow6/distribution
+        run: pytest -v --durations 0
         env:
           GITHUB_TOKEN: ${{ github.token }}
-        run: pytest -v --durations 0
 
-      - name: Test scripts (Windows)
+      - name: Test dist scripts (Windows)
         if: runner.os == 'Windows'
         working-directory: modflow6/distribution
         shell: pwsh
+        run: pytest -v --durations 0
         env:
           GITHUB_TOKEN: ${{ github.token }}
-        run: pytest -v --durations 0

.github/workflows/docs.yml

@@ -71,8 +71,7 @@ jobs:
 
       - name: Test building files from dfn's for LaTeX
         working-directory: modflow6/autotest
-        run: |
-          pytest -v build_mfio_tex.py
+        run: pytest -v build_mfio_tex.py
 
       - name: Setup GNU Fortran ${{ env.GCC_V }}
         uses: awvwgk/setup-fortran@main
@@ -103,17 +102,17 @@ jobs:
 
       - name: Run benchmarks
         working-directory: modflow6/distribution
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
         run: python benchmark.py
 
       - name: Run sphinx
         working-directory: modflow6/.build_rtd_docs
-        run: |
-          make html
+        run: make html
 
       - name: Show results
         working-directory: modflow6/distribution
-        run: |
-          cat run-time-comparison.md
+        run: cat run-time-comparison.md
 
       - name: Upload comparison
         uses: actions/upload-artifact@v3

.github/workflows/large.yml .github/workflows/examples.yml.github/workflows/large.yml renamed to .github/workflows/examples.yml

@@ -1,4 +1,4 @@
-name: MODFLOW 6 large models
+name: MODFLOW 6 example models
 on:
   schedule:
     - cron: '0 6 * * *' # run at 6 AM UTC every day
@@ -20,7 +20,7 @@ jobs:
       fail-fast: false
       matrix:
         fc: [ ifort, gfortran ]
-        repo: [ examples, largetestmodels ]
+        repo: [ examples, testmodels, largetestmodels ]
     defaults:
       run:
         shell: bash -l {0}
@@ -93,15 +93,16 @@ jobs:
 
       - name: Get executables
         working-directory: modflow6/autotest
-        run: |
-          pytest -v --durations 0 get_exes.py
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: pytest -v --durations 0 get_exes.py
 
       - name: Update flopy
         working-directory: modflow6/autotest
-        run: |
-          python update_flopy.py
+        run: python update_flopy.py
 
       - name: Run tests
         working-directory: modflow6/autotest
-        run: |
-          pytest -v -n auto --durations 0 test_z03_${{ matrix.repo }}.py
+        run: pytest -v -n auto --durations 0 test_${{ matrix.repo }}.py
+        env:
+          REPOS_PATH: ${{ github.workspace }}