General maintenance (#1320)

hoxbro · web-flow · commit 93211592bbbe · 2024-04-09T16:49:10.000+02:00
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -2,113 +2,146 @@ name: packages
 on:
   push:
     tags:
-    - 'v[0-9]+.[0-9]+.[0-9]+'
-    - 'v[0-9]+.[0-9]+.[0-9]+a[0-9]+'
-    - 'v[0-9]+.[0-9]+.[0-9]+b[0-9]+'
-    - 'v[0-9]+.[0-9]+.[0-9]+rc[0-9]+'
+      - "v[0-9]+.[0-9]+.[0-9]+"
+      - "v[0-9]+.[0-9]+.[0-9]+a[0-9]+"
+      - "v[0-9]+.[0-9]+.[0-9]+b[0-9]+"
+      - "v[0-9]+.[0-9]+.[0-9]+rc[0-9]+"
   # Dry-run only
   workflow_dispatch:
   schedule:
-    - cron: '0 16 * * SUN'
+    - cron: "0 16 * * SUN"
+
+defaults:
+  run:
+    shell: bash -l {0}
+
+env:
+  PYTHON_VERSION: "3.11"
+  PACKAGE: "datashader"
+  MPLBACKEND: "Agg"
+  DASK_DATAFRAME__QUERY_PLANNING: false
 
 jobs:
+  waiting_room:
+    name: Waiting Room
+    runs-on: ubuntu-latest
+    needs: [conda_build, pip_install]
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    environment:
+      name: publish
+    steps:
+      - run: echo "All builds have finished, have been approved, and ready to publish"
+
   conda_build:
-    name: Build Conda Packages
-    runs-on: 'ubuntu-latest'
-    timeout-minutes: 60
-    defaults:
-      run:
-        shell: bash -l {0}
-    env:
-      PKG_TEST_PYTHON: "--test-python=py39"
-      MPLBACKEND: "Agg"
-      CONDA_UPLOAD_TOKEN: ${{ secrets.CONDA_UPLOAD_TOKEN }}
+    name: Build Conda
+    runs-on: "ubuntu-latest"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: "100"
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          miniconda-version: "latest"
-          python-version: 3.9
-          channels: pyviz/label/dev,conda-forge,nodefaults
       - name: Fetch unshallow
         run: git fetch --prune --tags --unshallow -f
-      - name: Set output
-        id: vars
-        run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniconda-version: "latest"
       - name: conda setup
         run: |
-          conda install "pyctdev>=0.5"
-          doit ecosystem_setup
-          # FIXME: downgrade urllib3 until this issue is fixed:
-          # https://github.com/Anaconda-Platform/anaconda-client/issues/654
-          conda install -c conda-forge "urllib3<2.0.0"
-      - name: doit env_capture
-        run: |
-          doit env_capture
+          # pyct is for running setup.py
+          conda install -y conda-build build pyct -c pyviz/label/dev
       - name: conda build
-        run: doit package_build $PKG_TEST_PYTHON --no-pkg-tests
-      - name: dev deploy
-        if: (github.event_name == 'push' && (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
         run: |
-          doit package_upload --token=$CONDA_UPLOAD_TOKEN --label=dev
-      - name: main deploy
-        if: (github.event_name == 'push' && !(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
-        run: |
-          doit package_upload --token=$CONDA_UPLOAD_TOKEN --label=dev --label=main
-  pip_build:
-    name: Build PyPI Packages
-    runs-on: 'ubuntu-latest'
-    timeout-minutes: 120
-    defaults:
-      run:
-        shell: bash -l {0}
-    env:
-      PKG_TEST_PYTHON: "--test-python=py39"
-      PYTHON_VERSION: "3.9"
-      MPLBACKEND: "Agg"
-      PPU: ${{ secrets.PPU }}
-      PPP: ${{ secrets.PPP }}
-      PYPI: "https://upload.pypi.org/legacy/"
-      MAMBA_NO_BANNER: 1
+          source ./scripts/conda/build.sh
+          echo "CONDA_FILE="$CONDA_PREFIX/conda-bld/noarch/$PACKAGE-$VERSION-py_0.tar.bz2"" >> $GITHUB_ENV
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: conda
+          path: ${{ env.CONDA_FILE }}
+          if-no-files-found: error
+
+  conda_publish:
+    name: Publish Conda
+    runs-on: ubuntu-latest
+    needs: [conda_build, waiting_room]
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v4
         with:
-          fetch-depth: "100"
-      - uses: conda-incubator/setup-miniconda@v2
+          name: conda
+          path: dist/
+      - name: Set environment variables
+        run: |
+          echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+          echo "CONDA_FILE=$(ls dist/*.tar.bz2)" >> $GITHUB_ENV
+      - uses: conda-incubator/setup-miniconda@v3
         with:
-          channel-priority: strict
           miniconda-version: "latest"
-          channels: pyviz/label/dev,conda-forge,nodefaults
-          mamba-version: "*"
-      - name: Fetch unshallow
-        run: git fetch --prune --tags --unshallow -f
       - name: conda setup
         run: |
-          conda install "pyctdev>=0.5"
-          doit ecosystem_setup
-          conda create -n test-environment python=$PYTHON_VERSION pyctdev
-      - name: env setup
-        run: |
-          eval "$(conda shell.bash hook)"
-          conda activate test-environment
-          doit develop_install -o tests --conda-mode=mamba
-          pip uninstall -y datashader
-          doit pip_on_conda
-      - name: doit env_capture
+          conda install -y anaconda-client
+      - name: conda dev upload
+        if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc')
         run: |
-          eval "$(conda shell.bash hook)"
-          conda activate test-environment
-          doit env_capture
-      - name: pip build
+          anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE
+      - name: conda main upload
+        if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc')))
         run: |
-          eval "$(conda shell.bash hook)"
-          conda activate test-environment
-          doit ecosystem=pip package_build $PKG_TEST_PYTHON --test-group=unit
-      - name: pip upload
-        if: github.event_name == 'push'
+          anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE
+
+  pip_build:
+    name: Build PyPI
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: "100"
+      - name: Fetch unshallow
+        run: git fetch --prune --tags --unshallow -f
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install build
         run: |
-          eval "$(conda shell.bash hook)"
-          conda activate test-environment
-          doit ecosystem=pip package_upload -u $PPU -p $PPP -r $PYPI
+          python -m pip install build
+      - name: Build package
+        run: python -m build .
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: pip
+          path: dist/
+          if-no-files-found: error
+
+  pip_install:
+    name: Install PyPI
+    runs-on: "ubuntu-latest"
+    needs: [pip_build]
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: pip
+          path: dist/
+      - name: Install package
+        run: python -m pip install dist/*.whl
+      - name: Test package
+        run: python -c "import $PACKAGE; print($PACKAGE.__version__)"
+
+  pip_publish:
+    name: Publish PyPI
+    runs-on: ubuntu-latest
+    needs: [pip_build, waiting_room]
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: pip
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: ${{ secrets.PPU }}
+          password: ${{ secrets.PPP }}
+          repository-url: "https://upload.pypi.org/legacy/"
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -30,6 +30,7 @@ jobs:
         shell: bash -l {0}
     env:
       DESC: "Documentation build"
+      DASK_DATAFRAME__QUERY_PLANNING: false
     steps:
       - uses: holoviz-dev/holoviz_tasks/install@v0
         with:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@ exclude: (\.(js|svg)$)
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-builtin-literals
       - id: check-case-conflict
@@ -13,18 +13,18 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.292
+    rev: v0.3.5
     hooks:
       - id: ruff
         files: datashader/
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.6
     hooks:
-    - id: codespell
-      additional_dependencies:
-        - tomli
+      - id: codespell
+        additional_dependencies:
+          - tomli
   - repo: https://github.com/hoxbro/clean_notebook
-    rev: v0.1.13
+    rev: v0.1.14
     hooks:
       - id: clean-notebook
 
diff --git a/scripts/conda/build.sh b/scripts/conda/build.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+git status
+
+python -m build -w .
+
+git diff --exit-code
+
+VERSION=$(find dist -name "*.whl" -exec basename {} \; | cut -d- -f2)
+export VERSION
+
+# Note: pyct is needed in the same environment as conda-build!
+conda build scripts/conda/recipe --no-anaconda-upload --no-verify
diff --git a/scripts/conda/recipe/meta.yaml b/scripts/conda/recipe/meta.yaml
@@ -1,15 +1,16 @@
-{% set sdata = load_setup_py_data() %}
+{% set sdata = load_setup_py_data(setup_file="../../../setup.py", from_recipe_dir=True) %}
+{% set pyproject = load_file_data('../../../pyproject.toml', from_recipe_dir=True) %}
 
 package:
-  name: datashader
-  version: {{ sdata['version'] }}
+  name: {{ sdata['name'] }}
+  version: {{ VERSION }}
 
 source:
-  path: ..
+  url: ../../../dist/{{ sdata['name'] }}-{{ VERSION }}-py2.py3-none-any.whl
 
 build:
   noarch: python
-  script: python setup.py install --single-version-externally-managed --record=record.txt
+  script: {{ PYTHON }} -m pip install -vv {{ sdata['name'] }}-{{ VERSION }}-py2.py3-none-any.whl
   entry_points:
     {% for group,epoints in sdata.get("entry_points",{}).items() %}
     {% for entry_point in epoints %}
@@ -18,18 +19,16 @@ build:
     {% endfor %}
 
 requirements:
-  build:
+  host:
     - python {{ sdata['python_requires'] }}
-    - param >=1.7.0
-    - pyct-core >=0.4.4
-    - setuptools >30.3.0
+    {% for dep in pyproject['build-system']['requires'] %}
+    - {{ dep }}
+    {% endfor %}
   run:
     - python {{ sdata['python_requires'] }}
     {% for dep in sdata.get('install_requires',{}) %}
     - {{ dep if dep != 'dask' else 'dask-core'}}
     {% endfor %}
-    # adding pyct here is temporary (this conda recipe template will disappear when new pyctdev is released)
-    - pyct
   run_constrained:
     - cudf >=0.10.0
 test:
