Datafusion upstream merge

.github/cluster-upstream.yml

@@ -0,0 +1,21 @@
+# Docker-compose setup used during tests
+version: '3'
+services:
+    dask-scheduler:
+        container_name: dask-scheduler
+        image: daskdev/dask:dev-py3.9
+        command: dask-scheduler
+        environment:
+            USE_MAMBA: "true"
+            EXTRA_CONDA_PACKAGES: "dask/label/dev::dask cloudpickle>=2.1.0"
+        ports:
+            - "8786:8786"
+    dask-worker:
+        container_name: dask-worker
+        image: daskdev/dask:dev-py3.9
+        command: dask-worker dask-scheduler:8786
+        environment:
+            USE_MAMBA: "true"
+            EXTRA_CONDA_PACKAGES: "dask/label/dev::dask cloudpickle>=2.1.0 pyarrow>=3.0.0 libstdcxx-ng>=12.1.0"
+        volumes:
+            - /tmp:/tmp

.github/docker-compose.yaml → .github/cluster.yml

@@ -3,7 +3,7 @@ version: '3'
 services:
     dask-scheduler:
         container_name: dask-scheduler
-        image: daskdev/dask:dev
+        image: daskdev/dask:dev-py3.9
         command: dask-scheduler
         environment:
             USE_MAMBA: "true"
@@ -12,10 +12,10 @@ services:
             - "8786:8786"
     dask-worker:
         container_name: dask-worker
-        image: daskdev/dask:dev
+        image: daskdev/dask:dev-py3.9
         command: dask-worker dask-scheduler:8786
         environment:
             USE_MAMBA: "true"
-            EXTRA_CONDA_PACKAGES: "pyarrow>=4.0.0"  # required for parquet IO
+            EXTRA_CONDA_PACKAGES: "cloudpickle>=2.1.0 pyarrow>=3.0.0 libstdcxx-ng>=12.1.0"
         volumes:
             - /tmp:/tmp

.github/workflows/datafusion-sync.yml

@@ -0,0 +1,30 @@
+name: Keep datafusion branch up to date
+on:
+  push:
+    branches:
+      - main
+
+# When this workflow is queued, automatically cancel any previous running
+# or pending jobs
+concurrency:
+  group: datafusion-sync
+  cancel-in-progress: true
+
+jobs:
+  sync-branches:
+    runs-on: ubuntu-latest
+    if: github.repository == 'dask-contrib/dask-sql'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: 12
+      - name: Opening pull request
+        id: pull
+        uses: tretuna/[email protected]
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          FROM_BRANCH: main
+          TO_BRANCH: datafusion-sql-planner

.github/workflows/test-upstream.yml

@@ -4,6 +4,11 @@ on:
     - cron: "0 0 * * *" # Daily “At 00:00” UTC
   workflow_dispatch: # allows you to trigger the workflow run manually
 
+# Required shell entrypoint to have properly activated conda environments
+defaults:
+  run:
+    shell: bash -l {0}
+
 jobs:
   test-dev:
     name: "Test upstream dev (${{ matrix.os }}, python: ${{ matrix.python }})"
@@ -29,6 +34,7 @@ jobs:
           use-mamba: true
           python-version: ${{ matrix.python }}
           channel-priority: strict
+          channels: dask/label/dev,conda-forge,nodefaults
           activate-environment: dask-sql
           environment-file: ${{ env.CONDA_FILE }}
       - name: Install hive testing dependencies for Linux
@@ -39,23 +45,110 @@ jobs:
           docker pull bde2020/hive-metastore-postgresql:2.3.0
       - name: Install upstream dev Dask / dask-ml
         run: |
-          python -m pip install --no-deps git+https://github.com/dask/dask
-          python -m pip install --no-deps git+https://github.com/dask/distributed
+          mamba update dask
           python -m pip install --no-deps git+https://github.com/dask/dask-ml
       - name: Test with pytest
         run: |
           pytest --junitxml=junit/test-results.xml --cov-report=xml -n auto tests --dist loadfile
 
+  cluster-dev:
+    name: "Test upstream dev in a dask cluster"
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }}
+      - name: Set up Python
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          use-mamba: true
+          python-version: "3.9"
+          channel-priority: strict
+          channels: dask/label/dev,conda-forge,nodefaults
+          activate-environment: dask-sql
+          environment-file: continuous_integration/environment-3.9-jdk11-dev.yaml
+      - name: Download the pre-build jar
+        uses: actions/download-artifact@v1
+        with:
+          name: jar
+          path: dask_sql/jar/
+      - name: Install cluster dependencies
+        run: |
+          mamba install python-blosc lz4 -c conda-forge
+
+          which python
+          pip list
+          mamba list
+      - name: Install upstream dev dask-ml
+        run: |
+          mamba update dask
+          python -m pip install --no-deps git+https://github.com/dask/dask-ml
+      - name: run a dask cluster
+        run: |
+          docker-compose -f .github/cluster-upstream.yml up -d
+
+          # periodically ping logs until a connection has been established; assume failure after 2 minutes
+          timeout 2m bash -c 'until docker logs dask-worker 2>&1 | grep -q "Starting established connection"; do sleep 1; done'
+
+          docker logs dask-scheduler
+          docker logs dask-worker
+      - name: Test with pytest while running an independent dask cluster
+        run: |
+          DASK_SQL_TEST_SCHEDULER="tcp://127.0.0.1:8786" pytest --junitxml=junit/test-cluster-results.xml --cov-report=xml -n auto tests --dist loadfile
+
+  import-dev:
+    name: "Test importing with bare requirements and upstream dev"
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }}
+      - name: Set up Python
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          python-version: "3.8"
+          mamba-version: "*"
+          channels: dask/label/dev,conda-forge,nodefaults
+          channel-priority: strict
+      - name: Download the pre-build jar
+        uses: actions/download-artifact@v1
+        with:
+          name: jar
+          path: dask_sql/jar/
+      - name: Install upstream dev Dask / dask-ml
+        if: needs.detect-ci-trigger.outputs.triggered == 'true'
+        run: |
+          mamba update dask
+          python -m pip install --no-deps git+https://github.com/dask/dask-ml
+      - name: Install dependencies and nothing else
+        run: |
+          pip install -e .
+
+          which python
+          pip list
+          mamba list
+      - name: Try to import dask-sql
+        run: |
+          python -c "import dask_sql; print('ok')"
+
   report-failures:
     name: Open issue for upstream dev failures
-    needs: test-dev
+    needs: [test-dev, cluster-dev]
     if: |
       always()
-      && needs.test-dev.result == 'failure'
+      && (
+        needs.test-dev.result == 'failure' || needs.cluster-dev.result == 'failure'
+      )
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
     steps:
       - uses: actions/checkout@v2
       - name: Report failures

.github/workflows/test.yml

@@ -57,6 +57,7 @@ jobs:
           use-mamba: true
           python-version: ${{ matrix.python }}
           channel-priority: strict
+          channels: ${{ needs.detect-ci-trigger.outputs.triggered == 'true' && 'dask/label/dev,conda-forge,nodefaults' || 'conda-forge,nodefaults' }}
           activate-environment: dask-sql
           environment-file: ${{ env.CONDA_FILE }}
       - name: Setup Rust Toolchain
@@ -77,8 +78,7 @@ jobs:
       - name: Optionally install upstream dev Dask / dask-ml
         if: needs.detect-ci-trigger.outputs.triggered == 'true'
         run: |
-          python -m pip install --no-deps git+https://github.com/dask/dask
-          python -m pip install --no-deps git+https://github.com/dask/distributed
+          mamba update dask
           python -m pip install --no-deps git+https://github.com/dask/dask-ml
       - name: Test with pytest
         run: |
@@ -107,10 +107,11 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          python-version: "3.8"
+          python-version: "3.9"
           channel-priority: strict
+          channels: ${{ needs.detect-ci-trigger.outputs.triggered == 'true' && 'dask/label/dev,conda-forge,nodefaults' || 'conda-forge,nodefaults' }}
           activate-environment: dask-sql
-          environment-file: continuous_integration/environment-3.8-dev.yaml
+          environment-file: continuous_integration/environment-3.9-dev.yaml
       - name: Setup Rust Toolchain
         uses: actions-rs/toolchain@v1
         id: rust-toolchain
@@ -127,18 +128,23 @@ jobs:
           which python
           pip list
           mamba list
-      - name: Optionally install upstream dev Dask / dask-ml
+      - name: Optionally install upstream dev dask-ml
         if: needs.detect-ci-trigger.outputs.triggered == 'true'
         run: |
-          python -m pip install --no-deps git+https://github.com/dask/dask
-          python -m pip install --no-deps git+https://github.com/dask/distributed
+          mamba update dask
           python -m pip install --no-deps git+https://github.com/dask/dask-ml
       - name: run a dask cluster
+        env:
+          UPSTREAM: ${{ needs.detect-ci-trigger.outputs.triggered }}
         run: |
-          docker-compose -f .github/docker-compose.yaml up -d
+          if [[ $UPSTREAM == "true" ]]; then
+            docker-compose -f .github/cluster-upstream.yml up -d
+          else
+            docker-compose -f .github/cluster.yml up -d
+          fi
 
-          # Wait for installation
-          sleep 40
+          # periodically ping logs until a connection has been established; assume failure after 2 minutes
+          timeout 2m bash -c 'until docker logs dask-worker 2>&1 | grep -q "Starting established connection"; do sleep 1; done'
 
           docker logs dask-scheduler
           docker logs dask-worker
@@ -157,7 +163,7 @@ jobs:
         with:
           python-version: "3.8"
           mamba-version: "*"
-          channels: conda-forge,defaults
+          channels: ${{ needs.detect-ci-trigger.outputs.triggered == 'true' && 'dask/label/dev,conda-forge,nodefaults' || 'conda-forge,nodefaults' }}
           channel-priority: strict
       - name: Install dependencies and nothing else
         run: |
@@ -167,12 +173,6 @@ jobs:
           which python
           pip list
           mamba list
-      - name: Optionally install upstream dev Dask / dask-ml
-        if: needs.detect-ci-trigger.outputs.triggered == 'true'
-        run: |
-          python -m pip install --no-deps git+https://github.com/dask/dask
-          python -m pip install --no-deps git+https://github.com/dask/distributed
-          python -m pip install --no-deps git+https://github.com/dask/dask-ml
       - name: Try to import dask-sql
         run: |
           python -c "import dask_sql; print('ok')"

continuous_integration/environment-3.10-dev.yaml

@@ -3,40 +3,41 @@ channels:
 - conda-forge
 - nodefaults
 dependencies:
-- adagio>=0.2.3
-- antlr4-python3-runtime>=4.9.2, <4.10.0 # Remove max pin after qpd(fugue dependency) updates their conda recipe
-- black=22.3.0
-- ciso8601>=2.2.0
 - dask-ml>=2022.1.22
 - dask>=2022.3.0
-- fastapi>=0.61.1
-- fs>=2.4.11
+- fastapi>=0.69.0
 - intake>=0.6.0
-- isort=5.7.0
-- jsonschema>=4.4.0
-- lightgbm>=3.2.1
-- mlflow>=1.19.0
-- mock>=4.0.3
-- nest-asyncio>=1.4.3
-- pandas>=1.0.0  # below 1.0, there were no nullable ext. types
-- pip=20.2.4
-- pre-commit>=2.11.1
-- prompt_toolkit>=3.0.8
-- psycopg2>=2.9.1
-- pygments>=2.7.1
-- pyhive>=0.6.4
-- pytest-cov>=2.10.1
+- jsonschema
+- lightgbm
+- maturin>=0.12.8
+- mlflow
+- mock
+- nest-asyncio
+- pandas>=1.1.2
+- pre-commit
+- prompt_toolkit
+- psycopg2
+- pyarrow>=3.0.0
+- pygments
+- pyhive
+- pytest-cov
 - pytest-xdist
-- pytest>=6.0.1
+- pytest
 - python=3.10
-- scikit-learn>=0.24.2
-- sphinx>=3.2.1
-- tpot>=0.11.7
-- triad>=0.5.4
+- rust>=1.60.0
+- scikit-learn>=1.0.0
+- setuptools-rust>=1.1.2
+- sphinx
+- tpot
 - tzlocal>=2.1
 - uvicorn>=0.11.3
-- maturin>=0.12.8
-- setuptools-rust>=1.1.2
-- rust>=1.60.0
+# fugue dependencies; remove when we conda install fugue
+- adagio
+- antlr4-python3-runtime<4.10
+- ciso8601
+- fs
+- pip
+- qpd
+- triad
 - pip:
   - fugue[sql]>=0.5.3