[REVIEW] Bump Arrow DataFusion Python dependency to 28.0.0

.github/CODEOWNERS

@@ -2,4 +2,7 @@
 *              @ayushdg @charlesbluca @galipremsagar
 
 # rust codeowners
-dask_planner/  @ayushdg @charlesbluca @galipremsagar @jdye64
+.cargo/        @ayushdg @charlesbluca @galipremsagar @jdye64
+src/           @ayushdg @charlesbluca @galipremsagar @jdye64
+Cargo.toml     @ayushdg @charlesbluca @galipremsagar @jdye64
+Cargo.lock     @ayushdg @charlesbluca @galipremsagar @jdye64

.github/workflows/conda.yml

@@ -6,10 +6,9 @@ on:
   pull_request:
     paths:
       - setup.py
-      - dask_planner/Cargo.toml
-      - dask_planner/Cargo.lock
-      - dask_planner/pyproject.toml
-      - dask_planner/rust-toolchain.toml
+      - Cargo.toml
+      - Cargo.lock
+      - pyproject.toml
       - continuous_integration/recipe/**
       - .github/workflows/conda.yml
   schedule:
@@ -34,7 +33,9 @@ jobs:
       fail-fast: false
       matrix:
         python: ["3.8", "3.9", "3.10"]
-        arch: ["linux-64", "linux-aarch64"]
+        # FIXME: aarch64 builds are consuming too much memory to run on GHA
+        # arch: ["linux-64", "linux-aarch64"]
+        arch: ["linux-64"]
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/release.yml

@@ -15,111 +15,159 @@ concurrency:
 env:
   upload: ${{ github.event_name == 'release' && github.repository == 'dask-contrib/dask-sql' }}
 
-# Required shell entrypoint to have properly activated conda environments
-defaults:
-  run:
-    shell: bash -l {0}
-
 jobs:
-  wheels:
-    name: Build and publish py3.${{ matrix.python }} wheels on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
+  linux:
+    name: Build and publish wheels for linux ${{ matrix.target }}
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        python: ["8", "9", "10"] # 3.x
+        target: [x86_64, aarch64]
     steps:
       - uses: actions/checkout@v3
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v1
+        if: matrix.target == 'aarch64'
+        with:
+          version: '3.x'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Build wheels for x86_64
+        if: matrix.target == 'x86_64'
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist
+          sccache: 'true'
+          manylinux: '2_17'
+          before-script-linux: >
+            DOWNLOAD_URL=$(curl --retry 6 --retry-delay 10 -s https://api.github.com/repos/protocolbuffers/protobuf/releases/latest | grep -o '"browser_download_url": "[^"]*' | cut -d'"' -f4 | grep "\linux-x86_64.zip$") &&
+            curl --retry 6 --retry-delay 10 -LO $DOWNLOAD_URL &&
+            unzip protoc-*-linux-x86_64.zip -d $HOME/.local
+          docker-options: --env PROTOC=/root/.local/bin/protoc
+      - name: Build wheels for aarch64
+        if: matrix.target == 'aarch64'
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --zig
+          sccache: 'true'
+          manylinux: '2_17'
+      - name: Check dist files
+        run: |
+          pip install twine
+
+          twine check dist/*
+          ls -lh dist/
+      - name: Upload binary wheels
+        uses: actions/upload-artifact@v3
         with:
-          fetch-depth: 0
+          name: wheels for linux ${{ matrix.target }}
+          path: dist/*
+      - name: Publish package
+        if: env.upload == 'true'
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: twine upload dist/*
+
+  windows:
+    name: Build and publish wheels for windows
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v3
       - name: Install Protoc
-        if: matrix.os != 'ubuntu-latest'
         uses: arduino/setup-protoc@v1
         with:
           version: '3.x'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Set up QEMU for linux-aarch64
-        if: matrix.os == 'ubuntu-latest'
-        uses: docker/setup-qemu-action@v2
+      - uses: actions/setup-python@v4
         with:
-          platforms: arm64
-      - name: Add rust toolchain target for macos-aarch64
-        if: matrix.os == 'macos-latest'
-        run: rustup target add aarch64-apple-darwin
+          python-version: '3.10'
+          architecture: x64
       - name: Build wheels
-        uses: pypa/[email protected]
+        uses: PyO3/maturin-action@v1
+        with:
+          target: x64
+          args: --release --out dist
+          sccache: 'true'
+      - name: Check dist files
+        run: |
+          pip install twine
+
+          twine check dist/*
+          ls dist/
+      - name: Upload binary wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels for windows
+          path: dist/*
+      - name: Publish package
+        if: env.upload == 'true'
         env:
-          CIBW_BUILD: 'cp3${{ matrix.python }}-*'
-          CIBW_SKIP: '*musllinux*'
-          CIBW_ARCHS_LINUX: 'aarch64 x86_64'
-          CIBW_ARCHS_WINDOWS: 'AMD64'
-          CIBW_ARCHS_MACOS: 'x86_64 arm64'
-          # Without CARGO_NET_GIT_FETCH_WITH_CLI we oom (https://github.com/rust-lang/cargo/issues/10583)
-          CIBW_ENVIRONMENT_LINUX: >
-            CARGO_NET_GIT_FETCH_WITH_CLI="true"
-            PATH="$HOME/.cargo/bin:$HOME/.local/bin:$PATH"
-          CIBW_ENVIRONMENT_WINDOWS: 'PATH="$UserProfile\.cargo\bin;$PATH"'
-          CIBW_BEFORE_BUILD: 'pip install -U setuptools-rust'
-          CIBW_BEFORE_BUILD_LINUX: >
-            ARCH=$([ $(uname -m) == x86_64 ] && echo x86_64 || echo aarch_64) &&
-            DOWNLOAD_URL=$(curl --retry 6 --retry-delay 10 -s https://api.github.com/repos/protocolbuffers/protobuf/releases/latest | grep -o '"browser_download_url": "[^"]*' | cut -d'"' -f4 | grep "\linux-${ARCH}.zip$") &&
-            curl --retry 6 --retry-delay 10 -LO $DOWNLOAD_URL &&
-            unzip protoc-*-linux-$ARCH.zip -d $HOME/.local &&
-            protoc --version &&
-            pip install -U setuptools-rust &&
-            pip list &&
-            curl --retry 6 --retry-delay 10 https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=stable --profile=minimal -y &&
-            rustup show
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: twine upload dist/*
+
+  macos:
+    name: Build and publish wheels for macos ${{ matrix.target }}
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64, aarch64]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v1
+        with:
+          version: '3.x'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/setup-python@v4
         with:
-          package-dir: .
-          output-dir: dist
-          config-file: "dask_planner/pyproject.toml"
-      - name: Set up Python
-        uses: conda-incubator/[email protected]
+          python-version: '3.10'
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
         with:
-          miniforge-variant: Mambaforge
-          use-mamba: true
-          python-version: "3.8"
-          channel-priority: strict
+          target: ${{ matrix.target }}
+          args: --release --out dist
+          sccache: 'true'
       - name: Check dist files
         run: |
-          mamba install twine
+          pip install twine
 
           twine check dist/*
           ls -lh dist/
       - name: Upload binary wheels
         uses: actions/upload-artifact@v3
         with:
-          name: wheels for py3.${{ matrix.python }} on ${{ matrix.os }}
+          name: wheels for macos ${{ matrix.target }}
           path: dist/*
       - name: Publish package
         if: env.upload == 'true'
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: twine upload dist/*
+
   sdist:
-    name: Build and publish source distribution
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
         with:
-          fetch-depth: 0
-      - name: Set up Python
-        uses: conda-incubator/setup-[email protected]
+          command: sdist
+          args: --out dist
+      - uses: actions/setup-python@v4
         with:
-          miniforge-variant: Mambaforge
-          use-mamba: true
-          python-version: "3.8"
-          channel-priority: strict
-      - name: Build source distribution
-        run: |
-          mamba install setuptools-rust twine
-
-          python setup.py sdist
+          python-version: '3.10'
       - name: Check dist files
         run: |
+          pip install twine
+
           twine check dist/*
           ls -lh dist/
       - name: Publish source distribution

.github/workflows/rust.yml

@@ -51,7 +51,6 @@ jobs:
       - name: Optionally update upstream dependencies
         if: needs.detect-ci-trigger.outputs.triggered == 'true'
         run: |
-          cd dask_planner
           bash update-dependencies.sh
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
@@ -60,11 +59,9 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Check workspace in debug mode
         run: |
-          cd dask_planner
           cargo check
       - name: Check workspace in release mode
         run: |
-          cd dask_planner
           cargo check --release
 
   # test the crate
@@ -84,7 +81,6 @@ jobs:
       - name: Optionally update upstream dependencies
         if: needs.detect-ci-trigger.outputs.triggered == 'true'
         run: |
-          cd dask_planner
           bash update-dependencies.sh
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
@@ -93,5 +89,4 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Run tests
         run: |
-          cd dask_planner
           cargo test

.github/workflows/test-upstream.yml

@@ -68,11 +68,10 @@ jobs:
       - name: Optionally update upstream cargo dependencies
         if: env.which_upstream == 'DataFusion'
         run: |
-          cd dask_planner
           bash update-dependencies.sh
       - name: Build the Rust DataFusion bindings
         run: |
-          python setup.py build install
+          maturin develop
       - name: Install hive testing dependencies
         if: matrix.os == 'ubuntu-latest'
         run: |
@@ -124,11 +123,9 @@ jobs:
         env:
           UPDATE_ALL_CARGO_DEPS: false
         run: |
-          cd dask_planner
           bash update-dependencies.sh
       - name: Install dependencies and nothing else
         run: |
-          mamba install setuptools-rust
           pip install -e . -vv
 
           which python

.github/workflows/test.yml

@@ -72,7 +72,7 @@ jobs:
           shared-key: test
       - name: Build the Rust DataFusion bindings
         run: |
-          python setup.py build install
+          maturin develop
       - name: Install hive testing dependencies
         if: matrix.os == 'ubuntu-latest'
         run: |
@@ -118,7 +118,6 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install dependencies and nothing else
         run: |
-          mamba install "setuptools-rust>=1.5.2"
           pip install -e . -vv
 
           which python

.gitignore

@@ -46,23 +46,15 @@ venv
 # IDE
 .idea
 .vscode
-planner/.classpath
-planner/.project
-planner/.settings/
-planner/.idea
-planner/*.iml
 *.swp
 
 # project specific
-planner/dependency-reduced-pom.xml
-planner/target/
-dask_sql/jar
-.next/
 dask-worker-space/
 node_modules/
 docs/source/_build/
 tests/unit/queries
 tests/unit/data
+target/*
 
 # Ignore development specific local testing files
 dev_tests

.pre-commit-config.yaml

@@ -20,9 +20,9 @@ repos:
     rev: v1.0
     hooks:
       - id: cargo-check
-        args: ['--manifest-path', './dask_planner/Cargo.toml', '--verbose', '--']
+        args: ['--manifest-path', './Cargo.toml', '--verbose', '--']
       - id: clippy
-        args: ['--manifest-path', './dask_planner/Cargo.toml', '--verbose', '--', '-D', 'warnings']
+        args: ['--manifest-path', './Cargo.toml', '--verbose', '--', '-D', 'warnings']
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.2.0
     hooks:
@@ -39,4 +39,4 @@ repos:
         entry: cargo +nightly fmt
         language: system
         types: [rust]
-        args: ['--manifest-path', './dask_planner/Cargo.toml', '--verbose', '--']
+        args: ['--manifest-path', './Cargo.toml', '--verbose', '--']