Drop python 3.8, add 3.11 & 3.12 to testing

.github/workflows/conda.yml

@@ -40,7 +40,7 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          python-version: "3.8"
+          python-version: "3.9"
           channel-priority: strict
       - name: Install dependencies
         run: |

.github/workflows/release.yml

@@ -16,7 +16,8 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python: ["8", "9", "10"] # 3.x
+        # corresponds to python 3.9, 3.10, 3.11
+        python: ["9", "10", "11"]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -54,7 +55,7 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          python-version: "3.8"
+          python-version: "3.9"
           channel-priority: strict
       - name: Check dist files
         run: |
@@ -84,7 +85,7 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          python-version: "3.8"
+          python-version: "3.9"
           channel-priority: strict
       - name: Build source distribution
         run: |

.github/workflows/test-upstream.yml

@@ -36,21 +36,21 @@ jobs:
     name: "Test upstream dev (${{ matrix.os }}, python: ${{ matrix.python }}, distributed: ${{ matrix.distributed }})"
     runs-on: ${{ matrix.os }}
     env:
-      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-dev.yaml
+      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}.yaml
       DASK_SQL_DISTRIBUTED_TESTS: ${{ matrix.distributed }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python: ["3.8", "3.9", "3.10"]
+        python: ["3.9", "3.10", "3.11"]
         distributed: [false]
         include:
           # run tests on a distributed client
           - os: "ubuntu-latest"
-            python: "3.8"
+            python: "3.9"
             distributed: true
           - os: "ubuntu-latest"
-            python: "3.10"
+            python: "3.11"
             distributed: true
     steps:
       - uses: actions/checkout@v3
@@ -110,7 +110,7 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          python-version: "3.8"
+          python-version: "3.9"
           channel-priority: strict
       - name: Install Protoc
         uses: arduino/setup-protoc@v1

.github/workflows/test.yml

@@ -37,21 +37,21 @@ jobs:
     needs: [detect-ci-trigger]
     runs-on: ${{ matrix.os }}
     env:
-      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-dev.yaml
+      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}.yaml
       DASK_SQL_DISTRIBUTED_TESTS: ${{ matrix.distributed }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python: ["3.8", "3.9", "3.10"]
+        python: ["3.9", "3.10", "3.11"]
         distributed: [false]
         include:
           # run tests on a distributed client
           - os: "ubuntu-latest"
-            python: "3.8"
+            python: "3.9"
             distributed: true
           - os: "ubuntu-latest"
-            python: "3.10"
+            python: "3.11"
             distributed: true
     steps:
       - uses: actions/checkout@v3
@@ -107,7 +107,7 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          python-version: "3.8"
+          python-version: "3.9"
           channel-priority: strict
       - name: Install Protoc
         uses: arduino/setup-protoc@v1

CONTRIBUTING.md

@@ -19,7 +19,7 @@ rustup update
 To initialize and activate the conda environment for a given Python version:
 
 ```
-conda env create -f dask-sql/continuous_integration/environment-{$PYTHON_VER}-dev.yaml
+conda env create -f dask-sql/continuous_integration/environment-{$PYTHON_VER}.yaml
 conda activate dask-sql
 ```
 

README.md

@@ -101,7 +101,7 @@ If you want to have the newest (unreleased) `dask-sql` version or if you plan to
 
 Create a new conda environment and install the development environment:
 
-    conda env create -f continuous_integration/environment-3.9-dev.yaml
+    conda env create -f continuous_integration/environment-3.9.yaml
 
 It is not recommended to use `pip` instead of `conda` for the environment setup.
 

continuous_integration/environment-3.10-dev.yaml → continuous_integration/environment-3.10.yaml

@@ -1,7 +1,6 @@
 name: dask-sql
 channels:
 - conda-forge
-- nodefaults
 dependencies:
 - c-compiler
 - dask>=2022.3.0
@@ -10,9 +9,10 @@ dependencies:
 - fugue>=0.7.3
 - intake>=0.6.0
 - jsonschema
+- libprotobuf=3
 - lightgbm
 - maturin>=0.12.8
-- mlflow
+- mlflow>=1.20
 - mock
 # tpot imports fail with numpy >=1.24.0
 # https://github.com/EpistasisLab/tpot/issues/1281
@@ -36,4 +36,3 @@ dependencies:
 - tpot
 - tzlocal>=2.1
 - uvicorn>=0.13.4
-- libprotobuf=3

continuous_integration/environment-3.9-dev.yaml → continuous_integration/environment-3.11.yaml

@@ -1,7 +1,6 @@
 name: dask-sql
 channels:
 - conda-forge
-- nodefaults
 dependencies:
 - c-compiler
 - dask>=2022.3.0
@@ -10,9 +9,10 @@ dependencies:
 - fugue>=0.7.3
 - intake>=0.6.0
 - jsonschema
+- libprotobuf=3
 - lightgbm
 - maturin>=0.12.8
-- mlflow
+- mlflow>=1.20
 - mock
 # tpot imports fail with numpy >=1.24.0
 # https://github.com/EpistasisLab/tpot/issues/1281
@@ -28,12 +28,11 @@ dependencies:
 - pytest-rerunfailures
 - pytest-xdist
 - pytest
-- python=3.9
+- python=3.11
 - scikit-learn>=1.0.0
 - setuptools-rust>=1.5.2
 - sphinx
 - sqlalchemy<2
 - tpot
 - tzlocal>=2.1
 - uvicorn>=0.13.4
-- libprotobuf=3

continuous_integration/environment-3.8-dev.yaml → continuous_integration/environment-3.9.yaml

@@ -1,17 +1,17 @@
-name: dask-sql
+name: dask-sql-py39
 channels:
 - conda-forge
-- nodefaults
 dependencies:
 - c-compiler
 - dask=2022.3.0
 - fastapi=0.69.0
 - fugue=0.7.3
 - intake=0.6.0
 - jsonschema
+- libprotobuf=3
 - lightgbm
 - maturin=0.12.8
-- mlflow
+- mlflow=1.20
 - mock
 # tpot imports fail with numpy >=1.24.0
 # https://github.com/EpistasisLab/tpot/issues/1281
@@ -27,12 +27,11 @@ dependencies:
 - pytest-rerunfailures
 - pytest-xdist
 - pytest
-- python=3.8
+- python=3.9
 - scikit-learn=1.0.0
 - setuptools-rust=1.5.2
 - sphinx
 - sqlalchemy<2
 - tpot
 - tzlocal=2.1
 - uvicorn=0.13.4
-- libprotobuf=3

continuous_integration/gpuci/environment-3.10.yaml

@@ -4,7 +4,6 @@ channels:
 - rapidsai-nightly
 - nvidia
 - conda-forge
-- nodefaults
 dependencies:
 - c-compiler
 - dask>=2022.3.0
@@ -13,9 +12,10 @@ dependencies:
 - fugue>=0.7.3
 - intake>=0.6.0
 - jsonschema
+- libprotobuf=3
 - lightgbm
 - maturin>=0.12.8
-- mlflow
+- mlflow>=1.20
 - mock
 - pandas>=1.4.0
 - pre-commit

continuous_integration/gpuci/environment-3.9.yaml

@@ -4,38 +4,37 @@ channels:
 - rapidsai-nightly
 - nvidia
 - conda-forge
-- nodefaults
 dependencies:
 - c-compiler
-- dask>=2022.3.0
-# FIXME: handling is needed for httpx-based fastapi>=0.87.0
-- fastapi>=0.69.0,<0.87.0
-- fugue>=0.7.3
-- intake>=0.6.0
+- dask=2022.3.0
+- fastapi=0.69.0
+- fugue=0.7.3
+- intake=0.6.0
 - jsonschema
+- libprotobuf=3
 - lightgbm
-- maturin>=0.12.8
-- mlflow
+- maturin=0.12.8
+- mlflow>=1.20
 - mock
-- pandas>=1.4.0
+- pandas=1.4.0
 - pre-commit
-- prompt_toolkit>=3.0.8
+- prompt_toolkit=3.0.8
 - psycopg2
-- pyarrow>=6.0.1
-- pygments>=2.7.1
+- pyarrow=6.0.1
+- pygments=2.7.1
 - pyhive
 - pytest-cov
 - pytest-rerunfailures
 - pytest-xdist
 - pytest
 - python=3.9
-- scikit-learn>=1.0.0
-- setuptools-rust>=1.5.2
+- scikit-learn=1.0.0
+- setuptools-rust=1.5.2
 - sphinx
 - sqlalchemy<2
 - tpot
-- tzlocal>=2.1
-- uvicorn>=0.13.4
+- tzlocal=2.1
+- uvicorn=0.13.4
 # GPU-specific requirements
 - cudatoolkit=11.5
 - cudf=23.06

continuous_integration/recipe/conda_build_config.yaml

@@ -1,7 +1,7 @@
 python:
-    - 3.8
     - 3.9
     - 3.10
+    - 3.11
 c_compiler_version:
     - 11
 rust_compiler_version:

dask_planner/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools", "wheel", "setuptools-rust"]
 
 [project]
 name = "datafusion_planner"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 classifiers = [
     "Programming Language :: Rust",
     "Programming Language :: Python :: Implementation :: CPython",

docker/conda.txt

@@ -1,4 +1,4 @@
-python>=3.8
+python>=3.9
 dask>=2022.3.0
 pandas>=1.4.0
 jpype1>=1.0.2

docs/environment.yml

@@ -1,7 +1,6 @@
 name: dask-sql-docs
 channels:
   - conda-forge
-  - nodefaults
 dependencies:
   - python=3.9
   - sphinx>=4.0.0

docs/source/installation.rst

@@ -74,7 +74,7 @@ Create a new conda environment and install the development environment:
 
 .. code-block:: bash
 
-    conda env create -f continuous_integration/environment-3.9-dev.yaml
+    conda env create -f continuous_integration/environment-3.9.yaml
 
 It is not recommended to use ``pip`` instead of ``conda``.
 

setup.py

@@ -39,7 +39,7 @@
             debug=debug_build,
         )
     ],
-    python_requires=">=3.8",
+    python_requires=">=3.9",
     setup_requires=sphinx_requirements,
     install_requires=[
         "dask[dataframe]>=2022.3.0",

tests/integration/test_fugue.py

@@ -40,7 +40,8 @@ def test_fugue_fsql(client):
     assert_eq(return_df, pd.DataFrame({"a": [1], "b": ["world"]}))
 
 
-@pytest.mark.flaky(reruns=4, condition="sys.version_info < (3, 9)")
+# TODO: uncomment if flaky failures persist on python 3.9
+# @pytest.mark.flaky(reruns=4, condition="sys.version_info < (3, 9)")
 def test_dask_fsql(client):
     def assert_fsql(df: pd.DataFrame) -> None:
         assert_eq(df, pd.DataFrame({"a": [1]}))