Fix pyarrow / cloudpickle failures in cluster testing

.github/cluster-upstream.yml

@@ -5,9 +5,6 @@ services:
         container_name: dask-scheduler
         image: daskdev/dask:dev
         command: dask-scheduler
-        environment:
-            USE_MAMBA: "true"
-            EXTRA_CONDA_PACKAGES: "dask/label/dev::dask"
         ports:
             - "8786:8786"
     dask-worker:
@@ -16,6 +13,6 @@ services:
         command: dask-worker dask-scheduler:8786
         environment:
             USE_MAMBA: "true"
-            EXTRA_CONDA_PACKAGES: "dask/label/dev::dask pyarrow>1.0.0"  # required for parquet IO
+            EXTRA_CONDA_PACKAGES: "dask/label/dev::dask cloudpickle>=2.1.0 pyarrow>=1.0.0 libstdcxx-ng>=12.1.0"
         volumes:
             - /tmp:/tmp

.github/cluster.yml

@@ -13,6 +13,6 @@ services:
         command: dask-worker dask-scheduler:8786
         environment:
             USE_MAMBA: "true"
-            EXTRA_CONDA_PACKAGES: "pyarrow>1.0.0"  # required for parquet IO
+            EXTRA_CONDA_PACKAGES: "cloudpickle>=2.1.0 pyarrow>=1.0.0 libstdcxx-ng>=12.1.0"
         volumes:
             - /tmp:/tmp

continuous_integration/environment-3.10-jdk11-dev.yaml

@@ -26,7 +26,7 @@ dependencies:
 - pre-commit>=2.11.1
 - prompt_toolkit>=3.0.8
 - psycopg2>=2.9.1
-- pyarrow>=0.15.1
+- pyarrow>=1.0.0
 - pygments>=2.7.1
 - pyhive>=0.6.4
 - pytest-cov>=2.10.1

continuous_integration/environment-3.10-jdk8-dev.yaml

@@ -26,7 +26,7 @@ dependencies:
 - pre-commit>=2.11.1
 - prompt_toolkit>=3.0.8
 - psycopg2>=2.9.1
-- pyarrow>=0.15.1
+- pyarrow>=1.0.0
 - pygments>=2.7.1
 - pyhive>=0.6.4
 - pytest-cov>=2.10.1

continuous_integration/environment-3.8-jdk11-dev.yaml

@@ -26,7 +26,7 @@ dependencies:
 - pre-commit>=2.11.1
 - prompt_toolkit>=3.0.8
 - psycopg2>=2.9.1
-- pyarrow>=0.15.1
+- pyarrow>=1.0.0
 - pygments>=2.7.1
 - pyhive>=0.6.4
 - pytest-cov>=2.10.1

continuous_integration/environment-3.8-jdk8-dev.yaml

@@ -26,7 +26,7 @@ dependencies:
 - pre-commit>=2.11.1
 - prompt_toolkit>=3.0.8
 - psycopg2>=2.9.1
-- pyarrow>=0.15.1
+- pyarrow>=1.0.0
 - pygments>=2.7.1
 - pyhive>=0.6.4
 - pytest-cov>=2.10.1

continuous_integration/environment-3.9-jdk11-dev.yaml

@@ -26,7 +26,7 @@ dependencies:
 - pre-commit>=2.11.1
 - prompt_toolkit>=3.0.8
 - psycopg2>=2.9.1
-- pyarrow>=0.15.1
+- pyarrow>=1.0.0
 - pygments>=2.7.1
 - pyhive>=0.6.4
 - pytest-cov>=2.10.1

continuous_integration/environment-3.9-jdk8-dev.yaml

@@ -26,7 +26,7 @@ dependencies:
 - pre-commit>=2.11.1
 - prompt_toolkit>=3.0.8
 - psycopg2>=2.9.1
-- pyarrow>=0.15.1
+- pyarrow>=1.0.0
 - pygments>=2.7.1
 - pyhive>=0.6.4
 - pytest-cov>=2.10.1

docker/conda.txt

@@ -13,7 +13,7 @@ tzlocal>=2.1
 fastapi>=0.61.1
 nest-asyncio>=1.4.3
 uvicorn>=0.11.3
-pyarrow>=0.15.1
+pyarrow>=1.0.0
 prompt_toolkit>=3.0.8
 pygments>=2.7.1
 dask-ml>=2022.1.22

docker/main.dockerfile

@@ -13,7 +13,7 @@ RUN conda config --add channels conda-forge \
     "tzlocal>=2.1" \
     "fastapi>=0.61.1" \
     "uvicorn>=0.11.3" \
-    "pyarrow>=0.15.1" \
+    "pyarrow>=1.0.0" \
     "prompt_toolkit>=3.0.8" \
     "pygments>=2.7.1" \
     "dask-ml>=2022.1.22" \

setup.py

@@ -104,7 +104,7 @@ def build(self):
             "pytest-cov>=2.10.1",
             "mock>=4.0.3",
             "sphinx>=3.2.1",
-            "pyarrow>=0.15.1",
+            "pyarrow>=1.0.0",
             "dask-ml>=2022.1.22",
             "scikit-learn>=0.24.2",
             "intake>=0.6.0",