diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b22b2c205..64d61e73c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -105,7 +105,8 @@ jobs: with: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} - test_independent: + + cluster: name: "Test in a dask cluster" needs: build runs-on: ubuntu-latest @@ -153,7 +154,8 @@ jobs: pytest tests env: DASK_SQL_TEST_SCHEDULER: tcp://127.0.0.1:8786 - test_import: + + import: name: "Test importing with bare requirements" needs: build runs-on: ubuntu-latest @@ -188,3 +190,43 @@ jobs: shell: bash -l {0} run: | python -c "import dask_sql; print('ok')" + + conda: + name: "Test building the conda package" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + use-mamba: true + python-version: 3.8 + - name: Install dependencies + shell: bash -l {0} + run: | + mamba install conda-build conda-verify + + which python + pip list + mamba list + - name: Build conda package + if: ${{ github.event_name != 'push' || github.repository != 'dask-contrib/dask-sql' }} + shell: bash -l {0} + run: | + # suffix for nightly package versions + export VERSION_SUFFIX=`date +%y%m%d` + + mamba build continuous_integration/recipe --no-anaconda-upload + - name: Build and upload conda package + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'dask-contrib/dask-sql' }} + shell: bash -l {0} + env: + ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }} + run: | + # suffix for nightly package versions + export VERSION_SUFFIX=`date +%y%m%d` + + mamba build continuous_integration/recipe --label dev diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 51786832f..e50a5f1d5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,4 +22,5 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml + exclude: ^continuous_integration/recipe/ - id: check-added-large-files diff --git a/continuous_integration/recipe/meta.yaml b/continuous_integration/recipe/meta.yaml new file mode 100644 index 000000000..7c7b2e210 --- /dev/null +++ b/continuous_integration/recipe/meta.yaml @@ -0,0 +1,49 @@ +{% set name = "dask-sql" %} +{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev') + environ.get('VERSION_SUFFIX', '') %} +{% set py_version=environ.get('CONDA_PY', 36) %} + + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + git_url: ../.. + +build: + noarch: python + number: {{ GIT_DESCRIBE_NUMBER }} + string: py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script: {{ PYTHON }} -m pip install . --no-deps -vv + +requirements: + build: + - maven >=3.6.0 + host: + - pip + - python >=3.6 + - setuptools_scm + run: + - python >=3.6 + - dask >=2.19.0,!=2021.3.0 + - pandas >=1.0.0 + - jpype1 >=1.0.2 + - openjdk >=8 + - fastapi >=0.61.1 + - uvicorn >=0.11.3 + - tzlocal >=2.1 + - prompt_toolkit >=3.0.8 + - pygments >=2.7.3 + - nest-asyncio >=1.0.0 + +test: + commands: + - pip check + requires: + - pip + +about: + home: https://github.com/dask-contrib/dask-sql + summary: SQL Query Layer for dask + license: MIT + license_file: LICENSE.txt diff --git a/continuous_integration/recipe/run_test.py b/continuous_integration/recipe/run_test.py new file mode 100644 index 000000000..0ca97261b --- /dev/null +++ b/continuous_integration/recipe/run_test.py @@ -0,0 +1,31 @@ +import dask.dataframe as dd +import pandas as pd + +from dask_sql import Context + +c = Context() + +data = """ +name,x +Alice,34 +Bob, +""" + +df = pd.DataFrame({"name": ["Alice", "Bob", "Chris"] * 100, "x": list(range(300))}) +ddf = dd.from_pandas(df, npartitions=10) +c.create_table("my_data", ddf) + +got = c.sql( + """ + SELECT + my_data.name, + SUM(my_data.x) AS "S" + FROM + my_data + GROUP BY + my_data.name +""" +) +expect = pd.DataFrame({"name": ["Alice", "Bob", "Chris"], "S": [14850, 14950, 15050]}) + +dd.assert_eq(got, expect)