diff --git a/.github/workflows/_test_python_source.yml b/.github/workflows/_test_python_source.yml index fae14016f5..1427993137 100644 --- a/.github/workflows/_test_python_source.yml +++ b/.github/workflows/_test_python_source.yml @@ -23,11 +23,15 @@ on: required: false default: 'ubuntu-latest' type: string - # Options: DF_LIB, DF_LIB_NO_JIT, SPAWN, NORMAL + # Options: DF_LIB, DF_LIB_NO_JIT, SPAWN, NORMAL, NARWHALS test-type: type: string description: The kind of tests to run e.g. spawn tests required: true + bodo-version: + type: string + description: The version string of Bodo. + required: true jobs: run: @@ -43,6 +47,8 @@ jobs: uses: ./.github/actions/build-source with: build-all: true + env: + SETUPTOOLS_SCM_PRETEND_VERSION: ${{ inputs.bodo-version }} - name: Load Hadoop from Cache if: runner.os != 'Windows' uses: actions/cache/restore@v4 @@ -66,6 +72,10 @@ jobs: run: | pip install transformers pip install --no-deps 'git+https://github.com/apache/polaris.git@release/1.0.x#subdirectory=client/python' + - name: Install Narwhals Test dependencies + if: ${{ inputs.test-type == 'NARWHALS' }} + run: | + pip install hypothesis polars sqlframe ibis-framework pyarrow-hotfix # Run Tests - name: Run Tests @@ -81,40 +91,50 @@ jobs: fi set +eo pipefail - # Bodo Tests: Coverage is collected - pytest -s -v -Wignore \ - --cov-report= --cov=bodo \ - --splits=${{ inputs.total-batches }} --group=${{ inputs.batch }} \ - --store-durations --clean-durations \ - --durations-path=buildscripts/github/test_dur_bodo.json \ - -m "$PYTEST_MARKER" $PYTEST_IGNORE bodo/tests/ - # Save Exit Code for Later - python_test_exit_code=$? + if [[ "${{ inputs.test-type }}" == "NARWHALS" ]]; then + # Checkout narwhals fork + git clone https://github.com/bodo-ai/narwhals.git + cd narwhals + python -c "import bodo; print('Detected Bodo version:', bodo.__version__)" + pytest -s -v -Wignore -k bodo + exit $? + else + # Bodo Tests: Coverage is collected + pytest -s -v -Wignore \ + --cov-report= --cov=bodo \ + --splits=${{ inputs.total-batches }} --group=${{ inputs.batch }} \ + --store-durations --clean-durations \ + --durations-path=buildscripts/github/test_dur_bodo.json \ + -m "$PYTEST_MARKER" $PYTEST_IGNORE bodo/tests/ - # BodoSQL Tests - cd BodoSQL - pytest -s -v -Wignore \ - --splits=${{ inputs.total-batches }} --group=${{ inputs.batch }} \ - --store-durations --clean-durations \ - --durations-path=../buildscripts/github/test_dur_bodosql.json \ - -m "$PYTEST_MARKER" bodosql/tests/ - sql_test_exit_code=$? + # Save Exit Code for Later + python_test_exit_code=$? - # Pytest exits code 5 if no tests are run. Some markers are only in python or sql - # so we suppress this. - if [ "$python_test_exit_code" -eq 5 ]; then - python_test_exit_code=0 - fi - if [ "$sql_test_exit_code" -eq 5 ]; then - sql_test_exit_code=0 - fi + # BodoSQL Tests + cd BodoSQL + pytest -s -v -Wignore \ + --splits=${{ inputs.total-batches }} --group=${{ inputs.batch }} \ + --store-durations --clean-durations \ + --durations-path=../buildscripts/github/test_dur_bodosql.json \ + -m "$PYTEST_MARKER" bodosql/tests/ + sql_test_exit_code=$? - # Merge Exit Codes. - # If neither Bodo nor BodoSQL fails, the exit code will be 0 - # If one fails, the exit code will be 1 - # If both fail, the exit code will be 2 - exit $((python_test_exit_code + sql_test_exit_code)) + # Pytest exits code 5 if no tests are run. Some markers are only in python or sql + # so we suppress this. + if [ "$python_test_exit_code" -eq 5 ]; then + python_test_exit_code=0 + fi + if [ "$sql_test_exit_code" -eq 5 ]; then + sql_test_exit_code=0 + fi + + # Merge Exit Codes. + # If neither Bodo nor BodoSQL fails, the exit code will be 0 + # If one fails, the exit code will be 1 + # If both fail, the exit code will be 2 + exit $((python_test_exit_code + sql_test_exit_code)) + fi shell: bash env: BODO_TESTING_ONLY_RUN_1D_VAR: true @@ -128,6 +148,7 @@ jobs: BODO_TEST_SPAWN_MODE: ${{ inputs.test-type == 'SPAWN' && '1' || '0' }} # Disabling the DataFrame library for spawn tests since some of the tests # create Pandas manager states for testing that are not fully functional. + BODO_DATAFRAME_LIBRARY_WARN: ${{ inputs.test-type == 'NARWHALS' && '0' || '1' }} BODO_ENABLE_DATAFRAME_LIBRARY: ${{ inputs.test-type != 'SPAWN' && '1' || '0' }} BODO_ENABLE_TEST_DATAFRAME_LIBRARY: ${{ startsWith(inputs.test-type, 'DF_LIB') && '1' || '0' }} BODOSQL_PY4J_GATEWAY_PORT: "auto" diff --git a/.github/workflows/pr_ci.yml b/.github/workflows/pr_ci.yml index a58279aff8..9331eaa9a3 100644 --- a/.github/workflows/pr_ci.yml +++ b/.github/workflows/pr_ci.yml @@ -16,6 +16,25 @@ concurrency: cancel-in-progress: true jobs: + get_version: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v5 + with: + # For setuptools_scm to be able to determine version, + # we need to fetch the entire history + fetch-depth: 0 + - name: Get Version + id: get_version + run: | + set -exo pipefail + pip install setuptools_scm + # Print to the logs + echo "bodo_version=$(python -m setuptools_scm)" + echo "bodo_version=$(python -m setuptools_scm)" >> "$GITHUB_OUTPUT" + outputs: + bodo_version: ${{ steps.get_version.outputs.bodo_version }} # 1) Validate changes and commit message validate: @@ -107,7 +126,7 @@ jobs: # 4) Actually run tests pr-ci: - needs: [compile-bodo] + needs: [compile-bodo, get_version] name: Test Compiler strategy: matrix: @@ -122,10 +141,11 @@ jobs: collect-coverage: true os: ${{ inputs.runner_os || 'ubuntu-latest' }} test-type: NORMAL + bodo-version: ${{ needs.get_version.outputs.bodo_version }} secrets: inherit spawn-ci: - needs: [compile-bodo] + needs: [compile-bodo, get_version] name: Test Spawn uses: ./.github/workflows/_test_python_source.yml with: @@ -135,10 +155,11 @@ jobs: collect-coverage: true os: ${{ inputs.runner_os || 'ubuntu-latest' }} test-type: SPAWN + bodo-version: ${{ needs.get_version.outputs.bodo_version }} secrets: inherit df-lib-non-jit-ci: - needs: [compile-bodo] + needs: [compile-bodo, get_version] name: Test DF Library Non-JIT uses: ./.github/workflows/_test_python_source.yml with: @@ -148,10 +169,11 @@ jobs: collect-coverage: true os: ${{ inputs.runner_os || 'ubuntu-latest' }} test-type: "DF_LIB_NO_JIT" + bodo-version: ${{ needs.get_version.outputs.bodo_version }} secrets: inherit df-lib-ci: - needs: [compile-bodo] + needs: [compile-bodo, get_version] name: Test DF Library strategy: matrix: @@ -166,10 +188,25 @@ jobs: collect-coverage: true os: ${{ inputs.runner_os || 'ubuntu-latest' }} test-type: "DF_LIB" + bodo-version: ${{ needs.get_version.outputs.bodo_version }} + secrets: inherit + + narwhals-ci: + needs: [compile-bodo, get_version] + name: Test Narwhals Bodo Integration + uses: ./.github/workflows/_test_python_source.yml + with: + batch: 1 + total-batches: 1 + pytest-marker: "" + collect-coverage: false + os: ${{ inputs.runner_os || 'ubuntu-latest' }} + test-type: NARWHALS + bodo-version: ${{ needs.get_version.outputs.bodo_version }} secrets: inherit bodosql-cpp-ci: - needs: [compile-bodo] + needs: [compile-bodo, get_version] name: Test BodoSQL C++ Backend uses: ./.github/workflows/_test_python_source.yml with: @@ -179,6 +216,7 @@ jobs: collect-coverage: true os: ${{ inputs.runner_os || 'ubuntu-latest' }} test-type: "BODOSQL_CPP" + bodo-version: ${{ needs.get_version.outputs.bodo_version }} secrets: inherit java-ci: @@ -268,6 +306,7 @@ jobs: - collect-results - df-lib-ci - df-lib-non-jit-ci + - narwhals-ci - bodosql-cpp-ci - validate runs-on: ubuntu-latest