LakeSoul/.github/workflows/python-ci.yml at main · lakesoul-io/LakeSoul · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
name: CI with Python Test
on:
  push:
    paths-ignore:
      - "javadoc/**"
      - "website/**"
      - "**.md"
    branches:
      - 'main'
      - 'release/**'
    tags:
      - 'py-v[0-9]+.[0-9]+.[0-9]+'
  pull_request:
    paths-ignore:
      - "javadoc/**"
      - "website/**"
      - "**.md"
    branches:
      - 'main'
      - 'release/**'
  workflow_dispatch:

jobs:
  run-pytest:
    runs-on: ubuntu-24.04
    services:
      # Label used to access the service container
      postgres:
        # Docker Hub image
        image: postgres:14.5
        # Provide the password for postgres
        env:
          POSTGRES_PASSWORD: lakesoul_test
          POSTGRES_USER: lakesoul_test
          POSTGRES_DB: lakesoul_test
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
          --name lakesoul-test-pg
        ports:
          # Maps tcp port 5432 on service container to the host
          - 5432:5432
    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: false
          docker-images: true
          swap-storage: true
      - uses: actions/checkout@v4
      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: stable
      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          java-version: '11'
          distribution: 'temurin'
          cache: maven
      - name: Install psql
        run: sudo apt-get install -y postgresql-client-16
      - name: Init PG
        run: |
          ./script/meta_init_for_local_test.sh -j 2
      - name: Install Protoc
        uses: arduino/setup-protoc@v2
        with:
          version: "23.x"
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true
          cache-suffix: "lakesoul-py-ci"
          cache-dependency-glob: |
            pyproject.toml
            uv.lock
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: ". -> rust/target"
          env-vars: "JAVA_HOME"
      - name: pytest
        env:
          LAKESOUL_SOURCE_DIR: ${{ github.workspace }}
        run: |
          mvn package -pl lakesoul-spark -am -DskipTests
          cd python
          uv sync
          source .venv/bin/activate
          export LAKESOUL_IO_USE_V2_MERGE=true
          cargo run --release -p lakesoul-console -- tpch-gen -p "file:///tmp/lakesoul/tpch_data" --scale-factor 0.1 -n 8
          cargo run --release -p lakesoul-console -- -f test.sql
          uv run pytest ./tests/ -s
  publish:
    runs-on: ubuntu-24.04
    if: startsWith(github.event.ref, 'refs/tags/py-v')
    needs: ["run-pytest"]
    env:
      PIP_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
    permissions:
      id-token: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Install uv
        uses: astral-sh/setup-uv@v6
      - name: build
        run: |
          cd python && uvx --from 'maturin[zig]' maturin build --release --zig --target x86_64-unknown-linux-gnu --auditwheel repair --compatibility manylinux2014 --out ../dist
      - name: Publish package distributions to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1