Update iceberg and iceberg-catalog-rest to 0.9 (#1170) #1560
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| branches: ["main"] | |
| push: | |
| branches: ["main"] | |
| tags: ["*"] | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| packages: write | |
| jobs: | |
| compute-base-build-tag: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| image_tag: ${{ steps.out.outputs.tag }} | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Create Tag | |
| id: out | |
| run: echo "tag=${{ hashFiles('Dockerfile','rust-toolchain.toml') }}" >> "$GITHUB_OUTPUT" | |
| build-base: | |
| needs: compute-base-build-tag | |
| runs-on: ubuntu-latest | |
| env: | |
| CACHE_HIT: false | |
| IMAGE: ghcr.io/${{ github.repository }}/base:${{ needs.compute-base-build-tag.outputs.image_tag }} | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Pull Cached base Image | |
| run: | | |
| if docker manifest inspect "$IMAGE" > /dev/null 2>&1; then | |
| echo "CACHE_HIT=true" >> $GITHUB_ENV | |
| else | |
| echo "CACHE_HIT=false" >> $GITHUB_ENV | |
| fi | |
| - name: Set up Docker Buildx | |
| if: env.CACHE_HIT == 'false' | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 | |
| - name: Build and Push base Image (if not cached) | |
| if: env.CACHE_HIT == 'false' | |
| uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| target: base | |
| push: true | |
| tags: ghcr.io/${{ github.repository }}/base:${{ needs.compute-base-build-tag.outputs.image_tag }} | |
| fmt: | |
| needs: [compute-base-build-tag, build-base] | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/${{ github.repository }}/base:${{ needs.compute-base-build-tag.outputs.image_tag }} | |
| options: --user 1001:1001 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-fmt | |
| cancel-in-progress: true | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Check formatting | |
| run: cargo fmt -- --check | |
| clippy: | |
| needs: [compute-base-build-tag, build-base] | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/${{ github.repository }}/base:${{ needs.compute-base-build-tag.outputs.image_tag }} | |
| options: --user 1001:1001 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-clippy | |
| cancel-in-progress: true | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Cache Cargo Target Directory | |
| uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: target | |
| key: clippy-base-${{ needs.compute-base-build-tag.outputs.image_tag }}-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| clippy-base-${{ needs.compute-base-build-tag.outputs.image_tag }}- | |
| - name: Clippy | |
| run: cargo clippy --all-targets --locked -- -Dclippy::all -D warnings | |
| generate-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.get-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Restore Cached Matrix | |
| id: cache-matrix | |
| uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: matrix.json | |
| key: matrix-${{ hashFiles('Cargo.toml', 'Cargo.lock') }} | |
| - name: Extract Cargo Workspace Members (if not cached) | |
| if: steps.cache-matrix.outputs.cache-hit != 'true' | |
| run: | | |
| set -e | |
| cargo metadata --format-version=1 | jq -c '[.workspace_members[] | split("#")[0] | split("/") | last | gsub("_"; "-") | select(. != "metrics")]' > matrix.json | |
| - name: Save Matrix to Output | |
| id: get-matrix | |
| run: echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT | |
| tests: | |
| needs: [compute-base-build-tag, build-base, generate-matrix] | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/${{ github.repository }}/base:${{ needs.compute-base-build-tag.outputs.image_tag }} | |
| options: --user 1001:1001 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| package: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-tests-${{ matrix.package }} | |
| cancel-in-progress: true | |
| services: | |
| postgres: | |
| image: postgres:14.17 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: POLARIS | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| rustfs: | |
| image: rustfs/rustfs:1.0.0-alpha.90 | |
| ports: | |
| - 9000:9000 | |
| - 4566:9000 | |
| env: | |
| RUSTFS_CONSOLE_ENABLE: false | |
| RUSTFS_ACCESS_KEY: admin | |
| RUSTFS_SECRET_KEY: admin | |
| polaris: | |
| image: apache/polaris:1.3.0-incubating | |
| env: | |
| POLARIS_PERSISTENCE_TYPE: in-memory | |
| POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,s3cr3t | |
| POLARIS_REALM_CONTEXT_REALMS: POLARIS | |
| QUARKUS_OTEL_SDK_DISABLED: "true" | |
| POLARIS_FEATURES__ALLOW_INSECURE_STORAGE_TYPES: "true" | |
| POLARIS_FEATURES__SUPPORTED_CATALOG_STORAGE_TYPES: '["S3"]' | |
| POLARIS_READINESS__IGNORE_SEVERE_ISSUES: "true" | |
| POLARIS_FEATURES__DROP_WITH_PURGE_ENABLED: "true" | |
| AWS_REGION: us-west-2 | |
| AWS_ACCESS_KEY_ID: admin | |
| AWS_SECRET_ACCESS_KEY: admin | |
| ports: | |
| - 8181:8181 | |
| trino: | |
| image: trinodb/trino:480 | |
| env: | |
| AWS_ACCESS_KEY_ID: admin | |
| AWS_SECRET_ACCESS_KEY: admin | |
| CATALOG_MANAGEMENT: dynamic | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Cache Cargo Target Directory | |
| # Anything above 4min run | |
| if: contains('["ingest", "mobile-packet-verifier", "mobile-verifier", "price", "solana"]', matrix.package) | |
| uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 | |
| with: | |
| path: target | |
| key: tests-${{ matrix.package }}-base-${{ needs.compute-base-build-tag.outputs.image_tag }}-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| tests-${{ matrix.package }}-base-${{ needs.compute-base-build-tag.outputs.image_tag }}- | |
| - name: Install AWS CLI (user install) | |
| run: | | |
| curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" | |
| unzip -q /tmp/awscliv2.zip -d /tmp | |
| /tmp/aws/install --install-dir $HOME/.local/aws-cli --bin-dir $HOME/.local/bin | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Rustfs init | |
| timeout-minutes: 1 | |
| env: | |
| AWS_ACCESS_KEY_ID: admin | |
| AWS_SECRET_ACCESS_KEY: admin | |
| run: | | |
| until curl -sf http://rustfs:9000/health > /dev/null 2>&1; do | |
| echo "...waiting for Rustfs..." | |
| sleep 2 | |
| done | |
| ./infra/rustfs/init.sh | |
| - name: Wait for Trino | |
| timeout-minutes: 1 | |
| run: | | |
| until curl -sf http://trino:8080/v1/info | grep -q '"starting":false'; do | |
| echo "...waiting for Trino..." | |
| sleep 2 | |
| done | |
| echo "Trino is up" | |
| - name: Run tests | |
| env: | |
| DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres # trufflehog:ignore | |
| AWSLOCAL_ENDPOINT: http://rustfs:9000 | |
| S3_QUALIFIED_HOST: rustfs | |
| S3_LOCAL_HOST: rustfs | |
| CATALOG_QUALIFIED_HOST: polaris | |
| CATALOG_LOCAL_HOST: polaris | |
| TRINO_HOST: trino | |
| run: cargo test -p ${{ matrix.package }} --locked | |
| build-mobile-images: | |
| if: startsWith(github.ref, 'refs/tags/') | |
| needs: [fmt, clippy, tests] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| package: | |
| [ | |
| "ingest", | |
| "mobile-config", | |
| "mobile-packet-verifier", | |
| "mobile-verifier", | |
| "price", | |
| "reward-index", | |
| ] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-build-image-${{ matrix.package }} | |
| cancel-in-progress: true | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 | |
| - name: Build and Push Base Image | |
| uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| target: runner | |
| push: true | |
| tags: ghcr.io/${{ github.repository }}/${{ matrix.package }}:${{ github.ref_name }} | |
| build-args: | | |
| PACKAGE=${{ matrix.package }} |