reusable_build_and_test_wheels.yml

name: Reusable Build and Test Wheels

on:
  workflow_call:
    inputs:
      CONCURRENCY:
        required: true
        type: string
      MATURIN_FEATURE_FLAGS:
        required: false
        type: string
        default: "--no-default-features --features pypi"
      PLATFORM:
        required: true
        type: string
      RELEASE_VERSION:
        required: false
        type: string
        default: "prerelease"
      RRD_ARTIFACT_NAME:
        required: false
        type: string
        default: ""
      WHEEL_ARTIFACT_NAME:
        required: false
        type: string
        default: ""
      RELEASE_COMMIT:
        required: false
        type: string
        default: ""

concurrency:
  group: ${{ inputs.CONCURRENCY }}-build-wheels
  cancel-in-progress: true

env:
  PYTHON_VERSION: "3.8"
  # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses
  # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html
  # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html

  # TODO(jleibs) --deny warnings causes installation of wasm-bindgen to fail on mac
  # RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings
  RUSTFLAGS: --cfg=web_sys_unstable_apis

  # See https://github.com/ericseppanen/cargo-cranky/issues/8
  RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs

  # See: https://github.com/marketplace/actions/sccache-action
  SCCACHE_GHA_ENABLED: "false"

permissions:
  contents: "read"
  id-token: "write"

jobs:
  # ---------------------------------------------------------------------------

  set-config:
    name: Set Config (${{ inputs.PLATFORM }})
    runs-on: ubuntu-latest-16-cores
    outputs:
      RUNNER: ${{ steps.set-config.outputs.runner }}
      TARGET: ${{ steps.set-config.outputs.target }}
      RUN_TESTS: ${{ steps.set-config.outputs.run_tests }}
      CONTAINER: ${{ steps.set-config.outputs.container }}
    steps:
      - name: Set runner and target based on platform
        id: set-config
        run: |
          case "${{ inputs.PLATFORM }}" in
            linux)
              runner="ubuntu-latest-16-cores"
              target="x86_64-unknown-linux-gnu"
              run_tests="true"
              container="{'image': 'rerunio/ci_docker:0.10.0'}"
              ;;
            windows)
              runner="windows-latest-8-cores"
              target="x86_64-pc-windows-msvc"
              run_tests="true"
              container="null"
              ;;
            macos-arm)
              runner="macos-latest"
              target="aarch64-apple-darwin"
              run_tests="false"
              container="null"
              ;;
            macos-intel)
              runner="macos-latest"
              target="x86_64-apple-darwin"
              run_tests="false"
              container="null"
              ;;
            *) echo "Invalid platform" && exit 1 ;;
          esac
          echo "runner=$runner" >> "$GITHUB_OUTPUT"
          echo "target=$target" >> "$GITHUB_OUTPUT"
          echo "run_tests=$run_tests" >> "$GITHUB_OUTPUT"
          echo "container=$container" >> "$GITHUB_OUTPUT"

  # ---------------------------------------------------------------------------

  build-wheels:
    name: Build Wheels (${{ needs.set-config.outputs.RUNNER }})

    needs: [set-config]

    runs-on: ${{ needs.set-config.outputs.RUNNER }}
    container: ${{ fromJson(needs.set-config.outputs.CONTAINER) }}

    steps:
      - name: Show context
        run: |
          echo "GITHUB_CONTEXT": $GITHUB_CONTEXT
          echo "JOB_CONTEXT": $JOB_CONTEXT
          echo "INPUTS_CONTEXT": $INPUTS_CONTEXT
          echo "ENV_CONTEXT": $ENV_CONTEXT
        env:
          ENV_CONTEXT: ${{ toJson(env) }}
          GITHUB_CONTEXT: ${{ toJson(github) }}
          JOB_CONTEXT: ${{ toJson(job) }}
          INPUTS_CONTEXT: ${{ toJson(inputs) }}

      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.ref) || '') }}

      - name: Set up Rust
        uses: ./.github/actions/setup-rust
        with:
          cache_key: "build-${{ inputs.PLATFORM }}"
          # Cache will be produced by `reusable_checks/rs-lints`
          save_cache: false
          workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
          service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}

      # The pip-cache setup logic doesn't work in the ubuntu docker container
      # That's probably fine since we bake these deps into the container already
      - name: Setup python
        if: ${{ inputs.PLATFORM != 'linux' }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: "pip"
          cache-dependency-path: "scripts/ci/requirements.txt"

      # These should already be in the docker container, but run for good measure. A no-op install
      # should be fast, and this way things don't break if we add new packages without rebuilding
      # docker
      - run: pip install -r scripts/ci/requirements.txt

      # We have a nice script for that: ./scripts/setup_web.sh
      # Unfortunately, we can't run that on Windows, because Windows doesn't come with
      # a package manager like grown-up OSes do (at least not the CI version of Windows).
      # Also we can't run it on linux because the 20.04 Docker container will install
      # an old version of binaryen/wasm-opt that barfs on the `--fast-math` flag
      # So we only run the script on macos, and then on Windows we do the parts of the script manually.
      # On ubuntu, the correct packages are pre-installed in our docker container.

      - name: Install prerequisites for building the web-viewer Wasm (non-Windows)
        if: (inputs.PLATFORM == 'macos-intel') || (inputs.PLATFORM == 'macos-arm')
        shell: bash
        run: ./scripts/setup_web.sh

      # The first steps of setup_web.sh, for Windows:
      - name: Install wasm32 cargo target for building the web-viewer Wasm on windows
        if: inputs.PLATFORM == 'windows'
        shell: bash
        run: rustup target add wasm32-unknown-unknown

      # The last step of setup_web.sh, for Windows.
      # Since 'winget' is not available within the GitHub runner, we download the package directly:
      # See: https://github.com/marketplace/actions/engineerd-configurator
      - name: Install binaryen for building the web-viewer Wasm on windows
        if: inputs.PLATFORM == 'windows'
        uses: engineerd/configurator@v0.0.9
        with:
          name: "wasm-opt.exe"
          url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz"
          pathInArchive: "binaryen-version_111/bin/wasm-opt.exe"

      - name: Build Wheel
        uses: PyO3/maturin-action@v1
        with:
          maturin-version: "0.14.17"
          manylinux: manylinux_2_31
          container: off
          command: build
          sccache: "true"
          args: |
            --manifest-path rerun_py/Cargo.toml
            --release
            --target ${{ needs.set-config.outputs.TARGET }}
            ${{ inputs.MATURIN_FEATURE_FLAGS }}
            --out dist

      - name: Save wheel artifact
        if: ${{ inputs.WHEEL_ARTIFACT_NAME != '' }}
        uses: actions/upload-artifact@v3
        with:
          name: ${{inputs.WHEEL_ARTIFACT_NAME}}
          path: dist

      # ---------------------------------------------------------------------------
      # Test the wheel

      - name: Install wheel dependencies
        if: needs.set-config.outputs.RUN_TESTS == 'true'
        # First we install the dependencies manually so we can use `--no-index` when installing the wheel.
        # This needs to be a separate step for some reason or the following step fails
        # TODO(jleibs): pull these deps from pyproject.toml
        # TODO(jleibs): understand why deps can't be installed in the same step as the wheel
        shell: bash
        run: |
          pip install attrs>=23.1.0 numpy>=1.23 pillow pyarrow==10.0.1 pytest==7.1.2 torch==2.1.0 typing_extensions>=4.5

      - name: Get version
        id: get-version
        if: needs.set-config.outputs.RUN_TESTS == 'true'
        shell: bash
        run: |
          echo "wheel_version=$(python3 scripts/ci/crates.py get-version)" >> "$GITHUB_OUTPUT"

      - name: Install built wheel
        if: needs.set-config.outputs.RUN_TESTS == 'true'
        # Now install the wheel using a specific version and --no-index to guarantee we get the version from
        # the pre-dist folder. Note we don't use --force-reinstall here because --no-index means it wouldn't
        # find the dependencies to reinstall them.
        shell: bash
        run: |
          pip uninstall rerun-sdk
          pip install rerun-sdk==${{ steps.get-version.outputs.wheel_version }} --no-index --find-links dist

      - name: Verify built wheel version
        if: needs.set-config.outputs.RUN_TESTS == 'true'
        shell: bash
        run: |
          python3 -m rerun --version
          which rerun
          rerun --version

      - name: Run unit tests
        if: needs.set-config.outputs.RUN_TESTS == 'true'
        shell: bash
        run: cd rerun_py/tests && pytest -c ../pyproject.toml

      - name: Run e2e test
        if: needs.set-config.outputs.RUN_TESTS == 'true'
        shell: bash
        run: RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed

      - name: Run tests/roundtrips.py
        if: ${{ (needs.set-config.outputs.RUN_TESTS == 'true') && (inputs.PLATFORM != 'windows') }}
        shell: bash
        # --release so we can inherit from some of the artifacts that maturin has just built before
        # --target x86_64-unknown-linux-gnu because otherwise cargo loses the target cache… even though this is the target anyhow…
        # --no-py-build because rerun-sdk is already built and installed
        run: |
          RUST_LOG=debug tests/roundtrips.py --release --target x86_64-unknown-linux-gnu --no-py-build

      - name: Run docs/code-examples/roundtrips.py
        if: ${{ (needs.set-config.outputs.RUN_TESTS == 'true') && (inputs.PLATFORM != 'windows') }}
        shell: bash
        # --release so we can inherit from some of the artifacts that maturin has just built before
        # --target x86_64-unknown-linux-gnu because otherwise cargo loses the target cache… even though this is the target anyhow…
        # --no-py-build because rerun-sdk is already built and installed
        run: |
          RUST_LOG=debug docs/code-examples/roundtrips.py --release --target x86_64-unknown-linux-gnu --no-py-build

      - name: Cache RRD dataset
        if: needs.set-config.outputs.RUN_TESTS == 'true'
        id: dataset
        uses: actions/cache@v3
        with:
          path: examples/python/structure_from_motion/dataset/
          # TODO(jleibs): Derive this key from the invocation below
          key: structure-from-motion-dataset-structure-from-motion-fiat-v1

      - name: Generate Embedded RRD file
        if: needs.set-config.outputs.RUN_TESTS == 'true'
        shell: bash
        # If you change the line below you should almost definitely change the `key:` line above by giving it a new, unique name
        run: |
          mkdir rrd
          pip install -r examples/python/structure_from_motion/requirements.txt
          python3 examples/python/structure_from_motion/main.py --dataset colmap_fiat --resize 800x600 --save rrd/colmap_fiat.rrd

      # All platforms are currently creating the same rrd file, upload one of them
      - name: Save RRD artifact
        if: ${{ (needs.set-config.outputs.RUN_TESTS == 'true') && (inputs.RRD_ARTIFACT_NAME != '') }}
        uses: actions/upload-artifact@v3
        with:
          name: ${{ inputs.RRD_ARTIFACT_NAME }}
          path: rrd