ci: faster python wheel builds using precompiled

.github/workflows/build-and-deploy.yml

@@ -92,9 +92,11 @@ jobs:
 
   build-wheels:
     if: ${{ github.event_name == 'schedule' || github.event.inputs.skipPython != 'true' }}
+    needs: [build-precompiled-bin]
     uses: ./.github/workflows/python-wheel-workflow.yml
     with:
       isNightly: ${{ github.event_name == 'schedule' || github.event.inputs.isNightly == 'true' }}
+      precompiledRunId: ${{ github.run_id }}
     secrets: inherit
 
   deploy-python:

.github/workflows/python-wheel-workflow.yml

@@ -2,12 +2,20 @@ name: Build Python Wheels
 
 on:
   workflow_dispatch:
+    inputs:
+      precompiledRunId:
+        description: "Run ID of the precompiled-bin job whose artifacts should be used"
+        required: true
+        type: string
   workflow_call:
     inputs:
       isNightly:
         type: boolean
         required: true
         default: false
+      precompiledRunId:
+        type: string
+        required: true
 
 jobs:
   package-python-sdist:
@@ -84,17 +92,21 @@ jobs:
     env:
       CIBW_PLATFORM: ${{ matrix.os }}
       CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-* cp314-*"
+      LBUG_PYTHON_PRECOMPILED_DIR: ${{ github.workspace }}/precompiled-liblbug
+      LBUG_PYTHON_CIBW_SOURCE_DIR: ${{ github.workspace }}/scripts/pip-package/cibw-source/sdist
 
       # Let CIBW detect the arch, except explicitly set Windows wheel arch per matrix
       CIBW_ARCHS: ${{ matrix.os == 'windows' && (matrix.platform == 'arm64' && 'ARM64' || 'AMD64') || 'auto' }}
 
       # Set the manylinux image based on the detected arch (Linux only)
       CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.os == 'linux' && format('manylinux{0}', matrix.manylinux) || '' }}
       CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.os == 'linux' && format('manylinux{0}', matrix.manylinux) || '' }}
+      CIBW_BEFORE_ALL_LINUX: "command -v dnf && dnf install -y ccache || apk add --no-cache ccache"
+      CIBW_ENVIRONMENT_LINUX: ${{ matrix.os == 'linux' && 'LBUG_API_PRECOMPILED_LIB_PATH=/project/precompiled-liblbug/liblbug.a CMAKE_C_COMPILER_LAUNCHER=ccache CMAKE_CXX_COMPILER_LAUNCHER=ccache CCACHE_DIR=/project/.ccache' || '' }}
 
       # Set deployment target for macOS, and Windows MSVC setup
-      CIBW_ENVIRONMENT_WINDOWS: ${{ matrix.os == 'windows' && format('DISTUTILS_USE_SDK=1 MSSdk=1 TMP={0} TEMP={0}', matrix.platform == 'arm64' && 'C:/t' || 'D:/t') || '' }}
-      CIBW_ENVIRONMENT_MACOS: ${{ matrix.os == 'macos' && format('MACOSX_DEPLOYMENT_TARGET={0}', matrix.deployment-target) || '' }}
+      CIBW_ENVIRONMENT_WINDOWS: ${{ matrix.os == 'windows' && format('DISTUTILS_USE_SDK=1 MSSdk=1 TMP={0} TEMP={0} LBUG_API_PRECOMPILED_LIB_PATH=$LBUG_API_PRECOMPILED_LIB_PATH', matrix.platform == 'arm64' && 'C:/t' || 'D:/t') || '' }}
+      CIBW_ENVIRONMENT_MACOS: ${{ matrix.os == 'macos' && format('MACOSX_DEPLOYMENT_TARGET={0} LBUG_API_PRECOMPILED_LIB_PATH={1}/precompiled-liblbug/liblbug.a CMAKE_C_COMPILER_LAUNCHER=ccache CMAKE_CXX_COMPILER_LAUNCHER=ccache CCACHE_DIR={1}/.ccache', matrix.deployment-target, github.workspace) || '' }}
 
     steps:
       - name: Check out repository
@@ -113,6 +125,13 @@ jobs:
         shell: bash
         run: echo "SDIST_FILE=$(ls scripts/pip-package/*.tar.gz)" >> $GITHUB_ENV
 
+      - name: Prepare wheel source tree
+        shell: bash
+        run: |
+          rm -rf scripts/pip-package/cibw-source
+          mkdir -p scripts/pip-package/cibw-source
+          tar -xzf "${SDIST_FILE}" -C scripts/pip-package/cibw-source
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -121,6 +140,28 @@ jobs:
       - name: Install uv
         run: python -m pip install uv
 
+      - name: Download precompiled liblbug
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          LBUG_LIB_KIND: static
+          LBUG_LINUX_VARIANT: compat
+          LBUG_PRECOMPILED_RUN_ID: ${{ inputs.precompiledRunId }}
+          LBUG_TARGET_DIR: ${{ env.LBUG_PYTHON_PRECOMPILED_DIR }}
+        run: |
+          rm -rf "$LBUG_TARGET_DIR"
+          ./scripts/download-liblbug.sh
+
+      - name: Inject precompiled lib into wheel source tree
+        shell: bash
+        run: |
+          rm -rf "${LBUG_PYTHON_CIBW_SOURCE_DIR}/precompiled-liblbug"
+          mkdir -p "${LBUG_PYTHON_CIBW_SOURCE_DIR}/precompiled-liblbug"
+          cp -R "${LBUG_PYTHON_PRECOMPILED_DIR}/." "${LBUG_PYTHON_CIBW_SOURCE_DIR}/precompiled-liblbug/"
+          if [ "$RUNNER_OS" = "Windows" ]; then
+            echo "LBUG_API_PRECOMPILED_LIB_PATH=${LBUG_PYTHON_CIBW_SOURCE_DIR}/precompiled-liblbug/lbug.lib" >> $GITHUB_ENV
+          fi
+
       - name: Update nightly version
         if: ${{ inputs.isNightly == true }}
         shell: bash
@@ -191,16 +232,29 @@ jobs:
           powershell.exe -Command "Add-MpPreference -ExclusionPath '${{ github.workspace }}'"
           call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
           ccache -s
-          pipx run --spec cibuildwheel cibuildwheel %SDIST_FILE% --output-dir scripts\pip-package\wheelhouse
+          pipx run --spec cibuildwheel cibuildwheel %LBUG_PYTHON_CIBW_SOURCE_DIR% --output-dir scripts\pip-package\wheelhouse
           if errorlevel 1 exit /b 1
           ccache -s
 
+      - name: Install ccache (macOS)
+        if: matrix.os == 'macos'
+        run: brew install ccache
+
+      - name: Cache ccache directory (Linux/macOS)
+        if: matrix.os != 'windows'
+        uses: actions/cache@v4
+        with:
+          path: ${{ github.workspace }}/.ccache
+          key: ${{ runner.os }}-${{ matrix.platform }}-ccache-wheel-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.platform }}-ccache-wheel-
+
       - name: Build wheels on ${{ matrix.platform }} (Linux/macOS)
         if: matrix.os != 'windows'
         shell: bash
         run: |
           source .venv/bin/activate
-          python -m cibuildwheel --output-dir scripts/pip-package/wheelhouse ${{ env.SDIST_FILE }}
+          python -m cibuildwheel --output-dir scripts/pip-package/wheelhouse "${LBUG_PYTHON_CIBW_SOURCE_DIR}"
 
       - name: Upload wheels
         uses: actions/upload-artifact@v4

CMakeLists.txt

@@ -478,7 +478,7 @@ add_definitions(-DLBUG_ROOT_DIRECTORY="${PROJECT_SOURCE_DIR}")
 add_definitions(-DLBUG_CMAKE_VERSION="${CMAKE_PROJECT_VERSION}")
 add_definitions(-DLBUG_EXTENSION_VERSION="0.15.0")
 
-if(BUILD_LBUG)
+if(BUILD_LBUG OR LBUG_API_USE_PRECOMPILED_LIB)
 include_directories(
     src/include
     ${CMAKE_CURRENT_BINARY_DIR}/src/include

scripts/pip-package/setup.py

@@ -84,6 +84,14 @@ def build_extension(self, ext: CMakeExtension) -> None:
 
         # Build the native extension.
         full_cmd = ['make', 'python', 'NUM_THREADS=%d' % num_cores]
+        precompiled_lib_path = os.environ.get('LBUG_API_PRECOMPILED_LIB_PATH', '').strip()
+        if precompiled_lib_path:
+            self.announce("Using precompiled liblbug from %s." % precompiled_lib_path)
+            full_cmd.append(
+                'EXTRA_CMAKE_FLAGS=-DBUILD_LBUG=FALSE -DBUILD_SHELL=FALSE '
+                '-DLBUG_API_USE_PRECOMPILED_LIB=TRUE '
+                '-DLBUG_API_PRECOMPILED_LIB_PATH=%s' % precompiled_lib_path
+            )
         subprocess.run(full_cmd, cwd=build_dir, check=True, env=env_vars)
         self.announce("Done building native extension.")
         self.announce("Copying native extension...")