From 162042c6d3cafd0ac90191212c40d1353e7b6779 Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Fri, 3 Apr 2026 17:24:00 +0800 Subject: [PATCH 1/3] Integrate ccache to accelerate compilation in local and CI environments - Add ccache auto-detection in CMakeLists.txt via CMAKE_C/CXX_COMPILER_LAUNCHER - Add ccache status display in build_and_package.sh - Configure ccache environment variables in ci/scripts/build_paimon.sh - Add ccache installation, actions/cache@v4 caching, and statistics steps to all GitHub CI workflows (gcc_test, clang_test, gcc8_test, test_with_sanitizer, build_release) --- .github/workflows/build_release.yaml | 38 ++++++++++++++++++++++ .github/workflows/clang_test.yaml | 19 +++++++++++ .github/workflows/gcc8_test.yaml | 19 ++++++++++- .github/workflows/gcc_test.yaml | 19 +++++++++++ .github/workflows/test_with_sanitizer.yaml | 19 +++++++++++ CMakeLists.txt | 10 ++++++ build_and_package.sh | 7 ++++ ci/scripts/build_paimon.sh | 22 +++++++++++++ 8 files changed, 152 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_release.yaml b/.github/workflows/build_release.yaml index f6357d0e..975a2f3b 100644 --- a/.github/workflows/build_release.yaml +++ b/.github/workflows/build_release.yaml @@ -40,12 +40,31 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: lfs: true + - name: Install ccache + run: sudo apt-get update && sudo apt-get install -y ccache + - name: Setup ccache + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ccache-clang-release-${{ runner.os }}-${{ github.ref_name }}-${{ hashFiles('**/CMakeLists.txt', '**/*.cmake') }} + restore-keys: | + ccache-clang-release-${{ runner.os }}-${{ github.ref_name }}- + ccache-clang-release-${{ runner.os }}- + - name: Configure ccache + run: | + ccache --set-config=max_size=2G + ccache --set-config=compress=true + ccache --set-config=compression_level=6 + ccache -z - name: Build Paimon shell: bash env: CC: clang CXX: clang++ run: ci/scripts/build_paimon.sh $(pwd) false false Release + - name: Show ccache statistics + if: always() + run: ccache -s gcc-release: runs-on: ubuntu-24.04 timeout-minutes: 120 @@ -56,9 +75,28 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: lfs: true + - name: Install ccache + run: sudo apt-get update && sudo apt-get install -y ccache + - name: Setup ccache + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ccache-gcc-release-${{ runner.os }}-${{ github.ref_name }}-${{ hashFiles('**/CMakeLists.txt', '**/*.cmake') }} + restore-keys: | + ccache-gcc-release-${{ runner.os }}-${{ github.ref_name }}- + ccache-gcc-release-${{ runner.os }}- + - name: Configure ccache + run: | + ccache --set-config=max_size=2G + ccache --set-config=compress=true + ccache --set-config=compression_level=6 + ccache -z - name: Build Paimon shell: bash env: CC: gcc-14 CXX: g++-14 run: ci/scripts/build_paimon.sh $(pwd) false false Release + - name: Show ccache statistics + if: always() + run: ccache -s diff --git a/.github/workflows/clang_test.yaml b/.github/workflows/clang_test.yaml index 3bcd2ccc..f6b8de1d 100644 --- a/.github/workflows/clang_test.yaml +++ b/.github/workflows/clang_test.yaml @@ -41,9 +41,28 @@ jobs: with: lfs: true fetch-depth: 0 # fetch all history for git diff in clang-tidy + - name: Install ccache + run: sudo apt-get update && sudo apt-get install -y ccache + - name: Setup ccache + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ccache-clang-test-${{ runner.os }}-${{ github.ref_name }}-${{ hashFiles('**/CMakeLists.txt', '**/*.cmake') }} + restore-keys: | + ccache-clang-test-${{ runner.os }}-${{ github.ref_name }}- + ccache-clang-test-${{ runner.os }}- + - name: Configure ccache + run: | + ccache --set-config=max_size=2G + ccache --set-config=compress=true + ccache --set-config=compression_level=6 + ccache -z - name: Build Paimon shell: bash env: CC: clang CXX: clang++ run: ci/scripts/build_paimon.sh $(pwd) false true + - name: Show ccache statistics + if: always() + run: ccache -s diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml index c8827cde..0e794dab 100644 --- a/.github/workflows/gcc8_test.yaml +++ b/.github/workflows/gcc8_test.yaml @@ -42,7 +42,7 @@ jobs: - name: Install dependencies run: | apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git git-lfs tar curl tzdata zip unzip pkg-config build-essential python3-dev gdb + DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git git-lfs tar curl tzdata zip unzip pkg-config build-essential python3-dev gdb ccache curl -L -O https://github.com/Kitware/CMake/releases/download/v3.28.3/cmake-3.28.3-linux-x86_64.tar.gz tar -zxvf cmake-3.28.3-linux-x86_64.tar.gz -C /usr/local --strip-components=1 rm cmake-3.28.3-linux-x86_64.tar.gz @@ -59,9 +59,26 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: lfs: true + - name: Setup ccache + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ccache-gcc8-test-${{ runner.os }}-${{ github.ref_name }}-${{ hashFiles('**/CMakeLists.txt', '**/*.cmake') }} + restore-keys: | + ccache-gcc8-test-${{ runner.os }}-${{ github.ref_name }}- + ccache-gcc8-test-${{ runner.os }}- + - name: Configure ccache + run: | + ccache --set-config=max_size=2G + ccache --set-config=compress=true + ccache --set-config=compression_level=6 + ccache -z - name: Build Paimon shell: bash env: CC: gcc-8 CXX: g++-8 run: ci/scripts/build_paimon.sh $(pwd) + - name: Show ccache statistics + if: always() + run: ccache -s diff --git a/.github/workflows/gcc_test.yaml b/.github/workflows/gcc_test.yaml index 8b76981d..cdf62ff9 100644 --- a/.github/workflows/gcc_test.yaml +++ b/.github/workflows/gcc_test.yaml @@ -40,9 +40,28 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: lfs: true + - name: Install ccache + run: sudo apt-get update && sudo apt-get install -y ccache + - name: Setup ccache + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ccache-gcc-test-${{ runner.os }}-${{ github.ref_name }}-${{ hashFiles('**/CMakeLists.txt', '**/*.cmake') }} + restore-keys: | + ccache-gcc-test-${{ runner.os }}-${{ github.ref_name }}- + ccache-gcc-test-${{ runner.os }}- + - name: Configure ccache + run: | + ccache --set-config=max_size=2G + ccache --set-config=compress=true + ccache --set-config=compression_level=6 + ccache -z - name: Build Paimon shell: bash env: CC: gcc-14 CXX: g++-14 run: ci/scripts/build_paimon.sh $(pwd) + - name: Show ccache statistics + if: always() + run: ccache -s diff --git a/.github/workflows/test_with_sanitizer.yaml b/.github/workflows/test_with_sanitizer.yaml index 1226c787..61dca57d 100644 --- a/.github/workflows/test_with_sanitizer.yaml +++ b/.github/workflows/test_with_sanitizer.yaml @@ -40,9 +40,28 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: lfs: true + - name: Install ccache + run: sudo apt-get update && sudo apt-get install -y ccache + - name: Setup ccache + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ccache-sanitizer-${{ runner.os }}-${{ github.ref_name }}-${{ hashFiles('**/CMakeLists.txt', '**/*.cmake') }} + restore-keys: | + ccache-sanitizer-${{ runner.os }}-${{ github.ref_name }}- + ccache-sanitizer-${{ runner.os }}- + - name: Configure ccache + run: | + ccache --set-config=max_size=2G + ccache --set-config=compress=true + ccache --set-config=compression_level=6 + ccache -z - name: Build Paimon shell: bash env: CC: clang CXX: clang++ run: ci/scripts/build_paimon.sh $(pwd) true + - name: Show ccache statistics + if: always() + run: ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f975e75..afbd3d8d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,16 @@ project(paimon VERSION 0.1.0 DESCRIPTION "Paimon C++ Project") +# Setup ccache to accelerate compilation if available +find_program(CCACHE_PROGRAM ccache) +if(CCACHE_PROGRAM) + message(STATUS "Found ccache: ${CCACHE_PROGRAM}") + set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_PROGRAM}") + set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}") +else() + message(STATUS "ccache not found, compiling without cache acceleration") +endif() + string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_BUILD_TYPE) set(CMAKE_CXX_STANDARD 17) diff --git a/build_and_package.sh b/build_and_package.sh index 048ebfa8..d6af6199 100755 --- a/build_and_package.sh +++ b/build_and_package.sh @@ -103,6 +103,13 @@ if [ -n "$JOBS" ]; then else echo "Parallel Jobs: auto-detect" fi +if command -v ccache &> /dev/null; then + echo "ccache: $(ccache --version | head -1)" + echo "ccache stats before build:" + ccache -s | head -5 +else + echo "ccache: not found (install ccache to accelerate repeated builds)" +fi if [ ${#CMAKE_OPTIONS[@]} -gt 0 ]; then echo "CMake Options: ${CMAKE_OPTIONS[*]}" else diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index cafc88a7..6aa96198 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -22,6 +22,22 @@ check_clang_tidy=${3:-false} build_type=${4:-Debug} build_dir=${1}/build +# Configure ccache if available +if command -v ccache &> /dev/null; then + echo "=== ccache found: $(ccache --version | head -1) ===" + export CCACHE_DIR="${CCACHE_DIR:-${HOME}/.ccache}" + export CCACHE_MAXSIZE="${CCACHE_MAXSIZE:-2G}" + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=6 + mkdir -p "${CCACHE_DIR}" + echo "ccache directory: ${CCACHE_DIR}" + echo "ccache max size: ${CCACHE_MAXSIZE}" + ccache -s | head -5 + ccache -z # Reset statistics for this build +else + echo "=== ccache not found, compiling without cache acceleration ===" +fi + mkdir ${build_dir} pushd ${build_dir} @@ -61,6 +77,12 @@ if [[ "${check_clang_tidy}" == "true" ]]; then cmake --build . --target check-clang-tidy fi +# Print ccache statistics after build +if command -v ccache &> /dev/null; then + echo "=== ccache statistics after build ===" + ccache -s +fi + popd rm -rf ${build_dir} From 3daeaea43bdfac79b233eeb46f085918122cd2c6 Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Fri, 3 Apr 2026 17:32:20 +0800 Subject: [PATCH 2/3] Fix ccache config: use 'compression' instead of 'compress' for ccache 4.x --- .github/workflows/build_release.yaml | 4 ++-- .github/workflows/clang_test.yaml | 2 +- .github/workflows/gcc8_test.yaml | 2 +- .github/workflows/gcc_test.yaml | 2 +- .github/workflows/test_with_sanitizer.yaml | 2 +- ci/scripts/build_paimon.sh | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_release.yaml b/.github/workflows/build_release.yaml index 975a2f3b..ce62af0d 100644 --- a/.github/workflows/build_release.yaml +++ b/.github/workflows/build_release.yaml @@ -53,7 +53,7 @@ jobs: - name: Configure ccache run: | ccache --set-config=max_size=2G - ccache --set-config=compress=true + ccache --set-config=compression=true ccache --set-config=compression_level=6 ccache -z - name: Build Paimon @@ -88,7 +88,7 @@ jobs: - name: Configure ccache run: | ccache --set-config=max_size=2G - ccache --set-config=compress=true + ccache --set-config=compression=true ccache --set-config=compression_level=6 ccache -z - name: Build Paimon diff --git a/.github/workflows/clang_test.yaml b/.github/workflows/clang_test.yaml index f6b8de1d..022747d4 100644 --- a/.github/workflows/clang_test.yaml +++ b/.github/workflows/clang_test.yaml @@ -54,7 +54,7 @@ jobs: - name: Configure ccache run: | ccache --set-config=max_size=2G - ccache --set-config=compress=true + ccache --set-config=compression=true ccache --set-config=compression_level=6 ccache -z - name: Build Paimon diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml index 0e794dab..393f28c0 100644 --- a/.github/workflows/gcc8_test.yaml +++ b/.github/workflows/gcc8_test.yaml @@ -70,7 +70,7 @@ jobs: - name: Configure ccache run: | ccache --set-config=max_size=2G - ccache --set-config=compress=true + ccache --set-config=compression=true ccache --set-config=compression_level=6 ccache -z - name: Build Paimon diff --git a/.github/workflows/gcc_test.yaml b/.github/workflows/gcc_test.yaml index cdf62ff9..5436f3cf 100644 --- a/.github/workflows/gcc_test.yaml +++ b/.github/workflows/gcc_test.yaml @@ -53,7 +53,7 @@ jobs: - name: Configure ccache run: | ccache --set-config=max_size=2G - ccache --set-config=compress=true + ccache --set-config=compression=true ccache --set-config=compression_level=6 ccache -z - name: Build Paimon diff --git a/.github/workflows/test_with_sanitizer.yaml b/.github/workflows/test_with_sanitizer.yaml index 61dca57d..15edf014 100644 --- a/.github/workflows/test_with_sanitizer.yaml +++ b/.github/workflows/test_with_sanitizer.yaml @@ -53,7 +53,7 @@ jobs: - name: Configure ccache run: | ccache --set-config=max_size=2G - ccache --set-config=compress=true + ccache --set-config=compression=true ccache --set-config=compression_level=6 ccache -z - name: Build Paimon diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index 6aa96198..d9140bfc 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -27,7 +27,7 @@ if command -v ccache &> /dev/null; then echo "=== ccache found: $(ccache --version | head -1) ===" export CCACHE_DIR="${CCACHE_DIR:-${HOME}/.ccache}" export CCACHE_MAXSIZE="${CCACHE_MAXSIZE:-2G}" - export CCACHE_COMPRESS=1 + export CCACHE_COMPRESS=true export CCACHE_COMPRESSLEVEL=6 mkdir -p "${CCACHE_DIR}" echo "ccache directory: ${CCACHE_DIR}" From 3fa3eab012f67196c610c879dfb30d2c9d474130 Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Fri, 3 Apr 2026 18:22:02 +0800 Subject: [PATCH 3/3] chore(ci): optimize ccache config - increase max_size to 5G and remove redundant settings in build_paimon.sh --- .github/workflows/build_release.yaml | 4 ++-- .github/workflows/clang_test.yaml | 2 +- .github/workflows/gcc8_test.yaml | 2 +- .github/workflows/gcc_test.yaml | 2 +- .github/workflows/test_with_sanitizer.yaml | 2 +- ci/scripts/build_paimon.sh | 11 ++--------- 6 files changed, 8 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_release.yaml b/.github/workflows/build_release.yaml index ce62af0d..828b663c 100644 --- a/.github/workflows/build_release.yaml +++ b/.github/workflows/build_release.yaml @@ -52,7 +52,7 @@ jobs: ccache-clang-release-${{ runner.os }}- - name: Configure ccache run: | - ccache --set-config=max_size=2G + ccache --set-config=max_size=5G ccache --set-config=compression=true ccache --set-config=compression_level=6 ccache -z @@ -87,7 +87,7 @@ jobs: ccache-gcc-release-${{ runner.os }}- - name: Configure ccache run: | - ccache --set-config=max_size=2G + ccache --set-config=max_size=5G ccache --set-config=compression=true ccache --set-config=compression_level=6 ccache -z diff --git a/.github/workflows/clang_test.yaml b/.github/workflows/clang_test.yaml index 022747d4..f180e047 100644 --- a/.github/workflows/clang_test.yaml +++ b/.github/workflows/clang_test.yaml @@ -53,7 +53,7 @@ jobs: ccache-clang-test-${{ runner.os }}- - name: Configure ccache run: | - ccache --set-config=max_size=2G + ccache --set-config=max_size=5G ccache --set-config=compression=true ccache --set-config=compression_level=6 ccache -z diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml index 393f28c0..3f649a23 100644 --- a/.github/workflows/gcc8_test.yaml +++ b/.github/workflows/gcc8_test.yaml @@ -69,7 +69,7 @@ jobs: ccache-gcc8-test-${{ runner.os }}- - name: Configure ccache run: | - ccache --set-config=max_size=2G + ccache --set-config=max_size=5G ccache --set-config=compression=true ccache --set-config=compression_level=6 ccache -z diff --git a/.github/workflows/gcc_test.yaml b/.github/workflows/gcc_test.yaml index 5436f3cf..097ef1a5 100644 --- a/.github/workflows/gcc_test.yaml +++ b/.github/workflows/gcc_test.yaml @@ -52,7 +52,7 @@ jobs: ccache-gcc-test-${{ runner.os }}- - name: Configure ccache run: | - ccache --set-config=max_size=2G + ccache --set-config=max_size=5G ccache --set-config=compression=true ccache --set-config=compression_level=6 ccache -z diff --git a/.github/workflows/test_with_sanitizer.yaml b/.github/workflows/test_with_sanitizer.yaml index 15edf014..d8d04c20 100644 --- a/.github/workflows/test_with_sanitizer.yaml +++ b/.github/workflows/test_with_sanitizer.yaml @@ -52,7 +52,7 @@ jobs: ccache-sanitizer-${{ runner.os }}- - name: Configure ccache run: | - ccache --set-config=max_size=2G + ccache --set-config=max_size=5G ccache --set-config=compression=true ccache --set-config=compression_level=6 ccache -z diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index d9140bfc..f1d0423d 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -22,17 +22,10 @@ check_clang_tidy=${3:-false} build_type=${4:-Debug} build_dir=${1}/build -# Configure ccache if available +# Display ccache status if available if command -v ccache &> /dev/null; then echo "=== ccache found: $(ccache --version | head -1) ===" - export CCACHE_DIR="${CCACHE_DIR:-${HOME}/.ccache}" - export CCACHE_MAXSIZE="${CCACHE_MAXSIZE:-2G}" - export CCACHE_COMPRESS=true - export CCACHE_COMPRESSLEVEL=6 - mkdir -p "${CCACHE_DIR}" - echo "ccache directory: ${CCACHE_DIR}" - echo "ccache max size: ${CCACHE_MAXSIZE}" - ccache -s | head -5 + ccache -p | grep -E "cache_dir|max_size|compression" || true ccache -z # Reset statistics for this build else echo "=== ccache not found, compiling without cache acceleration ==="