feat(core): Add support for building and running on manylinux_2_28; Add manylinux_2_28 dependency container images.

components/core/CMakeLists.txt

@@ -72,7 +72,12 @@ if (CLP_USE_STATIC_LIBS)
     if (APPLE)
         set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "macOS")
     elseif (EXISTS "/etc/centos-release")
-        set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS")
+        # For now, only manylinux_2_28 (AlmaLinux 8 based) supports static linking
+        file(READ "/etc/centos-release" CENTOS_RELEASE_CONTENT)
+        string(FIND "${CENTOS_RELEASE_CONTENT}" "AlmaLinux" ALMALINUX_FOUND)
+        if(ALMALINUX_FOUND EQUAL -1)
+            set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS")
+        endif()
     endif()
 
     if (DEFINED CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM)
@@ -212,7 +217,7 @@ endif()
 # Find and setup libcurl
 # By default, CURL does not provide static libraries
 if(CLP_NEED_CURL)
-    find_package(CURL 7.68.0 REQUIRED)
+    find_package(CURL 7.61.1 REQUIRED)
     if(CURL_FOUND)
         message(STATUS "Found CURL ${CURL_VERSION_STRING}")
     else()

components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile

@@ -0,0 +1,23 @@
+FROM quay.io/pypa/manylinux_2_28_aarch64
+
+# Binaries built on manylinux_2_28 (AlmaLinux 8 based) is expected to be
+# compatible with other distros using glibc 2.28 or later, including:
+# - Debian 10+
+# - Ubuntu 18.10+
+# - Fedora 29+
+# - CentOS/RHEL 8+
+
+WORKDIR /root
+
+RUN mkdir -p ./tools/scripts/lib_install
+ADD ./tools/scripts/lib_install ./tools/scripts/lib_install
+
+RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh
+
+# Remove cached files
+RUN dnf clean all && rm -rf /var/cache/dnf /tmp/* /var/tmp/*
+
+# DO NOT FLATTEN THE IMAGE!!!
+# The consequence of flattening the image is that we lose shell startup scripts
+# and environment modification from /etc/profile.d, /etc/bashrc, etc.
+# These are important for manylinux, otherwise, default system compilers are used, etc.

components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/build.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Get the directory this script is in
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+component_root=${script_dir}/../../../
+
+# Build aarch64 image, will automatically use QEMU emulate if not on native platform
+build_cmd=(
+    docker buildx build
+    --platform linux/arm64
+    --tag clp-core-dependencies-manylinux_2_28_aarch64:dev
+    "$component_root"
+    --file "${script_dir}/Dockerfile"
+    --load
+)
+
+# If in a git repo, add labels for revision and source
+if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ;
+then
+    build_cmd+=(
+        --label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)"
+        --label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)"
+    )
+fi
+
+echo "Running: ${build_cmd[*]}"
+
+# Execute the docker buildx command
+"${build_cmd[@]}"

components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile

@@ -0,0 +1,23 @@
+FROM quay.io/pypa/manylinux_2_28_x86_64
+
+# Binaries built on manylinux_2_28 (AlmaLinux 8 based) is expected to be
+# compatible with other distros using glibc 2.28 or later, including:
+# - Debian 10+
+# - Ubuntu 18.10+
+# - Fedora 29+
+# - CentOS/RHEL 8+
+
+WORKDIR /root
+
+RUN mkdir -p ./tools/scripts/lib_install
+ADD ./tools/scripts/lib_install ./tools/scripts/lib_install
+
+RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh
+
+# Remove cached files
+RUN dnf clean all && rm -rf /var/cache/dnf /tmp/* /var/tmp/*
+
+# DO NOT FLATTEN THE IMAGE!!!
+# The consequence of flattening the image is that we lose shell startup scripts
+# and environment modification from /etc/profile.d, /etc/bashrc, etc.
+# These are important for manylinux, otherwise, default system compilers are used, etc.

components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/build.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Get the directory this script is in
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+component_root=${script_dir}/../../../
+
+# Build x86_64 image, will automatically use QEMU emulate if not on native platform
+build_cmd=(
+    docker buildx build
+    --platform linux/amd64
+    --tag clp-core-dependencies-manylinux_2_28_x86_64:dev
+    "$component_root"
+    --file "${script_dir}/Dockerfile"
+    --load
+)
+
+# If in a git repo, add labels for revision and source
+if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ;
+then
+    build_cmd+=(
+        --label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)"
+        --label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)"
+    )
+fi
+
+echo "Running: ${build_cmd[*]}"
+
+# Execute the docker buildx command
+"${build_cmd[@]}"

components/core/tools/scripts/lib_install/libarchive.sh

@@ -63,8 +63,8 @@ fi
 cd ${extracted_dir}
 mkdir -p cmake-build-release
 cd cmake-build-release
-# NOTE: Disable expat so the static libarchive doesn't look for it at link time
-cmake -DENABLE_EXPAT=OFF ../
+# NOTE: Disable expat and openssl  so the static libarchive doesn't look for it at link time
+cmake -DENABLE_EXPAT=OFF -DENABLE_OPENSSL=OFF ../
 make -j${num_cpus}
 
 # Check if checkinstall is installed

components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# Exit on any error
+set -e
+
+# Error on undefined variable
+set -u
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+"${script_dir}/install-prebuilt-packages.sh"
+"${script_dir}/install-packages-from-source.sh"
+
+# TODO: https://github.com/y-scope/clp/issues/795
+"${script_dir}/../check-cmake-version.sh"

components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# Exit on any error
+set -e
+
+# Error on undefined variable
+set -u
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+lib_install_scripts_dir="${script_dir}/.."
+
+# NOTE: The remaining installation scripts depend on boost, so we install it beforehand.
+"${lib_install_scripts_dir}"/install-boost.sh 1.87.0
+
+"$lib_install_scripts_dir"/liblzma.sh 5.8.1
+"$lib_install_scripts_dir"/lz4.sh 1.10.0
+"$lib_install_scripts_dir"/zstandard.sh 1.5.7
+"$lib_install_scripts_dir"/libarchive.sh 3.8.0
+"${lib_install_scripts_dir}"/msgpack.sh 7.0.0

components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# Exit on any error
+set -e
+
+# Error on undefined variable
+set -u
+
+dnf install -y \
+    gcc-c++ \
+    java-11-openjdk \
+    libcurl-devel \
+    mariadb-connector-c-devel \
+    openssl-devel \
+    zlib-devel \
+    zlib-static \
+    jq
+
+# Determine architecture for `task` release to install
+rpm_arch=$(rpm --eval "%{_arch}")
+case "$rpm_arch" in
+    "x86_64")
+        task_pkg_arch="amd64"
+        ;;
+    "aarch64")
+        task_pkg_arch="arm64"
+        ;;
+    *)
+        echo "Error: Unsupported architecture - $rpm_arch"
+        exit 1
+        ;;
+esac
+
+# Install `task`
+# NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872
+task_pkg_path="$(mktemp -t --suffix ".rpm" task-pkg.XXXXXXXXXX)"
+curl \
+    --fail \
+    --location \
+    --output "$task_pkg_path" \
+    --show-error \
+    "https://github.com/go-task/task/releases/download/v3.42.1/task_linux_${task_pkg_arch}.rpm"
+dnf install --assumeyes "$task_pkg_path"
+rm "$task_pkg_path"
+
+# The bundled CMake 4.x version is too new; downgrading to CMake 3.27 is
+# necessary to resolve build issues with third-party libraries. Even after
+# we added the "-DCMAKE_POLICY_VERSION_MINIMUM=3.5" flag, compilation issues
+# in libraries such as googletest-src and yaml-cpp still persist.
+# See sample error below:
+# CMake Error at _deps/googletest-src/CMakeLists.txt:4 (cmake_minimum_required):
+#   Support for CMake versions below 3.5 has been dropped.
+#
+#   Adjust the VERSION argument’s minimum value, or use the <min>...<max> syntax
+#   to indicate the required CMake version range.
+#
+#   Alternatively, you can try adding -DCMAKE_POLICY_VERSION_MINIMUM=3.5
+export PLATFORM=$(uname -m)
+export CMAKE_VERSION=3.27.9
+export CMAKE_RELEASE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz
+echo "Downgrading CMAKE to 3.27.9"
+curl -L ${CMAKE_RELEASE_URL} | tar -xzf -
+mv cmake-${CMAKE_VERSION}-linux-${PLATFORM} /opt/cmake-${CMAKE_VERSION}
+ln -sf /opt/cmake-${CMAKE_VERSION}/bin/cmake /usr/local/bin/cmake

components/core/tools/scripts/utils/README.md

@@ -1,8 +1,15 @@
 This directory contains uncategorized utility scripts.
 
-
-* `build-and-run-unit-tests.sh` can be used to build all executables and run
-  the unit tests.
+* `build.py` can be used to trigger a local platform build
+* `build-with-docker.py` can be used to build all targets within a docker 
+  image for x86_64 and arm64
+    * manylinux_2_28 based images are used and the output binaries are 
+      expected to be compatible with Debian 10+, Ubuntu 18.10+, Fedora 29+,
+      CentOS/RHEL 8+ and other platforms.
+    * Command to enable docker emulation: 
+      `docker run --rm --privileged multiarch/qemu-user-static --reset -p yes`
+* `build-and-run-unit-tests.py` can be used to perform a local platform 
+  build of all executables and run the unit tests.
 * `run-in-container.sh` can be used to run a command (e.g., `make`), in a 
   container containing the core component's dependencies.
   * All commands are run from the root of the core component.

components/core/tools/scripts/utils/build-and-run-unit-tests.py

@@ -1,5 +1,6 @@
 import argparse
 import logging
+import os
 import subprocess
 import sys
 from pathlib import Path
@@ -21,45 +22,13 @@
 logger = logging.getLogger(__name__)
 
 
-def _config_cmake_project(src_dir: Path, build_dir: Path, use_shared_libs: bool):
-    cmd = [
-        "cmake",
-        "-S",
-        str(src_dir),
-        "-B",
-        str(build_dir),
-    ]
-    if use_shared_libs:
-        cmd.append("-DCLP_USE_STATIC_LIBS=OFF")
-    subprocess.run(cmd, check=True)
-
-
-def _build_project(build_dir: Path, num_jobs: Optional[int]):
-    """
-    :param build_dir:
-    :param num_jobs: Max number of jobs to run when building.
-    """
-
-    cmd = [
-        "cmake",
-        "--build",
-        str(build_dir),
-        "--parallel",
-    ]
-    if num_jobs is not None:
-        cmd.append(str(num_jobs))
-    subprocess.run(cmd, check=True)
-
-
 def _run_unit_tests(build_dir: Path, test_spec: Optional[str]):
     """
     :param build_dir:
     :param test_spec: Catch2 test specification.
     """
 
-    cmd = [
-        "./unitTest",
-    ]
+    cmd = ["./unitTest"]
     if test_spec is not None:
         cmd.append(test_spec)
     subprocess.run(cmd, cwd=build_dir, check=True)
@@ -69,31 +38,40 @@ def main(argv: List[str]) -> int:
     args_parser = argparse.ArgumentParser(
         description="Builds the CLP-core's binaries and runs its unit tests."
     )
-    args_parser.add_argument(
-        "--source-dir", required=True, help="Directory containing the main CMakeLists.txt."
-    )
     args_parser.add_argument("--build-dir", required=True, help="Build output directory.")
     args_parser.add_argument(
         "--use-shared-libs",
         action="store_true",
         help="Build targets by linking against shared libraries.",
     )
     args_parser.add_argument(
-        "--num-jobs", type=int, help="Max number of jobs to run when building."
+        "--num-jobs",
+        type=int,
+        default=os.cpu_count(),
+        help="Max number of jobs to run when building.",
     )
     args_parser.add_argument("--test-spec", help="Catch2 test specification.")
 
     parsed_args = args_parser.parse_args(argv[1:])
-    src_dir: Path = Path(parsed_args.source_dir)
     build_dir: Path = Path(parsed_args.build_dir)
-    use_shared_libs: bool = parsed_args.use_shared_libs
-    num_jobs: Optional[int] = parsed_args.num_jobs
     test_spec: Optional[str] = parsed_args.test_spec
 
-    _config_cmake_project(src_dir, build_dir, use_shared_libs)
-    _build_project(build_dir, num_jobs)
-    _run_unit_tests(build_dir, test_spec)
+    build_cmd = [
+        "python3",
+        "build.py",
+        "--build-dir",
+        str(build_dir),
+        "--num-jobs",
+        str(parsed_args.num_jobs),
+    ]
+    if parsed_args.use_shared_libs:
+        build_cmd.append("--use-shared-libs")
 
+    logger.info("Starting build process...")
+    subprocess.run(build_cmd, check=True)
+    logger.info("Build completed. Running unit tests...")
+    _run_unit_tests(build_dir, test_spec)
+    logger.info("All unit tests completed successfully.")
     return 0