diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 8f94b3b263..274925ebff 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -227,6 +227,36 @@ blocks: - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz + - name: 'Linux s390x: release artifact cross-compilation builds' + dependencies: [] + run: + # TODO: Remove branch condition after testing - only keep tag condition for production + when: "tag =~ '^v[0-9]\\.' or branch = 'feat/s390x-cross-compilation'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + prologue: + commands: + # Setup QEMU for s390x emulation (for verification only) + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # Setup buildx for cross-compilation + - docker buildx create --name s390x-builder --use || docker buildx use s390x-builder + - docker buildx inspect --bootstrap + - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY' + epilogue: + commands: + - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/' + jobs: + # TODO: For testing only - running single job. Add back both variants for production. + - name: 'Build: cross-compiled s390x (test)' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-linux__dist-ubuntu__arch-s390x__lnk-all + commands: + - packaging/tools/build-s390x-cross.sh artifacts/librdkafka.tgz + + - name: 'Windows x64: MinGW-w64' dependencies: [] run: @@ -313,6 +343,7 @@ blocks: - 'OSX x64' - 'Linux x64: release artifact docker builds' - 'Linux arm64: release artifact docker builds' + - 'Linux s390x: release artifact cross-compilation builds' - 'Windows x64: MinGW-w64' - 'Windows x64: Windows SDK 10.0 / MSVC v142 / VS 2019' run: diff --git a/packaging/tools/Dockerfile.s390x b/packaging/tools/Dockerfile.s390x new file mode 100644 index 0000000000..c31de834a8 --- /dev/null +++ b/packaging/tools/Dockerfile.s390x @@ -0,0 +1,80 @@ +# syntax=docker/dockerfile:1 +# +# Cross-compilation Dockerfile for building librdkafka for s390x architecture +# Uses BuildKit cross-compilation with tonistiigi/xx toolkit to avoid QEMU slowness +# + +ARG BASE_IMAGE=ubuntu:24.04 + +# Cross-compilation helpers +FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.5.0 AS xx + +# Build stage runs on native platform (AMD64), outputs s390x binaries +FROM --platform=$BUILDPLATFORM ${BASE_IMAGE} AS builder + +COPY --from=xx / / + +ARG TARGETPLATFORM +ARG LIBRDKAFKA_VERSION + +# Install native build tools and cross-compilation toolchain +RUN NEEDRESTART_MODE=a apt-get update && apt-get install -y \ + wget curl python3 git build-essential \ + clang lld pkg-config \ + && rm -rf /var/lib/apt/lists/* + +# Install cross-compilation toolchain for s390x +RUN xx-apt-get install -y \ + gcc g++ libc6-dev \ + libssl-dev \ + libsasl2-dev \ + libcurl4-openssl-dev \ + zlib1g-dev \ + libzstd-dev \ + liblz4-dev + +# Set up cross-compilation environment +ENV PKG_CONFIG_PATH=/usr/lib/$(xx-info)-linux-gnu/pkgconfig + +WORKDIR /build + +# Copy source code +COPY . /build/ + +# Configure for cross-compilation +# --host tells configure we're cross-compiling for s390x +# --enable-static builds static libraries +# We use system packages for dependencies (installed via xx-apt-get above) +# Must explicitly set CC, AR, etc. as configure doesn't pick up ENV vars reliably +RUN CC=$(xx-info)-gcc \ + CXX=$(xx-info)-g++ \ + AR=$(xx-info)-ar \ + RANLIB=$(xx-info)-ranlib \ + STRIP=$(xx-info)-strip \ + ./configure \ + --host=$(xx-info) \ + --prefix=/usr/local \ + --enable-static \ + --enable-strip \ + --enable-ssl \ + --enable-sasl \ + --enable-zstd \ + --enable-lz4-ext + +# Build librdkafka +RUN make -j$(nproc) + +# Run examples to verify build (using QEMU for this step) +RUN xx-verify --static src/librdkafka.a +RUN xx-verify src/librdkafka.so.1 + +# Install to staging directory +RUN DESTDIR=/artifacts make install + +# Show what was built +RUN ls -lh /artifacts/usr/local/lib/ +RUN file /artifacts/usr/local/lib/librdkafka.so.1 + +# Final stage: Package the artifacts +FROM scratch AS artifacts +COPY --from=builder /artifacts / diff --git a/packaging/tools/build-s390x-cross.sh b/packaging/tools/build-s390x-cross.sh new file mode 100755 index 0000000000..f048163359 --- /dev/null +++ b/packaging/tools/build-s390x-cross.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# Build librdkafka for s390x using cross-compilation +# +# This script uses Docker BuildKit with cross-compilation to build +# s390x binaries on AMD64 hosts, avoiding QEMU emulation issues. +# +# Usage: +# packaging/tools/build-s390x-cross.sh +# +# Requirements: +# - Docker with BuildKit support +# - QEMU binfmt support (for verification only) +# + +set -e + +if [ $# -ne 1 ]; then + echo "Usage: $0 " + echo "" + echo "Example:" + echo " $0 artifacts/librdkafka-s390x.tgz" + exit 1 +fi + +output="$1" +output_dir=$(dirname "$output") +output_file=$(basename "$output") + +# Ensure output directory exists +mkdir -p "$output_dir" + +# Get absolute path for output +output_abs=$(cd "$output_dir" && pwd)/$output_file + +echo "Building librdkafka for s390x using cross-compilation..." +echo "Output will be: $output_abs" + +# Enable Docker BuildKit +export DOCKER_BUILDKIT=1 + +# Build using cross-compilation Dockerfile +docker buildx build \ + --platform linux/s390x \ + --file packaging/tools/Dockerfile.s390x \ + --target artifacts \ + --output type=local,dest=/tmp/librdkafka-s390x-build \ + . + +# Package the artifacts +echo "Packaging artifacts..." +cd /tmp/librdkafka-s390x-build +tar czf "$output_abs" . +cd - + +# Clean up +rm -rf /tmp/librdkafka-s390x-build + +# Emit SHA256 for verification +echo "" +echo "Build complete!" +echo "SHA256 $output:" +sha256sum "$output"