Skip to content

Build

Build #2449

Workflow file for this run

name: Build
on:
schedule:
# Build apps daily at 6am UTC
- cron: '0 6 * * *'
workflow_dispatch:
inputs:
packages:
description: 'Package domains to build (e.g., "redis.io curl.se"), or leave empty for apps-only'
required: false
type: string
platform:
description: 'Platform to build on'
required: false
type: choice
options:
- all
- unix
- darwin-arm64
- darwin-x86-64
- linux-x86-64
- linux-arm64
- windows-x64
default: all
force:
description: 'Force re-upload even if already in S3'
required: false
type: boolean
default: true
apps-only:
description: 'Only build apps / GUI applications (ignores packages input)'
required: false
type: boolean
default: false
stripe:
description: 'Build only an even slice of the needs-build set, as "i/n" (e.g. "0/8"). Lets you fan out N parallel runners per platform without computing package lists. Empty = whole set.'
required: false
type: string
parallel:
description: 'Workers to run concurrently WITHIN this runner (build-all-packages has no in-process parallelism). Each splits the runner stripe into a sub-stripe, so effective concurrency = runners x parallel. Empty/1 = one worker. Use ~3 on Linux, ~2 on macOS (RAM).'
required: false
type: string
multi_version:
description: 'Build multiple important versions instead of only latest'
required: false
type: boolean
default: false
env:
# Build-status dashboard reporting is authenticated; without this token every
# report-build POST 401s and the build is invisible on pantry.dev/packages.
PANTRY_REGISTRY_TOKEN: ${{ secrets.PANTRY_TOKEN }}
# Object storage provider. Set the repo variable STORAGE_PROVIDER=hetzner (or
# backblaze) plus the S3_* vars and the S3_ACCESS_KEY_ID/S3_SECRET_ACCESS_KEY
# secrets below to publish binaries there. Defaults to AWS S3 (unchanged).
STORAGE_PROVIDER: ${{ vars.STORAGE_PROVIDER || 'aws' }}
S3_BUCKET: ${{ vars.S3_BUCKET || 'pantry-registry' }}
S3_REGION: ${{ vars.S3_REGION || 'us-east-1' }}
S3_ENDPOINT: ${{ vars.S3_ENDPOINT }}
AWS_REGION: ${{ vars.S3_REGION || 'us-east-1' }}
# Provider-agnostic S3-compatible credentials (Hetzner Object Storage,
# Backblaze B2, etc.). Used for any non-AWS provider; unset ⇒ AWS path unaffected.
S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
permissions:
contents: read
jobs:
build:
name: Build (${{ matrix.platform.name }})
strategy:
fail-fast: false
matrix:
# "all" = darwin + linux + windows
# "unix" = darwin + linux (for system packages that don't support Windows)
# apps-only = darwin + windows (GUI apps)
platform: ${{
(github.event_name == 'schedule' || github.event.inputs.apps-only == 'true')
&& fromJson('[{"os":"macos-15","name":"darwin-arm64"},{"os":"macos-15-intel","name":"darwin-x86-64"},{"os":"windows-latest","name":"windows-x64"}]')
|| github.event.inputs.platform == 'unix'
&& fromJson('[{"os":"macos-15","name":"darwin-arm64"},{"os":"macos-15-intel","name":"darwin-x86-64"},{"os":"ubuntu-latest","name":"linux-x86-64"},{"os":"ubuntu-24.04-arm","name":"linux-arm64"}]')
|| github.event.inputs.platform == 'darwin-arm64'
&& fromJson('[{"os":"macos-15","name":"darwin-arm64"}]')
|| github.event.inputs.platform == 'darwin-x86-64'
&& fromJson('[{"os":"macos-15-intel","name":"darwin-x86-64"}]')
|| github.event.inputs.platform == 'linux-x86-64'
&& fromJson('[{"os":"ubuntu-latest","name":"linux-x86-64"}]')
|| github.event.inputs.platform == 'linux-arm64'
&& fromJson('[{"os":"ubuntu-24.04-arm","name":"linux-arm64"}]')
|| github.event.inputs.platform == 'windows-x64'
&& fromJson('[{"os":"windows-latest","name":"windows-x64"}]')
|| fromJson('[{"os":"macos-15","name":"darwin-arm64"},{"os":"macos-15-intel","name":"darwin-x86-64"},{"os":"ubuntu-latest","name":"linux-x86-64"},{"os":"ubuntu-24.04-arm","name":"linux-arm64"},{"os":"windows-latest","name":"windows-x64"}]')
}}
runs-on: ${{ matrix.platform.os }}
permissions:
contents: read
# macOS runners bill 10x — cap them tighter so a hung mac job can't burn money.
# Latest-only builds never need anywhere near this; the cap is a runaway backstop.
timeout-minutes: ${{ contains(matrix.platform.os, 'macos') && 70 || 120 }}
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Setup Bun (Windows)
if: runner.os == 'Windows'
uses: oven-sh/setup-bun@v2
with:
bun-version: latest
- name: Setup Pantry
if: runner.os != 'Windows'
uses: ./packages/action
env:
GH_TOKEN: ${{ github.token }}
- name: Install dependencies
run: bun install
- name: Preflight source build
id: source-preflight
if: runner.os != 'Windows' && github.event_name != 'schedule' && github.event.inputs.apps-only != 'true'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
shell: bash
run: |
set -o pipefail
cd packages/ts-pantry
FORCE="${{ github.event.inputs.force || 'true' }}"
if [ "$FORCE" = "true" ]; then
echo "Force enabled; build tools are required"
echo "needs_build=true" >> "$GITHUB_OUTPUT"
exit 0
fi
ARGS="-b $S3_BUCKET -r $AWS_REGION --platform ${{ matrix.platform.name }}"
# Latest-only by default to drain the UNBUILT backlog (breadth-first,
# pre-teardown). Set the 'multi_version' dispatch input to 'true' to
# build multiple versions per package again.
if [ "${{ github.event.inputs.multi_version }}" = "true" ]; then
ARGS="$ARGS --multi-version --max-versions 5"
fi
PACKAGES="${{ github.event.inputs.packages }}"
PACKAGES=$(echo "$PACKAGES" | tr -cd 'a-zA-Z0-9._\-/, ')
if [ -n "$PACKAGES" ]; then
PACKAGES="${PACKAGES// /,}"
ARGS="$ARGS -p $PACKAGES"
fi
STRIPE_INPUT="${{ github.event.inputs.stripe || '' }}"
STRIPE=""
if [ -n "$STRIPE_INPUT" ]; then
if [[ "$STRIPE_INPUT" =~ ^[0-9]+/[0-9]+$ ]]; then
STRIPE="$STRIPE_INPUT"
stripe_i="${STRIPE%%/*}"
stripe_n="${STRIPE##*/}"
if [ "$stripe_n" -le 0 ] || [ "$stripe_i" -ge "$stripe_n" ]; then
echo "::error::Invalid stripe '$STRIPE_INPUT' (expected i/n with 0<=i<n, e.g. 0/6)"
exit 1
fi
else
echo "::error::Invalid stripe '$STRIPE_INPUT' (use a numeric i/n like 0/6; dispatch separate runs for 0/6, 1/6, ...)"
exit 1
fi
fi
if [ -n "$STRIPE" ]; then
ARGS="$ARGS --stripe $STRIPE"
fi
NEEDS_BUILD=$(bun scripts/build-all-packages.ts $ARGS --needs-build | tail -1)
echo "needs_build=$NEEDS_BUILD" >> "$GITHUB_OUTPUT"
if [ "$NEEDS_BUILD" = "false" ]; then
echo "All selected source artifacts already exist in S3; skipping toolchain setup"
fi
# Heavy toolchain setup — skipped for apps-only builds (they just need curl/hdiutil/brew)
- name: Free disk space (Linux)
if: runner.os == 'Linux' && github.event_name != 'schedule' && github.event.inputs.apps-only != 'true' && steps.source-preflight.outputs.needs_build != 'false'
run: |
# Reclaim ~25-30GB of unused preinstalled SDKs and move heavy build
# trees to the large /mnt volume so source builds don't exhaust the
# ~20GB root volume ("No space left on device").
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
/opt/hostedtoolcache/CodeQL /usr/local/.ghcup \
/usr/local/share/boost /usr/share/swift \
/opt/hostedtoolcache/PyPy /opt/hostedtoolcache/Ruby \
/usr/local/lib/node_modules /usr/share/miniconda \
/usr/local/graalvm /usr/local/share/chromium /usr/local/share/powershell || true
sudo docker image prune --all --force || true
sudo mkdir -p /mnt/buildkit && sudo chown "$(id -u):$(id -g)" /mnt/buildkit
echo "BUILDKIT_ROOT=/mnt/buildkit" >> "$GITHUB_ENV"
df -h / /mnt
- name: Install build tools (Linux)
if: runner.os == 'Linux' && github.event_name != 'schedule' && github.event.inputs.apps-only != 'true' && steps.source-preflight.outputs.needs_build != 'false'
run: |
sudo apt-get update
sudo apt-get install -y \
build-essential libreadline-dev zlib1g-dev libssl-dev \
cmake meson ninja-build patchelf pkg-config \
autoconf automake libtool gettext intltool \
libffi-dev libexpat1-dev libbz2-dev liblzma-dev \
libncurses5-dev libsqlite3-dev \
python3-venv python3-pip python3-dev \
libcurl4-openssl-dev libxml2-dev libonig-dev \
ruby ruby-dev scons \
gperf bison flex texinfo nasm yasm \
libgmp-dev libmpfr-dev libmpc-dev \
libuv1-dev libpcre2-dev libevent-dev \
libglib2.0-dev libpixman-1-dev \
libgit2-dev libssh2-1-dev libhttp-parser-dev \
libasound2-dev libdbus-1-dev libsystemd-dev \
protobuf-compiler libprotobuf-dev \
nettle-dev libp11-kit-dev \
libboost-all-dev libsndfile1-dev \
libjpeg-dev libpng-dev libtiff-dev libwebp-dev \
libfreetype-dev libharfbuzz-dev \
libltdl-dev libacl1-dev libattr1-dev \
liblz4-dev libzstd-dev libxxhash-dev \
libyaml-dev libjansson-dev \
libedit-dev libelf-dev \
tcl-dev tk-dev \
libunwind-dev libdw-dev \
libzip-dev liblapack-dev gfortran \
python3-lxml python3-libxml2 \
libx11-dev libxext-dev libxrender-dev libxrandr-dev \
libxinerama-dev libxcomposite-dev libxdamage-dev \
libxcursor-dev libxi-dev libxtst-dev \
libxmu-dev libxt-dev libxaw7-dev libxres-dev \
libsm-dev libice-dev libxpm-dev \
libxxf86vm-dev libxshmfence-dev libxv-dev libxss-dev \
x11proto-dev xtrans-dev xutils-dev xauth \
libxkbcommon-dev libxkbfile-dev \
libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev \
libxcb-image0-dev libxcb-util-dev libxcb-keysyms1-dev \
libxcb-randr0-dev libxcb-shm0-dev \
libwayland-dev libfontconfig1-dev \
gobject-introspection libgirepository1.0-dev \
libatk1.0-dev libcairo2-dev libcairo-gobject2 \
libpango1.0-dev libgtk-3-dev \
librsvg2-dev libsecret-1-dev libjson-glib-dev \
libsoup2.4-dev libgdk-pixbuf-2.0-dev \
libva-dev libegl-dev libgbm-dev libdrm-dev \
libgl-dev libglu1-mesa-dev \
clang llvm-dev libclang-dev \
libgnutls28-dev liblua5.4-dev \
libcap-dev libcap-ng-dev libslirp-dev libssh-dev \
libspeexdsp-dev libmaxminddb-dev libsmi2-dev \
libc-ares-dev libre2-dev libpcap-dev \
libnl-3-dev libnl-genl-3-dev \
libpam0g-dev libudev-dev \
libarchive-dev libusb-1.0-0-dev \
libjemalloc-dev libvterm-dev \
libxslt1-dev \
libsdl2-dev libplist-dev \
libassuan-dev libgpg-error-dev \
libfido2-dev \
libdouble-conversion-dev libgoogle-glog-dev libgflags-dev \
libsodium-dev liblzo2-dev libsnappy-dev libbrotli-dev \
doxygen xsltproc docbook-xsl xmlto
- name: Setup Xcode (macOS)
if: runner.os == 'macOS' && github.event_name != 'schedule' && github.event.inputs.apps-only != 'true' && steps.source-preflight.outputs.needs_build != 'false'
run: |
XCODE_PATH=$(ls -d /Applications/Xcode_26*.app 2>/dev/null | sort -V | tail -1)
if [ -n "$XCODE_PATH" ]; then
sudo xcode-select -s "$XCODE_PATH"
echo "Selected Xcode: $XCODE_PATH"
else
echo "No Xcode 26.x found, using default"
fi
swift --version
xcodebuild -version
- name: Cache Homebrew packages
if: runner.os == 'macOS' && github.event_name != 'schedule' && github.event.inputs.apps-only != 'true' && steps.source-preflight.outputs.needs_build != 'false'
uses: actions/cache@v5
with:
path: |
~/Library/Caches/Homebrew
/opt/homebrew/Cellar
/opt/homebrew/opt
/opt/homebrew/lib
/opt/homebrew/include
key: brew-build-tools-${{ runner.os }}-v2
restore-keys: |
brew-build-tools-${{ runner.os }}-
- name: Install build tools (macOS)
if: runner.os == 'macOS' && github.event_name != 'schedule' && github.event.inputs.apps-only != 'true' && steps.source-preflight.outputs.needs_build != 'false'
env:
HOMEBREW_NO_AUTO_UPDATE: 1
HOMEBREW_NO_INSTALL_CLEANUP: 1
run: |
brew install cmake meson ninja pkg-config autoconf automake libtool gettext gnu-sed scons \
gcc go boost libsndfile jpeg-turbo libpng libtiff webp freetype harfbuzz \
libyaml jansson libedit libzip lapack nasm libogg libvorbis flac \
gobject-introspection cairo pango atk gtk+3 glib gdk-pixbuf librsvg json-glib libsoup \
gnutls libgcrypt libgpg-error libassuan libksba libusb pcre2 \
libarchive jemalloc lua protobuf grpc c-ares re2 capnp \
libsecret libfido2 double-conversion glog gflags \
libsodium lzo snappy brotli \
doxygen docbook-xsl sdl2 sdl2_ttf sdl2_mixer sdl2_gfx || true
brew link --force libtool autoconf automake meson gnu-sed scons 2>/dev/null || true
- name: Fix Python setuptools (Linux)
if: runner.os == 'Linux' && github.event_name != 'schedule' && github.event.inputs.apps-only != 'true' && steps.source-preflight.outputs.needs_build != 'false'
run: |
sudo rm -rf /usr/lib/python3/dist-packages/setuptools* \
/usr/lib/python3/dist-packages/wheel* \
/usr/lib/python3/dist-packages/pkg_resources* \
/usr/lib/python3/dist-packages/_distutils_hack* 2>/dev/null || true
sudo pip3 install --break-system-packages setuptools wheel 2>/dev/null || true
- name: Setup Rust toolchain
if: runner.os != 'Windows' && github.event_name != 'schedule' && github.event.inputs.apps-only != 'true' && steps.source-preflight.outputs.needs_build != 'false'
run: |
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
source "$HOME/.cargo/env"
rustup update stable
cargo install cargo-c 2>/dev/null || true
echo "Rust: $(rustc --version), Cargo: $(cargo --version)"
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
python3 -m pip install --break-system-packages setuptools meson 2>/dev/null || pip3 install --break-system-packages setuptools meson 2>/dev/null || true
- name: Verify toolchains
if: runner.os != 'Windows' && github.event_name != 'schedule' && github.event.inputs.apps-only != 'true' && steps.source-preflight.outputs.needs_build != 'false'
run: |
echo "=== Toolchain versions ==="
cargo --version 2>/dev/null || echo "cargo: not found"
rustc --version 2>/dev/null || echo "rustc: not found"
go version 2>/dev/null || echo "go: not found"
python3 --version 2>/dev/null || echo "python3: not found"
node --version 2>/dev/null || echo "node: not found"
java -version 2>/dev/null || echo "java: not found"
- name: Build packages
if: runner.os == 'Windows' || github.event_name == 'schedule' || github.event.inputs.apps-only == 'true' || steps.source-preflight.outputs.needs_build != 'false'
shell: bash
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
cd packages/ts-pantry
FORCE="${{ github.event.inputs.force || 'true' }}"
ARGS="-b $S3_BUCKET -r $AWS_REGION --platform ${{ matrix.platform.name }}"
# Download-first: mirror pkgx's official prebuilt (seconds) instead of
# compiling; falls back to source build when pkgx lacks it or it's a
# custom build (php/postgres). Makes even macOS runs cheap.
ARGS="$ARGS --pkgx-mirror"
# Latest-only by default to drain the UNBUILT backlog (breadth-first,
# pre-teardown). Set the 'multi_version' dispatch input to 'true' to
# build multiple versions per package again.
if [ "${{ github.event.inputs.multi_version }}" = "true" ]; then
ARGS="$ARGS --multi-version --max-versions 5"
fi
if [ "$FORCE" = "true" ]; then
ARGS="$ARGS --force"
fi
# Desktop-only mode: scheduled runs or explicit flag
IS_DESKTOP="${{ github.event.inputs.apps-only || '' }}"
if [ "${{ github.event_name }}" = "schedule" ] || [ "$IS_DESKTOP" = "true" ]; then
ARGS="$ARGS --apps-only"
echo "Building apps only"
else
# Pass specific packages if provided
PACKAGES="${{ github.event.inputs.packages }}"
# Validate package input (only allow safe characters)
PACKAGES=$(echo "$PACKAGES" | tr -cd 'a-zA-Z0-9._\-/, ')
if [ -n "$PACKAGES" ]; then
PACKAGES="${PACKAGES// /,}"
ARGS="$ARGS -p $PACKAGES"
fi
# Even slice of the needs-build set for parallel fan-out (e.g. "0/8").
# Validate instead of stripping characters: a literal "i/8" would
# otherwise become "/8" and fail after more runner setup.
STRIPE_INPUT="${{ github.event.inputs.stripe || '' }}"
STRIPE=""
if [ -n "$STRIPE_INPUT" ]; then
if [[ "$STRIPE_INPUT" =~ ^[0-9]+/[0-9]+$ ]]; then
STRIPE="$STRIPE_INPUT"
stripe_i="${STRIPE%%/*}"
stripe_n="${STRIPE##*/}"
if [ "$stripe_n" -le 0 ] || [ "$stripe_i" -ge "$stripe_n" ]; then
echo "::error::Invalid stripe '$STRIPE_INPUT' (expected i/n with 0<=i<n, e.g. 0/8)"
exit 1
fi
else
echo "::error::Invalid stripe '$STRIPE_INPUT' (use a numeric i/n like 0/8; dispatch separate runs for 0/8, 1/8, ...)"
exit 1
fi
fi
fi
# Internal parallelism: build-all-packages has no in-process concurrency
# (the Hetzner fleet runs K separate processes per box to use all cores).
# Mirror that on a runner — run PAR workers, each on a sub-stripe of this
# runner's stripe, with its own BUILDKIT_ROOT so they don't clash. So the
# runner uses all its cores and effective concurrency = runners x PAR.
PAR=$(echo "${{ github.event.inputs.parallel }}" | tr -cd '0-9'); PAR=${PAR:-1}
[ "${PAR:-1}" -lt 1 ] && PAR=1
if [ -z "${STRIPE:-}" ] && [ "$PAR" -gt 1 ]; then
STRIPE="0/1"
fi
if [ -n "$STRIPE" ] && [ "$PAR" -gt 1 ]; then
i=${STRIPE%%/*}; n=${STRIPE##*/}; tot=$(( n * PAR ))
echo "Internal parallelism: $PAR workers, sub-stripes of $STRIPE over $tot total"
pids=""
for w in $(seq 0 $((PAR - 1))); do
sub=$(( i * PAR + w ))
echo " worker $w -> --stripe $sub/$tot"
BUILDKIT_ROOT="/tmp/pb-$w" bun scripts/build-all-packages.ts $ARGS --stripe "$sub/$tot" > "/tmp/worker-$w.log" 2>&1 &
pids="$pids $!"
done
tail -f /tmp/worker-*.log & TAILPID=$!
rc=0; for p in $pids; do wait "$p" || rc=1; done
kill "$TAILPID" 2>/dev/null || true
exit $rc
fi
[ -n "$STRIPE" ] && ARGS="$ARGS --stripe $STRIPE"
echo "Running: bun scripts/build-all-packages.ts $ARGS"
bun scripts/build-all-packages.ts $ARGS
timeout-minutes: ${{ contains(matrix.platform.os, 'macos') && 60 || 110 }}
notify:
name: discord-notification
needs: build
if: always()
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: ./.github/actions/discord-notify
with:
title: "Build — ${{ needs.build.result == 'success' && 'Success' || needs.build.result == 'cancelled' && 'Cancelled' || 'Failed' }}"
description: "${{ github.event_name == 'schedule' && 'Daily app sync' || format('Packages: {0}', github.event.inputs.packages || 'apps-only') }}"
status: ${{ needs.build.result == 'success' && 'success' || needs.build.result == 'cancelled' && 'warning' || 'failure' }}
fields: |
[
{"name": "Platform", "value": "${{ github.event.inputs.platform || 'darwin-arm64' }}", "inline": true},
{"name": "Mode", "value": "${{ (github.event_name == 'schedule' || github.event.inputs.apps-only == 'true') && 'Apps' || 'System Packages' }}", "inline": true},
{"name": "Actor", "value": "${{ github.actor }}", "inline": true}
]
webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }}