Draft PGO+LTO+BOLT build + test CI

.buildkite/hooks/post-checkout

@@ -4,8 +4,8 @@ set -euo pipefail
 
 # Buildkite users can trigger new builds with these values overridden to
 # test specific julia gitshas
-UPSTREAM_URL="${UPSTREAM_URL:-https://github.com/JuliaLang/julia.git}"
-UPSTREAM_BRANCH="${UPSTREAM_BRANCH:-master}"
+UPSTREAM_URL="${UPSTREAM_URL:-https://github.com/Zentrik/julia.git}"
+UPSTREAM_BRANCH="${UPSTREAM_BRANCH:-try_fix_pgo_lto_bolt}"
 
 # All of our workers are required to provide a default for the julia cache dir
 # We're going to cache repositories in here, just like buildkite itself would:

pipelines/main/launch_unsigned_jobs.yml

@@ -41,6 +41,12 @@ steps:
               bash .buildkite/utilities/arches_pipeline_upload.sh \
               .buildkite/pipelines/main/platforms/build_linux.arches \
               .buildkite/pipelines/main/platforms/build_linux.yml
+          # Launch Linux PGO+LTO+BOLT build jobs
+          GROUP="Build" \
+              ALLOW_FAIL="false" \
+              bash .buildkite/utilities/arches_pipeline_upload.sh \
+              .buildkite/pipelines/main/platforms/build_linux.pgo_lto.arches \
+              .buildkite/pipelines/main/platforms/build_linux.yml
           # PowerPC (only for Julia prior to 1.12):
           GROUP="Build" \
               ALLOW_FAIL="false" \
@@ -120,6 +126,11 @@ steps:
               bash .buildkite/utilities/arches_pipeline_upload.sh \
               .buildkite/pipelines/main/platforms/test_linux.i686.arches \
               .buildkite/pipelines/main/platforms/test_linux.i686.yml
+          # Launch Linux PGO+LTO+BOLT test jobs
+          GROUP="Test" \
+              bash .buildkite/utilities/arches_pipeline_upload.sh \
+              .buildkite/pipelines/main/platforms/test_linux.pgo_lto.arches \
+              .buildkite/pipelines/main/platforms/test_linux.yml
           ### Launch macOS test jobs:
           GROUP="Test" \
               bash .buildkite/utilities/arches_pipeline_upload.sh \

pipelines/main/platforms/build_linux.pgo_lto.arches

@@ -0,0 +1,8 @@
+# ROOTFS_IMAGE_NAME  TRIPLET                          ARCH     ARCH_ROOTFS    USE_JULIA_PGO_LTO_BOLT    MAKE_FLAGS    TIMEOUT      ROOTFS_TAG   ROOTFS_HASH
+llvm_passes          x86_64-linux-gnu-pgo-lto-bolt    x86_64   x86_64         PGO_LTO_BOLT              .             .            v7.6         ebdfa2d40c0e27842cccbfdd43710144f9adf9d8
+
+# These special lines allow us to embed default values for the columns above.
+# Any column without a default mapping here will simply substitute a `.` to the empty string
+
+# Builds should generally finish in much less time than this, but from-source builds can take longer
+#default TIMEOUT 240

pipelines/main/platforms/build_linux.yml

@@ -34,3 +34,4 @@ steps:
           TRIPLET: "${TRIPLET?}"
           MAKE_FLAGS: "${MAKE_FLAGS?}"
           ROOTFS_IMAGE_NAME: "${ROOTFS_IMAGE_NAME?}"
+          USE_JULIA_PGO_LTO_BOLT: "${USE_JULIA_PGO_LTO_BOLT}" 

pipelines/main/platforms/test_linux.pgo_lto.arches

@@ -0,0 +1,10 @@
+# ROOTFS_IMAGE_NAME    TRIPLET                         ARCH           ARCH_ROOTFS    TIMEOUT    USE_RR     USE_JULIA_PGO_LTO_BOLT     ROOTFS_TAG    ROOTFS_HASH
+tester_linux           x86_64-linux-gnu-pgo-lto-bolt   x86_64         x86_64         .          .          PGO_LTO_BOLT               v6.00         770c0240be788cfeb9654e1980ea929d3ed98d1f
+
+# These special lines allow us to embed default values for the columns above.
+# Any column without a default mapping here will simply substitute a `.` to the empty string
+
+# Most tests should finish within ~45 minutes, barring exceptionally slow hardware
+# We double that to a default of 90 minutes, with an extra 45 minutes for cleanup,
+# including things like `rr` trace compression,
+#default TIMEOUT 135

utilities/build_julia.sh

@@ -26,7 +26,6 @@ if [[ "${ROOTFS_IMAGE_NAME-}" == "llvm_passes" ]]; then
     contrib/download_cmake.sh
 fi
 
-echo "--- Collect make options"
 # These are the flags we'll provide to `make`
 MFLAGS=()
 
@@ -55,19 +54,61 @@ else
 fi
 MFLAGS+=( "JULIA_CPU_TARGET=${JULIA_CPU_TARGET}" )
 
+if [[ ! -z "${USE_JULIA_PGO_LTO_BOLT-}" ]]; then
+    STAGE2_BUILD=$PWD
+    MFLAGS+=( "STAGE2_BUILD=$STAGE2_BUILD" )
+
+    echo "--- Collect make options"
+    echo "Make Options:"
+    for FLAG in "${MFLAGS[@]}"; do
+        echo " -> ${FLAG}"
+    done
+
+    echo "--- Build Julia Stage 1 - with instrumentation"
+
+    cd contrib/pgo-lto-bolt
+    ${MAKE} "${MFLAGS[@]}" stage1
+    # Building stage1 collects profiling data which we use instead of collecting our own
+fi
+
 # Finish off with any extra make flags from the `.arches` file
 IFS=',' read -ra ARCHES_FLAGS <<<"${MAKE_FLAGS}"
 MFLAGS+=( "${ARCHES_FLAGS[@]}" )
 
+echo "--- Collect make options"
 echo "Make Options:"
 for FLAG in "${MFLAGS[@]}"; do
     echo " -> ${FLAG}"
 done
 
-echo "--- Build Julia"
-echo "Note: The log stream is filtered. [buildroot] replaces pwd $(pwd)"
-${MAKE} "${MFLAGS[@]}" 2>&1 | sed "s|$(pwd)|[buildroot]|g"
+if [[ ! -z "${USE_JULIA_PGO_LTO_BOLT-}" ]]; then
+    echo "--- Build Julia Stage 2 - PGO + LTO optimised"
+    ${MAKE} "${MFLAGS[@]}" "$STAGE2_BUILD"
+    ${MAKE} "${MFLAGS[@]}" -C "$STAGE2_BUILD/deps" install-csl
+    ${MAKE} "${MFLAGS[@]}" stage2
+
+    echo "--- Copying original shared libraries"
+    ${MAKE} "${MFLAGS[@]}" copy_originals
 
+    echo "--- Instrumenting with BOLT"
+    ${MAKE} "${MFLAGS[@]}" bolt_instrument
+
+    echo "--- Finishing and Profiling Julia Stage 2"
+    ${MAKE} "${MFLAGS[@]}" finish_stage2
+
+    echo "--- Merging BOLT Profiles"
+    ${MAKE} "${MFLAGS[@]}" merge_data
+
+    echo "--- Optimize Julia Stage 2 with BOLT"
+    ${MAKE} "${MFLAGS[@]}" bolt
+
+    cd ../..
+    rm base/default.profraw
+else
+    echo "--- Build Julia"
+    echo "Note: The log stream is filtered. [buildroot] replaces pwd $(pwd)"
+    ${MAKE} "${MFLAGS[@]}" 2>&1 | sed "s|$(pwd)|[buildroot]|g"
+fi
 
 echo "--- Check that the working directory is clean"
 if [ -n "$(git status --short)" ]; then
@@ -101,3 +142,9 @@ fi
 
 echo "--- Upload build artifacts to buildkite"
 buildkite-agent artifact upload "${UPLOAD_FILENAME}.tar.gz"
+
+# Upload the profile data to allow for reproducible builds
+if [[ ! -z "${USE_JULIA_PGO_LTO_BOLT-}" ]]; then
+    buildkite-agent artifact upload "contrib/pgo-lto-bolt/profiles/merged.prof"
+    buildkite-agent artifact upload "contrib/pgo-lto-bolt/profiles-bolt/*.merged.fdata"
+fi

utilities/extract_triplet.sh

@@ -30,6 +30,9 @@ case "${TRIPLET}" in
     *-gnunogpl) # builds that use `USE_GPL_LIBS=0`
         OS="linuxnogpl"
         ;;
+    *-gnu-pgo-lto-bolt) # PGO + LTO + BOLT builds
+        OS="linuxpgoltobolt"
+        ;;
     *-musl)
         OS="musl"
         ;;