From ba55b78a94227c9f145a93668f32c9aaa608cfa1 Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Thu, 14 Aug 2025 13:15:57 +0200 Subject: [PATCH 1/2] Upload to binaries to Cloudflare R2 This patch adds the `julialangnightlies` bucket on Cloudflare R2 as an additional upload target mirroring the layout of the `julialangnightlies` bucket on AWS S3. --- pipelines/main/platforms/upload_freebsd.yml | 2 + pipelines/main/platforms/upload_linux.yml | 2 + pipelines/main/platforms/upload_macos.yml | 2 + pipelines/main/platforms/upload_windows.yml | 2 + .../platforms/upload_linux.no_gpl.yml | 2 + .../platforms/upload_macos.no_gpl.yml | 2 + .../platforms/upload_windows.no_gpl.yml | 2 + utilities/build_envs.sh | 35 ++++++------ utilities/upload_julia.sh | 57 ++++++++++++++++++- 9 files changed, 86 insertions(+), 20 deletions(-) diff --git a/pipelines/main/platforms/upload_freebsd.yml b/pipelines/main/platforms/upload_freebsd.yml index 987456256..5b18b8547 100644 --- a/pipelines/main/platforms/upload_freebsd.yml +++ b/pipelines/main/platforms/upload_freebsd.yml @@ -21,6 +21,8 @@ steps: variables: - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" + - R2_ACCESS_KEY_ID="U2FsdGVkX18HwnN9t7y7uYG17hD1HKNbIic0VkEXKK7fl583LSCVug1ZHFj4/ZzE7FKILpuk0ISnM4rZ7yaQcg==" + - R2_SECRET_ACCESS_KEY="U2FsdGVkX19gMi8SkWCcvn5/mIJNRduyyiRRX6IlYge8PowwT+vD+OIg1acIZhag8FnZn2y+tzj+4kk/3qkm/E6hMAcc8hia5rhch7fEeCpak5F/KiV5CLBEXI8s+aHx" files: - .buildkite/secrets/tarball_signing.gpg timeout_in_minutes: ${TIMEOUT?} diff --git a/pipelines/main/platforms/upload_linux.yml b/pipelines/main/platforms/upload_linux.yml index 1888e9e13..d976d43a0 100644 --- a/pipelines/main/platforms/upload_linux.yml +++ b/pipelines/main/platforms/upload_linux.yml @@ -33,6 +33,8 @@ steps: variables: - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" + - R2_ACCESS_KEY_ID="U2FsdGVkX18HwnN9t7y7uYG17hD1HKNbIic0VkEXKK7fl583LSCVug1ZHFj4/ZzE7FKILpuk0ISnM4rZ7yaQcg==" + - R2_SECRET_ACCESS_KEY="U2FsdGVkX19gMi8SkWCcvn5/mIJNRduyyiRRX6IlYge8PowwT+vD+OIg1acIZhag8FnZn2y+tzj+4kk/3qkm/E6hMAcc8hia5rhch7fEeCpak5F/KiV5CLBEXI8s+aHx" files: - .buildkite/secrets/tarball_signing.gpg timeout_in_minutes: ${TIMEOUT?} diff --git a/pipelines/main/platforms/upload_macos.yml b/pipelines/main/platforms/upload_macos.yml index 1d14b3da4..98f8dd17d 100644 --- a/pipelines/main/platforms/upload_macos.yml +++ b/pipelines/main/platforms/upload_macos.yml @@ -21,6 +21,8 @@ steps: variables: - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" + - R2_ACCESS_KEY_ID="U2FsdGVkX18HwnN9t7y7uYG17hD1HKNbIic0VkEXKK7fl583LSCVug1ZHFj4/ZzE7FKILpuk0ISnM4rZ7yaQcg==" + - R2_SECRET_ACCESS_KEY="U2FsdGVkX19gMi8SkWCcvn5/mIJNRduyyiRRX6IlYge8PowwT+vD+OIg1acIZhag8FnZn2y+tzj+4kk/3qkm/E6hMAcc8hia5rhch7fEeCpak5F/KiV5CLBEXI8s+aHx" - NOTARIZATION_APPLE_ID="U2FsdGVkX1/b2eyuYolwIaW0DI2YE5tPcMYJ7X7/J5Qz6T/oV6bZEuQqUEet6Jvx" - NOTARIZATION_APPLE_KEY="U2FsdGVkX1+z0XUSEnj0vLawcT1/U2wx1YDN6s3V9U5+qhbGwRQuXHNJXPmlNe/o" files: diff --git a/pipelines/main/platforms/upload_windows.yml b/pipelines/main/platforms/upload_windows.yml index 09183cd95..49dfcea98 100644 --- a/pipelines/main/platforms/upload_windows.yml +++ b/pipelines/main/platforms/upload_windows.yml @@ -21,6 +21,8 @@ steps: variables: - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" + - R2_ACCESS_KEY_ID="U2FsdGVkX18HwnN9t7y7uYG17hD1HKNbIic0VkEXKK7fl583LSCVug1ZHFj4/ZzE7FKILpuk0ISnM4rZ7yaQcg==" + - R2_SECRET_ACCESS_KEY="U2FsdGVkX19gMi8SkWCcvn5/mIJNRduyyiRRX6IlYge8PowwT+vD+OIg1acIZhag8FnZn2y+tzj+4kk/3qkm/E6hMAcc8hia5rhch7fEeCpak5F/KiV5CLBEXI8s+aHx" - AZURE_TENANT_ID="U2FsdGVkX1+c2HbeSTq1c0fEep5riO3+PSQCiovdRT5akVpyy5hC311cXUCUB+A7ivIezAdJ7oz3dt19t60iuw==" - AZURE_CLIENT_ID="U2FsdGVkX195QeT0ywNdXSa2ctMb9DFXyFVo0zG0/GtZUw95PTycqSZ+qEwXZbRZhGxXjGMVzcP2XZ0f/OuNnQ==" - AZURE_CLIENT_SECRET="U2FsdGVkX1/905nz82Iy5toukRXEO8QS2etwfFCocofgnpBB62wJRDrWClPCNEX6ICo0jQzgoSbwDr2+C1ETSA==" diff --git a/pipelines/scheduled/platforms/upload_linux.no_gpl.yml b/pipelines/scheduled/platforms/upload_linux.no_gpl.yml index b0d8b4b2b..442bbefa8 100644 --- a/pipelines/scheduled/platforms/upload_linux.no_gpl.yml +++ b/pipelines/scheduled/platforms/upload_linux.no_gpl.yml @@ -38,6 +38,8 @@ steps: variables: - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" + - R2_ACCESS_KEY_ID="U2FsdGVkX18HwnN9t7y7uYG17hD1HKNbIic0VkEXKK7fl583LSCVug1ZHFj4/ZzE7FKILpuk0ISnM4rZ7yaQcg==" + - R2_SECRET_ACCESS_KEY="U2FsdGVkX19gMi8SkWCcvn5/mIJNRduyyiRRX6IlYge8PowwT+vD+OIg1acIZhag8FnZn2y+tzj+4kk/3qkm/E6hMAcc8hia5rhch7fEeCpak5F/KiV5CLBEXI8s+aHx" files: - .buildkite/secrets/tarball_signing.gpg timeout_in_minutes: ${TIMEOUT?} diff --git a/pipelines/scheduled/platforms/upload_macos.no_gpl.yml b/pipelines/scheduled/platforms/upload_macos.no_gpl.yml index 4fc0d5e15..a3a02b0d9 100644 --- a/pipelines/scheduled/platforms/upload_macos.no_gpl.yml +++ b/pipelines/scheduled/platforms/upload_macos.no_gpl.yml @@ -26,6 +26,8 @@ steps: variables: - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" + - R2_ACCESS_KEY_ID="U2FsdGVkX18HwnN9t7y7uYG17hD1HKNbIic0VkEXKK7fl583LSCVug1ZHFj4/ZzE7FKILpuk0ISnM4rZ7yaQcg==" + - R2_SECRET_ACCESS_KEY="U2FsdGVkX19gMi8SkWCcvn5/mIJNRduyyiRRX6IlYge8PowwT+vD+OIg1acIZhag8FnZn2y+tzj+4kk/3qkm/E6hMAcc8hia5rhch7fEeCpak5F/KiV5CLBEXI8s+aHx" - NOTARIZATION_APPLE_ID="U2FsdGVkX1/b2eyuYolwIaW0DI2YE5tPcMYJ7X7/J5Qz6T/oV6bZEuQqUEet6Jvx" - NOTARIZATION_APPLE_KEY="U2FsdGVkX1+z0XUSEnj0vLawcT1/U2wx1YDN6s3V9U5+qhbGwRQuXHNJXPmlNe/o" files: diff --git a/pipelines/scheduled/platforms/upload_windows.no_gpl.yml b/pipelines/scheduled/platforms/upload_windows.no_gpl.yml index 3a73192c5..b28c35335 100644 --- a/pipelines/scheduled/platforms/upload_windows.no_gpl.yml +++ b/pipelines/scheduled/platforms/upload_windows.no_gpl.yml @@ -25,6 +25,8 @@ steps: variables: - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" + - R2_ACCESS_KEY_ID="U2FsdGVkX18HwnN9t7y7uYG17hD1HKNbIic0VkEXKK7fl583LSCVug1ZHFj4/ZzE7FKILpuk0ISnM4rZ7yaQcg==" + - R2_SECRET_ACCESS_KEY="U2FsdGVkX19gMi8SkWCcvn5/mIJNRduyyiRRX6IlYge8PowwT+vD+OIg1acIZhag8FnZn2y+tzj+4kk/3qkm/E6hMAcc8hia5rhch7fEeCpak5F/KiV5CLBEXI8s+aHx" - AZURE_TENANT_ID="U2FsdGVkX1+c2HbeSTq1c0fEep5riO3+PSQCiovdRT5akVpyy5hC311cXUCUB+A7ivIezAdJ7oz3dt19t60iuw==" - AZURE_CLIENT_ID="U2FsdGVkX195QeT0ywNdXSa2ctMb9DFXyFVo0zG0/GtZUw95PTycqSZ+qEwXZbRZhGxXjGMVzcP2XZ0f/OuNnQ==" - AZURE_CLIENT_SECRET="U2FsdGVkX1/905nz82Iy5toukRXEO8QS2etwfFCocofgnpBB62wJRDrWClPCNEX6ICo0jQzgoSbwDr2+C1ETSA==" diff --git a/utilities/build_envs.sh b/utilities/build_envs.sh index e3adde5ee..3e3415086 100755 --- a/utilities/build_envs.sh +++ b/utilities/build_envs.sh @@ -149,37 +149,39 @@ S3_BUCKET="${S3_BUCKET:-julialangnightlies}" S3_BUCKET_PREFIX="${S3_BUCKET_PREFIX:-bin}" # We generally upload to multiple upload targets -UPLOAD_TARGETS=() +CANONICAL_UPLOAD_TARGETS=() # The canonical, fully qualified, target and various julia-latest-* targets +LEGACY_UPLOAD_TARGETS=() # Legacy upload targets kept for compatibility with old URLs +EPHEMERAL_UPLOAD_TARGETS=() # Pull request upload targets if [[ "${BUILDKITE_BRANCH}" == master ]] || [[ "${BUILDKITE_BRANCH}" == release-* ]] || [[ "${BUILDKITE_TAG:-}" == v* ]] || [[ "${BUILDKITE_PIPELINE_SLUG}" == "julia-buildkite" ]]; then # First, we have the canonical fully-specified upload target - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}" ) + CANONICAL_UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}" ) # Next, we have the "majmin/latest" upload target - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}" ) + CANONICAL_UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}" ) # The absolute latest upload target is only for if we're on the `master` branch if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/julia-latest-${OS?}-${ARCH?}" ) + CANONICAL_UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/julia-latest-${OS?}-${ARCH?}" ) fi # Finally, for compatibility, we keep on uploading x86_64 and i686 targets to folders called `x64` # and `x86`, and ending in `-linux64` and `-linux32`, although I would very much like to stop doing that. if [[ "${ARCH}" == "x86_64" ]]; then - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}64" ) - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/${MAJMIN?}/julia-latest-${OS?}64" ) + LEGACY_UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}64" ) + LEGACY_UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/${MAJMIN?}/julia-latest-${OS?}64" ) # Only upload to absolute latest if we're on `master` if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/julia-latest-${OS?}64" ) + LEGACY_UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/julia-latest-${OS?}64" ) fi elif [[ "${ARCH}" == "i686" ]]; then - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}32" ) - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/${MAJMIN?}/julia-latest-${OS?}32" ) + LEGACY_UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}32" ) + LEGACY_UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/${MAJMIN?}/julia-latest-${OS?}32" ) # Only upload to absolute latest if we're on `master` if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/julia-latest-${OS?}32" ) + LEGACY_UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/julia-latest-${OS?}32" ) fi fi @@ -209,21 +211,21 @@ if [[ "${BUILDKITE_BRANCH}" == master ]] || [[ "${BUILDKITE_BRANCH}" == release- fi # First, we have the canonical fully-specified upload target - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${FOLDER_OS}/${FOLDER_ARCH}/${MAJMIN?}/julia-${TAR_VERSION?}-${SHORT_OS}${SHORT_ARCH}" ) + LEGACY_UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${FOLDER_OS}/${FOLDER_ARCH}/${MAJMIN?}/julia-${TAR_VERSION?}-${SHORT_OS}${SHORT_ARCH}" ) # Next, we have the "majmin/latest" upload target - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${FOLDER_OS}/${FOLDER_ARCH}/${MAJMIN?}/julia-latest-${SHORT_OS}${SHORT_ARCH}" ) + LEGACY_UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${FOLDER_OS}/${FOLDER_ARCH}/${MAJMIN?}/julia-latest-${SHORT_OS}${SHORT_ARCH}" ) # If we're on `master` and we're uploading, we consider ourselves "absolute latest" if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${FOLDER_OS}/${FOLDER_ARCH}/julia-latest-${SHORT_OS}${SHORT_ARCH}" ) + LEGACY_UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${FOLDER_OS}/${FOLDER_ARCH}/julia-latest-${SHORT_OS}${SHORT_ARCH}" ) fi fi fi # If we're a pull request build, upload to a special `-prXXXX` location if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/julia-pr${BUILDKITE_PULL_REQUEST}-${OS?}-${ARCH?}" ) + EPHEMERAL_UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/julia-pr${BUILDKITE_PULL_REQUEST}-${OS?}-${ARCH?}" ) fi # This is the "main" filename that is used. We technically don't need this for uploading, @@ -236,8 +238,3 @@ echo "The short commit is: ${SHORT_COMMIT}" echo "Julia will be installed to: ${JULIA_BINARY}" echo "Detected Julia version: ${MAJMIN} (${JULIA_VERSION})" echo "Detected build platform: ${TRIPLET} (${ARCH}, ${OS})" -echo "Julia will be uploaded to: s3://${UPLOAD_TARGETS[0]}.tar.gz" -echo "With additional upload targets:" -for UPLOAD_TARGET in "${UPLOAD_TARGETS[@]:1}"; do - echo " -> s3://${UPLOAD_TARGET}.tar.gz" -done diff --git a/utilities/upload_julia.sh b/utilities/upload_julia.sh index 030730292..045205581 100644 --- a/utilities/upload_julia.sh +++ b/utilities/upload_julia.sh @@ -10,6 +10,26 @@ set -euo pipefail # shellcheck source=SCRIPTDIR/build_envs.sh source .buildkite/utilities/build_envs.sh +echo "--- Print upload target information" +if (( ${#CANONICAL_UPLOAD_TARGETS[@]} )); then + echo "Canonical upload target(s):" + for target in "${CANONICAL_UPLOAD_TARGETS[@]}"; do + echo " s3://${target}.tar.gz" + done +fi +if (( ${#LEGACY_UPLOAD_TARGETS[@]} )); then + echo "Legacy upload target(s):" + for target in "${LEGACY_UPLOAD_TARGETS[@]}"; do + echo " s3://${target}.tar.gz" + done +fi +if (( ${#EPHEMERAL_UPLOAD_TARGETS[@]} )); then + echo "Ephemeral upload target(s):" + for target in "${EPHEMERAL_UPLOAD_TARGETS[@]}"; do + echo " s3://${target}.tar.gz" + done +fi + echo "--- Download ${UPLOAD_FILENAME}.tar.gz to ." buildkite-agent artifact download "${UPLOAD_FILENAME}.tar.gz" . @@ -137,6 +157,12 @@ for EXT in "${UPLOAD_EXTENSIONS[@]}"; do done wait_pids "${PIDS[@]}" +# We upload all targets to S3 +# S3_UPLOAD_TARGETS=("${CANONICAL_UPLOAD_TARGETS[@]}" "${LEGACY_UPLOAD_TARGETS[@]}" "${EPHEMERAL_UPLOAD_TARGETS[@]}") +UPLOAD_TARGETS=("${CANONICAL_UPLOAD_TARGETS[@]}" "${LEGACY_UPLOAD_TARGETS[@]}" "${EPHEMERAL_UPLOAD_TARGETS[@]}") +# Only non-legacy targets are uploaded to R2 +R2_UPLOAD_TARGETS=("${CANONICAL_UPLOAD_TARGETS[@]}" "${EPHEMERAL_UPLOAD_TARGETS[@]}") + # Next, upload primary files to S3 echo "--- Upload primary products to S3" PIDS=() @@ -157,10 +183,39 @@ for SECONDARY_TARGET in "${UPLOAD_TARGETS[@]:1}"; do done wait_pids "${PIDS[@]}" +# Upload primary files to R2 +echo "--- Upload primary products to R2" +R2_ACCOUNT_ID=2f1a87f641da882a3f856da54947be1a +R2_ENDPOINT_URL="https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" +PIDS=() +for EXT in "${UPLOAD_EXTENSIONS[@]}"; do + AWS_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY}" \ + aws s3 cp --endpoint-url "${R2_ENDPOINT_URL}" "${UPLOAD_FILENAME}.${EXT}" "s3://${R2_UPLOAD_TARGETS[0]}.${EXT}" & + PIDS+=( "$!" ) +done +wait_pids "${PIDS[@]}" + +echo "--- Copy to secondary R2 upload targets" +PIDS=() +for SECONDARY_TARGET in "${R2_UPLOAD_TARGETS[@]:1}"; do + for EXT in "${UPLOAD_EXTENSIONS[@]}"; do + AWS_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY}" \ + aws s3 cp --endpoint-url "${R2_ENDPOINT_URL}" "s3://${R2_UPLOAD_TARGETS[0]}.${EXT}" "s3://${SECONDARY_TARGET}.${EXT}" \ + --copy-props metadata-directive & + PIDS+=( "$!" ) + done +done +wait_pids "${PIDS[@]}" + # Report to the user some URLs that they can use to download this from echo "+++ Uploaded to targets" for UPLOAD_TARGET in "${UPLOAD_TARGETS[@]}"; do for EXT in "${UPLOAD_EXTENSIONS[@]}"; do - echo " -> s3://${UPLOAD_TARGET}.${EXT}" + echo " -> s3://${UPLOAD_TARGET}.${EXT} (AWS S3)" + done +done +for UPLOAD_TARGET in "${R2_UPLOAD_TARGETS[@]}"; do + for EXT in "${UPLOAD_EXTENSIONS[@]}"; do + echo " -> s3://${UPLOAD_TARGET}.${EXT} (Cloudflare R2)" done done From 650840883d0f204bd5f2cf8f4823a6194b61d251 Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Thu, 14 Aug 2025 16:13:30 +0200 Subject: [PATCH 2/2] sign treehashes --- pipelines/main/launch_signed_jobs.yml.signature | 2 +- pipelines/main/launch_upload_jobs.yml.signature | Bin 96 -> 96 bytes .../scheduled/coverage/coverage.yml.signature | 2 +- .../scheduled/launch_signed_jobs.yml.signature | Bin 96 -> 96 bytes .../scheduled/launch_upload_jobs.yml.signature | Bin 96 -> 96 bytes 5 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/main/launch_signed_jobs.yml.signature b/pipelines/main/launch_signed_jobs.yml.signature index 4ef3ded05..9e42862f0 100644 --- a/pipelines/main/launch_signed_jobs.yml.signature +++ b/pipelines/main/launch_signed_jobs.yml.signature @@ -1 +1 @@ -Salted__�m~N㱭��$�5�V�/��u�([�k��q�L�벧|<�k��i��#x��Co��!� ��z/��/Ev������O .19�Z \ No newline at end of file +Salted__0_�]� �f��_Rlގq�K ��e���Fw4��B��IR��!��l�?.�@Z�5��������30��fj�[\y�7ߓ�!fK�U�" \ No newline at end of file diff --git a/pipelines/main/launch_upload_jobs.yml.signature b/pipelines/main/launch_upload_jobs.yml.signature index b278e60826eeaa5129a24372ac55d5f6965a40af..0c9c2ef0f75090baa3c3ed2a023729789a875d83 100644 GIT binary patch literal 96 zcmV-m0H6O;VQh3|WM5wjp3jXQ4hubI{>oK7)YKGFqk*)f28L8-DCgP&{K@M8aCU{D z@GgT1+cRM_qQX*M#@|AfM}k?U9cRZ|Rz-9zt6F2Yo+b0P8!Gyrb}q1Tn`6+PZm!i& Ctu5RD literal 96 zcmV-m0H6O;VQh3|WM5z3b1|kXfCT>0z9uXD{k%{LU70BioH!Tshj@U8;!D?^e|TG6 z+6ZtWwT5l0o6!0AcR+nCiOL@2JSJ0=c@pcKpR+8+Vd74jLAJNvveV|uYCyhP5$Imi C0WiG) diff --git a/pipelines/scheduled/coverage/coverage.yml.signature b/pipelines/scheduled/coverage/coverage.yml.signature index ed74784f1..8d292491f 100644 --- a/pipelines/scheduled/coverage/coverage.yml.signature +++ b/pipelines/scheduled/coverage/coverage.yml.signature @@ -1 +1 @@ -Salted__��|5y�5�O|��-�mO�o�E��m�S ������y�8�[��x�`�8�^G���77q,j�q��m�N�`?��Hh��Ć� \ No newline at end of file +Salted__���D&�)[�1)t�$���;A׷�&Rjrϋ8��H��,<�현�܎�f�������#4}ְ��a�F��_37z?J�xT��G \ No newline at end of file diff --git a/pipelines/scheduled/launch_signed_jobs.yml.signature b/pipelines/scheduled/launch_signed_jobs.yml.signature index 9d0b6d2f36cd5f97b9b64fccaa56eec7375ace1b..de0b3fa9aec2f7c76b58cd69f000e6ece614f9e0 100644 GIT binary patch literal 96 zcmV-m0H6O;VQh3|WM5y80KAQ>vpBF{3R1aOA#oaZo$R9-70a+1#lj>(X3GOx_ai=% zR(#$s5*Znwm1rU*JPtulRra0uz> literal 96 zcmV-m0H6O;VQh3|WM5x=oqUhD4$1K)z#S?Jvl+yyvI|O9imzqv6So8|s=^B`1xuF_ z^~s0fMpQuZTPW@8*Q23p!WAF-YRBlY?!LT1r4+ z1p64hX7?q^nAx`vRm&q)?3ZrzoAkV1bpBpnl1F+h>a!9vkrM-cg(tL9pSchvfvlN- C;w@zW