Skip to content

[MONGOCRYPT-838] Upload build artifacts to a restricted bucket on release branch builds #1126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kevinAlbs merged 10 commits into from
Mar 17, 2026
Merged

[MONGOCRYPT-838] Upload build artifacts to a restricted bucket on release branch builds #1126

Changes from 3 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
bf7b996
Switch the upload target bucket based on the project/patch status
vector-of-bool Feb 4, 2026
832bdb1
Restrict access to files posted in S3
vector-of-bool Feb 5, 2026
7f60088
Python steps use the unrestricted bucket
vector-of-bool Feb 10, 2026
610561e
Use the same role to download files for the Python release script
vector-of-bool Feb 5, 2026
dc8bcc5
maybe fix package URL
kevinAlbs Mar 12, 2026
10f2386
use `libmongocrypt`, not `${project}`
kevinAlbs Mar 13, 2026
7c8b709
add skip_existing:true to s3.put
kevinAlbs Mar 17, 2026
f6ae98e
note expected download path
kevinAlbs Mar 17, 2026
4acaf63
fix comparison
kevinAlbs Mar 17, 2026
932d1ec
fix package url
kevinAlbs Mar 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 95 additions & 77 deletions .evergreen/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,20 +80,20 @@ functions:
include: [./**]
- command: s3.put
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: '${project}/${build_variant}/${branch_name}/${libmongocrypt_s3_suffix}/libmongocrypt.tar.gz'
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
local_file: 'libmongocrypt.tar.gz'
content_type: '${content_type|application/x-gzip}'
- command: s3.put
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: '${project}/${build_variant}/${branch_name}/${libmongocrypt_s3_suffix_copy}/libmongocrypt.tar.gz'
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
local_file: 'libmongocrypt.tar.gz'
content_type: '${content_type|application/x-gzip}'

Expand Down Expand Up @@ -138,11 +138,11 @@ functions:
include: [./**]
- command: s3.put
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: '${project}/${build_variant}/${branch_name}/${libmongocrypt_s3_suffix}/libmongocrypt-distro-packages.tar.gz'
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
local_file: 'libmongocrypt-distro-packages.tar.gz'
content_type: '${content_type|application/x-gzip}'
optional: true
Expand Down Expand Up @@ -224,10 +224,9 @@ functions:
"download tarball":
- command: s3.get
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: '${project}/${variant_name}/${branch_name}/${libmongocrypt_s3_suffix}/libmongocrypt.tar.gz'
bucket: mciuploads
bucket: ${upload_bucket}
extract_to: all/${variant_name}

"setup packaging credentials":
Expand Down Expand Up @@ -323,12 +322,15 @@ functions:
- "*"
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
# The upload of this component uses the less restricted bucket because it is only
Comment thread
kevinAlbs marked this conversation as resolved.
Outdated
# used for transferring temporary files until they are later merged in the next build step
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
local_file: release-files.tgz
remote_file: '${project}/python-release/${branch_name}/${libmongocrypt_s3_suffix}/${task_id}-${execution}-release-files.tar.gz'
bucket: mciuploads
permissions: public-read
permissions: private
visibility: signed
content_type: ${content_type|application/gzip}
display_name: Release Python files

Expand All @@ -355,6 +357,8 @@ functions:
- command: shell.exec
params:
shell: "bash"
# This script downloads from the less restricted bucket to the location that was pushed by the
# `upload python release` step
script: |
set -o xtrace
# Download all the release files.
Expand All @@ -373,12 +377,13 @@ functions:
- "*"
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
role_arn: ${upload_arn}
local_file: release-files-all.tgz
remote_file: '${project}/python-release/${branch_name}/${libmongocrypt_s3_suffix}/${task_id}-${execution}-release-files-all.tar.gz'
bucket: mciuploads
permissions: public-read
# The merged results are placed in the CDN bucket for releases
bucket: ${upload_bucket}
permissions: private
visibility: signed
content_type: ${content_type|application/gzip}
display_name: Release Python files all
earthly:
Expand Down Expand Up @@ -432,12 +437,12 @@ functions:
type: test
params:
display_name: Augmented SBOM
aws_key: ${aws_key}
aws_secret: ${aws_secret}
bucket: mciuploads
role_arn: ${upload_arn}
bucket: ${upload_bucket}
content_type: application/json
local_file: libmongocrypt/cyclonedx.augmented.sbom.json
permissions: public-read
permissions: private
visibility: signed
remote_file: ${project}/${build_variant}/${branch_name}/${libmongocrypt_s3_suffix}/sbom/cyclonedx.augmented.sbom.json

tasks:
Expand Down Expand Up @@ -640,20 +645,20 @@ tasks:
fi
- command: s3.put
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: 'libmongocrypt/java/${revision}/libmongocrypt-java.tar.gz'
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
local_file: 'libmongocrypt-java.tar.gz'
content_type: '${content_type|application/x-gzip}'
- command: s3.put
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: 'libmongocrypt/java/${tag_upload_location}/libmongocrypt-java.tar.gz'
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
optional: true
display_name: 'libmongocrypt-java-${tag_upload_location}.tar.gz'
local_file: 'libmongocrypt-java-${tag_upload_location}.tar.gz'
Expand Down Expand Up @@ -835,51 +840,51 @@ tasks:
fi
- command: s3.put
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: 'libmongocrypt/all/${branch_name}/${libmongocrypt_s3_suffix}/libmongocrypt-all.tar.gz'
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
local_file: 'libmongocrypt-all.tar.gz'
content_type: '${content_type|application/x-gzip}'
- command: s3.put
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: 'libmongocrypt/all/${branch_name}/${libmongocrypt_s3_suffix_copy}/libmongocrypt-all.tar.gz'
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
local_file: 'libmongocrypt-all.tar.gz'
content_type: '${content_type|application/x-gzip}'
- command: s3.put
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: 'libmongocrypt/all/${tag_upload_location}/libmongocrypt-all.tar.gz'
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
optional: true # Do not fail task if `local_file` does not exist. `local_file` only exists for tagged release.
display_name: 'libmongocrypt-all-${tag_upload_location}.tar.gz'
local_file: 'libmongocrypt-all-${tag_upload_location}.tar.gz'
content_type: '${content_type|application/x-gzip}'
- command: s3.put
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: 'libmongocrypt/all/latest/stable/libmongocrypt-all.tar.gz'
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
optional: true # Do not fail task if `local_file` does not exist. `local_file` only exists for stable release.
display_name: 'stable/libmongocrypt-all-${tag_upload_location}.tar.gz'
local_file: 'stable/libmongocrypt-all-${tag_upload_location}.tar.gz'
content_type: '${content_type|application/x-gzip}'
- command: s3.put
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: 'libmongocrypt/all/latest/unstable/libmongocrypt-all.tar.gz'
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
optional: true # Do not fail task if `local_file` does not exist. `local_file` only exists for unstable release.
display_name: 'unstable/libmongocrypt-all-${tag_upload_location}.tar.gz'
local_file: 'unstable/libmongocrypt-all-${tag_upload_location}.tar.gz'
Expand Down Expand Up @@ -931,10 +936,9 @@ tasks:
file: libmongocrypt/expansions.yml
- command: s3.get # Download Windows build.
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: '${project}/windows-test/${branch_name}/${libmongocrypt_s3_suffix}/libmongocrypt.tar.gz'
bucket: mciuploads
bucket: ${upload_bucket}
extract_to: libmongocrypt_download
- command: shell.exec
params:
Expand All @@ -960,22 +964,22 @@ tasks:
# Documentation now refers to the GitHub release page, which includes the per-release tarball.
# The fixed URL upload is kept to avoid possibly breaking expectations. Consider removing in the future.
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: 'libmongocrypt/windows/latest_release/libmongocrypt${upload_suffix}.tar.gz'
display_name: (Deprecated) libmongocrypt${upload_suffix}.tar.gz
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
local_file: 'libmongocrypt_upload.tar.gz'
content_type: 'application/x-gzip'
- command: s3.put # Upload tarball for GitHub Release.
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: '${project}/${build_variant}/${branch_name}/${revision}/${version_id}/libmongocrypt-windows-x86_64-${libmongocrypt_release_version}.tar.gz'
display_name: libmongocrypt-windows-x86_64-${libmongocrypt_release_version}.tar.gz
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
local_file: 'libmongocrypt_upload.tar.gz'
content_type: 'application/x-gzip'
- command: shell.exec
Expand All @@ -990,12 +994,12 @@ tasks:
args: --secret garasign_username=${garasign_username} --secret garasign_password=${garasign_password} +sign --file_to_sign=libmongocrypt_upload.tar.gz --output_file=libmongocrypt_upload.asc
- command: s3.put # Upload signature for GitHub Release.
params:
aws_key: '${aws_key}'
aws_secret: '${aws_secret}'
role_arn: ${upload_arn}
remote_file: '${project}/${build_variant}/${branch_name}/${revision}/${version_id}/libmongocrypt-windows-x86_64-${libmongocrypt_release_version}.asc'
display_name: libmongocrypt-windows-x86_64-${libmongocrypt_release_version}.asc
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
local_file: 'libmongocrypt/libmongocrypt_upload.asc'
content_type: 'application/pgp-signature'

Expand All @@ -1017,12 +1021,12 @@ tasks:
bash .evergreen/debian_package_build.sh --is-patch=${is_patch}
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
role_arn: ${upload_arn}
local_file: deb.tar.gz
remote_file: libmongocrypt/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/debian-packages.tar.gz
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
content_type: ${content_type|application/x-gzip}
display_name: "deb.tar.gz"

Expand All @@ -1042,12 +1046,12 @@ tasks:
bash .evergreen/debian_package_build.sh --arch=i386 --is-patch=${is_patch}
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
role_arn: ${upload_arn}
local_file: deb.tar.gz
remote_file: libmongocrypt/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/debian-packages-i386.tar.gz
bucket: mciuploads
permissions: public-read
bucket: ${upload_bucket}
permissions: private
visibility: signed
content_type: ${content_type|application/x-gzip}
display_name: "deb.tar.gz"

Expand Down Expand Up @@ -1142,15 +1146,29 @@ pre:
REMOTE_SUFFIX_COPY="latest-${branch_name}"
fi

# If we are a non-patch build in the libmongocrypt-release project, we upload to a restricted
# CDN S3 bucket. Otherwise, we upload to a less restricted bucket for convenience. The corresponding
# role_arn_... values come from EVG project configuration variables stored on the EVG server
if test "${is_patch}" = 'true' || "${project}" != 'libmongocrypt-release'; then
upload_bucket='mciuploads'
upload_arn='${role_arn_for_mciuploads}'
else
upload_bucket='cdn-origin-libmongocrypt'
upload_arn='${role_arn_for_release}'
fi

PROJECT_DIRECTORY="$(pwd)"
echo "libmongocrypt_s3_suffix: $REMOTE_SUFFIX"
echo "libmongocrypt_s3_suffix_copy: $REMOTE_SUFFIX_COPY"
echo "project_directory: $PROJECT_DIRECTORY"
echo "Upload S3 bucket: $upload_bucket"

cat <<EOT > expansion.yml
libmongocrypt_s3_suffix: "$REMOTE_SUFFIX"
libmongocrypt_s3_suffix_copy: "$REMOTE_SUFFIX_COPY"
project_directory: "$PROJECT_DIRECTORY"
upload_bucket: "$upload_bucket"
upload_arn: "$upload_arn"
EOT
- command: expansions.update
params:
Expand Down