diff --git a/.github/workflows/velox-deps-upload.yml b/.github/workflows/velox-deps-upload.yml new file mode 100644 index 00000000..3806ea67 --- /dev/null +++ b/.github/workflows/velox-deps-upload.yml @@ -0,0 +1,79 @@ +name: Upload Velox Dependencies to S3 +run-name: Upload Velox Deps Image to S3 + +on: + workflow_dispatch: + inputs: + repository: + description: 'Velox repository' + type: string + required: false + default: 'facebookincubator/velox' + velox_commit: + description: 'Velox commit SHA or branch' + type: string + required: false + default: 'main' + + workflow_call: + inputs: + repository: + description: 'Velox repository' + type: string + required: false + default: 'facebookincubator/velox' + velox_commit: + description: 'Velox commit SHA or branch' + type: string + required: false + default: 'main' + +defaults: + run: + shell: bash + +jobs: + build-and-upload-deps: + runs-on: linux-amd64-cpu4 + + env: + GH_TOKEN: ${{ github.token }} + DOCKER_RUNTIME: runc + + steps: + - name: Checkout this repository for CI scripts + uses: actions/checkout@v4 + with: + path: velox-testing + + - name: Setup Velox + uses: ./velox-testing/.github/actions/velox-setup + with: + repository: ${{ inputs.repository }} + velox_commit: ${{ inputs.velox_commit }} + + - name: Build Velox Dependencies Container Image + working-directory: ${{ github.workspace }}/velox-testing/velox/scripts + run: ./build_centos_deps_image.sh + + - name: Upload Velox Dependencies Container Image to S3 + env: + AWS_ARN_STRING: ${{ secrets.AWS_ARN_STRING }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + S3_BUCKET_NAME: ${{ vars.S3_BUCKET_NAME }} + S3_BUCKET_REGION: ${{ vars.S3_BUCKET_REGION }} + working-directory: ${{ github.workspace }}/velox-testing/velox/scripts + run: ./upload_centos_deps_image.sh + + - name: Upload Summary + if: success() + run: | + echo "### ✅ Velox Dependencies Image Uploaded Successfully" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Repository:** ${{ inputs.repository }}" >> $GITHUB_STEP_SUMMARY + echo "**Velox Commit:** ${{ inputs.velox_commit }}" >> $GITHUB_STEP_SUMMARY + echo "**Architecture:** $(uname -m)" >> $GITHUB_STEP_SUMMARY + echo "**Image:** ghcr.io/facebookincubator/velox-dev:adapters" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "The dependencies image has been uploaded to S3 and is now available for use in CI workflows." >> $GITHUB_STEP_SUMMARY diff --git a/scripts/upload_docker_image_to_s3.sh b/scripts/upload_docker_image_to_s3.sh new file mode 100755 index 00000000..09ad1f1c --- /dev/null +++ b/scripts/upload_docker_image_to_s3.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# +# upload_docker_image_to_s3 +# +# saves Docker image to a tar.gz file +# and uploads it to s3://rapidsai-velox-testing// +# + +validate_docker_image() { + local IMAGE_NAME=$1 + echo "Validating Docker image ${IMAGE_NAME}..." + if [[ -z $(docker images -q ${IMAGE_NAME}) ]]; then + echo "ERROR: Docker image ${IMAGE_NAME} does not exist" + exit 1 + fi + echo "✓ Docker image exists" +} + +upload_docker_image_to_s3() { + # validate parameter count + if [[ "$#" -ne 3 ]]; then + echo "Usage: upload_docker_image_to_s3 " >&2 + exit 2 + fi + + # expected parameters + local IMAGE_NAME=$1 + local BUCKET_SUBDIR=$2 + local IMAGE_FILE_NAME=$3 + + # these env vars are required regardless of what creds are used + echo "Validating incoming environment..." + if [ -z "${AWS_ACCESS_KEY_ID}" ] || [ -z "${AWS_SECRET_ACCESS_KEY}" ] || [ -z "${S3_BUCKET_NAME}" ] || [ -z "${S3_BUCKET_REGION}" ]; then + echo "ERROR: The following values must be set in the environment:" + echo " AWS_ARN_STRING (optional)" + echo " AWS_ACCESS_KEY_ID" + echo " AWS_SECRET_ACCESS_KEY" + echo " S3_BUCKET_NAME" + echo " S3_BUCKET_REGION" + echo "Keys must either be valid for direct access to the bucket, or valid for an assume-role operation if AWS_ARN_STRING is set" + exit 1 + fi + + # validate image exists before proceeding + validate_docker_image ${IMAGE_NAME} + + # construct full S3 path + local IMAGE_FILE_PATH="s3://${S3_BUCKET_NAME}/${BUCKET_SUBDIR}/${IMAGE_FILE_NAME}" + + # ensure region is set + export AWS_REGION=${S3_BUCKET_REGION} + + # if AWS_ARN_STRING is set in the environment, use environment creds to request new + # temporary rolling creds for the private bucket, otherwise use environment creds directly + if [ ! -z "${AWS_ARN_STRING}" ]; then + # ask for temporary credentials for file access + echo "Requesting temporary S3 credentials..." + local TEMP_CREDS_JSON=$(aws sts assume-role \ + --role-arn ${AWS_ARN_STRING} \ + --role-session-name "UploadVeloxContainerImage" \ + --query "Credentials" \ + --output json) + + # override environment with full temporary credentials + export AWS_ACCESS_KEY_ID=$(echo "$TEMP_CREDS_JSON" | jq -r '.AccessKeyId') + export AWS_SECRET_ACCESS_KEY=$(echo "$TEMP_CREDS_JSON" | jq -r '.SecretAccessKey') + export AWS_SESSION_TOKEN=$(echo "$TEMP_CREDS_JSON" | jq -r '.SessionToken') + fi + + # save the Docker image to a tar.gz file + echo "Saving Docker image to file..." + docker save ${IMAGE_NAME} | gzip > /tmp/${IMAGE_FILE_NAME} + + # get file size for progress reporting + local FILE_SIZE=$(du -h /tmp/${IMAGE_FILE_NAME} | cut -f1) + echo "Image file size: ${FILE_SIZE}" + + # upload to S3 + echo "Uploading image file to S3..." + echo "Destination: ${IMAGE_FILE_PATH}" + aws s3 cp --no-progress /tmp/${IMAGE_FILE_NAME} ${IMAGE_FILE_PATH} + + # clean up + echo "Cleaning up temporary file..." + rm -f /tmp/${IMAGE_FILE_NAME} + + echo "✓ Successfully uploaded Docker image to S3" +} + +# if executed directly, run with provided args +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + upload_docker_image_to_s3 "$@" +fi + diff --git a/velox/scripts/upload_centos_deps_image.sh b/velox/scripts/upload_centos_deps_image.sh new file mode 100755 index 00000000..f5531ae6 --- /dev/null +++ b/velox/scripts/upload_centos_deps_image.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e + +source ./config.sh +source ../../scripts/upload_docker_image_to_s3.sh + +IMAGE_NAME="ghcr.io/facebookincubator/velox-dev:adapters" + +ARCH=$(uname -m) +BUCKET_SUBDIR="velox-docker-images" +IMAGE_FILE="velox_adapters_deps_image_centos9_${ARCH}.tar.gz" + +# +# validate that the container image exists +# + +validate_docker_image ${IMAGE_NAME} + +# +# upload container image to S3 bucket +# + +upload_docker_image_to_s3 ${IMAGE_NAME} ${BUCKET_SUBDIR} ${IMAGE_FILE} + +if [[ $? -eq 0 ]]; then + echo "Successfully uploaded Velox dependencies/run-time container image to S3" + exit 0 +else + echo "Failed to upload Velox dependencies/run-time container image to S3" + exit 1 +fi +