Skip to content
Docker Image CI and Test

Inspect Docker image size before Apptainer conversion #102

Sign in to view logs

Inspect Docker image size before Apptainer conversion

Inspect Docker image size before Apptainer conversion #102

Workflow file for this run

.github/workflows/docker-test.yml at dd3cb37
name: Docker Image CI and Test
on:
push:
branches: [ docker-test ]
pull_request:
branches: [ master ]
env:
# see the Dockerfile for the definition of some of the following environment variables
# status: pre # pre, alpha, beta, final
status: alpha
CINC2025_REVENGER_TEST: 1
download_data_dir: ${{ github.workspace }}/download_data
mount_data_dir: /challenge/data
revenger_data_dir: /challenge/cache/revenger_data_dir
revenger_model_cache_dir: /challenge/cache/revenger_model_dir
revenger_model_dir: /challenge/model
revenger_test_dir: /challenge/test
revenger_output_dir: /challenge/output
docker_main_filename: Dockerfile
DOCKER_HUB_USERNAME: wenh06
DOCKER_HUB_REPOSITORY: deeppsp-cinc2025-docker-image
DOCKER_IMAGE_TAG: latest
# host-side writable directories to bind-mount into the container
host_output_dir: ${{ github.workspace }}/output
host_test_dir: ${{ github.workspace }}/test
host_model_dir: ${{ github.workspace }}/model
jobs:
build:
if: contains(fromJson('["wenh06", "kjs11", "DeepPSP"]'), github.repository_owner)
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v6
with:
submodules: recursive
- name: Clear unnecessary system components
run: |
echo "Free space before cleanup:"
df -h
# remove large pre-installed toolchains and SDKs
sudo rm -rf /usr/local/lib/android
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf /usr/local/share/boost
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
# remove large apt packages not needed for our build
sudo apt-get remove -y '^aspnetcore-.*' '^dotnet-.*' '^llvm-.*' \
'php.*' '^mongodb-.*' '^mysql-.*' \
azure-cli google-cloud-cli google-chrome-stable \
firefox powershell mono-devel libgl1-mesa-dri --fix-missing || true
sudo apt-get autoremove -y || true
sudo apt-get clean || true
# remove docker images already pulled (we will build our own)
docker rmi $(docker images -q) 2>/dev/null || true
# remove swap file to reclaim space
sudo swapoff -a || true
sudo rm -f /mnt/swapfile || true
echo "Free space after cleanup:"
df -h
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker
# - name: Login to Docker Hub
# uses: docker/login-action@v3
# with:
# username: ${{ secrets.DOCKERHUB_USERNAME }}
# password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push (optional) the Docker image
uses: docker/build-push-action@v6
with:
context: .
file: ${{ env.docker_main_filename }}
push: false
# load: true
tags: |
${{ env.DOCKER_HUB_USERNAME }}/${{ env.DOCKER_HUB_REPOSITORY }}:${{ env.DOCKER_IMAGE_TAG }}
- name: Install Apptainer from official PPA
run: |
sudo add-apt-repository -y ppa:apptainer/ppa
sudo apt update
sudo apt install -y apptainer
apptainer --version
- name: Convert Docker image to Apptainer SIF
run: |
docker image inspect ${{ env.DOCKER_HUB_USERNAME }}/${{ env.DOCKER_HUB_REPOSITORY }}:${{ env.DOCKER_IMAGE_TAG }} \
--format='{{.Size}}'
apptainer build container.sif \
docker-daemon://${{ env.DOCKER_HUB_USERNAME }}/${{ env.DOCKER_HUB_REPOSITORY }}:${{ env.DOCKER_IMAGE_TAG }}
- name: Download the training_subset data
if: ${{ env.status != 'pre' }}
# TODO: adjust this step for CinC2025 challenge
# this step is done before running the test of challenge entry
# in the Docker image in the next step since the data directory
# is mounted as read-only in the challenge settings
# run: |
# mkdir -p ${{ env.download_data_dir }}
# docker run \
# -e PYTHONUNBUFFERED=1 \
# -e CINC2025_REVENGER_TEST=${{ env.CINC2025_REVENGER_TEST }} \
# -e revenger_data_dir=${{ env.revenger_data_dir }} \
# -e revenger_model_dir=${{ env.revenger_model_dir }} \
# -e revenger_test_dir=${{ env.revenger_test_dir }} \
# -e revenger_output_dir=${{ env.revenger_output_dir }} \
# -e mount_data_dir=${{ env.mount_data_dir }} \
# -v ${{ env.download_data_dir }}:${{ env.mount_data_dir }} \
# ${{ env.DOCKER_HUB_USERNAME }}/${{ env.DOCKER_HUB_REPOSITORY }}:${{ env.DOCKER_IMAGE_TAG }} \
# bash -c "python data_reader.py download --db-dir ${{ env.mount_data_dir }} --files code-15-exams-part17,sami-trop,ptb-xl-subset,labels,chagas-labels --convert --remove"
# docker ps -a
# docker stop $(docker ps -a -q)
run: |
mkdir -p ${{ env.download_data_dir }}
apptainer exec \
--env PYTHONUNBUFFERED=1 \
--env CINC2025_REVENGER_TEST=${{ env.CINC2025_REVENGER_TEST }} \
--env revenger_data_dir=${{ env.revenger_data_dir }} \
--env revenger_model_dir=${{ env.revenger_model_dir }} \
--env revenger_test_dir=${{ env.revenger_test_dir }} \
--env revenger_output_dir=${{ env.revenger_output_dir }} \
--env mount_data_dir=${{ env.mount_data_dir }} \
--bind ${{ env.download_data_dir }}:${{ env.mount_data_dir }} \
container.sif \
python data_reader.py download \
--db-dir ${{ env.mount_data_dir }} \
--files code-15-exams-part17,sami-trop,ptb-xl-subset,labels,chagas-labels \
--convert --remove
echo "=== model dir contents after data download ==="
apptainer exec container.sif \
tree ${{ env.revenger_model_cache_dir }}
# - name: Run the Docker image
# # TODO: adjust this step for CinC2025 challenge
# # NOTE: mount env.download_data_dir to env.mount_data_dir
# # and set it read-only as in the CinC2025 challenge settings.
# # Also, the docker image is run without any internet access.
# if: ${{ env.status != 'pre' }}
# run: |
# docker run \
# -e PYTHONUNBUFFERED=1 \
# -e CINC2025_REVENGER_TEST=${{ env.CINC2025_REVENGER_TEST }} \
# -e revenger_data_dir=${{ env.revenger_data_dir }} \
# -e revenger_model_dir=${{ env.revenger_model_dir }} \
# -e revenger_test_dir=${{ env.revenger_test_dir }} \
# -e revenger_output_dir=${{ env.revenger_output_dir }} \
# -e mount_data_dir=${{ env.mount_data_dir }} \
# -v ${{ env.download_data_dir }}:${{ env.mount_data_dir }}:ro \
# --network none \
# ${{ env.DOCKER_HUB_USERNAME }}/${{ env.DOCKER_HUB_REPOSITORY }}:${{ env.DOCKER_IMAGE_TAG }} \
# bash -c "bash test_run_challenge.sh"
# docker ps -a
# docker stop $(docker ps -a -q)
- name: Create writable host directories for container output
if: ${{ env.status != 'pre' }}
run: |
mkdir -p ${{ env.host_output_dir }}
mkdir -p ${{ env.host_test_dir }}
mkdir -p ${{ env.host_model_dir }}
- name: Run the Apptainer container
if: ${{ env.status != 'pre' }}
run: |
echo "=== model dir contents before run ==="
apptainer exec container.sif \
tree ${{ env.revenger_model_cache_dir }}
apptainer exec \
--env PYTHONUNBUFFERED=1 \
--env CINC2025_REVENGER_TEST=${{ env.CINC2025_REVENGER_TEST }} \
--env revenger_data_dir=${{ env.revenger_data_dir }} \
--env revenger_model_dir=${{ env.revenger_model_dir }} \
--env revenger_test_dir=${{ env.revenger_test_dir }} \
--env revenger_output_dir=${{ env.revenger_output_dir }} \
--env mount_data_dir=${{ env.mount_data_dir }} \
--bind ${{ env.download_data_dir }}:${{ env.mount_data_dir }}:ro \
--bind ${{ env.host_output_dir }}:${{ env.revenger_output_dir }} \
--bind ${{ env.host_test_dir }}:${{ env.revenger_test_dir }} \
--bind ${{ env.host_model_dir }}:${{ env.revenger_model_dir }} \
--network none \
container.sif \
bash test_run_challenge.sh