[CI] Fixed docker build GHA (#3101)

* [CI] Fixed docker build GHA

* Fixed HVD build, vision image issue and apex build issue

Author: vfdev-5

.github/workflows/docker-build.yml

@@ -91,5 +91,5 @@ echo "Show installed packages:"
 docker run --rm -i pytorchignite/${image_name}:${image_tag} pip list
 
 echo "Test pytorchignite/${image_name}:${image_tag}"
-python test_image.py pytorchignite/${image_name}:${image_tag}
+docker run --rm -i -v $PWD:/ws -w /ws -e HVD_VERSION=${HVD_VERSION:-} -e MSDP_VERSION=${MSDP_VERSION:-} pytorchignite/${image_name}:${image_tag} /bin/bash -c "python test_image.py pytorchignite/${image_name}:${image_tag}"
 echo "OK"

.github/workflows/docker-publish.yml

@@ -1,4 +1,4 @@
 [DEFAULT]
-build_docker_image_pytorch_version = 2.0.0-cuda11.7-cudnn8
-build_docker_image_hvd_version = v0.27.0
+build_docker_image_pytorch_version = 2.1.0-cuda12.1-cudnn8
+build_docker_image_hvd_version = v0.28.1
 build_docker_image_msdp_version = v0.8.1

docker/build.sh

@@ -6,8 +6,6 @@ ARG PTH_VERSION
 # 1/Building apex with pytorch:*-devel
 FROM pytorch/pytorch:${PTH_VERSION}-devel AS apex-hvd-builder
 
-ARG ARG_TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1 7.0+PTX 8.0 8.6"
-ENV TORCH_CUDA_ARCH_LIST=$ARG_TORCH_CUDA_ARCH_LIST
 ENV CUDA_HOME=/usr/local/cuda
 
 # Install git
@@ -21,7 +19,7 @@ RUN echo "Setup NVIDIA Apex" && \
     git clone https://github.com/NVIDIA/apex $tmp_apex_path && \
     cd $tmp_apex_path && \
     pip install packaging && \
-    pip wheel -v --disable-pip-version-check --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" .
+    pip wheel -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" .
 
 ARG HVD_VERSION
 
@@ -30,6 +28,9 @@ RUN apt-get update && apt-get install -y git && \
         git clone --recursive --depth 1 --branch ${HVD_VERSION} https://github.com/horovod/horovod.git /horovod && \
         conda install -y cmake nccl -c conda-forge && \
         cd /horovod && \
+        # temporary -std=c++17 fix
+        sed -i "s/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g" CMakeLists.txt && \
+        sed -i "s/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g" horovod/torch/CMakeLists.txt && \
         HOROVOD_GPU_OPERATIONS=NCCL HOROVOD_NCCL_LINK=SHARED HOROVOD_WITHOUT_MPI=1 HOROVOD_WITH_PYTORCH=1 pip wheel --no-cache-dir . && \
         rm -rf /var/lib/apt/lists/*
 

docker/docker.cfg

@@ -4,4 +4,5 @@ FROM pytorchignite/hvd-apex:latest
 # Ignite NLP dependencies
 RUN pip install --upgrade --no-cache-dir transformers \
                                          spacy \
-                                         nltk
+                                         nltk \
+                                         torchtext

docker/hvd/Dockerfile.hvd-apex

@@ -1,15 +1,6 @@
 # Dockerfile.hvd-apex-vision
 FROM pytorchignite/hvd-apex:latest
 
-# Install opencv dependencies
-RUN apt-get update && \
-    apt-get -y install --no-install-recommends libglib2.0 \
-                                               libsm6 \
-                                               libxext6 \
-                                               libxrender-dev \
-                                               libgl1-mesa-glx && \
-    rm -rf /var/lib/apt/lists/*
-
 # Ignite vision dependencies
 RUN pip install --upgrade --no-cache-dir albumentations \
                                          image-dataset-viz \

docker/hvd/Dockerfile.hvd-apex-nlp

@@ -12,6 +12,9 @@ RUN apt-get update && apt-get install -y git && \
         git clone --recursive --depth 1 --branch ${HVD_VERSION} https://github.com/horovod/horovod.git /horovod && \
         conda install -y cmake nccl -c conda-forge && \
         cd /horovod && \
+        # temporary -std=c++17 fix
+        sed -i "s/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g" CMakeLists.txt && \
+        sed -i "s/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g" horovod/torch/CMakeLists.txt && \
         HOROVOD_GPU_OPERATIONS=NCCL HOROVOD_NCCL_LINK=SHARED HOROVOD_WITHOUT_MPI=1 HOROVOD_WITH_PYTORCH=1 pip wheel --no-cache-dir . && \
         rm -rf /var/lib/apt/lists/*
 

docker/hvd/Dockerfile.hvd-apex-vision

@@ -4,4 +4,5 @@ FROM pytorchignite/hvd-base:latest
 # Ignite NLP dependencies
 RUN pip install --upgrade --no-cache-dir transformers \
                                          spacy \
-                                         nltk
+                                         nltk \
+                                         torchtext

docker/hvd/Dockerfile.hvd-base

@@ -1,15 +1,6 @@
 # Dockerfile.hvd-vision
 FROM pytorchignite/hvd-base:latest
 
-# Install opencv dependencies
-RUN apt-get update && \
-    apt-get -y install --no-install-recommends libglib2.0 \
-                                               libsm6 \
-                                               libxext6 \
-                                               libxrender-dev \
-                                               libgl1-mesa-glx && \
-    rm -rf /var/lib/apt/lists/*
-
 # Ignite vision dependencies
 RUN pip install --upgrade --no-cache-dir albumentations \
                                          image-dataset-viz \