feat: enhance CI scripts for improved logging and DPF-reader support

.ci/collect_mapdl_logs_remote.sh

@@ -15,6 +15,11 @@ mkdir "$LOG_NAMES" && echo "Successfully generated directory $LOG_NAMES"
 echo "Collecting MAPDL logs..."
 
 (docker exec "$MAPDL_INSTANCE" /bin/bash -c "mkdir -p /mapdl_logs && echo 'Successfully created directory inside docker container'") || echo "Failed to create a directory inside docker container for logs."
+
+(docker exec "$MAPDL_INSTANCE" /bin/bash -c "mv ./*.log /mapdl_logs") || echo "Failed to move the logs files."
+
+(docker exec "$MAPDL_INSTANCE" /bin/bash -c "ps aux > /mapdl_logs/docker_processes_end.log") || echo "Failed to get the processes from the docker container"
+
 (docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$FILE*.out' > /dev/null ;then mv -f /file*.out /mapdl_logs && echo 'Successfully moved out files.'; fi") || echo "Failed to move the 'out' files into a local file"
 (docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$FILE*.err' > /dev/null ;then mv -f /file*.err /mapdl_logs && echo 'Successfully moved err files.'; fi") || echo "Failed to move the 'err' files into a local file"
 (docker exec "$MAPDL_INSTANCE" /bin/bash -c "if compgen -G '$FILE*.log' > /dev/null ;then mv -f /file*.log /mapdl_logs && echo 'Successfully moved log files.'; fi") || echo "Failed to move the 'log' files into a local file"
@@ -24,14 +29,12 @@ docker cp "$MAPDL_INSTANCE":/mapdl_logs/. ./"$LOG_NAMES"/. || echo "Failed to co
 
 ####
 echo "Collecting local build logs..."
+ls -la
 
-echo "Collecting docker run log..."
-mv log.txt ./"$LOG_NAMES"/log.txt || echo "MAPDL run docker log not found."
-mv log_dpf.txt ./"$LOG_NAMES"/log_dpf.txt || echo "DPF run docker log not found."
+docker ps > /"$LOG_NAMES"/docker_ps_end.log || echo "Failed to print the docker ps"
 
-echo "Moving docker launch log..."
-mv mapdl_launch_0.log ./"$LOG_NAMES"/mapdl_launch_0.log || echo "MAPDL launch docker log not found."
-mv mapdl_launch_1.log ./"$LOG_NAMES"/mapdl_launch_1.log || echo "MAPDL launch docker log not found."
+echo "Collecting logs..."
+mv ./*.log ./"$LOG_NAMES"/ || echo "MAPDL run docker log not found."
 
 echo "Moving the profiling files..."
 mkdir -p ./"$LOG_NAMES"/prof

.ci/start_mapdl.sh

@@ -1,4 +1,35 @@
 #!/bin/bash
+# This script is used to start a MAPDL instance in a Docker container.
+# 
+# Usage:
+# ------
+# This script is intended to be run in a CI/CD environment where the necessary environment variables are set.
+#
+# Required environment variables:
+# -------------------------------
+#
+# - DISTRIBUTED_MODE: The mode of operation for MAPDL (e.g., "smp", "dmp").
+# - DPF_PORT: The port for the DPF service (e.g., 50055).
+# - INSTANCE_NAME: The name of the MAPDL instance (e.g., "MAPDL_0").
+# - LICENSE_SERVER: The address of the license server (e.g., "123.123.123.123").
+# - MAPDL_PACKAGE: The Docker image package name (e.g., ansys/mapdl).
+# - MAPDL_VERSION: The version of the MAPDL image to use (e.g., "v25.2-ubuntu-cicd").
+# - PYMAPDL_DB_PORT: The port for the PyMAPDL database service (e.g., 50056).
+# - PYMAPDL_PORT: The port for the PyMAPDL service (e.g., 50052).
+#
+# Example:
+# --------
+#
+#   export DISTRIBUTED_MODE="smp"
+#   export DPF_PORT=50055
+#   export INSTANCE_NAME=MAPDL_0
+#   export LICENSE_SERVER="123.123.123.123"
+#   export MAPDL_PACKAGE=ghcr.io/ansys/mapdl
+#   export MAPDL_VERSION=v25.2-ubuntu-cicd
+#   export PYMAPDL_DB_PORT=50056
+#   export PYMAPDL_PORT=50052
+#   ./start_mapdl.sh
+#   
 echo "MAPDL Instance name: $INSTANCE_NAME"
 echo "MAPDL_VERSION: $MAPDL_VERSION"
 
@@ -29,29 +60,51 @@ else
     export P_SCHEMA=/ansys_inc/ansys/ac4/schema
 fi;
 
+if [[ $MAPDL_VERSION == *"cicd"* ]] ; then
+    echo "It is a CICD version, binding DPF port too"
+    export DPF_ARG="-p ${DPF_PORT}:50055"
+    export DB_INT_PORT=50056
+
+    echo "DPF_ARG: $DPF_ARG"
+    echo "DB_INT_PORT: $DB_INT_PORT"
+else
+    export DPF_ARG=""
+    export DB_INT_PORT=50055
+fi;
+
 echo "EXEC_PATH: $EXEC_PATH"
 echo "P_SCHEMA: $P_SCHEMA"
 
-docker run \
-    --entrypoint "/bin/bash" \
-    --name "$INSTANCE_NAME" \
-    --restart always \
-    --health-cmd="ps aux | grep \"[/]ansys_inc/.*ansys\.e.*grpc\" -q && echo 0 || echo 1" \
-    --health-interval=0.5s \
-    --health-retries=4 \
-    --health-timeout=0.5s \
-    --health-start-period=10s \
-    -e ANSYSLMD_LICENSE_FILE=1055@"$LICENSE_SERVER" \
-    -e ANSYS_LOCK="OFF" \
-    -p "$PYMAPDL_PORT":50052 \
-    -p "$PYMAPDL_DB_PORT":50055 \
-    --shm-size=2gb \
-    -e I_MPI_SHM_LMT=shm \
-    -e P_SCHEMA="$P_SCHEMA" \
-    -w /jobs \
-    -u=0:0 \
-    --memory=6656MB \
-    --memory-swap=16896MB \
-    "$MAPDL_IMAGE" "$EXEC_PATH" -grpc -dir /jobs -"$DISTRIBUTED_MODE" -np 2 -db -5000 -m -5000 - > log.txt &
-
-grep -q 'Server listening on' <(timeout 60 tail -f log.txt)
+# Building docker command
+CMD=$(cat <<-_EOT_
+run \
+  --entrypoint /bin/bash \
+  --name ${INSTANCE_NAME} \
+  --restart always \
+  --health-interval=0.5s \
+  --health-retries=4 \
+  --health-timeout=0.5s \
+  --health-start-period=10s \
+  -e ANSYSLMD_LICENSE_FILE=1055@${LICENSE_SERVER} \
+  -e ANSYS_LOCK="OFF" \
+  -p ${PYMAPDL_PORT}:50052 \
+  -p ${PYMAPDL_DB_PORT}:${DB_INT_PORT} \
+  ${DPF_ARG} \
+  --shm-size=2gb \
+  -e I_MPI_SHM_LMT=shm \
+  -e P_SCHEMA="$P_SCHEMA" \
+  -w /jobs \
+  -u=0:0 \
+  --memory=6656MB \
+  --memory-swap=16896MB \
+  ${MAPDL_IMAGE} ${EXEC_PATH} -grpc -dir /jobs -${DISTRIBUTED_MODE} -np 2 -db -5000 -m -5000 -
+_EOT_
+)
+
+echo "Running docker command: "
+echo "docker ${CMD}"
+docker ${CMD} > ${INSTANCE_NAME}.log &
+grep -q 'Server listening on' <(timeout 60 tail -f ${INSTANCE_NAME}.log)
+
+echo "Content of ${INSTANCE_NAME}.log:"
+cat ${INSTANCE_NAME}.log

.ci/waiting_services.sh

@@ -1,16 +1,20 @@
 #!/bin/bash
+echo "::group:: Docker services" && docker ps && echo "::endgroup::" 
+
 echo "Waiting for the MAPDL service to be up..."
 nc -v -z localhost "$PYMAPDL_PORT"
+
 echo "::group:: ps aux Output" && ps aux && echo "::endgroup::" 
 
-echo "Waiting for MAPDL port is open..."
+
 echo "::group:: Waiting for the MAPDL port to be open..." 
 while ! nc -z localhost "$PYMAPDL_PORT"; do
-sleep 0.1
+    sleep 0.1
 done
 echo "::endgroup::"
 echo "MAPDL service is up!"
 
+
 echo "::group:: Waiting for the DPF port to be open..." 
 while ! nc -z localhost "$DPF_PORT"; do
     sleep 0.1

.github/workflows/ci.yml

@@ -182,7 +182,7 @@ jobs:
       username: ${{ github.actor }}
     with:
       mapdl-version: "${{ matrix.mapdl-version }}"
-      file-name: "${{ matrix.mapdl-version }}-remote"
+      file-name: "remote-${{ matrix.mapdl-version }}"
 
 
   build-test-local-matrix:
@@ -221,7 +221,7 @@ jobs:
       testing-minimal: false
       pytest-arguments: '--reset_only_failed  --add_missing_images'
       mapdl-version: ${{ matrix.mapdl-version }}
-      file-name: "${{ matrix.mapdl-version }}-local"
+      file-name: "local-${{ matrix.mapdl-version }}"
       tags: "local"
       latest-version: "252"
       test_dpf: ${{ contains(matrix.mapdl-version, 'cicd') }}
@@ -317,7 +317,7 @@ jobs:
       contents: write
     steps:
       - name: "Download the library artifacts from build-library step"
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
           name: ${{ env.PACKAGE_NAME }}-artifacts
           path: ${{ env.PACKAGE_NAME }}-artifacts

.github/workflows/test-local.yml

@@ -155,6 +155,11 @@ jobs:
             else export ON_STUDENT=false; export TAG_STUDENT="non-student";
           fi
 
+          if [[ "${{ inputs.mapdl-version }}" == *"cicd"* ]]; then
+            echo "CICD MAPDL version detected, testing DPF backend for results module.";
+            echo "TEST_DPF_BACKEND=true" >> $GITHUB_ENV;
+          fi
+
           echo "ON_STUDENT: $ON_STUDENT"
           echo "TAG_STUDENT: $TAG_STUDENT"
           echo "ON_STUDENT=$(echo $ON_STUDENT)" >> $GITHUB_OUTPUT

.github/workflows/test-remote.yml

@@ -62,10 +62,12 @@ jobs:
       PYMAPDL_DB_PORT: 21002  # default won't work on GitHub runners
       PYMAPDL_DB_PORT2: 21003  # default won't work on GitHub runners
       DPF_DOCKER_IMAGE: ghcr.io/ansys/mapdl:v25.2-rocky-dpf-standalone
-      DPF_PORT: 21004
+      DPF_PORT: 21014
+      DPF_PORT2: 21015
       DPF_START_SERVER: False
       HAS_DPF: True
-      PYTEST_ARGUMENTS: '-vvv -ra --color=yes --durations=30 --random-order --random-order-bucket=class --maxfail=10 --reruns 3 --reruns-delay 4 --cov=ansys.mapdl.core --cov-report=html --timeout=180 --profile-svg --profile --report-log-exclude-logs-on-passed-tests --strict-markers'
+      TEST_DPF_BACKEND: false
+      PYTEST_ARGUMENTS: '-vvv -ra --color=yes --durations=30 --random-order --random-order-bucket=class --maxfail=5 --reruns 3 --reruns-delay 4 --cov=ansys.mapdl.core --cov-report=html --timeout=180 --profile-svg --profile --report-log-exclude-logs-on-passed-tests --strict-markers'
       MAPDL_PACKAGE: ghcr.io/ansys/mapdl
 
     steps:
@@ -105,11 +107,20 @@ jobs:
         shell: bash
         run: |
           export ON_SAME_CONTAINER=false
+
           if [[ "${{ inputs.mapdl-version }}" == *"ubuntu"* ]];
             then export ON_UBUNTU=true; export TAG_UBUNTU="ubuntu";
             else export ON_UBUNTU=false; export TAG_UBUNTU="centos";
-            if [[ "${{ inputs.mapdl-version }}" == *"cicd"* ]]; then export ON_SAME_CONTAINER="true"; fi
           fi
+
+          if [[ "${{ inputs.mapdl-version }}" == *"cicd"* ]]; then
+            echo "CICD MAPDL version detected, testing DPF backend for results module.";
+            echo "TEST_DPF_BACKEND=true" >> $GITHUB_ENV;
+
+            echo "It should be run on the same container as MAPDL";
+            export ON_SAME_CONTAINER=true;
+          fi
+
           echo "ON_UBUNTU: $ON_UBUNTU"
           echo "TAG_UBUNTU: $TAG_UBUNTU"
           echo "ON_SAME_CONTAINER: $ON_SAME_CONTAINER"
@@ -143,11 +154,14 @@ jobs:
           echo "Launching first MAPDL instance..."
           export INSTANCE_NAME=MAPDL_0
           .ci/start_mapdl.sh &> mapdl_launch_0.log & export DOCKER_PID_0=$!
+
           echo "Launching a second instance for MAPDL pool testing..."
           export PYMAPDL_PORT=${{ env.PYMAPDL_PORT2 }}
           export PYMAPDL_DB_PORT=${{ env.PYMAPDL_DB_PORT2 }}
           export INSTANCE_NAME=MAPDL_1
+          export DPF_PORT=${{ env.DPF_PORT2 }}
           .ci/start_mapdl.sh &> mapdl_launch_1.log & export DOCKER_PID_1=$!
+
           echo "Launching MAPDL service 0 at PID: $DOCKER_PID_0"
           echo "Launching MAPDL service 1 at PID: $DOCKER_PID_1"
           echo "DOCKER_PID_0=$(echo $DOCKER_PID_0)" >> $GITHUB_OUTPUT
@@ -159,14 +173,7 @@ jobs:
         env:
           ANSYS_DPF_ACCEPT_LA: Y
         run: |
-          docker pull $DPF_DOCKER_IMAGE && docker run -d --name dpfserver --env ANSYS_DPF_ACCEPT_LA=Y -p ${{ env.DPF_PORT }}:50052 $DPF_DOCKER_IMAGE && echo "DPF Server active on port ${{ env.DPF_PORT }}." > log_dpf.txt &
-
-      - name: "Start DPF server on same container as MAPDL"
-        if: ${{ steps.ubuntu_check.outputs.ON_SAME_CONTAINER == 'true' }}
-        shell: bash
-        run: |
-          echo "Starting DPF server on same MAPDL container..."
-          docker exec MAPDL_0 /bin/bash -c "/ansys_inc/v252/aisol/bin/linx64/Ans.Dpf.Grpc.sh --port ${{ env.DPF_PORT }} &" &> log_dpf.txt &
+          docker pull $DPF_DOCKER_IMAGE && docker run -d --name dpfserver --env ANSYS_DPF_ACCEPT_LA=Y -p ${{ env.DPF_PORT }}:50052 $DPF_DOCKER_IMAGE && echo "DPF Server active on port ${{ env.DPF_PORT }}." > log_dpf.log &
 
       - name: "Getting files change filters"
         uses: dorny/paths-filter@v3
@@ -215,12 +222,27 @@ jobs:
         run: |
           python -m pip install .[tests]
 
+      - name: "Start DPF server on same container as MAPDL"
+        if: ${{ steps.ubuntu_check.outputs.ON_SAME_CONTAINER == 'true' }}
+        shell: bash
+        env:
+          MAPDL_INSTANCE: "MAPDL_0"
+          DPF_PORT: "${{ env.DPF_PORT }}"
+        run: |
+          docker ps
+          echo "Starting DPF server on same MAPDL container: ${MAPDL_INSTANCE}"
+          docker exec ${MAPDL_INSTANCE} /bin/bash -c "/ansys_inc/v252/aisol/bin/linx64/Ans.Dpf.Grpc.sh --port 50055 &" > log_dpf.log &
+
       - name: "Waiting for the services to be up"
         shell: bash
         env:
           PYMAPDL_PORT: ${{ env.PYMAPDL_PORT }}
           PYMAPDL_PORT2: ${{ env.PYMAPDL_PORT2 }}
+          MAPDL_INSTANCE: MAPDL_0
         run: |
+          docker ps > docker_ps_start.log || echo "Failed to get the docker images from the docker container"
+          (docker exec "${MAPDL_INSTANCE}" /bin/bash -c "ps aux > docker_processes_start.log") || echo "Failed to get the processes from the docker container"
+
           .ci/waiting_services.sh
 
       - name: "Unit testing"
@@ -229,6 +251,7 @@ jobs:
           ON_UBUNTU: ${{ steps.ubuntu_check.outputs.ON_UBUNTU }}
           ON_STUDENT: ${{ steps.student_check.outputs.ON_STUDENT }}
           file_name: "${{ inputs.file-name }}"
+          MAPDL_VERSION: "${{ inputs.mapdl-version }}"
         shell: bash
         run: |
           echo "ON_UBUNTU: $ON_UBUNTU"

doc/changelog.d/4040.miscellaneous.md

@@ -0,0 +1 @@
+Feat: enhance ci scripts for improved logging and dpf-reader support