Workbench edit4juwels gnu compiler

env/juwels/shell

@@ -1,20 +1,54 @@
 #!/usr/bin/bash
-# ENVIRONMENT used in new_test15_compute_20000101-20000101.run
 # Use this file to source the environment in your
 # preprocessing or postprocessing scripts
-
-module --force purge
-module use $OTHERSTAGES
 module --force purge
-module load Stages/2023
-module load GCC/11.3.0
-module load OpenMPI/4.1.4
-module load CMake/3.26.3
-module load Python/3.10.4
-module load imkl/2022.1.0
-module load Perl/5.34.1
+
+#_______________________________________________________________________________
+# module load Stages/2023
+# module load Stages/2024
+module load Stages/2025
+# 
+# #_______________________________________________________________________________
+# # use GCC + openMPI
+# module load GCC OpenMPI
+# export FC=gfortran
+# export F77=gfortran
+# export MPIFC=gfortran
+# export FCFLAGS=-free
+# export CC=cc
+# export CXX=c++
+
+#_______________________________________________________________________________
+# use INTEL + MPI
+module load Intel ParaStationMPI
+# module load Intel ParaStationMPI/5.10.0-1-mt # -mt ... Multi-threaded Version
+export FC=mpifort
+export F77=mpifort
+export MPIFC=mpifort
+export FCFLAGS=-free
+export CC=mpicc
+export CXX=mpic++
+
+#_______________________________________________________________________________
+# Intel oneAPI Math Kernel Library
+module load imkl
+
+#_______________________________________________________________________________
+# NetCDF (network Common Data Form) software libraries
+module load netCDF netCDF-Fortran
+
+#_______________________________________________________________________________
+# Others libraries
+module load CMake
 module load git
-module load CDO
-module unload ecCodes
-module load libaec FFTW imkl cURL netCDF netCDF-Fortran
-module list
+
+#_______________________________________________________________________________
+#module load Python
+#module load Perl
+#module load CDO
+#module unload ecCodes
+#module load libaec FFTW cURL
+
+#_______________________________________________________________________________
+# show list of now loaded module environment
+module list

src/CMakeLists.txt

@@ -24,6 +24,11 @@ if(DEFINED ENV{ENABLE_ALBEDO_INTELMPI_WORKAROUNDS}) # be able to set the initial
 else()
    option(ALBEDO_INTELMPI_WORKAROUNDS "workaround for performance issues on albedo" OFF)
 endif()
+if(DEFINED ENV{ENABLE_JUWELS_GNUOPENMPI_WORKAROUNDS}) # be able to set the initial cache value from our env settings for aleph, not only via cmake command
+   option(JUWELS_GNUOPENMPI_WORKAROUNDS "workaround for performance issues on juwels" ON)
+else()
+   option(JUWELS_GNUOPENMPI_WORKAROUNDS "workaround for performance issues on juwels" OFF)
+endif()
 
 #TODO: these machine specific changes must be done using cmake toolchain then here otherwise will make cmake another raps. 
 
@@ -41,26 +46,32 @@ if(ALBEDO_INTELMPI_WORKAROUNDS)
    add_compile_options(-DENABLE_ALBEDO_INTELMPI_WORKAROUNDS)
    add_compile_options(-DDISABLE_PARALLEL_RESTART_READ)
 endif()
+if(JUWELS_GNUOPENMPI_WORKAROUNDS)
+   add_compile_options(-DENABLE_JUWELS_GNUOPENMPI_WORKAROUNDS)
+endif()
 
 #TODO: this machine specific block can easyly go in  cmake toolchain.
-if(${FESOM_PLATFORM_STRATEGY} STREQUAL  levante.dkrz.de )
-   message(STATUS "multithreading disabled for Levante") # multithreading suddenly produces an error, disable it until a fix is found. issue #413
+if    (${FESOM_PLATFORM_STRATEGY} STREQUAL levante.dkrz.de )
    option(DISABLE_MULTITHREADING "disable asynchronous operations" ON)
-elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL  leo-dcgp )
-   message(STATUS "multithreading disabled for Leonardo") # multithreading suddenly produces an error, disable it until a fix is found. issue #413
+elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL leo-dcgp )
    option(DISABLE_MULTITHREADING "disable asynchronous operations" ON)
-elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL  mn5-gpp )
-   message(STATUS "multithreading disabled for MN5 GPP") # multithreading suddenly produces an error, disable it until a fix is found. issue #413
+elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL mn5-gpp )
    option(DISABLE_MULTITHREADING "disable asynchronous operations" ON)
-elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL  albedo )
-   message(STATUS "multithreading disabled for Albedo") # multithreading suddenly produces an error, disable it until a fix is found. issue #413
+elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL albedo )
    option(DISABLE_MULTITHREADING "disable asynchronous operations" ON)
 elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL atosecmwf )
-   message(STATUS "multithreading disabled for Atos ECMWF")
    option(DISABLE_MULTITHREADING "disable asynchronous operations" ON)
+elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL juwels )
+   option(DISABLE_MULTITHREADING "disable asynchronous operations" ON)   
 else()
    option(DISABLE_MULTITHREADING "disable asynchronous operations" ON)
 endif()
+if (DISABLE_MULTITHREADING)
+   message(STATUS "MULTITHREADING @ ${FESOM_PLATFORM_STRATEGY}: OFF")
+else()
+   message(STATUS "MULTITHREADING @ ${FESOM_PLATFORM_STRATEGY}: ON")
+endif()
+
 
 option(ENABLE_OPENACC "compile with OpenACC support" OFF)
 message(STATUS "ENABLE_OPENACC: ${ENABLE_OPENACC}")
@@ -242,38 +253,105 @@ if(${CISO_COUPLED})
 endif()
 
 # CMAKE_Fortran_COMPILER_ID will also work if a wrapper is being used (e.g. mpif90 wraps ifort -> compiler id is Intel)
-if(${CMAKE_Fortran_COMPILER_ID} STREQUAL  Intel )
-   target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -ip -init=zero -no-wrap-margin -fpe0) # add -fpe0 for RAPS environment
-   if(${FESOM_PLATFORM_STRATEGY} STREQUAL  levante.dkrz.de )
-      target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2)
+if(${CMAKE_Fortran_COMPILER_ID} STREQUAL  Intel OR ${CMAKE_Fortran_COMPILER_ID} STREQUAL  IntelLLVM )
+
+   # Base compiler flags
+   target_compile_options(${PROJECT_NAME} PRIVATE -O3 -r8 -i4 -fp-model precise -no-prec-div -fimf-use-svml -init=zero -no-wrap-margin -fpe0)
+
+   # compiler flags not supported by IntelLLVM
+   if(${CMAKE_Fortran_COMPILER_ID} STREQUAL  Intel )
+      target_compile_options(${PROJECT_NAME} PRIVATE -no-prec-sqrt -ip )
+   endif()   
+
+   if(ENABLE_OPENMP)
+      target_compile_options(${PROJECT_NAME} PRIVATE -qopenmp)
+   endif()   
+
+   # platform specific compiler flags
+   # add -fpe0 for RAPS environment
+   if    (${FESOM_PLATFORM_STRATEGY} STREQUAL levante.dkrz.de )
+      # PROCESSORS:  2x AMD 7763 CPU; 128 cores in total, 256 GB main memory
+      #target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2)
+      target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2 -qopt-zmm-usage=low -align array64byte -unroll-aggressive -qopt-malloc-options=2)
    elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL leo-dcgp )
-      target_compile_options(${PROJECT_NAME} PRIVATE -O3 -xCORE-AVX512 -qopt-zmm-usage=high -align array64byte -ipo)
+      target_compile_options(${PROJECT_NAME} PRIVATE -xCORE-AVX512 -qopt-zmm-usage=high -align array64byte -ipo)
    elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL mn5-gpp )
-      target_compile_options(${PROJECT_NAME} PRIVATE -O3 -xCORE-AVX512 -qopt-zmm-usage=high -align array64byte -ipo)
-   elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL  albedo)
-      target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -O3 -ip -fPIC -qopt-malloc-options=2 -qopt-prefetch=5 -unroll-aggressive) # -g -traceback -check) #NEC mpi option
+      target_compile_options(${PROJECT_NAME} PRIVATE -xCORE-AVX512 -qopt-zmm-usage=high -align array64byte -ipo)
+   elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL albedo)
+      # PROCESSORS: 2x AMD Rome Epyc 7702 (64 cores each), 128 cores, 256 GB RAM
+      target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -fPIC -qopt-malloc-options=2 -qopt-prefetch=5 -unroll-aggressive) # -g -traceback -check) #NEC mpi option
    elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL atosecmwf )
       target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2)
+   elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL juwels )
+      # PROCESSOR: 2× Intel Xeon Platinum 8168 CPU, 2× 24 cores, 2.7 GHz, 96 (12× 8) GB DDR4, 2666 MHz
+      # --> compiler flag performance test: core2, 192CPUs, 1mon. simulat.
+      # | Intel/Para...MPI    | nothing                                                   | 309s |
+      # | Intel/Para...MPI    | -O2                                                       | 114s |
+      # | Intel/Para...MPI    | -O3                                                       | 115s |
+      # | Intel/Para...MPI    | -O3 -xCORE-AVX512                                         | 117s |
+      # | Intel/Para...MPI    | -O3 -xCORE-AVX512 -qopt-zmm-usage=high -align array64byte | 119s |
+      # | Intel/Para...MPI    | -O2 -xCORE-AVX2                                           | 114s |
+      # |*Intel/Para...MPI    | -O3 -xCORE-AVX2                                           | 112s |
+      # | Intel/Para...MPI    | -O3 -xCORE-AVX2 -qopt-streaming-stores=always             | 126s |
+      # | Intel/Para...MPI    | -O3 -xCORE-AVX2 -qopt-prefetch=5                          | 116s |
+      # | Intel/Para...MPI    | -O3 -xCORE-AVX2 -funroll-loops                            | 113s |
+      # |*Intel/Para...MPI-mt | -O3 -xCORE-AVX2                                           | 113s |
+      target_compile_options(${PROJECT_NAME} PRIVATE -xCORE-AVX2)
    else()
       target_compile_options(${PROJECT_NAME} PRIVATE -xHost)
    endif()
-#    target_compile_options(${PROJECT_NAME} PRIVATE -g -traceback ) #-check all,noarg_temp_created,bounds,uninit ) #-ftrapuv ) #-init=zero)
-#    target_compile_options(${PROJECT_NAME} PRIVATE -qopenmp -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -xHost -ip -g -traceback -check all,noarg_temp_created,bounds,uninit ) #-ftrapuv ) #-init=zero)
-#    target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -ip -g -traceback -check all,noarg_temp_created,bounds,uninit ) #-ftrapuv ) #-init=zero)
-# CMAKE_Fortran_COMPILER_ID will also work if a wrapper is being used (e.g. mpif90 wraps ifort -> compiler id is Intel)
-elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL  IntelLLVM )
-     target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -fimf-use-svml -init=zero -no-wrap-margin) 
+
+   # # debugging related compiler flags
+   # target_compile_options(${PROJECT_NAME} PRIVATE -qopenmp -g -traceback -check all,noarg_temp_created,bounds,uninit ) #-ftrapuv ) #-init=zero)
+   # target_compile_options(${PROJECT_NAME} PRIVATE -g -traceback -check all,noarg_temp_created,bounds,uninit ) #-ftrapuv ) #-init=zero)
+
+
 elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL  GNU )
 #    target_compile_options(${PROJECT_NAME} PRIVATE -O3 -finit-local-zero  -finline-functions -fimplicit-none  -fdefault-real-8 -ffree-line-length-none)
    if(${FESOM_PLATFORM_STRATEGY} STREQUAL ubuntu )
       message(STATUS "Allowing type mismatches on Ubuntu for CI Testing" )  # NOTE(PG): Would be nicer to grab the CI=True from the env variable
       target_compile_options(${PROJECT_NAME} PRIVATE -O2 -g -fbacktrace -ffloat-store -finit-local-zero  -finline-functions -fimplicit-none  -fdefault-real-8 -fdefault-double-8 -ffree-line-length-none -fallow-argument-mismatch)
    else()
-      target_compile_options(${PROJECT_NAME} PRIVATE -O2 -g -fbacktrace -ffloat-store -finit-local-zero  -finline-functions -fimplicit-none  -fdefault-real-8 -fdefault-double-8 -ffree-line-length-none)
+      target_compile_options(${PROJECT_NAME} PRIVATE -ffloat-store -finit-local-zero  -finline-functions -fimplicit-none  -fdefault-real-8 -fdefault-double-8 -ffree-line-length-none)
    endif()
    if(CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10 )
       target_compile_options(${PROJECT_NAME} PRIVATE -fallow-argument-mismatch) # gfortran v10 is strict about erroneous API calls: "Rank mismatch between actual argument at (1) and actual argument at (2) (scalar and rank-1)"
    endif()
+
+   # platform specific compiler flags
+   if(${FESOM_PLATFORM_STRATEGY} STREQUAL juwels )
+      # PROCESSOR: 2× Intel Xeon Platinum 8168 CPU, 2× 24 cores, 2.7 GHz, 96 (12× 8) GB DDR4, 2666 MHz
+      # --> compiler flag performance test: core2, 192CPUs, 1mon. simulat.
+      # | GCC/openMPI | nothing                                                           | 390s |
+      # | GCC/openMPI | -O2                                                               | 192s |
+      # | GCC/openMPI | -O3 -march=skylake-avx512 -mtune=skylake-avx512                   | 173s |
+      # |             | -mprefer-vector-width=512 -falign-loops=64 -falign-functions=64   |      |
+      # |             | -falign-jumps=64  (chatGPT recomendation)                         |      |
+      target_compile_options(${PROJECT_NAME} PRIVATE -O3 -march=skylake-avx512 -mtune=skylake-avx512 -mprefer-vector-width=512 -falign-loops=64 -falign-functions=64 -falign-jumps=64)      
+   else()
+      if(NOT (${FESOM_PLATFORM_STRATEGY} STREQUAL ubuntu))
+         target_compile_options(${PROJECT_NAME} PRIVATE -native)
+      endif()    
+   endif()
+
+   # # debugging related compiler flags
+   #target_compile_options(${PROJECT_NAME} PRIVATE 
+                           #-fallow-argument-mismatch 
+                           #-fno-fast-math -fno-signed-zeros 
+                           #-freciprocal-math 
+                           #-mveclibabi=svml 
+                           #-flto
+                           #-pg 
+                           #-fbacktrace 
+                           #-fcheck=all,bounds 
+                           #-finit-real=snan 
+                           #-finit-integer=-9999 
+                           #-fsanitize=undefined,address
+                        #)
+   ## if use -fsanitize=undefined,address you also need ... PRIVATE -lubsan)           
+   #target_link_libraries(${PROJECT_NAME} PRIVATE -lubsan)
+
+
 elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray )
 	#target_compile_options(${PROJECT_NAME} PRIVATE -c -emf -hbyteswapio -hflex_mp=conservative -hfp1 -hadd_paren -Ounroll0 -hipa0 -r am -s real64 -N 1023 -g -G2 -O3)
    target_compile_options(${PROJECT_NAME} PRIVATE -c -emf -hbyteswapio -hflex_mp=conservative -hfp1 -hadd_paren -Ounroll0 -hipa0 -r am -s real64 -N 1023 -g -G2 -O2 -hnoacc -M878)  #-hnoacc is a workaround for cray automatically activate -hacc, -M878 is to suppress ftn-878 warning
@@ -315,6 +393,7 @@ target_link_libraries(${PROJECT_NAME} PRIVATE ${MCT_Fortran_LIBRARIES} ${MPEU_Fo
 target_link_libraries(${PROJECT_NAME} PRIVATE ${YACUTILS_Fortran_LIBRARIES} ${YACCORE_Fortran_LIBRARIES})
 target_link_libraries(${PROJECT_NAME} PRIVATE ${YAXT_Fortran_LIBRARIES} ${YAXTC_Fortran_LIBRARIES})
 
+
 #set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE Fortran)
 
 #target_include_directories(${PROJECT_NAME} PRIVATE ${NETCDF_Fortran_INCLUDE_DIRECTORIES} ${OASIS_Fortran_INCLUDE_DIRECTORIES})
@@ -334,6 +413,10 @@ add_executable(${PROJECT_NAME}.x ${src_home}/fesom_main.F90)
 target_link_libraries(${PROJECT_NAME}.x PUBLIC ${PROJECT_NAME})
 
 
+get_target_property(FLAGS ${PROJECT_NAME} COMPILE_OPTIONS)
+message(STATUS " --> Final Compile options for ${PROJECT_NAME}: ${FLAGS}")
+
+
 ### Export and installation
 
 fesom_export(TARGETS ${PROJECT_NAME} fesom.x)

src/associate_mesh_ass.h

@@ -25,7 +25,9 @@ nod_in_elem2D_num(1:myDim_nod2D+eDim_nod2D)                => mesh%nod_in_elem2D
 depth(1:myDim_nod2D+eDim_nod2D)                            => mesh%depth(:)
 gradient_vec(1:6,1:myDim_elem2D)                           => mesh%gradient_vec(:,:)
 gradient_sca(1:6,1:myDim_elem2D)                           => mesh%gradient_sca(:,:)
-bc_index_nod2D(1:myDim_nod2D+eDim_nod2D)                   => mesh%bc_index_nod2D(:)
+if (allocated(mesh%bc_index_nod2D)) then
+    bc_index_nod2D(1:myDim_nod2D+eDim_nod2D)               => mesh%bc_index_nod2D(:)
+end if 
 zbar(1:mesh%nl)                                            => mesh%zbar(:)
 Z(1:mesh%nl-1)                                             => mesh%Z(:)
 elem_depth         => mesh%elem_depth      ! never used, not even allocated
@@ -38,18 +40,24 @@ area_inv(1:mesh%nl,1:myDim_nod2d+eDim_nod2D)               => mesh%area_inv(:,:)
 areasvol_inv(1:mesh%nl,1:myDim_nod2d+eDim_nod2D)           => mesh%areasvol_inv(:,:)
 mesh_resolution(1:myDim_nod2d+eDim_nod2D)                  => mesh%mesh_resolution(:)
 ssh_stiff                                                  => mesh%ssh_stiff
-lump2d_north(1:myDim_nod2d)                                => mesh%lump2d_north(:)
-lump2d_south(1:myDim_nod2d)                                => mesh%lump2d_south(:)
-cavity_flag_n(1:myDim_nod2D+eDim_nod2D)                    => mesh%cavity_flag_n(:)
-cavity_flag_e(1:myDim_elem2D+eDim_elem2D+eXDim_elem2D)     => mesh%cavity_flag_e(:)
+if (allocated(mesh%lump2d_north)) then
+    lump2d_north(1:myDim_nod2d)                            => mesh%lump2d_north(:)
+    lump2d_south(1:myDim_nod2d)                            => mesh%lump2d_south(:)
+end if 
+if (allocated(mesh%cavity_flag_n)) then
+    cavity_flag_n(1:myDim_nod2D+eDim_nod2D)                => mesh%cavity_flag_n(:)
+    cavity_flag_e(1:myDim_elem2D+eDim_elem2D+eXDim_elem2D) => mesh%cavity_flag_e(:)
 !!$cavity_lev_nod2D(1:myDim_nod2D+eDim_nod2D)                 => mesh%cavity_lev_nod2D
 !!$cavity_lev_elem2D(1:myDim_elem2D+eDim_elem2D+eXDim_elem2D) => mesh%cavity_lev_elem2D
-cavity_depth(1:myDim_nod2D+eDim_nod2D)                     => mesh%cavity_depth(:)
+    cavity_depth(1:myDim_nod2D+eDim_nod2D)                 => mesh%cavity_depth(:)
+end if 
 ulevels(1:myDim_elem2D+eDim_elem2D+eXDim_elem2D)           => mesh%ulevels(:)
 ulevels_nod2D(1:myDim_nod2D+eDim_nod2D)                    => mesh%ulevels_nod2D(:)
 ulevels_nod2D_max(1:myDim_nod2D+eDim_nod2D)                => mesh%ulevels_nod2D_max(:)
-nn_num(1:myDim_nod2D)                                      => mesh%nn_num(:)
-nn_pos(1:mesh%nn_size, 1:myDim_nod2D)                      => mesh%nn_pos(:,:)
+if (allocated(mesh%nn_num)) then
+    nn_num(1:myDim_nod2D)                                      => mesh%nn_num(:)
+    nn_pos(1:mesh%nn_size, 1:myDim_nod2D)                      => mesh%nn_pos(:,:)
+end if 
 hnode(1:mesh%nl-1, 1:myDim_nod2D+eDim_nod2D)               => mesh%hnode(:,:)
 hnode_new(1:mesh%nl-1, 1:myDim_nod2D+eDim_nod2D)           => mesh%hnode_new(:,:)
 zbar_3d_n(1:mesh%nl, 1:myDim_nod2D+eDim_nod2D)             => mesh%zbar_3d_n(:,:)