diff --git a/lib/metis-5.1.0/CMakeLists.txt b/lib/metis-5.1.0/CMakeLists.txt index 01df5c8ee..f314f5c99 100644 --- a/lib/metis-5.1.0/CMakeLists.txt +++ b/lib/metis-5.1.0/CMakeLists.txt @@ -3,7 +3,7 @@ project(METIS) #set(GKLIB_PATH "GKlib" CACHE PATH "path to GKlib") #set(GKLIB_PATH "${CMAKE_SOURCE_DIR}/lib/metis-5.1.0/GKlib" CACHE PATH "path to GKlib") -set(GKLIB_PATH "${CMAKE_SOURCE_DIR}/../lib/metis-5.1.0/GKlib" CACHE PATH "path to GKlib") +set(GKLIB_PATH "${CMAKE_CURRENT_SOURCE_DIR}/GKlib" CACHE PATH "path to GKlib") set(SHARED FALSE CACHE BOOL "build a shared library") #if(MSVC) @@ -26,4 +26,4 @@ include_directories(include) # Recursively look for CMakeLists.txt in subdirs. add_subdirectory("include") add_subdirectory("libmetis") -add_subdirectory("programs") +add_subdirectory("programs") \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 083bab1ff..0038d32ec 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -87,6 +87,11 @@ endif() option(USE_ICEPACK "Use ICEPACK" OFF) message(STATUS "USE_ICEPACK: ${USE_ICEPACK}") + +option(BUILD_MESHPARTITIONER "build mesh partitioner" ON) +message(STATUS "BUILD_MESHPARTITIONER: ${BUILD_MESHPARTITIONER}") + + # option to trigger building a library version of FESOM # we do not always build the library along with the executable to avoid having two targets here in the CMakeLists.txt # two targets would allow e.g. setting different compiler options or preprocessor definition, which would be error prone. @@ -441,6 +446,39 @@ get_target_property(FLAGS ${PROJECT_NAME} COMPILE_OPTIONS) message(STATUS " --> Final Compile options for ${PROJECT_NAME}: ${FLAGS}") + + +# mesh partitioner + +if(BUILD_MESHPARTITIONER) +block() +set(CMAKE_Fortran_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/fesom_meshpart) + set(fesom_meshpart_sources_Fortran ${src_home}/MOD_MESH.F90 ${src_home}/oce_modules.F90 ${src_home}/gen_modules_config.F90 ${src_home}/gen_modules_partitioning.F90 ${src_home}/gen_modules_rotate_grid.F90 ${src_home}/fvom_init.F90 ${src_home}/oce_local.F90 ${src_home}/gen_comm.F90 ${src_home}/MOD_READ_BINARY_ARRAYS.F90 ${src_home}/MOD_WRITE_BINARY_ARRAYS.F90 ${src_home}/MOD_PARTIT.F90) + set(fesom_meshpart_sources_C ${src_home}/fort_part.c) + + add_subdirectory(${src_home}/../lib/metis-5.1.0 ${PROJECT_BINARY_DIR}/metis) + include_directories(${src_home}/../lib/metis-5.1.0/include) + + add_library(fesom_meshpart_C ${fesom_meshpart_sources_C}) + target_compile_definitions(fesom_meshpart_C PRIVATE USE_MPI REAL=double DBL HAS_BLAS FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8 SGI LINUX UNDER_ MPI2) + target_link_libraries(fesom_meshpart_C metis) + + add_executable(fesom_meshpart ${fesom_meshpart_sources_Fortran}) + # CMAKE_Fortran_COMPILER_ID will also work if a wrapper is being used (e.g. mpif90 wraps ifort -> compiler id is Intel) + if(${CMAKE_Fortran_COMPILER_ID} STREQUAL Intel ) + target_compile_options(fesom_meshpart PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fast-transcendentals -xHost -ip) + # target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fast-transcendentals -xHost -ip -g -traceback -check all,noarg_temp_created,bounds,uninit) + elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL GNU ) + target_compile_options(fesom_meshpart PRIVATE -fdefault-real-8 -ffree-line-length-none) + if(CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10 ) + target_compile_options(fesom_meshpart PRIVATE -fallow-argument-mismatch) # gfortran v10 is strict about erroneous API calls: "Rank mismatch between actual argument at (1) and actual argument at (2) (scalar and rank-1)" + endif() + endif() + target_link_libraries(fesom_meshpart fesom_meshpart_C MPI::MPI_Fortran) + set_target_properties(fesom_meshpart PROPERTIES LINKER_LANGUAGE Fortran) +endblock() +endif() + ### Export and installation -fesom_export(TARGETS ${PROJECT_NAME} fesom.x) +fesom_export(TARGETS ${PROJECT_NAME} fesom.x fesom_meshpart) diff --git a/src/fvom_init.F90 b/src/fvom_init.F90 index 30ee8f5f6..76a8891c0 100755 --- a/src/fvom_init.F90 +++ b/src/fvom_init.F90 @@ -17,6 +17,7 @@ program MAIN use MOD_PARTIT use g_CONFIG use g_rotate_grid + use iso_fortran_env, only: error_unit implicit none @@ -90,6 +91,9 @@ end subroutine find_levels_cavity character(len=MAX_PATH) :: nmlfile !> name of configuration namelist file integer :: start_t, interm_t, finish_t, rate_t + integer :: file_unit + integer :: ierr + character(512) :: errmsg type(t_mesh), target, save :: mesh type(t_partit), target, save :: partit @@ -97,12 +101,22 @@ end subroutine find_levels_cavity interm_t = start_t nmlfile ='namelist.config' - open (20,file=nmlfile) - read (20,NML=paths) ! We need MeshPath - read (20,NML=geometry) ! We need cyclic_length and cartesian - read (20,NML=run_config) ! We need use_cavity=true/false - read (20,NML=machine) ! We need partitioning hierarchy - close (20) + + open(newunit=file_unit, file=nmlfile, action='read', & + status='old', iostat=ierr, iomsg=errmsg) + if (ierr /= 0) then + write (unit=error_unit, fmt='(3A)') & + '### error: can not open file ', trim(nmlfile), & + ', error: ' // trim(errmsg) + error stop + end if + + read (file_unit, NML=paths) ! We need MeshPath + read (file_unit, NML=geometry) ! We need cyclic_length and cartesian + read (file_unit, NML=run_config) ! We need use_cavity=true/false + read (file_unit, NML=machine) ! We need partitioning hierarchy + close (file_unit) + cyclic_length=cyclic_length*rad alphaEuler=alphaEuler*rad betaEuler=betaEuler*rad @@ -1900,4 +1914,3 @@ subroutine check_partitioning(partit, mesh) end subroutine check_partitioning - diff --git a/work/job_ini_albedo b/work/job_ini_albedo index d610663d7..122699a10 100755 --- a/work/job_ini_albedo +++ b/work/job_ini_albedo @@ -14,7 +14,7 @@ ulimit -s unlimited source ../env/albedo/shell -ln -s ../bin/fesom_ini.x . # cp -n ../bin/fvom_ini.x +ln -s ../bin/fesom_meshpart . cp -n ../config/namelist.config . #cp -n ../config/namelist.forcing . #cp -n ../config/namelist.oce . @@ -27,6 +27,5 @@ fname="fesom2_${SLURM_JOB_NAME}_${jobid}.out" #___PUT JOB IN QUEUE____________________________________________________________ date -srun --mpi=pmi2 --ntasks=1 ./fesom_ini.x >> ${fname} +srun --mpi=pmi2 --ntasks=1 ./fesom_meshpart >> ${fname} date - diff --git a/work/job_ini_juwels b/work/job_ini_juwels index b53e1c235..5bd08be73 100755 --- a/work/job_ini_juwels +++ b/work/job_ini_juwels @@ -15,18 +15,18 @@ ulimit -s unlimited # determine JOBID JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` -ln -s ../bin/fesom_ini.x . # cp -n ../bin/fesom.x +ln -s ../bin/fesom_meshpart . cp -n ../config/namelist.config . cp -n ../config/namelist.forcing . cp -n ../config/namelist.oce . cp -n ../config/namelist.ice . date -srun --mpi=pmi2 ./fesom_ini.x > "fesom_ini.out" +srun --mpi=pmi2 ./fesom_meshpart > "fesom_meshpart.out" date #qstat -f $PBS_JOBID #export EXITSTATUS=$? #if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then #sbatch job_ollie -#fi +#fi \ No newline at end of file diff --git a/work/job_ini_levante b/work/job_ini_levante index c84232fab..3eba37450 100755 --- a/work/job_ini_levante +++ b/work/job_ini_levante @@ -19,7 +19,7 @@ squeue -u $USER # determine JOBID JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` -ln -s ../bin/fesom_ini.x . # cp -n ../bin/fesom.x +ln -s ../bin/fesom_meshpart . cp -n ../config/namelist.config . cp -n ../config/namelist.forcing . cp -n ../config/namelist.oce . @@ -27,7 +27,7 @@ cp -n ../config/namelist.ice . cp -n ../config/namelist.icepack . date -srun -l fesom_ini.x > "fesom_ini.out" +srun -l fesom_meshpart > "fesom_meshpart.out" date # qstat -f $PBS_JOBID diff --git a/work/job_ini_lumi b/work/job_ini_lumi index 09e9557bb..2cfc42e26 100755 --- a/work/job_ini_lumi +++ b/work/job_ini_lumi @@ -13,7 +13,7 @@ ulimit -s unlimited source ../env/lumi/shell -ln -s ../bin/fesom_ini.x . # cp -n ../bin/fvom_ini.x +ln -s ../bin/fesom_meshpart . cp -n ../config/namelist.config . #cp -n ../config/namelist.forcing . #cp -n ../config/namelist.oce . @@ -26,6 +26,6 @@ fname="fesom2_${SLURM_JOB_NAME}_${jobid}.out" #___PUT JOB IN QUEUE____________________________________________________________ date -srun --mpi=pmi2 --ntasks=1 ./fesom_ini.x >> ${fname} +srun --mpi=pmi2 --ntasks=1 ./fesom_meshpart >> ${fname} date diff --git a/work/job_ini_mistral b/work/job_ini_mistral index 16acdb48a..041d3fd2b 100755 --- a/work/job_ini_mistral +++ b/work/job_ini_mistral @@ -15,12 +15,12 @@ ulimit -s unlimited # determine JOBID JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` -ln -s ../bin/fesom_ini.x . # cp -n ../bin/fvom_ini.x +ln -s ../bin/fesom_meshpart . cp -n ../config/namelist.config . cp -n ../config/namelist.forcing . cp -n ../config/namelist.oce . cp -n ../config/namelist.ice . date -srun --mpi=pmi2 --ntasks=1 ./fesom_ini.x > "fesom_ini.out" +srun --mpi=pmi2 --ntasks=1 ./fesom_meshpart > "fesom_meshpart.out" date