Profile: Draco (MPCDF)

docs/source/install/profile.rst

@@ -100,3 +100,11 @@ Judge (FZJ)
 -----------
 
 (example missing)
+
+Draco (MPCDF)
+-------------
+
+For this profile to work, you need to download the :ref:`PIConGPU source code <install-dependencies-picongpu>` and install :ref:`libpng, PNGwriter and libSplash <install-dependencies>` manually.
+
+.. literalinclude:: profiles/draco-mpcdf/picongpu.profile.example
+   :language: bash

etc/picongpu/draco-mpcdf/general.tpl

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Copyright 2013-2017 Axel Huebl, Richard Pausch, Rene Widera
+#
+# This file is part of PIConGPU.
+#
+# PIConGPU is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# PIConGPU is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PIConGPU.
+# If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+# PIConGPU batch script for taurus' SLURM batch system
+
+#SBATCH --partition=!TBG_queue
+#SBATCH --time=!TBG_wallTime
+# Sets batch job's name
+#SBATCH --job-name=!TBG_jobName
+#SBATCH --nodes=!TBG_nodes
+#SBATCH --ntasks=!TBG_tasks
+#SBATCH --mincpus=!TBG_mpiTasksPerNode
+#SBATCH --cpus-per-task=!TBG_coresPerGPU
+#SBATCH --exclusive
+# send me mails on BEGIN, END, FAIL, REQUEUE, ALL,
+# TIME_LIMIT, TIME_LIMIT_90, TIME_LIMIT_80 and/or TIME_LIMIT_50
+#SBATCH --mail-type=!TBG_mailSettings
+#SBATCH --mail-user=!TBG_mailAddress
+#SBATCH --workdir=!TBG_dstPath
+
+#SBATCH -o stdout
+#SBATCH -e stderr
+
+
+## calculations will be performed by tbg ##
+.TBG_queue="general"
+
+# settings that can be controlled by environment variables before submit
+.TBG_mailSettings=${MY_MAILNOTIFY:-"ALL"}
+.TBG_mailAddress=${MY_MAIL:-"someone@example.com"}
+.TBG_author=${MY_NAME:+--author \"${MY_NAME}\"}
+.TBG_profile=${PIC_PROFILE:-"~/picongpu.profile"}
+
+# 1 gpus per node
+.TBG_gpusPerNode=2
+
+# number of cores to block per GPU - we got 6 cpus per gpu
+#   and we will be accounted 6 CPUs per GPU anyway
+.TBG_coresPerGPU=32
+
+# We only start 2 MPI task per GPU
+.TBG_mpiTasksPerNode="!TBG_gpusPerNode"
+
+# use ceil to caculate nodes
+.TBG_nodes="$((( TBG_tasks + TBG_gpusPerNode -1 ) / TBG_gpusPerNode))"
+
+## end calculations ##
+
+echo 'Running program...'
+
+cd !TBG_dstPath
+
+export MODULES_NO_OUTPUT=1
+source !TBG_profile
+if [ $? -ne 0 ] ; then
+  echo "Error: PIConGPU environment profile under \"!TBG_profile\" not found!"
+  exit 1
+fi
+unset MODULES_NO_OUTPUT
+
+#set user rights to u=rwx;g=r-x;o=---
+umask 0027
+
+mkdir simOutput 2> /dev/null
+cd simOutput
+
+#wait that all nodes see ouput folder
+sleep 1
+
+# Run PIConGPU
+srun -K1 !TBG_dstPath/tbg/cpuNumaStarter.sh !TBG_dstPath/input/bin/picongpu !TBG_author !TBG_programParams | tee output
+
+

etc/picongpu/draco-mpcdf/picongpu.profile.example

@@ -0,0 +1,66 @@
+# User Information ############################################################
+#   - edit those lines!
+#   - automatically add your name and contact to output file meta data
+#   - send me a mail on batch system jobs: BEGIN, END, FAIL, REQUEUE, ALL,
+#     TIME_LIMIT, TIME_LIMIT_90, TIME_LIMIT_80 and/or TIME_LIMIT_50
+export MY_MAILNOTIFY="ALL"
+export MY_MAIL="someone@example.com"
+export MY_NAME="$(whoami) <$MY_MAIL>"
+
+# Name and Path of this Script ################################################
+#   - do NOT change this line!
+export PIC_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE)
+
+# Text Editor for Tools #######################################################
+#   - edit those lines!
+#   - examples: "nano", "vim", "emacs -nw", "vi" or without terminal: "gedit"
+#export EDITOR="nano"
+
+# General Modules #############################################################
+#
+module purge
+
+module load git/2.14
+module load gcc/6.3
+module load cmake/3.7
+module load boost/gcc/1.64
+module load impi/2017.3
+module load hdf5-mpi/gcc/1.8.18
+
+# Other Software ##############################################################
+#
+# needs to be compiled by the user
+export PNGWRITER_ROOT=$HOME/lib/pngwriter-0.6.0
+export SPLASH_ROOT=$HOME/lib/splash-1.6.0
+
+export LD_LIBRARY_PATH=$PNGWRITER_ROOT/lib:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$SPLASH_ROOT/lib:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$BOOST_HOME/lib:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$HDF5_HOME/lib:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$I_MPI_ROOT/lib64:$LD_LIBRARY_PATH
+
+export HDF5_ROOT=$HDF5_HOME
+
+export CXX=$(which g++)
+export CC=$(which gcc)
+
+# PIConGPU Helper Variables ###################################################
+#
+export PICSRC=$HOME/src/picongpu
+export PIC_BACKEND="omp2b:haswell"
+
+export PATH=$PATH:$PICSRC
+export PATH=$PATH:$PICSRC/src/tools/bin
+
+export PYTHONPATH=$PICSRC/lib/python:$PYTHONPATH
+
+# "tbg" default options #######################################################
+#   - SLURM (sbatch)
+#   - "normal" queue
+export TBG_SUBMIT="sbatch"
+export TBG_TPLFILE="etc/picongpu/draco-mpcdf/general.tpl"
+
+# helper tools ################################################################
+
+# allocate an interactive shell for one hour
+alias getNode='salloc --time=1:00:00 --nodes=1 --exclusive --ntasks-per-node=2 --cpus-per-task=32 --partition general'