ComputationalRadiationPhysics · psychocoderHPC · Jun 24, 2020 · Apr 9, 2019 · May 18, 2020 · May 18, 2020
diff --git a/INSTALL.rst b/INSTALL.rst
@@ -325,6 +325,24 @@ ADIOS
   - ``export ADIOS_ROOT=$HOME/lib/adios``
   - ``export LD_LIBRARY_PATH=$ADIOS_ROOT/lib:$LD_LIBRARY_PATH``
 
+openPMD API
+"""""""""""
+- 0.12.0+ (yet to be released, requires *MPI*)
+- *Spack*: ``spack install openpmd-api``
+- *from source:*
+
+  - ``mkdir -p ~/src ~/lib``
+  - ``cd ~/src``
+  - ``git clone https://github.com/openPMD/openPMD-api.git``
+  - ``cd openPMD-api``
+  - ``mkdir build && cd build``
+  - ``cmake .. -DopenPMD_USE_MPI=ON -DCMAKE_INSTALL_PREFIX=~/lib/openPMD-api``
+  - ``make -j $(nproc) install``
+- environment:* (assumes install from source in ``$HOME/lib/openPMD-api``)
+
+  - ``export CMAKE_PREFIX_PATH="$HOME/lib/openPMD-api:$CMAKE_PREFIX_PATH"``
+  - ``export LD_LIBRARY_PATH="$HOME/lib/openPMD-api/lib:$LD_LIBRARY_PATH"``
+
 ISAAC
 """""
 - 1.4.0+

diff --git a/docs/TBG_macros.cfg b/docs/TBG_macros.cfg
@@ -1,4 +1,5 @@
-# Copyright 2014-2020 Felix Schmitt, Axel Huebl, Richard Pausch, Heiko Burau
+# Copyright 2014-2020 Felix Schmitt, Axel Huebl, Richard Pausch, Heiko Burau,
+#                     Franz Poeschel
 #
 # This file is part of PIConGPU.
 #
@@ -205,6 +206,22 @@ TBG_adios="--adios.period 100 --adios.file simData --adios.source 'species_all,f
 # select data sources for the dump
 #   --adios.source <comma_separated_list_of_data_sources>
 
+# Dump simulation data (fields and particles) via the openPMD API.
+# Data is dumped every .period steps to the fileset .file.
+TBG_openPMD="--openPMD.period 100   \
+             --openPMD.file simOutput \
+             --openPMD.ext bp \
+             --openPMD.json '{ \"adios2\": { \"engine\": { \"type\": \"file\", \"parameters\": { \"BufferGrowthFactor\": \"1.2\", \"InitialBufferSize\": \"2GB\" } } } }'"
+# Further control over the backends used in the openPMD plugins is available
+# through the mechanisms exposed by the openPMD API:
+# * environment variables
+# * JSON-formatted configuration string
+# Further information on both is retrieved from the official documentation
+# https://openpmd-api.readthedocs.io
+# Notice that specifying compression settings via --openPMD.compression
+# is considered legacy and backend-specific settings via the JSON string are
+# preferred if available for a backend.
+
 # Create a checkpoint that is restartable every --checkpoint.period steps
 #   http://git.io/PToFYg
 TBG_checkpoint="--checkpoint.period 1000"

diff --git a/docs/source/usage/plugins.rst b/docs/source/usage/plugins.rst
@@ -7,6 +7,7 @@ Plugins
 Plugin name                                                                          short description
 ==================================================================================== =================================================================================
 :ref:`ADIOS <usage-plugins-ADIOS>` [#f2]_ [#f7]_                                     stores simulation data as openPMD flavoured ADIOS files [Huebl2017]_
+:ref:`openPMD <usage-plugins-openPMD>` [#f2]_ [#f7]_                                 outputs simulation data via the openPMD API
 :ref:`energy histogram <usage-plugins-energyHistogram>` [#f7]_                       energy histograms for electrons and ions
 :ref:`charge conservation <usage-plugins-chargeConservation>` [#f6]_                 maximum difference between electron charge density and div E
 :ref:`checkpoint <usage-plugins-checkpoint>` [#f2]_                                  stores the primary data of the simulation for restarts.

diff --git a/docs/source/usage/plugins/checkpoint.rst b/docs/source/usage/plugins/checkpoint.rst
@@ -21,18 +21,18 @@ What is the format of the created files?
 
 We write our fields and particles in an open markup called :ref:`openPMD <pp-openPMD>`.
 
-For further details, see the according sections in :ref:`HDF5 <usage-plugins-HDF5>` and :ref:`ADIOS <usage-plugins-ADIOS>`.
+For further details, see the according sections in :ref:`the openPMD API <usage-plugins-openPMD>`, :ref:`HDF5 <usage-plugins-HDF5>` and :ref:`ADIOS <usage-plugins-ADIOS>`.
 
 External Dependencies
 ^^^^^^^^^^^^^^^^^^^^^
 
-The plugin is available as soon as the :ref:`libSplash (HDF5) or ADIOS libraries <install-dependencies>` are compiled in.
+The plugin is available as soon as the :ref:`openPMD API, libSplash (HDF5) or ADIOS libraries <install-dependencies>` are compiled in.
 
 .cfg file
 ^^^^^^^^^
 
 You can use ``--checkpoint.period`` to specify the output period of the created checkpoints.
-Note that this plugin will only be available if libSplash (HDF5) or ADIOS is found during compile configuration.
+Note that this plugin will only be available if the openPMD API, libSplash (HDF5) or ADIOS is found during compile configuration.
 
 ============================================= ======================================================================================
 PIConGPU command line option                  Description
@@ -61,6 +61,7 @@ PIConGPU command line option                  Description
 
 Depending on the available external dependencies (see above), the options for the ``<IO-backend>`` are:
 
+* :ref:`openPMD <usage-plugins-openPMD>`
 * :ref:`hdf5 <usage-plugins-HDF5>`
 * :ref:`adios <usage-plugins-ADIOS>` (keep in mind the :ref:`note on meta-files <usage-plugins-ADIOS-meta>` for restarts)
 

diff --git a/docs/source/usage/plugins/openPMD.cfg b/docs/source/usage/plugins/openPMD.cfg
@@ -0,0 +1,21 @@
+TBG_openPMD="--openPMD.period 100   \
+             --openPMD.file simOutput \
+             --openPMD.ext bp \
+             --openPMD.json '{ \
+                 \"adios2\": { \
+                   \"dataset\": { \
+                     \"operators\": [ \
+                       { \
+                         \"type\": \"bzip2\" \
+                       } \
+                     ] \
+                   }, \
+                   \"engine\": { \
+                     \"type\": \"file\", \
+                     \"parameters\": { \
+                       \"BufferGrowthFactor\": \"1.2\", \
+                       \"InitialBufferSize\": \"2GB\" \
+                     } \
+                   } \
+                 } \
+               }'"
diff --git a/docs/source/usage/plugins/openPMD.rst b/docs/source/usage/plugins/openPMD.rst
@@ -0,0 +1,126 @@
+.. _usage-plugins-openPMD:
+
+openPMD
+------
+
+Stores simulation data such as fields and particles according to the `openPMD standard <https://github.com/openPMD/openPMD-standard>`_ using the `openPMD API <https://openpmd-api.readthedocs.io>`_.
+
+External Dependencies
+^^^^^^^^^^^^^^^^^^^^^
+
+The plugin is available as soon as the :ref:`openPMD API <install-dependencies>` is compiled in.
+
+.param file
+^^^^^^^^^^^
+
+The corresponding ``.param`` file is :ref:`fileOutput.param <usage-params-plugins>`.
+
+One can e.g. disable the output of particles by setting:
+
+.. code-block:: cpp
+
+   /* output all species */
+   using FileOutputParticles = VectorAllSpecies;
+   /* disable */
+   using FileOutputParticles = MakeSeq_t< >;
+
+.cfg file
+^^^^^^^^^
+
+You can use ``--openPMD.period`` to specify the output period.
+The base filename is specified via ``--openPMD.file``.
+The openPMD API will parse the file name to decide the chosen backend and iteration layout:
+
+* The filename extension will determine the backend.
+* The openPMD will either create one file encompassing all iterations (group-based iteration layout) or one file per iteration (file-based iteration layout).
+  The filename will be searched for a pattern describing how to derive a concrete iteration's filename.
+  If no such pattern is found, the group-based iteration layout will be chosen.
+  Please refer to the documentation of the openPMD API for further information.
+
+In order to set defaults for these value, two further options control the filename:
+
+* ``--openPMD.ext`` sets the filename extension.
+  Possible extensions include ``.bp`` for the ADIOS backends (default).
+  If the openPMD API has been built with support for the ADIOS1 and ADIOS2 backends, ADIOS2 will take precedence over ADIOS1.
+  This behavior can be overridden by setting the environment variable ``OPENPMD_BP_BACKEND=ADIOS1``.
+  The extension for the HDF5 backend is ``.h5``.
+  (The version of ADIOS will depend on the compile-time configuration of the openPMD API.)
+* ``--openPMD.infix`` sets the filename pattern that controls the iteration layout, default is "_06T" for a six-digit number specifying the iteration.
+  Leave empty to pick group-based iteration layout.
+  Since passing an empty string may be tricky in some workflows, specifying ``--openPMD.infix=NULL`` is also possible.
+
+For example, ``--openPMD.period 128 --openPMD.file simData --openPMD.source 'species_all'`` will write only the particle species data to files of the form ``simData_000000.bp``, ``simData_000128.bp`` in the default simulation output directory every 128 steps.
+Note that this plugin will only be available if the openPMD API is found during compile configuration.
+
+openPMD backend-specific settings may be controlled via two mechanisms:
+* Environment variables.
+  Please refer to the backends' documentations for information on environment variables understood by the backends.
+* Backend-specific runtime parameters may be set via JSON in the openPMD API.
+  PIConGPU exposes this via the command line option ``--openPMD.json``.
+  Please refer to the openPMD API's documentation for further information.
+
+Specifying a JSON-formatted string from within a ``.cfg`` file can be tricky due to colliding escape mechanisms.
+An example for a well-escaped JSON string as part of a ``.cfg`` file is found below.
+
+.. literalinclude:: openPMD.cfg
+
+Two data preparation strategies are available for downloading particle data off compute devices.
+
+* Set ``--openPMD.dataPreparationStrategy doubleBuffer`` for use of the strategy that has been optimized for use with ADIOS-based backends.
+  The alias ``openPMD.dataPreparationStrategy adios`` may be used.
+  This strategy requires at least 2x the GPU main memory on the host side.
+  This is the default.
+* Set ``--openPMD.dataPreparationStrategy mappedMemory`` for use of the strategy that has been optimized for use with HDF5-based backends.
+  This strategy has a small host-side memory footprint (<< GPU main memory).
+  The alias ``openPMD.dataPreparationStrategy hdf5`` may be used.
+
+============================ ==================================================================================================================================================================
+PIConGPU command line option description
+============================ ==================================================================================================================================================================
+``--openPMD.period``                  Period after which simulation data should be stored on disk.
+``--openPMD.source``                  Select data sources to dump. Default is ``species_all,fields_all``, which dumps all fields and particle species.
+``--openPMD.compression``             Legacy parameter to set data transform compression method to be used for ADIOS1 backend until it implements setting compression from JSON config.
+``--openPMD.file``                    Relative or absolute openPMD file prefix for simulation data. If relative, files are stored under ``simOutput``. 
+``--openPMD.ext``                     openPMD filename extension (this controls thebackend picked by the openPMD API).
+``--openPMD.infix``                   openPMD filename infix (use to pick file- or group-based layout in openPMD). Set to NULL to keep empty (e.g. to pick group-based iteration layout).
+``--openPMD.json``                    Set backend-specific parameters for openPMD backends in JSON format.
+``--openPMD.dataPreparationStrategy`` Strategy for preparation of particle data ('doubleBuffer' or 'mappedMemory'). Aliases 'adios' and 'hdf5' may be used respectively.
+============================ ==================================================================================================================================================================
+
+.. note::
+
+   This plugin is a multi plugin. 
+   Command line parameter can be used multiple times to create e.g. dumps with different dumping period.
+   In the case where an optional parameter with a default value is explicitly defined, the parameter will always be passed to the instance of the multi plugin where the parameter is not set.
+   e.g.
+
+   .. code-block:: bash
+
+      --openPMD.period 128 --openPMD.file simData1 --openPMD.source 'species_all' 
+      --openPMD.period 1000 --openPMD.file simData2 --openPMD.source 'fields_all' --openPMD.ext h5
+
+   creates two plugins:
+
+   #. dump all species data each 128th time step, use HDF5 backend.
+   #. dump all field data each 1000th time step, use the default ADIOS backend.
+
+Memory Complexity
+^^^^^^^^^^^^^^^^^
+
+Accelerator
+"""""""""""
+
+no extra allocations.
+
+Host
+""""
+
+As soon as the openPMD plugin is compiled in, one extra ``mallocMC`` heap for the particle buffer is permanently reserved.
+During I/O, particle attributes are allocated one after another.
+Using ``--openPMD.dataPreparationStrategy doubleBuffer`` (default) will require at least 2x the GPU memory on the host side.
+For a smaller host side memory footprint (<< GPU main memory) pick ``--openPMD.dataPreparationStrategy mappedMemory``.
+
+Additional Tools
+^^^^^^^^^^^^^^^^
+
+See our :ref:`openPMD <pp-openPMD>` chapter.
diff --git a/include/picongpu/CMakeLists.txt b/include/picongpu/CMakeLists.txt
@@ -1,5 +1,6 @@
 # Copyright 2013-2020 Axel Huebl, Benjamin Schneider, Felix Schmitt, Heiko Burau,
-#                     Rene Widera, Alexander Grund, Alexander Matthes
+#                     Rene Widera, Alexander Grund, Alexander Matthes,
+#                     Franz Poeschel
 #
 # This file is part of PIConGPU.
 #
@@ -42,6 +43,7 @@ list(APPEND CMAKE_PREFIX_PATH "$ENV{CUDA_ROOT}")
 list(APPEND CMAKE_PREFIX_PATH "$ENV{BOOST_ROOT}")
 list(APPEND CMAKE_PREFIX_PATH "$ENV{HDF5_ROOT}")
 list(APPEND CMAKE_PREFIX_PATH "$ENV{ADIOS_ROOT}")
+list(APPEND CMAKE_PREFIX_PATH "$ENV{OPENPMD_ROOT}")
 # Add from environment after specific env vars
 list(APPEND CMAKE_PREFIX_PATH "$ENV{CMAKE_PREFIX_PATH}")
 
@@ -233,21 +235,6 @@ set(PIC_VERBOSE "1" CACHE STRING
 add_definitions(-DPIC_VERBOSE_LVL=${PIC_VERBOSE})
 
 
-################################################################################
-# ADIOS
-################################################################################
-
-# find adios installation
-#   set(ADIOS_USE_STATIC_LIBS ON) # force static linking
-find_package(ADIOS 1.13.1)
-
-if(ADIOS_FOUND)
-    add_definitions(-DENABLE_ADIOS=1)
-    include_directories(SYSTEM ${ADIOS_INCLUDE_DIRS})
-    set(LIBS ${LIBS} ${ADIOS_LIBRARIES})
-endif(ADIOS_FOUND)
-
-
 ################################################################################
 # Additional defines for PIConGPU outputs
 ################################################################################
@@ -293,6 +280,34 @@ elseif(MSVC)
 endif()
 
 
+################################################################################
+# openPMD
+################################################################################
+
+# find openPMD installation
+find_package(openPMD 0.11.0 CONFIG COMPONENTS MPI)
+
+if(openPMD_FOUND)
+    add_definitions(-DENABLE_OPENPMD=1)
+    set(LIBS ${LIBS} openPMD::openPMD)
+endif(openPMD_FOUND)
+
+
+################################################################################
+# ADIOS
+################################################################################
+
+# find adios installation
+#   set(ADIOS_USE_STATIC_LIBS ON) # force static linking
+find_package(ADIOS 1.13.1)
+
+if(ADIOS_FOUND)
+    add_definitions(-DENABLE_ADIOS=1)
+    include_directories(SYSTEM ${ADIOS_INCLUDE_DIRS})
+    set(LIBS ${LIBS} ${ADIOS_LIBRARIES})
+endif(ADIOS_FOUND)
+
+
 ################################################################################
 # libSplash (+ hdf5 due to required headers)
 ################################################################################

diff --git a/include/picongpu/plugins/Checkpoint.hpp b/include/picongpu/plugins/Checkpoint.hpp
@@ -1,4 +1,4 @@
-/* Copyright 2017-2020 Rene Widera
+/* Copyright 2017-2020 Rene Widera, Franz Poeschel
  *
  * This file is part of PIConGPU.
  *
@@ -30,6 +30,9 @@
 #if(ENABLE_HDF5 == 1)
 #   include "picongpu/plugins/hdf5/HDF5Writer.hpp"
 #endif
+#if (ENABLE_OPENPMD == 1)
+#   include "picongpu/plugins/openPMD/openPMDWriter.hpp"
+#endif
 #include <pmacc/pluginSystem/PluginConnector.hpp>
 
 #include <string>
@@ -52,6 +55,9 @@ namespace picongpu
             checkpointFilename( "checkpoint" ),
             restartChunkSize( 0u )
         {
+#if(ENABLE_OPENPMD == 1)
+            ioBackendsHelp[ "openPMD" ] = std::shared_ptr< plugins::multi::IHelp >( openPMD::openPMDWriter::getHelp( ) );
+#endif
 #if(ENABLE_ADIOS == 1)
             ioBackendsHelp[ "adios" ] = std::shared_ptr< plugins::multi::IHelp >( adios::ADIOSWriter::getHelp() );
 #endif
@@ -255,7 +261,7 @@ namespace picongpu
          */
         uint32_t restartChunkSize;
 
-        // can be "adios" and "hdf5"
+        // can be "adios", "hdf5" and "openPMD"
         std::map<
             std::string,
             std::shared_ptr< IIOBackend >

diff --git a/include/picongpu/plugins/PluginController.hpp b/include/picongpu/plugins/PluginController.hpp
@@ -1,6 +1,7 @@
 /* Copyright 2013-2020 Axel Huebl, Benjamin Schneider, Felix Schmitt,
  *                     Heiko Burau, Rene Widera, Richard Pausch,
- *                     Benjamin Worpitz, Erik Zenker, Finn-Ole Carstens
+ *                     Benjamin Worpitz, Erik Zenker, Finn-Ole Carstens,
+ *                     Franz Poeschel
  *
  * This file is part of PIConGPU.
  *
@@ -45,6 +46,10 @@
 #   include "picongpu/plugins/adios/ADIOSWriter.hpp"
 #endif
 
+#if (ENABLE_OPENPMD == 1)
+#   include "picongpu/plugins/openPMD/openPMDWriter.hpp"
+#endif
+
 #if( PMACC_CUDA_ENABLED == 1 )
 #   include "picongpu/plugins/PositionsParticles.hpp"
 #   include "picongpu/plugins/ChargeConservation.hpp"
@@ -168,6 +173,10 @@ class PluginController : public ILightweightPlugin
         , plugins::multi::Master< adios::ADIOSWriter >
 #endif
 
+#if (ENABLE_OPENPMD == 1)
+        , plugins::multi::Master< openPMD::openPMDWriter >
+#endif
+
 #if( PMACC_CUDA_ENABLED == 1 )
         , SumCurrents
         , ChargeConservation