Added new option <diag>.openpmd_encoding (#1979)

* Added new option <diag>.openpmd_encoding, which can be either file/group/variable
With the encoding being group/variable, there will be one file generated for all
iterations.

* fixed tab

* fixed style

* backward compatibility for openpmd-api versions < 0.14

* eol

* Update Docs/source/usage/parameters.rst

Co-authored-by: Axel Huebl <[email protected]>

* used inline instead of lamba, encoding decision is moved up at FlushFormatOpenPMD

* making file based the default so scripts can work as is

* Update Docs/source/usage/parameters.rst

Co-authored-by: Axel Huebl <[email protected]>

* Unify defaults of variables

Co-authored-by: Junmin Gu <[email protected]>
Co-authored-by: Junmin Gu <[email protected]>
Co-authored-by: Axel Huebl <[email protected]>

Author: 4 people

Docs/source/usage/parameters.rst

@@ -1597,8 +1597,10 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a
     ``json`` only works with serial/single-rank jobs.
     When WarpX is compiled with openPMD support, the first available backend in the order given above is taken.
 
-* ``<diag_name>.openpmd_tspf`` (`bool`, optional, default ``true``) only read if ``<diag_name>.format = openpmd``.
-    Whether to write one file per timestep.
+* ``<diag_name>.openpmd_encoding`` (optional, ``v`` (variable based), ``f`` (file based) or ``g`` (group based) ) only read if ``<diag_name>.format = openpmd``.
+     openPMD file output encoding (file based will write one file per timestep).
+     `variable based` is not supported for back-transformed diagnostics.
+     Default: ``f`` (full diagnostics)
 
 * ``<diag_name>.fields_to_plot`` (list of `strings`, optional)
     Fields written to output.

Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp

@@ -12,13 +12,49 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name)
     // Which backend to use (ADIOS, ADIOS2 or HDF5). Default depends on what is available
     std::string openpmd_backend {"default"};
     // one file per timestep (or one file for all steps)
-    bool openpmd_tspf = true;
+    std::string  openpmd_encoding {"f"};
     pp_diag_name.query("openpmd_backend", openpmd_backend);
-    pp_diag_name.query("openpmd_tspf", openpmd_tspf);
-    auto & warpx = WarpX::GetInstance();
-    m_OpenPMDPlotWriter = std::make_unique<WarpXOpenPMDPlot>(
-        openpmd_tspf, openpmd_backend, warpx.getPMLdirections()
-    );
+    bool encodingDefined = pp_diag_name.query("openpmd_encoding", openpmd_encoding);
+
+    openPMD::IterationEncoding encoding = openPMD::IterationEncoding::groupBased;
+    if ( 0 == openpmd_encoding.compare("v") )
+#if OPENPMDAPI_VERSION_GE(0, 14, 0)
+      encoding = openPMD::IterationEncoding::variableBased;
+#else
+      encoding = openPMD::IterationEncoding::groupBased;
+#endif
+    else if ( 0 == openpmd_encoding.compare("f") )
+      encoding = openPMD::IterationEncoding::fileBased;
+
+    std::string diag_type_str;
+    pp_diag_name.get("diag_type", diag_type_str);
+    if (diag_type_str == "BackTransformed")
+    {
+      if ( ( openPMD::IterationEncoding::fileBased != encoding ) &&
+           ( openPMD::IterationEncoding::groupBased != encoding ) )
+      {
+        std::string warnMsg = diag_name+" Unable to support BTD with streaming. Using GroupBased ";
+        amrex::Warning(warnMsg);
+        encoding = openPMD::IterationEncoding::groupBased;
+      }
+    }
+
+  //
+  // if no encoding is defined, then check to see if tspf is defined.
+  // (backward compatibility)
+  //
+  if ( !encodingDefined )
+    {
+      bool openpmd_tspf = false;
+      bool tspfDefined = pp_diag_name.query("openpmd_tspf", openpmd_tspf);
+      if ( tspfDefined && openpmd_tspf )
+          encoding = openPMD::IterationEncoding::fileBased;
+    }
+
+  auto & warpx = WarpX::GetInstance();
+  m_OpenPMDPlotWriter = std::make_unique<WarpXOpenPMDPlot>(
+                               encoding, openpmd_backend, warpx.getPMLdirections()
+                               );
 }
 
 void

Source/Diagnostics/WarpXOpenPMD.H

@@ -93,11 +93,11 @@ public:
 
   /** Initialize openPMD I/O routines
    *
-   * @param oneFilePerTS write one file per timestep
+   * @param ie  iteration encoding from openPMD: "group, file, variable"
    * @param filetype file backend, e.g. "bp" or "h5"
    * @param fieldPMLdirections PML field solver, @see WarpX::getPMLdirections()
    */
-  WarpXOpenPMDPlot (bool oneFilePerTS, std::string filetype, std::vector<bool> fieldPMLdirections);
+  WarpXOpenPMDPlot (openPMD::IterationEncoding ie, std::string filetype, std::vector<bool> fieldPMLdirections);
 
   ~WarpXOpenPMDPlot ();
 
@@ -129,6 +129,23 @@ public:
 private:
   void Init (openPMD::Access access, bool isBTD);
 
+
+  inline openPMD::Iteration& GetIteration(int iteration) const
+  {
+    // so BTD will be able to revisit previous steps, so we do not use steps with these two encodings,
+    if (  (openPMD::IterationEncoding::fileBased == m_Encoding ) ||
+          (openPMD::IterationEncoding::groupBased == m_Encoding )  )
+    {
+        openPMD::Iteration& it = m_Series->iterations[iteration];
+        return it;
+    } else {
+        auto iterations = m_Series->writeIterations();
+        openPMD::Iteration& it = iterations[iteration];
+        return it;
+    }
+  };
+
+
   /** This function does initial setup for the fields when interation is newly created
    *  @param[in] meshes   The meshes in a series
    *  @param[in] full_geom The geometry
@@ -243,7 +260,7 @@ private:
   //int m_NumSoAIntAttributes = PIdx::nattribs; //! WarpX' additional int particle attributes in SoA
   int m_NumAoSIntAttributes = 0; //! WarpX definition: no additional int attributes in particle AoS
 
-  bool m_OneFilePerTS = true;  //! write in openPMD fileBased manner for individual time steps
+  openPMD::IterationEncoding m_Encoding = openPMD::IterationEncoding::fileBased;
   std::string m_OpenPMDFileType = "bp"; //! MPI-parallel openPMD backend: bp or h5
   int m_CurrentStep  = -1;
 

Source/Diagnostics/WarpXOpenPMD.cpp

@@ -195,10 +195,10 @@ namespace detail
 }
 
 #ifdef WARPX_USE_OPENPMD
-WarpXOpenPMDPlot::WarpXOpenPMDPlot(bool oneFilePerTS,
+WarpXOpenPMDPlot::WarpXOpenPMDPlot(openPMD::IterationEncoding ie,
     std::string openPMDFileType, std::vector<bool> fieldPMLdirections)
   :m_Series(nullptr),
-   m_OneFilePerTS(oneFilePerTS),
+   m_Encoding(ie),
    m_OpenPMDFileType(std::move(openPMDFileType)),
    m_fieldPMLdirections(std::move(fieldPMLdirections))
 {
@@ -232,7 +232,7 @@ WarpXOpenPMDPlot::GetFileName (std::string& filepath)
   //
   // OpenPMD supports timestepped names
   //
-  if (m_OneFilePerTS)
+  if (m_Encoding == openPMD::IterationEncoding::fileBased)
       filename = filename.append("_%06T");
   filename.append(".").append(m_OpenPMDFileType);
   filepath.append(filename);
@@ -256,6 +256,7 @@ void WarpXOpenPMDPlot::SetStep (int ts, const std::string& dirPrefix,
             amrex::Warning(warnMsg);
         }
     }
+
     m_CurrentStep = ts;
     Init(openPMD::Access::CREATE, isBTD);
 }
@@ -267,8 +268,9 @@ void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush)
     // close BTD file only when isLastBTDFlush is true
     if (isBTD and !isLastBTDFlush) callClose = false;
     if (callClose) {
-        if (m_Series)
-            m_Series->iterations[m_CurrentStep].close();
+        if (m_Series) {
+            GetIteration(m_CurrentStep).close();
+        }
 
         // create a little helper file for ParaView 5.9+
         if (amrex::ParallelDescriptor::IOProcessor())
@@ -297,7 +299,10 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD)
 
     // close a previously open series before creating a new one
     // see ADIOS1 limitation: https://github.com/openPMD/openPMD-api/pull/686
-    m_Series = nullptr;
+    if ( m_Encoding == openPMD::IterationEncoding::fileBased )
+        m_Series = nullptr;
+    else if ( m_Series != nullptr )
+        return;
 
     if (amrex::ParallelDescriptor::NProcs() > 1) {
 #if defined(AMREX_USE_MPI)
@@ -316,6 +321,8 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD)
         m_MPIRank = 1;
     }
 
+    m_Series->setIterationEncoding( m_Encoding );
+
     // input file / simulation setup author
     if( WarpX::authors.size() > 0u )
         m_Series->setAuthor( WarpX::authors );
@@ -429,8 +436,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc,
   AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_Series != nullptr, "openPMD: series must be initialized");
 
   WarpXParticleCounter counter(pc);
-
-  openPMD::Iteration currIteration = m_Series->iterations[iteration];
+  openPMD::Iteration& currIteration = GetIteration(iteration);
 
   openPMD::ParticleSpecies currSpecies = currIteration.particles[name];
   // meta data for ED-PIC extension
@@ -938,7 +944,7 @@ WarpXOpenPMDPlot::WriteOpenPMDFieldsAll ( //const std::string& filename,
   bool const first_write_to_iteration = ! m_Series->iterations.contains( iteration );
 
   // meta data
-  openPMD::Iteration series_iteration = m_Series->iterations[iteration];
+  openPMD::Iteration& series_iteration = GetIteration(m_CurrentStep);
 
   auto meshes = series_iteration.meshes;
   if (first_write_to_iteration) {