ADIOS2: more fine-grained control for file endings (#1218)

* Frontend implementation

* Backend implementation

* Fix config bug: JSON has higher priority than env vars

* Testing

Adapt tests to BP5

Don't use parentheses in attribute names

Enable BP5 dtype tests

Add test for AppendAfterSteps in BP5

* Documentation

* CI debugging

* Revert CI Debugging

Author: franzpoeschel

docs/source/backends/adios2.rst

@@ -10,12 +10,43 @@ For further information, check out the :ref:`installation guide <install>`,
 :ref:`build dependencies <development-dependencies>` and the :ref:`build options <development-buildoptions>`.
 
 
-I/O Method
-----------
+I/O Method and Engine Selection
+-------------------------------
 
 ADIOS2 has several engines for alternative file formats and other kinds of backends, yet natively writes to ``.bp`` files.
-The openPMD API uses the BP4 engine as the default file engine and the SST engine for streaming support.
-We currently leverage the default ADIOS2 transport parameters, i.e. ``POSIX`` on Unix systems and ``FStream`` on Windows.
+The openPMD API uses the File meta engine as the default file engine and the SST engine for streaming support.
+
+The ADIOS2 engine can be selected in different ways:
+
+1. Automatic detection via the selected file ending
+2. Explicit selection of an engine by specifying the environment variable ``OPENPMD_ADIOS2_ENGINE`` (case-independent).
+   This overrides the automatically detected engine.
+3. Explicit selection of an engine by specifying the JSON/TOML key ``adios2.engine.type`` as a case-independent string.
+   This overrides both previous options.
+
+Automatic engine detection supports the following extensions:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Extension
+     - Selected ADIOS2 Engine
+   * - ``.bp``
+     - ``"file"``
+   * - ``.bp4``
+     - ``"bp4"``
+   * - ``.bp5``
+     - ``"bp5"``
+   * - ``.sst``
+     - ``"sst"``
+   * - ``.ssc``
+     - ``"ssc"``
+
+Specifying any of these extensions will automatically activate the ADIOS2 backend.
+The ADIOS2 backend will create file system paths exactly as they were specified and not change file extensions.
+Exceptions to this are the BP3 and SST engines which require their endings ``.bp`` and ``.sst`` respectively.
+
+For file engines, we currently leverage the default ADIOS2 transport parameters, i.e. ``POSIX`` on Unix systems and ``FStream`` on Windows.
 
 Steps
 -----

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -132,12 +132,16 @@ class ADIOS2IOHandlerImpl
         AbstractIOHandler *,
         MPI_Comm,
         json::TracingJSON config,
-        std::string engineType);
+        std::string engineType,
+        std::string specifiedExtension);
 
 #endif // openPMD_HAVE_MPI
 
     explicit ADIOS2IOHandlerImpl(
-        AbstractIOHandler *, json::TracingJSON config, std::string engineType);
+        AbstractIOHandler *,
+        json::TracingJSON config,
+        std::string engineType,
+        std::string specifiedExtension);
 
     ~ADIOS2IOHandlerImpl() override;
 
@@ -231,6 +235,11 @@ class ADIOS2IOHandlerImpl
      * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine
      */
     std::string m_engineType;
+    /*
+     * The filename extension specified by the user.
+     */
+    std::string m_userSpecifiedExtension;
+
     ADIOS2Schema::schema_t m_schema = ADIOS2Schema::schema_0000_00_00;
 
     enum class UseSpan : char
@@ -1320,15 +1329,17 @@ class ADIOS2IOHandler : public AbstractIOHandler
         Access,
         MPI_Comm,
         json::TracingJSON options,
-        std::string engineType);
+        std::string engineType,
+        std::string specifiedExtension);
 
 #endif
 
     ADIOS2IOHandler(
         std::string path,
         Access,
         json::TracingJSON options,
-        std::string engineType);
+        std::string engineType,
+        std::string specifiedExtension);
 
     std::string backendName() const override
     {

include/openPMD/IO/AbstractIOHandlerHelper.hpp

@@ -35,6 +35,8 @@ namespace openPMD
  * @param   access      Access mode describing desired operations and
  permissions of the desired handler.
  * @param   format      Format describing the IO backend of the desired handler.
+ * @param   originalExtension The filename extension as it was originally
+ *                            specified by the user.
  * @param   comm        MPI communicator used for IO.
  * @param   options     JSON-formatted option string, to be interpreted by
  *                      the backend.
@@ -47,6 +49,7 @@ std::shared_ptr<AbstractIOHandler> createIOHandler(
     std::string path,
     Access access,
     Format format,
+    std::string originalExtension,
     MPI_Comm comm,
     JSON options);
 #endif
@@ -58,6 +61,8 @@ std::shared_ptr<AbstractIOHandler> createIOHandler(
  * @param   access      Access describing desired operations and permissions
  * of the desired handler.
  * @param   format      Format describing the IO backend of the desired handler.
+ * @param   originalExtension The filename extension as it was originally
+ *                            specified by the user.
  * @param   options     JSON-formatted option string, to be interpreted by
  *                      the backend.
  * @tparam  JSON        Substitute for nlohmann::json. Templated to avoid
@@ -66,9 +71,16 @@ std::shared_ptr<AbstractIOHandler> createIOHandler(
  */
 template <typename JSON>
 std::shared_ptr<AbstractIOHandler> createIOHandler(
-    std::string path, Access access, Format format, JSON options = JSON());
+    std::string path,
+    Access access,
+    Format format,
+    std::string originalExtension,
+    JSON options = JSON());
 
 // version without configuration to use in AuxiliaryTest
-std::shared_ptr<AbstractIOHandler>
-createIOHandler(std::string path, Access access, Format format);
+std::shared_ptr<AbstractIOHandler> createIOHandler(
+    std::string path,
+    Access access,
+    Format format,
+    std::string originalExtension);
 } // namespace openPMD

include/openPMD/IO/Format.hpp

@@ -30,7 +30,9 @@ enum class Format
 {
     HDF5,
     ADIOS1,
-    ADIOS2,
+    ADIOS2_BP,
+    ADIOS2_BP4,
+    ADIOS2_BP5,
     ADIOS2_SST,
     ADIOS2_SSC,
     JSON,

include/openPMD/Series.hpp

@@ -108,6 +108,11 @@ namespace internal
          * Filename after the expansion pattern without filename extension.
          */
         std::string m_filenamePostfix;
+        /**
+         * Filename extension as specified by the user.
+         * (Not necessarily the backend's default suffix)
+         */
+        std::string m_filenameExtension;
         /**
          * The padding in file-based iteration encoding.
          * 0 if no padding is given (%T pattern).

src/Format.cpp

@@ -45,7 +45,7 @@ Format determineFormat(std::string const &filename)
         );
 
         if (bp_backend == "ADIOS2")
-            return Format::ADIOS2;
+            return Format::ADIOS2_BP;
         else if (bp_backend == "ADIOS1")
             return Format::ADIOS1;
         else
@@ -54,6 +54,10 @@ Format determineFormat(std::string const &filename)
                 "neither ADIOS1 nor ADIOS2: " +
                 bp_backend);
     }
+    if (auxiliary::ends_with(filename, ".bp4"))
+        return Format::ADIOS2_BP4;
+    if (auxiliary::ends_with(filename, ".bp5"))
+        return Format::ADIOS2_BP5;
     if (auxiliary::ends_with(filename, ".sst"))
         return Format::ADIOS2_SST;
     if (auxiliary::ends_with(filename, ".ssc"))
@@ -72,8 +76,12 @@ std::string suffix(Format f)
     case Format::HDF5:
         return ".h5";
     case Format::ADIOS1:
-    case Format::ADIOS2:
+    case Format::ADIOS2_BP:
         return ".bp";
+    case Format::ADIOS2_BP4:
+        return ".bp4";
+    case Format::ADIOS2_BP5:
+        return ".bp5";
     case Format::ADIOS2_SST:
         return ".sst";
     case Format::ADIOS2_SSC: