Properly deal with undefined datasets

Author: franzpoeschel

examples/14_toml_template.cpp

@@ -1,6 +1,7 @@
+#include <openPMD/auxiliary/TemplateFile.hpp>
 #include <openPMD/openPMD.hpp>
 
-int main()
+void write()
 {
     std::string config = R"(
 {
@@ -24,7 +25,11 @@ int main()
     auto E = iteration.meshes["E"];
     E["x"].resetDataset(ds);
     E["y"].resetDataset(ds);
-    E["z"].resetDataset(ds);
+    /*
+     * Don't specify datatype and extent for this one to indicate that this
+     * information is not yet known.
+     */
+    E["z"]; //.resetDataset(ds);
 
     ds.extent = {10};
 
@@ -53,3 +58,35 @@ int main()
     electrons.particlePatches["extent"]["y"].resetDataset(ds);
     electrons.particlePatches["extent"]["z"].resetDataset(ds);
 }
+
+void read()
+{
+    std::string config = R"(
+{
+  "iteration_encoding": "variable_based",
+  "toml": {
+    "mode": "template"
+  }
+}
+)";
+    openPMD::Series read(
+        "../samples/tomlTemplate.toml", openPMD::Access::READ_ONLY, config);
+
+    std::string jsonConfig = R"(
+{
+  "iteration_encoding": "variable_based",
+  "json": {
+    "mode": "template"
+  }
+}
+)";
+    openPMD::Series cloned(
+        "../samples/jsonTemplate.json", openPMD::Access::CREATE, jsonConfig);
+    openPMD::auxiliary::initializeFromTemplate(cloned, read, 0);
+}
+
+int main()
+{
+    write();
+    read();
+}

src/IO/JSON/JSONIOHandlerImpl.cpp

@@ -351,7 +351,15 @@ void JSONIOHandlerImpl::createDataset(
             break;
         }
         case IOMode::Template:
-            dset["extent"] = parameter.extent;
+            if (parameter.extent != Extent{0})
+            {
+                dset["extent"] = parameter.extent;
+            }
+            else
+            {
+                // no-op
+                // If extent is empty, don't bother writing it
+            }
             break;
         }
 
@@ -1471,7 +1479,14 @@ Extent JSONIOHandlerImpl::getExtent(nlohmann::json &j, IOMode mode)
     }
     break;
     case IOMode::Template:
-        res = j["extent"].get<Extent>();
+        if (j.contains("extent"))
+        {
+            res = j["extent"].get<Extent>();
+        }
+        else
+        {
+            res = {0};
+        }
         break;
     }
     return res;

src/RecordComponent.cpp

@@ -42,7 +42,7 @@ namespace internal
         RecordComponent impl{
             std::shared_ptr<RecordComponentData>{this, [](auto const *) {}}};
         impl.setUnitSI(1);
-        impl.resetDataset(Dataset(Datatype::CHAR, {1}));
+        m_dataset = Dataset(Datatype::UNDEFINED, {0});
     }
 } // namespace internal
 
@@ -83,18 +83,28 @@ RecordComponent &RecordComponent::resetDataset(Dataset d)
         rc.m_hasBeenExtended = true;
     }
 
-    if (d.dtype == Datatype::UNDEFINED)
-    {
-        throw error::WrongAPIUsage(
-            "[RecordComponent] Must set specific datatype.");
-    }
+    // if (d.dtype == Datatype::UNDEFINED)
+    // {
+    //     throw error::WrongAPIUsage(
+    //         "[RecordComponent] Must set specific datatype.");
+    // }
     // if( d.extent.empty() )
     //    throw std::runtime_error("Dataset extent must be at least 1D.");
     if (std::any_of(
             d.extent.begin(), d.extent.end(), [](Extent::value_type const &i) {
                 return i == 0u;
             }))
-        return makeEmpty(std::move(d));
+    {
+        if (d.dtype != Datatype::UNDEFINED)
+        {
+            return makeEmpty(std::move(d));
+        }
+        else
+        {
+            rc.m_dataset = std::move(d);
+            return *this;
+        }
+    }
 
     rc.m_isEmpty = false;
     if (written())
@@ -215,12 +225,18 @@ void RecordComponent::flush(
         /*
          * This catches when a user forgets to use resetDataset.
          */
-        if (rc.m_dataset.dtype == Datatype::UNDEFINED)
-        {
-            throw error::WrongAPIUsage(
-                "[RecordComponent] Must set specific datatype (Use "
-                "resetDataset call).");
-        }
+        /*
+         * Suggestion for keeping this check:
+         * Use a std::optional<Dataset> in RecordComponentData, throw an error
+         * if it is still empty here, let users explicitly set Undefined
+         * datatypes for backend-specific actions.
+         */
+        // if (rc.m_dataset.dtype == Datatype::UNDEFINED)
+        // {
+        //     throw error::WrongAPIUsage(
+        //         "[RecordComponent] Must set specific datatype (Use "
+        //         "resetDataset call).");
+        // }
         if (!written())
         {
             if (constant())