[WIP] Use JSON/TOML template for defining openPMD metadata in a config file

CMakeLists.txt

@@ -407,6 +407,7 @@ set(CORE_SOURCE
         src/auxiliary/Filesystem.cpp
         src/auxiliary/JSON.cpp
         src/auxiliary/Mpi.cpp
+        src/auxiliary/TemplateFile.cpp
         src/backend/Attributable.cpp
         src/backend/BaseRecordComponent.cpp
         src/backend/MeshRecordComponent.cpp

examples/14_toml_template.cpp

@@ -1,3 +1,5 @@
+#include "openPMD/Dataset.hpp"
+#include <openPMD/auxiliary/TemplateFile.hpp>
 #include <openPMD/openPMD.hpp>
 
 std::string backendEnding()
@@ -101,7 +103,24 @@ void read()
         "../samples/tomlTemplate." + backendEnding(),
         openPMD::Access::READ_LINEAR);
     read.parseBase();
-    openPMD::helper::listSeries(read);
+
+    std::string jsonConfig = R"(
+{
+  "iteration_encoding": "variable_based",
+  "json": {
+    "dataset": {"mode": "template"},
+    "attribute": {"mode": "short"}
+  }
+}
+)";
+    openPMD::Series cloned(
+        "../samples/jsonTemplate.json", openPMD::Access::CREATE, jsonConfig);
+    openPMD::auxiliary::initializeFromTemplate(cloned, read, 0);
+    // Have to define the dataset for E/z as it is not defined in the template
+    // @todo check that the dataset is defined only upon destruction, not at
+    // flushing already
+    cloned.writeIterations()[0].meshes["E"].at("z").resetDataset(
+        {openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}});
 }
 
 int main()

include/openPMD/Iteration.hpp

@@ -108,6 +108,15 @@ namespace internal
          * Otherwise empty.
          */
         std::optional<DeferredParseAccess> m_deferredParseAccess{};
+
+        enum TernaryBool
+        {
+            Undefined,
+            True,
+            False
+        };
+        TernaryBool hasMeshes = TernaryBool::Undefined;
+        TernaryBool hasParticles = TernaryBool::Undefined;
     };
 } // namespace internal
 /** @brief  Logical compilation of data from one snapshot (e.g. a single
@@ -245,6 +254,9 @@ class Iteration : public Attributable
     Container<Mesh> meshes{};
     Container<ParticleSpecies> particles{}; // particleSpecies?
 
+    bool hasMeshes() const;
+    bool hasParticles() const;
+
     virtual ~Iteration() = default;
 
 private:

include/openPMD/auxiliary/TemplateFile.hpp

@@ -0,0 +1,10 @@
+#pragma once
+
+#include "openPMD/Series.hpp"
+
+namespace openPMD::auxiliary
+{
+// @todo replace uint64_t with proper type after merging #1285
+Series &initializeFromTemplate(
+    Series &initializeMe, Series const &fromTemplate, uint64_t iteration);
+} // namespace openPMD::auxiliary

src/Iteration.cpp

@@ -45,6 +45,50 @@ namespace openPMD
 using internal::CloseStatus;
 using internal::DeferredParseAccess;
 
+bool Iteration::hasMeshes() const
+{
+    /*
+     * Currently defined at the Series level, but might be defined at the
+     * Iteration level in next standard iterations.
+     * Hence an Iteration:: method.
+     */
+
+    switch (get().hasMeshes)
+    {
+    case internal::IterationData::TernaryBool::True:
+        return true;
+    case internal::IterationData::TernaryBool::False:
+        return false;
+    case internal::IterationData::TernaryBool::Undefined: {
+        Series s = retrieveSeries();
+        return !meshes.empty() || s.containsAttribute("meshesPath");
+    };
+    }
+    throw std::runtime_error("Unreachable!");
+}
+
+bool Iteration::hasParticles() const
+{
+    /*
+     * Currently defined at the Series level, but might be defined at the
+     * Iteration level in next standard iterations.
+     * Hence an Iteration:: method.
+     */
+
+    switch (get().hasParticles)
+    {
+    case internal::IterationData::TernaryBool::True:
+        return true;
+    case internal::IterationData::TernaryBool::False:
+        return false;
+    case internal::IterationData::TernaryBool::Undefined: {
+        Series s = retrieveSeries();
+        return !particles.empty() || s.containsAttribute("particlesPath");
+    };
+    }
+    throw std::runtime_error("Unreachable!");
+}
+
 Iteration::Iteration() : Attributable(NoInit())
 {
     setData(std::make_shared<Data_t>());
@@ -352,7 +396,7 @@ void Iteration::flush(internal::FlushParams const &flushParams)
          * meshesPath and particlesPath are stored there */
         Series s = retrieveSeries();
 
-        if (!meshes.empty() || s.containsAttribute("meshesPath"))
+        if (hasMeshes())
         {
             if (!s.containsAttribute("meshesPath"))
             {
@@ -368,7 +412,7 @@ void Iteration::flush(internal::FlushParams const &flushParams)
             meshes.setDirty(false);
         }
 
-        if (!particles.empty() || s.containsAttribute("particlesPath"))
+        if (hasParticles())
         {
             if (!s.containsAttribute("particlesPath"))
             {
@@ -547,6 +591,12 @@ void Iteration::read_impl(std::string const &groupPath)
         hasMeshes = s.containsAttribute("meshesPath");
         hasParticles = s.containsAttribute("particlesPath");
     }
+    {
+        using TB = internal::IterationData::TernaryBool;
+        auto &data = get();
+        data.hasMeshes = hasMeshes ? TB::True : TB::False;
+        data.hasParticles = hasParticles ? TB::True : TB::False;
+    }
 
     if (hasMeshes)
     {

src/auxiliary/TemplateFile.cpp

@@ -0,0 +1,212 @@
+#include "openPMD/auxiliary/TemplateFile.hpp"
+#include "openPMD/DatatypeHelpers.hpp"
+
+#include <iostream>
+
+namespace openPMD::auxiliary
+{
+namespace
+{
+    // Some forward declarations
+    template <typename T>
+    void initializeFromTemplate(
+        Container<T> &initializeMe, Container<T> const &fromTemplate);
+
+    struct SetAttribute
+    {
+        template <typename T>
+        static void
+        call(Attributable &object, std::string const &name, Attribute &attr)
+        {
+            object.setAttribute(name, attr.get<T>());
+        }
+
+        template <unsigned n>
+        static void
+        call(Attributable &, std::string const &name, Attribute const &)
+        {
+            std::cerr << "Unknown datatype for template attribute '" << name
+                      << "'. Will skip it." << std::endl;
+        }
+    };
+
+    void copyAttributes(
+        Attributable &target,
+        Attributable const &source,
+        std::vector<std::string> ignore = {})
+    {
+#if 0 // leave this in for potential future debugging
+        std::cout << "COPYING ATTRIBUTES FROM '" << [&source]() -> std::string {
+            auto vec = source.myPath().group;
+            if (vec.empty())
+            {
+                return "[]";
+            }
+            std::stringstream sstream;
+            auto it = vec.begin();
+            sstream << "[" << *it++;
+            for (; it != vec.end(); ++it)
+            {
+                sstream << ", " << *it;
+            }
+            sstream << "]";
+            return sstream.str();
+        }() << "'"
+            << std::endl;
+#endif
+        auto shouldBeIgnored = [&ignore](std::string const &attrName) {
+            // `ignore` is empty by default and normally has only a handful of
+            // entries otherwise.
+            // So just use linear search.
+            for (auto const &ignored : ignore)
+            {
+                if (attrName == ignored)
+                {
+                    return true;
+                }
+            }
+            return false;
+        };
+
+        for (auto const &attrName : source.attributes())
+        {
+            if (shouldBeIgnored(attrName))
+            {
+                continue;
+            }
+            auto attr = source.getAttribute(attrName);
+            auto dtype = attr.dtype;
+            switchType<SetAttribute>(dtype, target, attrName, attr);
+        }
+    }
+
+    void initializeFromTemplate(
+        BaseRecordComponent &initializeMe,
+        BaseRecordComponent const &fromTemplate)
+    {
+        copyAttributes(initializeMe, fromTemplate);
+    }
+
+    void initializeFromTemplate(
+        RecordComponent &initializeMe, RecordComponent const &fromTemplate)
+    {
+        if (fromTemplate.getDatatype() != Datatype::UNDEFINED)
+        {
+            initializeMe.resetDataset(
+                Dataset{fromTemplate.getDatatype(), fromTemplate.getExtent()});
+        }
+        initializeFromTemplate(
+            static_cast<BaseRecordComponent &>(initializeMe),
+            static_cast<BaseRecordComponent const &>(fromTemplate));
+    }
+
+    void initializeFromTemplate(
+        PatchRecordComponent &initializeMe,
+        PatchRecordComponent const &fromTemplate)
+    {
+        if (fromTemplate.getDatatype() != Datatype::UNDEFINED)
+        {
+            initializeMe.resetDataset(
+                Dataset{fromTemplate.getDatatype(), fromTemplate.getExtent()});
+        }
+        initializeFromTemplate(
+            static_cast<BaseRecordComponent &>(initializeMe),
+            static_cast<BaseRecordComponent const &>(fromTemplate));
+    }
+
+    template <typename T>
+    void initializeFromTemplate(
+        BaseRecord<T> &initializeMe, BaseRecord<T> const &fromTemplate)
+    {
+        if (fromTemplate.scalar())
+        {
+            initializeMe[RecordComponent::SCALAR];
+            initializeFromTemplate(
+                static_cast<T &>(initializeMe),
+                static_cast<T const &>(fromTemplate));
+        }
+        else
+        {
+            initializeFromTemplate(
+                static_cast<Container<T> &>(initializeMe),
+                static_cast<Container<T> const &>(fromTemplate));
+        }
+    }
+
+    void initializeFromTemplate(
+        ParticleSpecies &initializeMe, ParticleSpecies const &fromTemplate)
+    {
+        if (!fromTemplate.particlePatches.empty())
+        {
+            initializeFromTemplate(
+                static_cast<Container<PatchRecord> &>(
+                    initializeMe.particlePatches),
+                static_cast<Container<PatchRecord> const &>(
+                    fromTemplate.particlePatches));
+        }
+        initializeFromTemplate(
+            static_cast<Container<Record> &>(initializeMe),
+            static_cast<Container<Record> const &>(fromTemplate));
+    }
+
+    template <typename T>
+    void initializeFromTemplate(
+        Container<T> &initializeMe, Container<T> const &fromTemplate)
+    {
+        copyAttributes(initializeMe, fromTemplate);
+        for (auto const &pair : fromTemplate)
+        {
+            initializeFromTemplate(initializeMe[pair.first], pair.second);
+        }
+    }
+
+    void initializeFromTemplate(
+        Iteration &initializeMe, Iteration const &fromTemplate)
+    {
+        copyAttributes(initializeMe, fromTemplate, {"snapshot"});
+        if (fromTemplate.hasMeshes())
+        {
+            initializeFromTemplate(initializeMe.meshes, fromTemplate.meshes);
+        }
+        if (fromTemplate.hasParticles())
+        {
+            initializeFromTemplate(
+                initializeMe.particles, fromTemplate.particles);
+        }
+    }
+} // namespace
+
+Series &initializeFromTemplate(
+    Series &initializeMe, Series const &fromTemplate, uint64_t iteration)
+{
+    if (!initializeMe.containsAttribute("from_template"))
+    {
+        copyAttributes(
+            initializeMe,
+            fromTemplate,
+            {"basePath", "iterationEncoding", "iterationFormat", "openPMD"});
+        initializeMe.setAttribute("from_template", fromTemplate.name());
+    }
+
+    uint64_t sourceIteration = iteration;
+    if (!fromTemplate.iterations.contains(sourceIteration))
+    {
+        if (fromTemplate.iterations.empty())
+        {
+            std::cerr << "Template file has no iterations, will only fill in "
+                         "global attributes."
+                      << std::endl;
+            return initializeMe;
+        }
+        else
+        {
+            sourceIteration = fromTemplate.iterations.begin()->first;
+        }
+    }
+
+    initializeFromTemplate(
+        initializeMe.iterations[iteration],
+        fromTemplate.iterations.at(sourceIteration));
+    return initializeMe;
+}
+} // namespace openPMD::auxiliary

test/ParallelIOTest.cpp

@@ -56,7 +56,7 @@ std::vector<std::string> testedFileExtensions()
             // sst and ssc need a receiver for testing
             // bp4 is already tested via bp
             return ext == "sst" || ext == "ssc" || ext == "bp4" ||
-                ext == "toml" || ext == "json";
+                ext == "json" || ext == "toml";
         });
     return {allExtensions.begin(), newEnd};
 }