diff --git a/CMakeLists.txt b/CMakeLists.txt index e5272ae2de..0fc268934a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -407,6 +407,7 @@ set(CORE_SOURCE src/auxiliary/Filesystem.cpp src/auxiliary/JSON.cpp src/auxiliary/Mpi.cpp + src/auxiliary/TemplateFile.cpp src/backend/Attributable.cpp src/backend/BaseRecordComponent.cpp src/backend/MeshRecordComponent.cpp diff --git a/examples/14_toml_template.cpp b/examples/14_toml_template.cpp index 29d19fb53a..10d775bf6f 100644 --- a/examples/14_toml_template.cpp +++ b/examples/14_toml_template.cpp @@ -1,3 +1,5 @@ +#include "openPMD/Dataset.hpp" +#include #include std::string backendEnding() @@ -101,7 +103,24 @@ void read() "../samples/tomlTemplate." + backendEnding(), openPMD::Access::READ_LINEAR); read.parseBase(); - openPMD::helper::listSeries(read); + + std::string jsonConfig = R"( +{ + "iteration_encoding": "variable_based", + "json": { + "dataset": {"mode": "template"}, + "attribute": {"mode": "short"} + } +} +)"; + openPMD::Series cloned( + "../samples/jsonTemplate.json", openPMD::Access::CREATE, jsonConfig); + openPMD::auxiliary::initializeFromTemplate(cloned, read, 0); + // Have to define the dataset for E/z as it is not defined in the template + // @todo check that the dataset is defined only upon destruction, not at + // flushing already + cloned.writeIterations()[0].meshes["E"].at("z").resetDataset( + {openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}}); } int main() diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 8c39c0a518..e5ec1e25cd 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -108,6 +108,15 @@ namespace internal * Otherwise empty. */ std::optional m_deferredParseAccess{}; + + enum TernaryBool + { + Undefined, + True, + False + }; + TernaryBool hasMeshes = TernaryBool::Undefined; + TernaryBool hasParticles = TernaryBool::Undefined; }; } // namespace internal /** @brief Logical compilation of data from one snapshot (e.g. a single @@ -245,6 +254,9 @@ class Iteration : public Attributable Container meshes{}; Container particles{}; // particleSpecies? + bool hasMeshes() const; + bool hasParticles() const; + virtual ~Iteration() = default; private: diff --git a/include/openPMD/auxiliary/TemplateFile.hpp b/include/openPMD/auxiliary/TemplateFile.hpp new file mode 100644 index 0000000000..95ea7e9cf3 --- /dev/null +++ b/include/openPMD/auxiliary/TemplateFile.hpp @@ -0,0 +1,10 @@ +#pragma once + +#include "openPMD/Series.hpp" + +namespace openPMD::auxiliary +{ +// @todo replace uint64_t with proper type after merging #1285 +Series &initializeFromTemplate( + Series &initializeMe, Series const &fromTemplate, uint64_t iteration); +} // namespace openPMD::auxiliary diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 931a1f9e3d..26d2bf4efa 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -45,6 +45,50 @@ namespace openPMD using internal::CloseStatus; using internal::DeferredParseAccess; +bool Iteration::hasMeshes() const +{ + /* + * Currently defined at the Series level, but might be defined at the + * Iteration level in next standard iterations. + * Hence an Iteration:: method. + */ + + switch (get().hasMeshes) + { + case internal::IterationData::TernaryBool::True: + return true; + case internal::IterationData::TernaryBool::False: + return false; + case internal::IterationData::TernaryBool::Undefined: { + Series s = retrieveSeries(); + return !meshes.empty() || s.containsAttribute("meshesPath"); + }; + } + throw std::runtime_error("Unreachable!"); +} + +bool Iteration::hasParticles() const +{ + /* + * Currently defined at the Series level, but might be defined at the + * Iteration level in next standard iterations. + * Hence an Iteration:: method. + */ + + switch (get().hasParticles) + { + case internal::IterationData::TernaryBool::True: + return true; + case internal::IterationData::TernaryBool::False: + return false; + case internal::IterationData::TernaryBool::Undefined: { + Series s = retrieveSeries(); + return !particles.empty() || s.containsAttribute("particlesPath"); + }; + } + throw std::runtime_error("Unreachable!"); +} + Iteration::Iteration() : Attributable(NoInit()) { setData(std::make_shared()); @@ -352,7 +396,7 @@ void Iteration::flush(internal::FlushParams const &flushParams) * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); - if (!meshes.empty() || s.containsAttribute("meshesPath")) + if (hasMeshes()) { if (!s.containsAttribute("meshesPath")) { @@ -368,7 +412,7 @@ void Iteration::flush(internal::FlushParams const &flushParams) meshes.setDirty(false); } - if (!particles.empty() || s.containsAttribute("particlesPath")) + if (hasParticles()) { if (!s.containsAttribute("particlesPath")) { @@ -547,6 +591,12 @@ void Iteration::read_impl(std::string const &groupPath) hasMeshes = s.containsAttribute("meshesPath"); hasParticles = s.containsAttribute("particlesPath"); } + { + using TB = internal::IterationData::TernaryBool; + auto &data = get(); + data.hasMeshes = hasMeshes ? TB::True : TB::False; + data.hasParticles = hasParticles ? TB::True : TB::False; + } if (hasMeshes) { diff --git a/src/auxiliary/TemplateFile.cpp b/src/auxiliary/TemplateFile.cpp new file mode 100644 index 0000000000..1ac672a524 --- /dev/null +++ b/src/auxiliary/TemplateFile.cpp @@ -0,0 +1,212 @@ +#include "openPMD/auxiliary/TemplateFile.hpp" +#include "openPMD/DatatypeHelpers.hpp" + +#include + +namespace openPMD::auxiliary +{ +namespace +{ + // Some forward declarations + template + void initializeFromTemplate( + Container &initializeMe, Container const &fromTemplate); + + struct SetAttribute + { + template + static void + call(Attributable &object, std::string const &name, Attribute &attr) + { + object.setAttribute(name, attr.get()); + } + + template + static void + call(Attributable &, std::string const &name, Attribute const &) + { + std::cerr << "Unknown datatype for template attribute '" << name + << "'. Will skip it." << std::endl; + } + }; + + void copyAttributes( + Attributable &target, + Attributable const &source, + std::vector ignore = {}) + { +#if 0 // leave this in for potential future debugging + std::cout << "COPYING ATTRIBUTES FROM '" << [&source]() -> std::string { + auto vec = source.myPath().group; + if (vec.empty()) + { + return "[]"; + } + std::stringstream sstream; + auto it = vec.begin(); + sstream << "[" << *it++; + for (; it != vec.end(); ++it) + { + sstream << ", " << *it; + } + sstream << "]"; + return sstream.str(); + }() << "'" + << std::endl; +#endif + auto shouldBeIgnored = [&ignore](std::string const &attrName) { + // `ignore` is empty by default and normally has only a handful of + // entries otherwise. + // So just use linear search. + for (auto const &ignored : ignore) + { + if (attrName == ignored) + { + return true; + } + } + return false; + }; + + for (auto const &attrName : source.attributes()) + { + if (shouldBeIgnored(attrName)) + { + continue; + } + auto attr = source.getAttribute(attrName); + auto dtype = attr.dtype; + switchType(dtype, target, attrName, attr); + } + } + + void initializeFromTemplate( + BaseRecordComponent &initializeMe, + BaseRecordComponent const &fromTemplate) + { + copyAttributes(initializeMe, fromTemplate); + } + + void initializeFromTemplate( + RecordComponent &initializeMe, RecordComponent const &fromTemplate) + { + if (fromTemplate.getDatatype() != Datatype::UNDEFINED) + { + initializeMe.resetDataset( + Dataset{fromTemplate.getDatatype(), fromTemplate.getExtent()}); + } + initializeFromTemplate( + static_cast(initializeMe), + static_cast(fromTemplate)); + } + + void initializeFromTemplate( + PatchRecordComponent &initializeMe, + PatchRecordComponent const &fromTemplate) + { + if (fromTemplate.getDatatype() != Datatype::UNDEFINED) + { + initializeMe.resetDataset( + Dataset{fromTemplate.getDatatype(), fromTemplate.getExtent()}); + } + initializeFromTemplate( + static_cast(initializeMe), + static_cast(fromTemplate)); + } + + template + void initializeFromTemplate( + BaseRecord &initializeMe, BaseRecord const &fromTemplate) + { + if (fromTemplate.scalar()) + { + initializeMe[RecordComponent::SCALAR]; + initializeFromTemplate( + static_cast(initializeMe), + static_cast(fromTemplate)); + } + else + { + initializeFromTemplate( + static_cast &>(initializeMe), + static_cast const &>(fromTemplate)); + } + } + + void initializeFromTemplate( + ParticleSpecies &initializeMe, ParticleSpecies const &fromTemplate) + { + if (!fromTemplate.particlePatches.empty()) + { + initializeFromTemplate( + static_cast &>( + initializeMe.particlePatches), + static_cast const &>( + fromTemplate.particlePatches)); + } + initializeFromTemplate( + static_cast &>(initializeMe), + static_cast const &>(fromTemplate)); + } + + template + void initializeFromTemplate( + Container &initializeMe, Container const &fromTemplate) + { + copyAttributes(initializeMe, fromTemplate); + for (auto const &pair : fromTemplate) + { + initializeFromTemplate(initializeMe[pair.first], pair.second); + } + } + + void initializeFromTemplate( + Iteration &initializeMe, Iteration const &fromTemplate) + { + copyAttributes(initializeMe, fromTemplate, {"snapshot"}); + if (fromTemplate.hasMeshes()) + { + initializeFromTemplate(initializeMe.meshes, fromTemplate.meshes); + } + if (fromTemplate.hasParticles()) + { + initializeFromTemplate( + initializeMe.particles, fromTemplate.particles); + } + } +} // namespace + +Series &initializeFromTemplate( + Series &initializeMe, Series const &fromTemplate, uint64_t iteration) +{ + if (!initializeMe.containsAttribute("from_template")) + { + copyAttributes( + initializeMe, + fromTemplate, + {"basePath", "iterationEncoding", "iterationFormat", "openPMD"}); + initializeMe.setAttribute("from_template", fromTemplate.name()); + } + + uint64_t sourceIteration = iteration; + if (!fromTemplate.iterations.contains(sourceIteration)) + { + if (fromTemplate.iterations.empty()) + { + std::cerr << "Template file has no iterations, will only fill in " + "global attributes." + << std::endl; + return initializeMe; + } + else + { + sourceIteration = fromTemplate.iterations.begin()->first; + } + } + + initializeFromTemplate( + initializeMe.iterations[iteration], + fromTemplate.iterations.at(sourceIteration)); + return initializeMe; +} +} // namespace openPMD::auxiliary diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 519a8749b2..f4288aa2ee 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -56,7 +56,7 @@ std::vector testedFileExtensions() // sst and ssc need a receiver for testing // bp4 is already tested via bp return ext == "sst" || ext == "ssc" || ext == "bp4" || - ext == "toml" || ext == "json"; + ext == "json" || ext == "toml"; }); return {allExtensions.begin(), newEnd}; } diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 9c06ac0e3e..6871946835 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -14,6 +14,7 @@ #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/auxiliary/TemplateFile.hpp" #include "openPMD/openPMD.hpp" #include @@ -1491,6 +1492,12 @@ inline void dtype_test( if (activateTemplateMode.has_value()) { + Series out( + "../samples/dtype_test_from_template." + backend, + Access::CREATE, + activateTemplateMode.value()); + auxiliary::initializeFromTemplate(out, s, 1000); + out.flush(); return; } // same implementation types (not necessary aliases) detection @@ -1563,7 +1570,15 @@ TEST_CASE("dtype_test", "[serial]") { for (auto const &t : testedFileExtensions()) { - dtype_test(t); + if (t == "json") + { + dtype_test(t); + dtype_test(t, R"(json.mode = "template")"); + } + else + { + dtype_test(t); + } } dtype_test("json", R"( { @@ -3029,21 +3044,21 @@ TEST_CASE("git_hdf5_legacy_picongpu", "[serial][hdf5]") TEST_CASE("git_hdf5_sample_attribute_test", "[serial][hdf5]") { - try - { - Series o = Series("../samples/git-sample/data%T.h5", Access::READ_ONLY); - + auto verifySeries = [](Series o, bool this_is_the_original_file) { REQUIRE(o.openPMD() == "1.1.0"); REQUIRE(o.openPMDextension() == 1); REQUIRE(o.basePath() == "/data/%T/"); REQUIRE(o.meshesPath() == "fields/"); REQUIRE(o.particlesPath() == "particles/"); - REQUIRE(o.iterationEncoding() == IterationEncoding::fileBased); - REQUIRE(o.iterationFormat() == "data%T.h5"); - REQUIRE(o.name() == "data%T"); + if (this_is_the_original_file) + { + REQUIRE(o.iterationEncoding() == IterationEncoding::fileBased); + REQUIRE(o.iterationFormat() == "data%T.h5"); + REQUIRE(o.name() == "data%T"); - REQUIRE(o.iterations.size() == 5); - REQUIRE(o.iterations.count(100) == 1); + REQUIRE(o.iterations.size() == 5); + REQUIRE(o.iterations.count(100) == 1); + } Iteration &iteration_100 = o.iterations[100]; REQUIRE(iteration_100.time() == 3.2847121452090077e-14); @@ -3273,6 +3288,30 @@ TEST_CASE("git_hdf5_sample_attribute_test", "[serial][hdf5]") REQUIRE(weighting_scalar.getDatatype() == Datatype::DOUBLE); REQUIRE(weighting_scalar.getDimensionality() == 1); REQUIRE(weighting_scalar.getExtent() == e); + }; + + try + { + { + Series o = + Series("../samples/git-sample/data%T.h5", Access::READ_ONLY); + verifySeries(o, true); + + Series fromTemplate( + "../samples/initialized_from_git_sample.json", + Access::CREATE, + R"(json.mode = "template")"); + auxiliary::initializeFromTemplate(fromTemplate, o, 100); + fromTemplate.flush(); + } + + { + Series o( + "../samples/initialized_from_git_sample.json", + Access::READ_ONLY, + R"(json.mode = "template")"); + verifySeries(o, false); + } } catch (error::ReadError &e) {