diff --git a/CMakeLists.txt b/CMakeLists.txt index bd888f84b5..1876cfa105 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -394,6 +394,7 @@ include(${openPMD_SOURCE_DIR}/cmake/dependencies/pybind11.cmake) set(CORE_SOURCE src/config.cpp src/ChunkInfo.cpp + src/CustomHierarchy.cpp src/Dataset.cpp src/Datatype.cpp src/Error.cpp @@ -578,6 +579,7 @@ if(openPMD_HAVE_PYTHON) src/binding/python/Attributable.cpp src/binding/python/BaseRecordComponent.cpp src/binding/python/ChunkInfo.cpp + src/binding/python/CustomHierarchy.cpp src/binding/python/Dataset.cpp src/binding/python/Datatype.cpp src/binding/python/Error.cpp @@ -746,6 +748,7 @@ set(openPMD_PYTHON_EXAMPLE_NAMES 11_particle_dataframe 12_span_write 13_write_dynamic_configuration + 14_custom_hierarchy 15_compression ) diff --git a/examples/14_custom_hierarchy.py b/examples/14_custom_hierarchy.py new file mode 100755 index 0000000000..b3eff208a9 --- /dev/null +++ b/examples/14_custom_hierarchy.py @@ -0,0 +1,48 @@ +import numpy as np +import openpmd_api as io + + +def main(): + if "bp" in io.file_extensions: + filename = "../samples/custom_hierarchy.bp" + else: + filename = "../samples/custom_hierarchy.json" + s = io.Series(filename, io.Access.create) + it = s.write_iterations()[100] + + # write openPMD part + temp = it.meshes["temperature"] + temp.axis_labels = ["x", "y"] + temp.unit_dimension = {io.Unit_Dimension.T: 1} + temp.position = [0.5, 0.5] + temp.grid_spacing = [1, 1] + temp.grid_global_offset = [0, 0] + temp.reset_dataset(io.Dataset(np.dtype("double"), [5, 5])) + temp[()] = np.zeros((5, 5)) + + # write NeXus part + nxentry = it["Scan"] + nxentry.set_attribute("NX_class", "NXentry") + nxentry.set_attribute("default", "data") + + data = nxentry["data"] + data.set_attribute("NX_class", "NXdata") + data.set_attribute("signal", "counts") + data.set_attribute("axes", ["two_theta"]) + data.set_attribute("two_theta_indices", [0]) + + counts = data.as_container_of_datasets()["counts"] + counts.set_attribute("units", "counts") + counts.set_attribute("long_name", "photodiode counts") + counts.reset_dataset(io.Dataset(np.dtype("int"), [15])) + counts[()] = np.zeros(15, dtype=np.dtype("int")) + + two_theta = data.as_container_of_datasets()["two_theta"] + two_theta.set_attribute("units", "degrees") + two_theta.set_attribute("long_name", "two_theta (degrees)") + two_theta.reset_dataset(io.Dataset(np.dtype("double"), [15])) + two_theta[()] = np.zeros(15) + + +if __name__ == "__main__": + main() diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp new file mode 100644 index 0000000000..dc23a97e64 --- /dev/null +++ b/include/openPMD/CustomHierarchy.hpp @@ -0,0 +1,256 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/backend/Container.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace openPMD +{ +class CustomHierarchy; +namespace internal +{ + enum class ContainedType + { + Group, + Mesh, + Particle + }; + struct MeshesParticlesPath + { + std::regex meshRegex; + std::set collectNewMeshesPaths; + std::regex particleRegex; + std::set collectNewParticlesPaths; + + /* + * These values decide which path will be returned upon use of the + * shorthand notation s.iterations[0].meshes or .particles. + * + */ + std::string m_defaultMeshesPath = "meshes"; + std::string m_defaultParticlesPath = "particles"; + + explicit MeshesParticlesPath() = default; + MeshesParticlesPath( + std::vector const &meshes, + std::vector const &particles); + MeshesParticlesPath(Series const &); + + [[nodiscard]] ContainedType + determineType(std::vector const &path) const; + [[nodiscard]] bool + isParticleContainer(std::vector const &path) const; + [[nodiscard]] bool + isMeshContainer(std::vector const &path) const; + }; + + struct CustomHierarchyData + : ContainerData + , ContainerData + , ContainerData + , ContainerData + { + explicit CustomHierarchyData(); + + void syncAttributables(); + +#if 0 + inline Container customHierarchiesWrapped() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } +#endif + inline Container embeddedDatasetsWrapped() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } + inline Container embeddedMeshesWrapped() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } + + inline Container embeddedParticlesWrapped() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } + +#if 0 + inline Container::InternalContainer & + customHierarchiesInternal() + { + return static_cast *>(this) + ->m_container; + } +#endif + inline Container::InternalContainer & + embeddedDatasetsInternal() + { + return static_cast *>(this) + ->m_container; + } + inline Container::InternalContainer &embeddedMeshesInternal() + { + return static_cast *>(this)->m_container; + } + + inline Container::InternalContainer & + embeddedParticlesInternal() + { + return static_cast *>(this) + ->m_container; + } + }; +} // namespace internal + +template +class ConversibleContainer : public Container +{ + template + friend class ConversibleContainer; + +protected: + using Container_t = Container; + using Data_t = internal::CustomHierarchyData; + static_assert( + std::is_base_of_v); + + ConversibleContainer(Attributable::NoInit) + : Container_t(Attributable::NoInit{}) + {} + + std::shared_ptr m_customHierarchyData; + + [[nodiscard]] Data_t &get() + { + return *m_customHierarchyData; + } + [[nodiscard]] Data_t const &get() const + { + return *m_customHierarchyData; + } + + inline void setData(std::shared_ptr data) + { + m_customHierarchyData = data; + Container_t::setData(std::move(data)); + } + +public: + template + auto asContainerOf() -> ConversibleContainer + { + if constexpr ( + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) + { + ConversibleContainer res(Attributable::NoInit{}); + res.setData(m_customHierarchyData); + return res; + } + else + { + static_assert( + auxiliary::dependent_false_v, + "[CustomHierarchy::asContainerOf] Type parameter must be " + "one of: CustomHierarchy, RecordComponent, Mesh, " + "ParticleSpecies."); + } + } +}; + +class CustomHierarchy : public ConversibleContainer +{ + friend class Iteration; + friend class Container; + +private: + using Container_t = Container; + using Parent_t = ConversibleContainer; + using Data_t = typename Parent_t::Data_t; + + using EraseStaleMeshes = internal::EraseStaleEntries>; + using EraseStaleParticles = + internal::EraseStaleEntries>; + void readNonscalarMesh(EraseStaleMeshes &map, std::string const &name); + void readScalarMesh(EraseStaleMeshes &map, std::string const &name); + void readParticleSpecies(EraseStaleParticles &map, std::string const &name); + +protected: + CustomHierarchy(); + CustomHierarchy(NoInit); + + void read(internal::MeshesParticlesPath const &); + void read( + internal::MeshesParticlesPath const &, + std::vector ¤tPath); + + void flush_internal( + internal::FlushParams const &, + internal::MeshesParticlesPath &, + std::vector currentPath); + void flush(std::string const &path, internal::FlushParams const &) override; + + /** + * @brief Link with parent. + * + * @param w The Writable representing the parent. + */ + void linkHierarchy(Writable &w) override; + +public: + CustomHierarchy(CustomHierarchy const &other) = default; + CustomHierarchy(CustomHierarchy &&other) = default; + + CustomHierarchy &operator=(CustomHierarchy const &) = default; + CustomHierarchy &operator=(CustomHierarchy &&) = default; +}; +} // namespace openPMD diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 7c02f38ddd..15ba26803c 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -399,7 +399,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // make sure that the given path exists in proper form in // the passed json value - static void ensurePath(nlohmann::json *json, std::string const &path); + static void + ensurePath(nlohmann::json *json, std::string const &path, Access); // In order not to insert the same file name into the data structures // with a new pointer (e.g. when reopening), search for a possibly diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 43ee1084bb..a79fc1a56d 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -20,6 +20,7 @@ */ #pragma once +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticleSpecies.hpp" @@ -98,7 +99,7 @@ namespace internal BeginStep beginStep = BeginStepTypes::DontBeginStep{}; }; - class IterationData : public AttributableData + class IterationData : public CustomHierarchyData { public: /* @@ -142,10 +143,22 @@ namespace internal * @see * https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#required-attributes-for-the-basepath */ -class Iteration : public Attributable +class Iteration : public CustomHierarchy { - template - friend class Container; +public: + using IterationIndex_t = uint64_t; + + /* + * Some old compilers have trouble with befriending the entire Container + * template here, so we restrict it + * to Container, more is not needed anyway. + * + * E.g. on gcc-7: + * > error: specialization of 'openPMD::Container' + * > after instantiation + * > friend class Container; + */ + friend class Container; friend class Series; friend class internal::AttributableData; template @@ -154,13 +167,17 @@ class Iteration : public Attributable friend class StatefulIterator; friend class StatefulSnapshotsContainer; -public: Iteration(Iteration const &) = default; Iteration(Iteration &&) = default; Iteration &operator=(Iteration const &) = default; Iteration &operator=(Iteration &&) = default; - using IterationIndex_t = uint64_t; + // These use the openPMD Container class mainly for consistency. + // But they are in fact only aliases that don't actually exist + // in the backend. + // Hence meshes.written() and particles.written() will always be false. + Container meshes{}; + Container particles{}; /** * @tparam T Floating point type of user-selected precision (e.g. float, @@ -268,9 +285,6 @@ class Iteration : public Attributable [[deprecated("This attribute is no longer set by the openPMD-api.")]] bool closedByWriter() const; - Container meshes{}; - Container particles{}; // particleSpecies? - virtual ~Iteration() = default; private: @@ -297,14 +311,25 @@ class Iteration : public Attributable inline void setData(std::shared_ptr data) { m_iterationData = std::move(data); - Attributable::setData(m_iterationData); + CustomHierarchy::setData(m_iterationData); } void flushFileBased( std::string const &, IterationIndex_t, internal::FlushParams const &); void flushGroupBased(IterationIndex_t, internal::FlushParams const &); void flushVariableBased(IterationIndex_t, internal::FlushParams const &); - void flush(internal::FlushParams const &); + /* + * Named flushIteration instead of flush to avoid naming + * conflicts with overridden virtual flush from CustomHierarchy + * class. + */ + void flushIteration(internal::FlushParams const &); + + void sync_meshes_and_particles_from_alias_to_subgroups( + internal::MeshesParticlesPath const &); + void sync_meshes_and_particles_from_subgroups_to_alias( + internal::MeshesParticlesPath const &); + void deferParseAccess(internal::DeferredParseAccess); /* * Control flow for runDeferredParseAccess(), readFileBased(), @@ -334,8 +359,6 @@ class Iteration : public Attributable void readGorVBased( std::string const &groupPath, internal::BeginStep const &beginStep); void read_impl(std::string const &groupPath); - void readMeshes(std::string const &meshesPath); - void readParticles(std::string const &particlesPath); /** * Status after beginning an IO step. Currently includes: @@ -416,12 +439,22 @@ class Iteration : public Attributable */ void setStepStatus(StepStatus); + /* + * @brief Check recursively whether this Iteration is dirty. + * It is dirty if any attribute or dataset is read from or written to + * the backend. + * + * @return true If dirty. + * @return false Otherwise. + */ + bool dirtyRecursive() const; + /** * @brief Link with parent. * * @param w The Writable representing the parent. */ - virtual void linkHierarchy(Writable &w); + void linkHierarchy(Writable &w); /** * @brief Access an iteration in read mode that has potentially not been diff --git a/include/openPMD/Mesh.hpp b/include/openPMD/Mesh.hpp index 77ef8b2886..edbd6ff6ff 100644 --- a/include/openPMD/Mesh.hpp +++ b/include/openPMD/Mesh.hpp @@ -41,6 +41,7 @@ class Mesh : public BaseRecord { friend class Container; friend class Iteration; + friend class CustomHierarchy; public: Mesh(Mesh const &) = default; diff --git a/include/openPMD/ParticleSpecies.hpp b/include/openPMD/ParticleSpecies.hpp index af7aa50375..9f454a0ed5 100644 --- a/include/openPMD/ParticleSpecies.hpp +++ b/include/openPMD/ParticleSpecies.hpp @@ -37,6 +37,7 @@ class ParticleSpecies : public Container friend class Iteration; template friend T &internal::makeOwning(T &self, Series); + friend class CustomHierarchy; public: ParticlePatches particlePatches; diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index f319e10cff..cdc5148c5b 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -128,6 +128,7 @@ class RecordComponent : public BaseRecordComponent friend class MeshRecordComponent; template friend T &internal::makeOwning(T &self, Series); + friend class CustomHierarchy; public: enum class Allocation @@ -483,8 +484,9 @@ class RecordComponent : public BaseRecordComponent static constexpr char const *const SCALAR = "\vScalar"; protected: - void flush(std::string const &, internal::FlushParams const &); - void read(bool require_unit_si); + void flush( + std::string const &, internal::FlushParams const &, bool set_defaults); + void read(bool read_defaults); private: /** @@ -532,7 +534,7 @@ OPENPMD_protected BaseRecordComponent::setData(m_recordComponentData); } - void readBase(bool require_unit_si); + void readBase(bool read_defaults); template void verifyChunk(Offset const &, Extent const &) const; diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 85387e6473..cdbd384da3 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -293,6 +293,7 @@ class Series : public Attributable friend class internal::SeriesData; friend class internal::AttributableData; friend class StatefulSnapshotsContainer; + friend class CustomHierarchy; public: explicit Series(); @@ -446,6 +447,7 @@ class Series : public Attributable * basePath. */ std::string meshesPath() const; + std::vector meshesPaths() const; /** Set the path to mesh * records, relative(!) to basePath. @@ -456,6 +458,7 @@ class Series : public Attributable * @return Reference to modified series. */ Series &setMeshesPath(std::string const &meshesPath); + Series &setMeshesPath(std::vector const &meshesPath); /** * @return True if there is a rankTable dataset defined for this Series. @@ -501,6 +504,7 @@ class Series : public Attributable * basePath. */ std::string particlesPath() const; + std::vector particlesPaths() const; /** Set the path to groups for each particle * species, relative(!) to basePath. @@ -511,6 +515,7 @@ class Series : public Attributable * @return Reference to modified series. */ Series &setParticlesPath(std::string const &particlesPath); + Series &setParticlesPath(std::vector const &particlesPath); /** * @throw no_such_attribute_error If optional attribute is not present. @@ -897,8 +902,6 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushMeshesPath(); - void flushParticlesPath(); void flushRankTable(); /* Parameter `read_only_this_single_iteration` used for reopening an * Iteration after closing it. diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index d34d5bb48f..93cf583f87 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -49,6 +49,7 @@ class AbstractFilePosition; class Attributable; class Iteration; class Series; +class CustomHierarchy; namespace internal { @@ -58,6 +59,7 @@ namespace internal class SharedAttributableData { friend class openPMD::Attributable; + friend class openPMD::CustomHierarchy; public: SharedAttributableData(AttributableData *); @@ -103,6 +105,7 @@ namespace internal class AttributableData : public std::shared_ptr { friend class openPMD::Attributable; + friend class openPMD::CustomHierarchy; using SharedData_t = std::shared_ptr; @@ -113,6 +116,17 @@ namespace internal AttributableData(AttributableData &&) = delete; virtual ~AttributableData() = default; + inline std::shared_ptr & + asSharedPtrOfAttributable() + { + return *this; + } + inline std::shared_ptr const & + asSharedPtrOfAttributable() const + { + return *this; + } + AttributableData &operator=(AttributableData const &) = delete; AttributableData &operator=(AttributableData &&) = delete; @@ -161,6 +175,7 @@ namespace internal class BaseRecordData; class RecordComponentData; + struct CustomHierarchyData; /* * Internal function to turn a handle into an owning handle that will keep @@ -213,6 +228,8 @@ class Attributable friend class StatefulSnapshotsContainer; friend class internal::AttributableData; friend class Snapshots; + friend class CustomHierarchy; + friend struct internal::CustomHierarchyData; protected: // tag for internal constructor diff --git a/include/openPMD/backend/Attribute.hpp b/include/openPMD/backend/Attribute.hpp index 216528204a..4f9c0a07e6 100644 --- a/include/openPMD/backend/Attribute.hpp +++ b/include/openPMD/backend/Attribute.hpp @@ -310,6 +310,25 @@ namespace detail } } } + // conversion cast: turn a 1-element vector into a single value + else if constexpr (auxiliary::IsVector_v) + { + if constexpr (std::is_convertible_v) + { + if (pv->size() != 1) + { + return {std::runtime_error( + "getCast: vector to scalar conversion requires " + "single-element vectors")}; + } + return {U(*pv->begin())}; + } + else + { + return {std::runtime_error( + "getCast: no vector to scalar conversion possible.")}; + } + } else { return {std::runtime_error("getCast: no cast possible.")}; diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index edb8ae5f8a..e1cad7b66a 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -186,6 +186,7 @@ class BaseRecord private: using T_Self = BaseRecord; + friend class CustomHierarchy; friend class Iteration; friend class ParticleSpecies; friend class PatchRecord; diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index afe0b25cc6..b5fec6cd39 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -57,11 +57,26 @@ namespace traits }; } // namespace traits +class CustomHierarchy; + namespace internal { + template + constexpr inline bool isDerivedFromAttributable = + std::is_base_of_v; + + /* + * Opt out from this check due to the recursive definition of + * class CustomHierarchy : public Container{ ... }; + * Cannot check this while CustomHierarchy is still an incomplete type. + */ + template <> + constexpr inline bool isDerivedFromAttributable = true; + class SeriesData; template class EraseStaleEntries; + struct CustomHierarchyData; template < typename T, @@ -103,7 +118,7 @@ template < class Container : virtual public Attributable { static_assert( - std::is_base_of::value, + internal::isDerivedFromAttributable, "Type of container element must be derived from Writable"); friend class Iteration; @@ -114,6 +129,9 @@ class Container : virtual public Attributable template friend class internal::EraseStaleEntries; friend class StatefulIterator; + friend class SeriesIterator; + friend struct internal::CustomHierarchyData; + friend class CustomHierarchy; protected: using ContainerData = internal::ContainerData; diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index 2d29bac983..5a81b7f76f 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -106,6 +106,7 @@ class Writable final friend void debug::printDirty(Series const &); friend struct Parameter; friend struct Parameter; + friend class CustomHierarchy; private: Writable(internal::AttributableData *); diff --git a/include/openPMD/binding/python/Common.hpp b/include/openPMD/binding/python/Common.hpp index b21d490070..0cdda471ff 100644 --- a/include/openPMD/binding/python/Common.hpp +++ b/include/openPMD/binding/python/Common.hpp @@ -9,6 +9,7 @@ #pragma once #include "openPMD/ChunkInfo.hpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticlePatches.hpp" @@ -60,6 +61,7 @@ using PyPatchRecordComponentContainer = Container; using PyBaseRecordRecordComponent = BaseRecord; using PyBaseRecordMeshRecordComponent = BaseRecord; using PyBaseRecordPatchRecordComponent = BaseRecord; +using PyCustomHierarchyContainer = Container; PYBIND11_MAKE_OPAQUE(PyIterationContainer) PYBIND11_MAKE_OPAQUE(PyMeshContainer) PYBIND11_MAKE_OPAQUE(PyPartContainer) @@ -71,3 +73,4 @@ PYBIND11_MAKE_OPAQUE(PyMeshRecordComponentContainer) PYBIND11_MAKE_OPAQUE(PyPatchRecordComponentContainer) PYBIND11_MAKE_OPAQUE(PyBaseRecordRecordComponent) PYBIND11_MAKE_OPAQUE(PyBaseRecordPatchRecordComponent) +PYBIND11_MAKE_OPAQUE(PyCustomHierarchyContainer) diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp new file mode 100644 index 0000000000..5a5d7d6884 --- /dev/null +++ b/src/CustomHierarchy.cpp @@ -0,0 +1,669 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/CustomHierarchy.hpp" + +#include "openPMD/Dataset.hpp" +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/Access.hpp" +#include "openPMD/IO/IOTask.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/Writable.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// @todo add handselected choice of [:punct:] characters to this +// using a macro here to make string interpolation simpler +#define OPENPMD_LEGAL_IDENTIFIER_CHARS "[:alnum:]_" +#define OPENPMD_SINGLE_GLOBBING_CHAR "%" +#define OPENPMD_DOUBLE_GLOBBING_CHAR "%%" + +namespace +{ +template +std::string +concatWithSep(Iterator &&begin, Iterator const &end, std::string const &sep) +{ + if (begin == end) + { + return ""; + } + std::stringstream res; + res << *(begin++); + for (; begin != end; ++begin) + { + res << sep << *begin; + } + return res.str(); +} + +std::string +concatWithSep(std::vector const &v, std::string const &sep) +{ + return concatWithSep(v.begin(), v.end(), sep); +} + +// Not specifying std::regex_constants::optimize here, only using it where +// it makes sense to. +constexpr std::regex_constants::syntax_option_type regex_flags = + std::regex_constants::egrep; + +template +void setDefaultMeshesParticlesPath( + std::vector const &meshes, + std::vector const &particles, + OutParam &writeTarget) +{ + std::regex is_default_path_specification( + "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "]+/", + regex_flags | std::regex_constants::optimize); + constexpr char const *default_default_mesh = "meshes"; + constexpr char const *default_default_particle = "particles"; + for (auto [vec, defaultPath, default_default] : + {std::make_tuple( + &meshes, &writeTarget.m_defaultMeshesPath, default_default_mesh), + std::make_tuple( + &particles, + &writeTarget.m_defaultParticlesPath, + default_default_particle)}) + { + bool set_default = true; + /* + * The first eligible path in meshesPath/particlesPath is used as + * the default, "meshes"/"particles" otherwise. + */ + for (auto const &path : *vec) + { + if (std::regex_match(path, is_default_path_specification)) + { + *defaultPath = openPMD::auxiliary::replace_last(path, "/", ""); + set_default = false; + break; + } + } + if (set_default) + { + *defaultPath = default_default; + } + } +} + +bool anyPathRegexMatches( + std::regex const ®ex, std::vector const &path) +{ + std::string pathToMatch = '/' + concatWithSep(path, "/") + '/'; + return std::regex_match(pathToMatch, regex); +} +} // namespace + +namespace openPMD +{ +namespace internal +{ + namespace + { + std::string globToRegexLongForm(std::string const &glob) + { + return auxiliary::replace_all( + auxiliary::replace_all( + glob, + OPENPMD_DOUBLE_GLOBBING_CHAR, + "([" OPENPMD_LEGAL_IDENTIFIER_CHARS "/]*)"), + OPENPMD_SINGLE_GLOBBING_CHAR, + "([" OPENPMD_LEGAL_IDENTIFIER_CHARS "]*)"); + } + + std::string globToRegexShortForm(std::string const &glob) + { + return "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "/]*/" + glob; + } + } // namespace + + MeshesParticlesPath::MeshesParticlesPath( + std::vector const &meshes, + std::vector const &particles) + { + /* + * /group/meshes/E is a mesh if the meshes path contains: + * + * 1) '/group/meshes/' (absolute path to mesh container) + * 2) 'meshes/' (relative path to mesh container) + * + * All this analogously for particles path. + */ + + // regex for detecting option 1) + // e.g. '/path/to/meshes/': The path to the meshes. Mandatory slashes at + // beginning and end, possibly slashes in + // between. Mandatory slash at beginning might + // be replaced with '%%' to enable paths like + // '%%/path/to/meshes'. + // resolves to: `(/|%%)[[:alnum:]_%/]+/` + std::regex is_legal_long_path_specification( + "(/|" OPENPMD_DOUBLE_GLOBBING_CHAR + ")[" OPENPMD_LEGAL_IDENTIFIER_CHARS OPENPMD_SINGLE_GLOBBING_CHAR + "/]+/", + regex_flags | std::regex_constants::optimize); + + // Regex for detecting option 2) + // e.g. 'meshes/': The name without path. One single mandatory slash + // at the end, no slashes otherwise. + // resolves to `[[:alnum:]_]+/` + std::regex is_legal_short_path_specification( + "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "]+/", + regex_flags | std::regex_constants::optimize); + + for (auto [target_regex, vec] : + {std::make_tuple(&this->meshRegex, &meshes), + std::make_tuple(&this->particleRegex, &particles)}) + { + std::stringstream build_regex; + // neutral element: empty language, regex doesn't match anything + build_regex << "(a^)"; + for (auto const &entry : *vec) + { + if (std::regex_match(entry, is_legal_short_path_specification)) + { + build_regex << "|(" << globToRegexShortForm(entry) << ')'; + } + else if (std::regex_match( + entry, is_legal_long_path_specification)) + { + build_regex << "|(" << globToRegexLongForm(entry) << ')'; + } + else + { + std::cerr + << "[WARNING] Not a legal meshes-/particles-path: '" + << entry << "'. Will skip." << std::endl; + } + } + auto regex_string = build_regex.str(); + // std::cout << "Using regex string: " << regex_string << std::endl; + *target_regex = std::regex( + regex_string, regex_flags | std::regex_constants::optimize); + } + setDefaultMeshesParticlesPath(meshes, particles, *this); + } + + ContainedType MeshesParticlesPath::determineType( + std::vector const &path) const + { + if (isMeshContainer(path)) + { + return ContainedType::Mesh; + } + else if (isParticleContainer(path)) + { + return ContainedType::Particle; + } + else + { + return ContainedType::Group; + } + } + + bool MeshesParticlesPath::isParticleContainer( + std::vector const &path) const + { + return anyPathRegexMatches(particleRegex, path); + } + bool MeshesParticlesPath::isMeshContainer( + std::vector const &path) const + { + return anyPathRegexMatches(meshRegex, path); + } + + CustomHierarchyData::CustomHierarchyData() + { + syncAttributables(); + } + + void CustomHierarchyData::syncAttributables() + { + /* + * m_embeddeddatasets and its friends should point to the same instance + * of Attributable. + * Not strictly necessary to do this explicitly due to virtual + * inheritance (all Attributable instances are the same anyway), + * but let's be explicit about this. + */ + for (auto p : std::initializer_list{ + static_cast *>(this), + static_cast *>(this), + static_cast *>(this), + static_cast *>(this)}) + { + p->asSharedPtrOfAttributable() = this->asSharedPtrOfAttributable(); + } + } +} // namespace internal + +// template +// class ConversibleContainer; + +CustomHierarchy::CustomHierarchy() : ConversibleContainer(NoInit{}) +{ + setData(std::make_shared()); +} +CustomHierarchy::CustomHierarchy(NoInit) : ConversibleContainer(NoInit{}) +{} + +void CustomHierarchy::readNonscalarMesh( + EraseStaleMeshes &map, std::string const &mesh_name) +{ + Parameter pOpen; + Parameter aList; + + Mesh &m = map[mesh_name]; + + pOpen.path = mesh_name; + aList.attributes->clear(); + IOHandler()->enqueue(IOTask(&m, pOpen)); + IOHandler()->enqueue(IOTask(&m, aList)); + IOHandler()->flush(internal::defaultFlushParams); + + auto att_begin = aList.attributes->begin(); + auto att_end = aList.attributes->end(); + auto value = std::find(att_begin, att_end, "value"); + auto shape = std::find(att_begin, att_end, "shape"); + if (value != att_end && shape != att_end) + { + MeshRecordComponent &mrc = m; + IOHandler()->enqueue(IOTask(&mrc, pOpen)); + IOHandler()->flush(internal::defaultFlushParams); + mrc.get().m_isConstant = true; + } + try + { + m.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } +} + +void CustomHierarchy::readScalarMesh( + EraseStaleMeshes &map, std::string const &mesh_name) +{ + Parameter pOpen; + Parameter pList; + + Parameter dOpen; + Mesh &m = map[mesh_name]; + dOpen.name = mesh_name; + MeshRecordComponent &mrc = m; + IOHandler()->enqueue(IOTask(&mrc, dOpen)); + IOHandler()->flush(internal::defaultFlushParams); + mrc.setWritten(false, Attributable::EnqueueAsynchronously::No); + mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + mrc.setWritten(true, Attributable::EnqueueAsynchronously::No); + try + { + m.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } +} + +void CustomHierarchy::readParticleSpecies( + EraseStaleParticles &map, std::string const &species_name) +{ + Parameter pOpen; + Parameter pList; + + ParticleSpecies &p = map[species_name]; + pOpen.path = species_name; + IOHandler()->enqueue(IOTask(&p, pOpen)); + IOHandler()->flush(internal::defaultFlushParams); + try + { + p.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle species with name '" << species_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(species_name); + } +} + +void CustomHierarchy::read(internal::MeshesParticlesPath const &mpp) +{ + std::vector currentPath; + read(mpp, currentPath); +} + +void CustomHierarchy::read( + internal::MeshesParticlesPath const &mpp, + std::vector ¤tPath) +{ + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + Parameter pList; + IOHandler()->enqueue(IOTask(this, pList)); + + Attributable::readAttributes(ReadMode::FullyReread); + Parameter dList; + IOHandler()->enqueue(IOTask(this, dList)); + IOHandler()->flush(internal::defaultFlushParams); + + std::deque constantComponentsPushback; + auto &data = get(); + auto embeddedMeshes = data.embeddedMeshesWrapped(); + auto embeddedParticles = data.embeddedParticlesWrapped(); + EraseStaleMeshes meshesMap(embeddedMeshes); + EraseStaleParticles particlesMap(embeddedParticles); + for (auto const &path : *pList.paths) + { + switch (mpp.determineType(currentPath)) + { + case internal::ContainedType::Group: { + Parameter pOpen; + pOpen.path = path; + auto &subpath = this->operator[](path); + IOHandler()->enqueue(IOTask(&subpath, pOpen)); + currentPath.emplace_back(path); + try + { + subpath.read(mpp, currentPath); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read subgroup '" << path << "' at path '" + << myPath().openPMDPath() + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + container().erase(path); + } + currentPath.pop_back(); + if (subpath.size() == 0 && subpath.containsAttribute("shape") && + subpath.containsAttribute("value")) + { + // This is not a group, but a constant record component + // Writable::~Writable() will deal with removing this from the + // backend again. + constantComponentsPushback.push_back(path); + container().erase(path); + } + break; + } + case internal::ContainedType::Mesh: { + try + { + readNonscalarMesh(meshesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + meshesMap.forget(path); + } + break; + } + case internal::ContainedType::Particle: { + try + { + readParticleSpecies(particlesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle species at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + particlesMap.forget(path); + } + break; + } + } + } + for (auto const &path : *dList.datasets) + { + switch (mpp.determineType(currentPath)) + { + + case internal::ContainedType::Particle: + std::cerr << "[Warning] Dataset found at '" + << (concatWithSep(currentPath, "/") + "/" + path) + << "' inside the particles path. A particle species is " + "always a group, never a dataset. Will parse as a " + "custom dataset. Storing custom datasets inside the " + "particles path is discouraged." + << std::endl; + [[fallthrough]]; + // Group is a bit of an internal misnomer here, it just means that + // it matches neither meshes nor particles path + case internal::ContainedType::Group: { + auto embeddedDatasets = data.embeddedDatasetsWrapped(); + auto &rc = embeddedDatasets[path]; + Parameter dOpen; + dOpen.name = path; + IOHandler()->enqueue(IOTask(&rc, dOpen)); + try + { + IOHandler()->flush(internal::defaultFlushParams); + rc.setWritten(false, Attributable::EnqueueAsynchronously::No); + rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + rc.setWritten(true, Attributable::EnqueueAsynchronously::No); + rc.read(/* read_defaults = */ false); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read contained custom dataset '" << path + << "' at path '" << myPath().openPMDPath() + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + embeddedDatasets.erase(path); + } + break; + } + case internal::ContainedType::Mesh: + try + { + readScalarMesh(meshesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read scalar mesh at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + meshesMap.forget(path); + } + break; + } + } + + for (auto const &path : constantComponentsPushback) + { + auto embeddedDatasets = data.embeddedDatasetsWrapped(); + auto &rc = embeddedDatasets[path]; + try + { + Parameter pOpen; + pOpen.path = path; + IOHandler()->enqueue(IOTask(&rc, pOpen)); + rc.get().m_isConstant = true; + rc.read(/* read_defaults = */ false); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read dataset at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + embeddedDatasets.erase(path); + } + } + setDirty(false); +} + +void CustomHierarchy::flush_internal( + internal::FlushParams const &flushParams, + internal::MeshesParticlesPath &mpp, + std::vector currentPath) +{ + if (!dirtyRecursive()) + { + return; + } + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + // No need to do anything in access::readOnly since meshes and particles + // are initialized as aliases for subgroups at parsing time + auto &data = get(); + if (access::write(IOHandler()->m_frontendAccess)) + { + flushAttributes(flushParams); + } + + Parameter pCreate; + for (auto &[name, subpath] : *this) + { + if (!subpath.written()) + { + pCreate.path = name; + IOHandler()->enqueue(IOTask(&subpath, pCreate)); + } + currentPath.emplace_back(name); + subpath.flush_internal(flushParams, mpp, currentPath); + currentPath.pop_back(); + } + for (auto &[name, mesh] : data.embeddedMeshesInternal()) + { + if (!mpp.isMeshContainer(currentPath)) + { + std::string extend_meshes_path; + // Check if this can be covered by shorthand notation + // (e.g. meshesPath == "meshes/") + if (!currentPath.empty() && + *currentPath.rbegin() == mpp.m_defaultMeshesPath) + { + extend_meshes_path = *currentPath.rbegin() + "/"; + } + else + { + // Otherwise use full path + extend_meshes_path = "/" + + (currentPath.empty() + ? "" + : concatWithSep(currentPath, "/") + "/"); + } + mpp.collectNewMeshesPaths.emplace(std::move(extend_meshes_path)); + } + mesh.flush(name, flushParams); + } + for (auto &[name, particleSpecies] : data.embeddedParticlesInternal()) + { + if (!mpp.isParticleContainer(currentPath)) + { + std::string extend_particles_path; + if (!currentPath.empty() && + *currentPath.rbegin() == mpp.m_defaultParticlesPath) + { + // Check if this can be covered by shorthand notation + // (e.g. particlesPath == "particles/") + extend_particles_path = *currentPath.rbegin() + "/"; + } + else + { + // Otherwise use full path + extend_particles_path = "/" + + (currentPath.empty() + ? "" + : concatWithSep(currentPath, "/") + "/"); + ; + } + mpp.collectNewParticlesPaths.emplace( + std::move(extend_particles_path)); + } + particleSpecies.flush(name, flushParams); + } + for (auto &[name, dataset] : get().embeddedDatasetsInternal()) + { + dataset.flush(name, flushParams, /* set_defaults = */ false); + } + + if (flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) + { + setDirty(false); + } +} + +void CustomHierarchy::flush( + std::string const & /* path */, internal::FlushParams const &) +{ + throw std::runtime_error( + "[CustomHierarchy::flush()] Don't use this method. Flushing should be " + "triggered via Iteration class."); +} + +void CustomHierarchy::linkHierarchy(Writable &w) +{ + Attributable::linkHierarchy(w); +} +} // namespace openPMD + +#undef OPENPMD_LEGAL_IDENTIFIER_CHARS +#undef OPENPMD_SINGLE_GLOBBING_CHAR +#undef OPENPMD_DOUBLE_GLOBBING_CHAR diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 5d3ce17f36..9d14be2fb9 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -628,13 +628,24 @@ ADIOS2IOHandlerImpl::flush(internal::ParsedFlushParams &flushParams) } } + std::vector sorted; + sorted.reserve(m_dirty.size()); for (auto const &file : m_dirty) { - auto file_data = m_fileData.find(file); + sorted.emplace_back(&file); + } + std::sort( + sorted.begin(), sorted.end(), [](auto const &left, auto const &right) { + return **left <= **right; + }); + + for (auto file : sorted) + { + auto file_data = m_fileData.find(*file); if (file_data == m_fileData.end()) { throw error::Internal( - "[ADIOS2 backend] No associated data found for file'" + *file + + "[ADIOS2 backend] No associated data found for file'" + **file + "'."); } file_data->second->flush( diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 88e221bea5..f8717af03e 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -24,6 +24,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/IO/Access.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSONMatcher.hpp" @@ -569,13 +570,13 @@ void JSONIOHandlerImpl::createPath( auto filepos = setAndGetFilePosition(writable, false); jsonVal = &(*jsonVal)[filepos->id]; - ensurePath(jsonVal, path); + ensurePath(jsonVal, path, m_handler->m_backendAccess); path = filepos->id.to_string() + "/" + path; } else { - ensurePath(jsonVal, path); + ensurePath(jsonVal, path, m_handler->m_backendAccess); } m_dirty.emplace(file); @@ -948,7 +949,10 @@ void JSONIOHandlerImpl::openPath( std::make_shared(json::json_pointer(path)); } - ensurePath(j, removeSlashes(parameters.path)); + ensurePath( + j, + removeSlashes(parameters.path), + /* Must not modify j */ Access::READ_ONLY); writable->written = true; } @@ -1850,18 +1854,45 @@ bool JSONIOHandlerImpl::hasKey(nlohmann::json const &j, KeyT &&key) } void JSONIOHandlerImpl::ensurePath( - nlohmann::json *jsonp, std::string const &path) + nlohmann::json *jsonp, std::string const &path, Access access) { auto groups = auxiliary::split(path, "/"); - for (std::string &group : groups) + if (access::readOnly(access)) { - // Enforce a JSON object - // the library will automatically create a list if the first - // key added to it is parseable as an int - jsonp = &(*jsonp)[group]; - if (jsonp->is_null()) + for (std::string const &group : groups) { - *jsonp = nlohmann::json::object(); + if (!jsonp->contains(group)) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::NotFound, + "JSON", + "Required group '" + path + "' not present."); + } + jsonp = &(*jsonp).at(group); + if (!jsonp->is_object()) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::UnexpectedContent, + "JSON", + "Required group '" + path + + "' is present, but not a JSON object."); + } + } + } + else + { + for (std::string const &group : groups) + { + // Enforce a JSON object + // the library will automatically create a list if the first + // key added to it is parseable as an int + jsonp = &(*jsonp)[group]; + if (jsonp->is_null()) + { + *jsonp = nlohmann::json::object(); + } } } } diff --git a/src/Iteration.cpp b/src/Iteration.cpp index ba04e73981..1f769a4ea0 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/Iteration.hpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/Error.hpp" @@ -36,12 +37,11 @@ #include "openPMD/backend/Writable.hpp" #include -#include -#include #include #include #include -#include +#include +#include #include namespace openPMD @@ -49,7 +49,7 @@ namespace openPMD using internal::CloseStatus; using internal::DeferredParseAccess; -Iteration::Iteration() : Attributable(NoInit()) +Iteration::Iteration() : CustomHierarchy(NoInit()) { setData(std::make_shared()); setTime(static_cast(0)); @@ -275,7 +275,7 @@ void Iteration::flushFileBased( case FlushLevel::SkeletonOnly: case FlushLevel::InternalFlush: case FlushLevel::UserFlush: - flush(flushParams); + flushIteration(flushParams); break; } } @@ -298,7 +298,7 @@ void Iteration::flushGroupBased( case FlushLevel::SkeletonOnly: case FlushLevel::InternalFlush: case FlushLevel::UserFlush: - flush(flushParams); + flushIteration(flushParams); break; } } @@ -321,7 +321,7 @@ void Iteration::flushVariableBased( case FlushLevel::SkeletonOnly: case FlushLevel::InternalFlush: case FlushLevel::UserFlush: - flush(flushParams); + flushIteration(flushParams); break; } @@ -347,74 +347,130 @@ void Iteration::flushVariableBased( } } -void Iteration::flush(internal::FlushParams const &flushParams) +void Iteration::flushIteration(internal::FlushParams const &flushParams) { Parameter touch; IOHandler()->enqueue(IOTask(&writable(), touch)); - if (access::readOnly(IOHandler()->m_frontendAccess)) + if (flushParams.flushLevel == FlushLevel::CreateOrOpenFiles) { - for (auto &m : meshes) - m.second.flush(m.first, flushParams); - for (auto &species : particles) - species.second.flush(species.first, flushParams); - setDirty(false); + return; } - else + + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + Series s = retrieveSeries(); + std::vector meshesPaths = s.meshesPaths(), + particlesPaths = s.particlesPaths(); + internal::MeshesParticlesPath mpp(meshesPaths, particlesPaths); + + sync_meshes_and_particles_from_alias_to_subgroups(mpp); + + std::vector currentPath; + CustomHierarchy::flush_internal(flushParams, mpp, currentPath); + + sync_meshes_and_particles_from_subgroups_to_alias(mpp); + + if (!mpp.collectNewMeshesPaths.empty() || + !mpp.collectNewParticlesPaths.empty()) { - /* Find the root point [Series] of this file, - * meshesPath and particlesPath are stored there */ - Series s = retrieveSeries(); + for (auto [newly_added_paths, vec] : + {std::make_pair(&mpp.collectNewMeshesPaths, &meshesPaths), + std::make_pair(&mpp.collectNewParticlesPaths, &particlesPaths)}) + { + std::transform( + newly_added_paths->begin(), + newly_added_paths->end(), + std::back_inserter(*vec), + [](auto const &pair) { return pair; }); + } + s.setMeshesPath(meshesPaths); + s.setParticlesPath(particlesPaths); + } - if (!meshes.empty() || s.containsAttribute("meshesPath")) + if (flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) + { + if (access::write(IOHandler()->m_frontendAccess)) { - if (!s.containsAttribute("meshesPath")) + flushAttributes(flushParams); + } + setDirty(false); + meshes.setDirty(false); + particles.setDirty(false); + } +} + +void Iteration::sync_meshes_and_particles_from_alias_to_subgroups( + internal::MeshesParticlesPath const &mpp) +{ + auto sync_meshes_and_particles = + [this](auto &m_or_p, std::string const &defaultPath) { + using type = + typename std::remove_reference_t::mapped_type; + + if (m_or_p.empty()) { - s.setMeshesPath("meshes/"); - s.flushMeshesPath(); + return; } - if (meshes.dirtyRecursive()) + auto container = (*this)[defaultPath].asContainerOf(); + + for (auto &[name, entry] : m_or_p) { - meshes.flush(s.meshesPath(), flushParams); - for (auto &m : meshes) + if (auxiliary::contains(name, '/')) + { + throw std::runtime_error( + "Unimplemented: Multi-level paths in " + "Iteration::meshes/Iteration::particles"); + } + if (auto it = container.find(name); it != container.end()) { - m.second.flush(m.first, flushParams); + if (it->second.m_attri->asSharedPtrOfAttributable() == + entry.m_attri->asSharedPtrOfAttributable()) + { + continue; // has been emplaced previously + } + else + { + throw std::runtime_error("asdfasdfasdfasd"); + } + } + else + { + container.emplace(name, entry); + entry.linkHierarchy(container.writable()); } } - } - else - { - meshes.setDirty(false); - } + }; - if (!particles.empty() || s.containsAttribute("particlesPath")) - { - if (!s.containsAttribute("particlesPath")) + sync_meshes_and_particles(meshes, mpp.m_defaultMeshesPath); + sync_meshes_and_particles(particles, mpp.m_defaultParticlesPath); +} + +void Iteration::sync_meshes_and_particles_from_subgroups_to_alias( + internal::MeshesParticlesPath const &mpp) +{ + auto sync_meshes_and_particles = + [this](auto &m_or_p, std::string const &defaultPath) { + using type = + typename std::remove_reference_t::mapped_type; + auto it = this->find(defaultPath); + if (it == this->end()) { - s.setParticlesPath("particles/"); - s.flushParticlesPath(); + return; } - if (particles.dirtyRecursive()) + auto container = it->second.asContainerOf(); + for (auto &[name, entry] : container) { - particles.flush(s.particlesPath(), flushParams); - for (auto &species : particles) - { - species.second.flush(species.first, flushParams); - } + m_or_p.emplace(name, entry); } - } - else - { - particles.setDirty(false); - } + }; - flushAttributes(flushParams); - } - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - meshes.setDirty(false); - particles.setDirty(false); - } + sync_meshes_and_particles(meshes, mpp.m_defaultMeshesPath); + sync_meshes_and_particles(particles, mpp.m_defaultParticlesPath); } void Iteration::deferParseAccess(DeferredParseAccess dr) @@ -572,68 +628,19 @@ void Iteration::read_impl(std::string const &groupPath) Attribute(Attribute::from_any, *aRead.m_resource).dtype) + ")"); - /* Find the root point [Series] of this file, - * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); Parameter pList; - auto version = IOHandler()->m_standard; - bool hasMeshes = false; - bool hasParticles = false; - if (version <= OpenpmdStandard::v_1_0_1) - { - IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(internal::defaultFlushParams); - hasMeshes = std::count( - pList.paths->begin(), - pList.paths->end(), - auxiliary::replace_last(s.meshesPath(), "/", "")) == 1; - hasParticles = - std::count( - pList.paths->begin(), - pList.paths->end(), - auxiliary::replace_last(s.particlesPath(), "/", "")) == 1; - pList.paths->clear(); - } - else - { - hasMeshes = s.containsAttribute("meshesPath"); - hasParticles = s.containsAttribute("particlesPath"); - } + IOHandler()->enqueue(IOTask(this, pList)); - if (hasMeshes) - { - try - { - readMeshes(s.meshesPath()); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read meshes in iteration " << groupPath - << " and will skip them due to read error:\n" - << err.what() << std::endl; - meshes.container().clear(); - } - } - meshes.setDirty(false); + // @todo restore compatibility with openPMD 1.0.*: + // hasMeshes <-> meshesPath is defined - if (hasParticles) - { - try - { - readParticles(s.particlesPath()); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read particles in iteration " << groupPath - << " and will skip them due to read error:\n" - << err.what() << std::endl; - particles.container().clear(); - } - } - particles.setDirty(false); + internal::MeshesParticlesPath mpp(s.meshesPaths(), s.particlesPaths()); + CustomHierarchy::read(mpp); + + sync_meshes_and_particles_from_subgroups_to_alias(mpp); - readAttributes(ReadMode::FullyReread); #ifdef openPMD_USE_INVASIVE_TESTS if (containsAttribute("__openPMD_internal_fail")) { @@ -650,125 +657,6 @@ void Iteration::read_impl(std::string const &groupPath) #endif } -void Iteration::readMeshes(std::string const &meshesPath) -{ - Parameter pOpen; - Parameter pList; - - pOpen.path = meshesPath; - IOHandler()->enqueue(IOTask(&meshes, pOpen)); - - meshes.readAttributes(ReadMode::FullyReread); - - internal::EraseStaleEntries map{meshes}; - - /* obtain all non-scalar meshes */ - IOHandler()->enqueue(IOTask(&meshes, pList)); - IOHandler()->flush(internal::defaultFlushParams); - - Parameter aList; - for (auto const &mesh_name : *pList.paths) - { - Mesh &m = map[mesh_name]; - pOpen.path = mesh_name; - aList.attributes->clear(); - IOHandler()->enqueue(IOTask(&m, pOpen)); - IOHandler()->enqueue(IOTask(&m, aList)); - IOHandler()->flush(internal::defaultFlushParams); - - auto att_begin = aList.attributes->begin(); - auto att_end = aList.attributes->end(); - auto value = std::find(att_begin, att_end, "value"); - auto shape = std::find(att_begin, att_end, "shape"); - if (value != att_end && shape != att_end) - { - MeshRecordComponent &mrc = m; - IOHandler()->enqueue(IOTask(&mrc, pOpen)); - IOHandler()->flush(internal::defaultFlushParams); - mrc.get().m_isConstant = true; - } - try - { - m.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read mesh with name '" << mesh_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(mesh_name); - } - } - - /* obtain all scalar meshes */ - Parameter dList; - IOHandler()->enqueue(IOTask(&meshes, dList)); - IOHandler()->flush(internal::defaultFlushParams); - - Parameter dOpen; - for (auto const &mesh_name : *dList.datasets) - { - Mesh &m = map[mesh_name]; - dOpen.name = mesh_name; - IOHandler()->enqueue(IOTask(&m, dOpen)); - IOHandler()->flush(internal::defaultFlushParams); - MeshRecordComponent &mrc = m; - IOHandler()->enqueue(IOTask(&mrc, dOpen)); - IOHandler()->flush(internal::defaultFlushParams); - mrc.setWritten(false, Attributable::EnqueueAsynchronously::No); - mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); - mrc.setWritten(true, Attributable::EnqueueAsynchronously::No); - try - { - m.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read mesh with name '" << mesh_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(mesh_name); - } - } -} - -void Iteration::readParticles(std::string const &particlesPath) -{ - Parameter pOpen; - Parameter pList; - - pOpen.path = particlesPath; - IOHandler()->enqueue(IOTask(&particles, pOpen)); - - particles.readAttributes(ReadMode::FullyReread); - - /* obtain all particle species */ - pList.paths->clear(); - IOHandler()->enqueue(IOTask(&particles, pList)); - IOHandler()->flush(internal::defaultFlushParams); - - internal::EraseStaleEntries map{particles}; - for (auto const &species_name : *pList.paths) - { - ParticleSpecies &p = map[species_name]; - pOpen.path = species_name; - IOHandler()->enqueue(IOTask(&p, pOpen)); - IOHandler()->flush(internal::defaultFlushParams); - try - { - p.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read particle species with name '" - << species_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(species_name); - } - } -} - auto Iteration::beginStep(bool reread) -> BeginStepStatus { BeginStepStatus res; @@ -944,6 +832,29 @@ void Iteration::setStepStatus(StepStatus status) } } +bool Iteration::dirtyRecursive() const +{ + if (dirty() || CustomHierarchy::dirtyRecursive()) + { + return true; + } + for (auto const &pair : particles) + { + if (!pair.second.written()) + { + return true; + } + } + for (auto const &pair : meshes) + { + if (!pair.second.written()) + { + return true; + } + } + return false; +} + void Iteration::linkHierarchy(Writable &w) { Attributable::linkHierarchy(w); diff --git a/src/ParticlePatches.cpp b/src/ParticlePatches.cpp index 491add8be7..c7fac4b654 100644 --- a/src/ParticlePatches.cpp +++ b/src/ParticlePatches.cpp @@ -100,7 +100,7 @@ void ParticlePatches::read() pr.setDirty(false); try { - prc.PatchRecordComponent::read(/* require_unit_si = */ false); + prc.PatchRecordComponent::read(/* read_defaults = */ false); } catch (error::ReadError const &err) { diff --git a/src/Record.cpp b/src/Record.cpp index c6baedec7c..ea65ce08ee 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -60,12 +60,14 @@ void Record::flush_impl( { if (scalar()) { - T_RecordComponent::flush(SCALAR, flushParams); + T_RecordComponent::flush( + SCALAR, flushParams, /* set_defaults = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } } else @@ -75,7 +77,7 @@ void Record::flush_impl( if (scalar()) { RecordComponent &rc = *this; - rc.flush(name, flushParams); + rc.flush(name, flushParams, /* set_defaults = */ true); } else { @@ -85,7 +87,8 @@ void Record::flush_impl( for (auto &comp : *this) { comp.second.parent() = getWritable(this); - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } } } @@ -94,12 +97,14 @@ void Record::flush_impl( if (scalar()) { - T_RecordComponent::flush(name, flushParams); + T_RecordComponent::flush( + name, flushParams, /* set_defaults = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } } @@ -114,7 +119,7 @@ void Record::read() /* using operator[] will incorrectly update parent */ try { - T_RecordComponent::read(/* require_unit_si = */ true); + T_RecordComponent::read(/* read_defaults = */ true); } catch (error::ReadError const &err) { @@ -138,7 +143,7 @@ void Record::read() rc.get().m_isConstant = true; try { - rc.read(/* require_unit_si = */ true); + rc.read(/* read_defaults = */ true); } catch (error::ReadError const &err) { @@ -165,7 +170,7 @@ void Record::read() rc.setWritten(true, Attributable::EnqueueAsynchronously::No); try { - rc.read(/* require_unit_si = */ true); + rc.read(/* read_defaults = */ true); } catch (error::ReadError const &err) { diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 6d11fbfa77..a1c583b649 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -253,7 +253,9 @@ bool RecordComponent::empty() const } void RecordComponent::flush( - std::string const &name, internal::FlushParams const &flushParams) + std::string const &name, + internal::FlushParams const &flushParams, + bool set_defaults) { if (!dirtyRecursive()) { @@ -296,7 +298,7 @@ void RecordComponent::flush( "RecordComponent::resetDataset())."); } } - if (!containsAttribute("unitSI")) + if (set_defaults && !containsAttribute("unitSI")) { setUnitSI(1); } @@ -380,9 +382,9 @@ void RecordComponent::flush( } } -void RecordComponent::read(bool require_unit_si) +void RecordComponent::read(bool read_defaults) { - readBase(require_unit_si); + readBase(read_defaults); } namespace @@ -407,7 +409,7 @@ namespace }; } // namespace -void RecordComponent::readBase(bool require_unit_si) +void RecordComponent::readBase(bool read_defaults) { using DT = Datatype; // auto & rc = get(); @@ -455,7 +457,7 @@ void RecordComponent::readBase(bool require_unit_si) readAttributes(ReadMode::FullyReread); - if (require_unit_si) + if (read_defaults) { if (!containsAttribute("unitSI")) { diff --git a/src/Series.cpp b/src/Series.cpp index 65ee793745..3f3f84da5f 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -191,7 +191,27 @@ Series &Series::setBasePath(std::string const &bp) std::string Series::meshesPath() const { - return getAttribute("meshesPath").get(); + auto res = meshesPaths(); + if (res.empty()) + { + throw no_such_attribute_error("meshesPath"); + } + /* + * @todo: Verify that meshesPath has canonical form + */ + return res.at(0); +} + +std::vector Series::meshesPaths() const +{ + if (containsAttribute("meshesPath")) + { + return getAttribute("meshesPath").get>(); + } + else + { + return {}; + } } Series &Series::setMeshesPath(std::string const &mp) @@ -214,6 +234,23 @@ Series &Series::setMeshesPath(std::string const &mp) setDirty(true); return *this; } +Series &Series::setMeshesPath(std::vector const &mp) +{ + // @todo if already written, then append + switch (mp.size()) + { + case 0: + return *this; + case 1: + setAttribute("meshesPath", *mp.begin()); + break; + default: + setAttribute("meshesPath", mp); + break; + } + setDirty(true); + return *this; +} std::vector Series::availableDatasets() { @@ -510,7 +547,27 @@ void Series::flushRankTable() std::string Series::particlesPath() const { - return getAttribute("particlesPath").get(); + auto res = particlesPaths(); + if (res.empty()) + { + throw no_such_attribute_error("particlesPath"); + } + /* + * @todo: Verify that particlesPath has canonical form + */ + return res.at(0); +} + +std::vector Series::particlesPaths() const +{ + if (containsAttribute("particlesPath")) + { + return getAttribute("particlesPath").get>(); + } + else + { + return {}; + } } Series &Series::setParticlesPath(std::string const &pp) @@ -533,6 +590,23 @@ Series &Series::setParticlesPath(std::string const &pp) setDirty(true); return *this; } +Series &Series::setParticlesPath(std::vector const &pp) +{ + // @todo if already written, then append + switch (pp.size()) + { + case 0: + return *this; + case 1: + setAttribute("particlesPath", *pp.begin()); + break; + default: + setAttribute("particlesPath", pp); + break; + } + setDirty(true); + return *this; +} std::string Series::author() const { @@ -1364,7 +1438,7 @@ void Series::flushFileBased( break; case IO::HasBeenOpened: // continue below - it->second.flush(flushParams); + it->second.flushIteration(flushParams); break; } @@ -1392,6 +1466,10 @@ void Series::flushFileBased( bool allDirty = dirty(); for (auto it = begin; it != end; ++it) { + /* reset the dirty bit for every iteration (i.e. file) + * otherwise only the first iteration will have updates attributes + */ + setDirty(allDirty); // Phase 1 switch (openIterationIfDirty(it->first, it->second)) { @@ -1437,12 +1515,7 @@ void Series::flushFileBased( IOHandler()->enqueue(IOTask(&it->second, std::move(fClose))); it->second.get().m_closed = internal::CloseStatus::Closed; } - /* reset the dirty bit for every iteration (i.e. file) - * otherwise only the first iteration will have updates attributes - */ - setDirty(allDirty); } - setDirty(false); // Phase 3 if (flushIOHandler) @@ -1483,7 +1556,7 @@ void Series::flushGorVBased( series.m_snapshotToStep.at(it->first)}; IOHandler()->enqueue(IOTask(this, std::move(param))); } - it->second.flush(flushParams); + it->second.flushIteration(flushParams); break; } @@ -1584,26 +1657,6 @@ void Series::flushGorVBased( } } -void Series::flushMeshesPath() -{ - Parameter aWrite; - aWrite.name = "meshesPath"; - Attribute a = getAttribute("meshesPath"); - aWrite.m_resource = a.getAny(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - -void Series::flushParticlesPath() -{ - Parameter aWrite; - aWrite.name = "particlesPath"; - Attribute a = getAttribute("particlesPath"); - aWrite.m_resource = a.getAny(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - void Series::readFileBased( std::optional read_only_this_single_iteration) { @@ -2373,7 +2426,7 @@ void Series::readBase() IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) - .getOptional(); + .getOptional>(); val.has_value()) { /* allow setting the meshes path after completed IO */ @@ -2415,7 +2468,7 @@ void Series::readBase() IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) - .getOptional(); + .getOptional>(); val.has_value()) { /* allow setting the meshes path after completed IO */ diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 6de68ffcd2..ab4d105a42 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -304,10 +304,10 @@ void Attributable::flushAttributes(internal::FlushParams const &flushParams) } } // Do this outside the if branch to also setDirty to dirtyRecursive - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + assert( + flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles); + setDirty(false); } void Attributable::readAttributes(ReadMode mode) diff --git a/src/backend/Container.cpp b/src/backend/Container.cpp index 7d421ac1a9..d8fe8bd5dd 100644 --- a/src/backend/Container.cpp +++ b/src/backend/Container.cpp @@ -21,6 +21,7 @@ #include "openPMD/backend/ContainerImpl.tpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticlePatches.hpp" @@ -33,6 +34,7 @@ namespace openPMD #define OPENPMD_COMMA , #define OPENPMD_INSTANTIATE(type) template class Container; +OPENPMD_INSTANTIATE(CustomHierarchy) OPENPMD_INSTANTIATE(Mesh) OPENPMD_INSTANTIATE(MeshRecordComponent) OPENPMD_INSTANTIATE(ParticlePatches) diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index 947db30f7c..dee94d0a85 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -66,7 +66,7 @@ void MeshRecordComponent::read() Attribute(Attribute::from_any, *aRead.m_resource).dtype) + ")"); - readBase(/* require_unit_si = */ true); + readBase(/* read_defaults = */ true); } void MeshRecordComponent::flush( @@ -81,7 +81,7 @@ void MeshRecordComponent::flush( { setPosition(std::vector{0}); } - RecordComponent::flush(name, params); + RecordComponent::flush(name, params, /* set_defaults = */ true); } template diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index e674828181..8fde033060 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -52,10 +52,11 @@ void PatchRecord::flush_impl( path, flushParams); // warning (clang-tidy-10): // bugprone-parent-virtual-call for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } else - T_RecordComponent::flush(path, flushParams); + T_RecordComponent::flush(path, flushParams, /* set_defaults = */ true); if (flushParams.flushLevel != FlushLevel::SkeletonOnly) { setDirty(false); @@ -101,7 +102,7 @@ void PatchRecord::read() prc.setWritten(true, Attributable::EnqueueAsynchronously::No); try { - prc.read(/* require_unit_si = */ false); + prc.read(/* read_defaults = */ false); } catch (error::ReadError const &err) { diff --git a/src/binding/python/Container.cpp b/src/binding/python/Container.cpp new file mode 100644 index 0000000000..d49185e57d --- /dev/null +++ b/src/binding/python/Container.cpp @@ -0,0 +1,71 @@ +/* Copyright 2018-2021 Axel Huebl + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + * + * The function `bind_container` is based on std_bind.h in pybind11 + * Copyright (c) 2016 Sergey Lyskov and Wenzel Jakob + * + * BSD-style license, see pybind11 LICENSE file. + */ + +#include + +#include "openPMD/Iteration.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticlePatches.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/Record.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/BaseRecordComponent.hpp" +#include "openPMD/backend/Container.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/PatchRecord.hpp" +#include "openPMD/backend/PatchRecordComponent.hpp" +#include "openPMD/binding/python/Container.hpp" + +#include "openPMD/binding/python/Common.hpp" + +void init_Container(py::module &m) +{ + ::detail::create_and_bind_container( + m, "Iteration_Container"); + ::detail::create_and_bind_container( + m, "Mesh_Container"); + ::detail::create_and_bind_container( + m, "Particle_Container"); + ::detail::create_and_bind_container( + m, "Particle_Patches_Container"); + ::detail::create_and_bind_container( + m, "Record_Container"); + ::detail::create_and_bind_container( + m, "Patch_Record_Container"); + ::detail:: + create_and_bind_container( + m, "Record_Component_Container"); + ::detail:: + create_and_bind_container( + m, "Mesh_Record_Component_Container"); + ::detail::create_and_bind_container< + PyPatchRecordComponentContainer, + Attributable>(m, "Patch_Record_Component_Container"); + ::detail:: + create_and_bind_container( + m, "Custom_Hierarchy_Container"); +} diff --git a/src/binding/python/CustomHierarchy.cpp b/src/binding/python/CustomHierarchy.cpp new file mode 100644 index 0000000000..b893d6a174 --- /dev/null +++ b/src/binding/python/CustomHierarchy.cpp @@ -0,0 +1,52 @@ + + +#include "openPMD/CustomHierarchy.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/backend/Attributable.hpp" +#include "openPMD/binding/python/Common.hpp" +#include "openPMD/binding/python/Container.H" +#include + +namespace py = pybind11; +using namespace openPMD; + +template +void define_conversible_container(py::module &m, std::string const &name) +{ + using CC = ConversibleContainer; + py::class_, Attributable>(m, name.c_str()) + .def( + "as_container_of_datasets", + &CC::template asContainerOf) + .def("as_container_of_meshes", &CC::template asContainerOf) + .def( + "as_container_of_particles", + &CC::template asContainerOf) + .def( + "as_container_of_custom_hierarchy", + &CC::template asContainerOf); +} + +void init_CustomHierarchy(py::module &m) +{ + auto py_ch_cont = + declare_container( + m, "Container_CustomHierarchy"); + + define_conversible_container( + m, "ConversibleContainer_CustomHierarchy"); + define_conversible_container( + m, "ConversibleContainer_ParticleSpecies"); + define_conversible_container( + m, "ConversibleContainer_RecordComponent"); + define_conversible_container(m, "ConversibleContainer_Mesh"); + + [[maybe_unused]] py::class_< + CustomHierarchy, + ConversibleContainer, + Container, + Attributable> custom_hierarchy(m, "CustomHierarchy"); + + finalize_container(py_ch_cont); +} diff --git a/src/binding/python/Iteration.cpp b/src/binding/python/Iteration.cpp index cd5fecacb0..4c92062e61 100644 --- a/src/binding/python/Iteration.cpp +++ b/src/binding/python/Iteration.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/Iteration.hpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/binding/python/Common.hpp" @@ -40,8 +41,11 @@ void init_Iteration(py::module &m) #define OPENPMD_AVOID_CLANG_FORMAT auto cl = OPENPMD_AVOID_CLANG_FORMAT #undef OPENPMD_AVOID_CLANG_FORMAT - - py::class_(m, "Iteration") + py::class_< + Iteration, + CustomHierarchy, + PyCustomHierarchyContainer, + Attributable>(m, "Iteration") .def(py::init()) .def( diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index f35ea1d301..833ad9b46f 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -24,6 +24,7 @@ #include "openPMD/Iteration.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/auxiliary/JSON.hpp" +#include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Container.H" @@ -37,6 +38,7 @@ #include #include #include +#include #if openPMD_USE_FILESYSTEM_HEADER #include @@ -440,13 +442,61 @@ this method. &Series::openPMDextension, &Series::setOpenPMDextension) .def_property("base_path", &Series::basePath, &Series::setBasePath) - .def_property( - "meshes_path", &Series::meshesPath, &Series::setMeshesPath) .def_property_readonly("has_rank_table_read", &Series::hasRankTableRead) .def("get_rank_table", &Series::rankTable, py::arg("collective")) .def("set_rank_table", &Series::setRankTable, py::arg("my_rank_info")) .def_property( - "particles_path", &Series::particlesPath, &Series::setParticlesPath) + "meshes_path", + [](Series &self) + -> std::variant> { + using res_t = + std::variant>; + auto res = self.meshesPaths(); + if (res.size() == 1) + { + return res_t{std::move(res[0])}; + } + else + { + return res_t{std::move(res)}; + } + }, + [](Series &self, + std::variant> const &arg) + -> Series & { + std::visit( + [&](auto const &arg_resolved) { + self.setMeshesPath(arg_resolved); + }, + arg); + return self; + }) + .def_property( + "particles_path", + [](Series &self) + -> std::variant> { + using res_t = + std::variant>; + auto res = self.particlesPaths(); + if (res.size() == 1) + { + return res_t{std::move(res[0])}; + } + else + { + return res_t{std::move(res)}; + } + }, + [](Series &self, + std::variant> const &arg) + -> Series & { + std::visit( + [&](auto const &arg_resolved) { + self.setParticlesPath(arg_resolved); + }, + arg); + return self; + }) .def_property("author", &Series::author, &Series::setAuthor) .def_property( "machine", @@ -489,8 +539,20 @@ this method. .def("set_openPMD", &Series::setOpenPMD) .def("set_openPMD_extension", &Series::setOpenPMDextension) .def("set_base_path", &Series::setBasePath) - .def("set_meshes_path", &Series::setMeshesPath) - .def("set_particles_path", &Series::setParticlesPath) + .def( + "set_meshes_path", + py::overload_cast(&Series::setMeshesPath)) + .def( + "set_meshes_path", + py::overload_cast const &>( + &Series::setMeshesPath)) + .def( + "set_particles_path", + py::overload_cast const &>( + &Series::setParticlesPath)) + .def( + "set_particles_path", + py::overload_cast(&Series::setParticlesPath)) .def("set_author", &Series::setAuthor) .def("set_date", &Series::setDate) .def("set_iteration_encoding", &Series::setIterationEncoding) diff --git a/src/binding/python/openPMD.cpp b/src/binding/python/openPMD.cpp index fe26bfced8..996208bd55 100644 --- a/src/binding/python/openPMD.cpp +++ b/src/binding/python/openPMD.cpp @@ -37,6 +37,7 @@ void init_Dataset(py::module &); void init_Datatype(py::module &); void init_Error(py::module &); void init_Helper(py::module &); +void init_CustomHierarchy(py::module &); void init_Iteration(py::module &); void init_IterationEncoding(py::module &); void init_Mesh(py::module &); @@ -101,6 +102,7 @@ PYBIND11_MODULE(openpmd_api_cxx, m) init_ParticleSpecies(m); init_Mesh(m); + init_CustomHierarchy(m); init_Iteration(m); init_IterationEncoding(m); init_Series(m); diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 6c62e1d82e..882c39be18 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -379,6 +379,252 @@ TEST_CASE("attribute_dtype_test", "[core]") } } +TEST_CASE("custom_hierarchies", "[core]") +{ + std::string filePath = "../samples/custom_hierarchies.json"; + Series write(filePath, Access::CREATE); + write.iterations[0]; + write.close(); + + Series read(filePath, Access::READ_ONLY); + REQUIRE(read.iterations[0].size() == 0); + read.close(); + + write = Series(filePath, Access::READ_WRITE); + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"].setAttribute("string", "attribute"); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + write.close(); + + read = Series(filePath, Access::READ_ONLY); + REQUIRE(read.iterations[0].size() == 2); + REQUIRE(read.iterations[0].count("custom") == 1); + REQUIRE(read.iterations[0].count("no_attributes") == 1); + REQUIRE(read.iterations[0]["custom"].size() == 1); + REQUIRE(read.iterations[0]["custom"].count("hierarchy") == 1); + REQUIRE(read.iterations[0]["custom"]["hierarchy"].size() == 0); + REQUIRE(read.iterations[0]["no_attributes"].size() == 0); + REQUIRE( + read.iterations[0]["custom"] + .getAttribute("string") + .get() == "attribute"); + REQUIRE( + read.iterations[0]["custom"]["hierarchy"] + .getAttribute("number") + .get() == 3); + read.close(); + + write = Series(filePath, Access::READ_WRITE); + { + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"] + .asContainerOf()["emptyDataset"] + .makeEmpty(Datatype::FLOAT, 3); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + auto iteration_level_ds = + write.iterations[0] + .asContainerOf()["iteration_level_dataset"]; + iteration_level_ds.resetDataset({Datatype::INT, {10}}); + std::vector data(10, 5); + iteration_level_ds.storeChunk(data); + + auto meshesViaAlias = write.iterations[0].meshes; + meshesViaAlias["E"]["x"].makeEmpty(2); + write.setMeshesPath(std::vector{"fields/", "%%/meshes/"}); + auto meshesManually = + write.iterations[0]["fields"].asContainerOf(); + REQUIRE(meshesManually.size() == 0); + write.flush(); // Synchronized upon flushing + REQUIRE(meshesManually.contains("E")); + REQUIRE(meshesManually.size() == 1); + meshesManually["B"]["x"].makeEmpty(2); + REQUIRE(meshesViaAlias.size() == 1); + write.flush(); + REQUIRE(meshesViaAlias.contains("B")); + REQUIRE(meshesViaAlias.size() == 2); + + write.setParticlesPath("species"); + auto particlesManually = + write.iterations[0]["species"].asContainerOf(); + particlesManually["e"]["position"]["x"].makeEmpty(1); + auto particlesViaAlias = write.iterations[0].particles; + particlesViaAlias["i"]["position"]["x"].makeEmpty(1); + + write.close(); + } + + read = Series(filePath, Access::READ_ONLY); + { + REQUIRE(read.iterations[0].size() == 4); + REQUIRE(read.iterations[0].count("custom") == 1); + REQUIRE(read.iterations[0].count("no_attributes") == 1); + REQUIRE(read.iterations[0].count("fields") == 1); + REQUIRE(read.iterations[0].count("species") == 1); + REQUIRE(read.iterations[0]["custom"].size() == 1); + REQUIRE(read.iterations[0]["custom"].count("hierarchy") == 1); + REQUIRE(read.iterations[0]["custom"]["hierarchy"].size() == 0); + REQUIRE(read.iterations[0]["no_attributes"].size() == 0); + REQUIRE(read.iterations[0]["fields"].asContainerOf().size() == 2); + REQUIRE( + read.iterations[0]["fields"].asContainerOf().contains("E")); + REQUIRE( + read.iterations[0]["fields"].asContainerOf().contains("B")); + REQUIRE(read.iterations[0].meshes.size() == 2); + REQUIRE(read.iterations[0].meshes.contains("E")); + REQUIRE(read.iterations[0].meshes.contains("B")); + REQUIRE( + read.iterations[0]["species"] + .asContainerOf() + .size() == 2); + REQUIRE(read.iterations[0]["species"] + .asContainerOf() + .contains("e")); + REQUIRE(read.iterations[0]["species"] + .asContainerOf() + .contains("i")); + REQUIRE(read.iterations[0].particles.size() == 2); + REQUIRE(read.iterations[0].particles.contains("e")); + REQUIRE(read.iterations[0].particles.contains("i")); + + REQUIRE( + read.iterations[0].asContainerOf().size() == 1); + REQUIRE( + read.iterations[0]["custom"] + .asContainerOf() + .size() == 1); + REQUIRE( + read.iterations[0]["custom"]["hierarchy"] + .asContainerOf() + .size() == 0); + REQUIRE( + read.iterations[0]["no_attributes"] + .asContainerOf() + .size() == 0); + + auto iteration_level_ds = + read.iterations[0] + .asContainerOf()["iteration_level_dataset"]; + REQUIRE(iteration_level_ds.getDatatype() == Datatype::INT); + REQUIRE(iteration_level_ds.getExtent() == Extent{10}); + auto loaded_chunk = iteration_level_ds.loadChunk(); + iteration_level_ds.seriesFlush(); + for (size_t i = 0; i < 10; ++i) + { + REQUIRE(loaded_chunk.get()[i] == 5); + } + + auto constant_dataset = + read.iterations[0]["custom"] + .asContainerOf()["emptyDataset"]; + REQUIRE(constant_dataset.getDatatype() == Datatype::FLOAT); + REQUIRE(constant_dataset.getExtent() == Extent{0, 0, 0}); + } + read.close(); + + write = Series(filePath, Access::READ_WRITE); + { + std::vector data(10, 3); + + auto E_x = write.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"]; + E_x.resetDataset({Datatype::INT, {10}}); + E_x.storeChunk(data, {0}, {10}); + + auto e_pos_x = + write.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"]; + e_pos_x.resetDataset({Datatype::INT, {10}}); + e_pos_x.storeChunk(data, {0}, {10}); + write.close(); + } + + read = Series(filePath, Access::READ_ONLY); + { + auto it0 = read.iterations[0]; + auto custom_meshes = it0["custom_meshes"]; + REQUIRE(custom_meshes["meshes"].asContainerOf().size() == 1); + REQUIRE( + read.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf() + .count("E") == 1); + auto E_x_loaded = read.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"] + .loadChunk(); + REQUIRE( + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf() + .size() == 1); + REQUIRE( + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf() + .count("e") == 1); + auto e_pos_x_loaded = + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"] + .loadChunk(); + read.flush(); + + for (size_t i = 0; i < 10; ++i) + { + REQUIRE(E_x_loaded.get()[i] == 3); + REQUIRE(e_pos_x_loaded.get()[i] == 3); + } + } +} + +TEST_CASE("custom_hierarchies_no_rw", "[core]") +{ + std::string filePath = "../samples/custom_hierarchies_no_rw.json"; + Series write(filePath, Access::CREATE); + write.setMeshesPath(std::vector{"%%/meshes/"}); + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"].setAttribute("string", "attribute"); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + + { + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"] + .asContainerOf()["emptyDataset"] + .makeEmpty(Datatype::FLOAT, 3); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + auto iteration_level_ds = + write.iterations[0] + .asContainerOf()["iteration_level_dataset"]; + iteration_level_ds.resetDataset({Datatype::INT, {10}}); + std::vector data(10, 5); + iteration_level_ds.storeChunk(data); + write.flush(); + } + + { + std::vector data(10, 3); + + auto E_x = write.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"]; + E_x.resetDataset({Datatype::INT, {10}}); + E_x.storeChunk(data, {0}, {10}); + + auto e_pos_x = + write.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"]; + e_pos_x.resetDataset({Datatype::INT, {10}}); + e_pos_x.storeChunk(data, {0}, {10}); + + auto gnihihi = write.iterations[0]["custom_particles"]["particles"] + .asContainerOf(); + auto dataset = gnihihi["custom_dataset"]; + dataset.resetDataset({Datatype::INT, {10}}); + dataset.storeChunk(std::unique_ptr(new int[10]{}), {0}, {10}); + write.close(); + } + + Series read(filePath, Access::READ_ONLY); +} + TEST_CASE("myPath", "[core]") { #if openPMD_USE_INVASIVE_TESTS @@ -409,7 +655,8 @@ TEST_CASE("myPath", "[core]") recordComponent.template makeConstant(5678); }; - REQUIRE(pathOf(iteration.meshes) == vec_t{"data", "1234", "meshes"}); + // iteration.meshes is only an alias without a path of its own + // REQUIRE(pathOf(iteration.meshes) == vec_t{"data", "1234", "meshes"}); auto scalarMesh = iteration.meshes["e_chargeDensity"]; REQUIRE( @@ -430,7 +677,11 @@ TEST_CASE("myPath", "[core]") pathOf(vectorMeshComponent) == vec_t{"data", "1234", "meshes", "E", "x"}); - REQUIRE(pathOf(iteration.particles) == vec_t{"data", "1234", "particles"}); + // iteration.particles is only an alias without a path of its own + // REQUIRE(pathOf(iteration.particles) == vec_t{"data", "1234", + // "particles"}); REQUIRE( + // pathOf(iteration.particles) == + // vec_t{"iterations", "1234", "particles"}); auto speciesE = iteration.particles["e"]; REQUIRE(pathOf(speciesE) == vec_t{"data", "1234", "particles", "e"}); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 432dd864f3..4331d19d13 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -2819,9 +2819,17 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") REQUIRE( o.iterations[100].meshes.parent() == getWritable(&o.iterations[100])); + REQUIRE( + getWritable( + &o.iterations[100]["fields"].asContainerOf()["E"]) == + getWritable(&o.iterations[100].meshes["E"])); + REQUIRE( + o.iterations[100]["fields"].asContainerOf()["E"].parent() == + &o.iterations[100]["fields"].asContainerOf().writable()); REQUIRE( o.iterations[100].meshes["E"].parent() == - getWritable(&o.iterations[100].meshes)); + // Iteration::meshes is only an alias, this is the actual parent + &o.iterations[100]["fields"].asContainerOf().writable()); REQUIRE( o.iterations[100].meshes["E"]["x"].parent() == getWritable(&o.iterations[100].meshes["E"])); @@ -2831,13 +2839,17 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") REQUIRE( o.iterations[100].meshes["E"]["z"].parent() == getWritable(&o.iterations[100].meshes["E"])); + REQUIRE( + getWritable(&o.iterations[100].meshes["rho"]) == + getWritable( + &o.iterations[100]["fields"].asContainerOf()["rho"])); REQUIRE( o.iterations[100].meshes["rho"].parent() == - getWritable(&o.iterations[100].meshes)); + getWritable(&o.iterations[100]["fields"])); REQUIRE( o.iterations[100] .meshes["rho"][MeshRecordComponent::SCALAR] - .parent() == getWritable(&o.iterations[100].meshes)); + .parent() == getWritable(&o.iterations[100]["fields"])); REQUIRE_THROWS_AS( o.iterations[100].meshes["cherries"], std::out_of_range); REQUIRE( @@ -2845,7 +2857,11 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") getWritable(&o.iterations[100])); REQUIRE( o.iterations[100].particles["electrons"].parent() == - getWritable(&o.iterations[100].particles)); + getWritable(&o.iterations[100]["particles"])); + REQUIRE( + getWritable(&o.iterations[100].particles["electrons"]) == + getWritable(&o.iterations[100]["particles"] + .asContainerOf()["electrons"])); REQUIRE( o.iterations[100].particles["electrons"]["charge"].parent() == getWritable(&o.iterations[100].particles["electrons"]));