Python: Fix ODR Violation (#1521)

In pybind11 (and nanobind), auxiliary headers and
`PYBIND11_MAKE_OPAQUE` definitions for distributed modules need to be
included in every translation unit. Otherwise, a one-definition-rule
violation occurs, which is undefined behavior.

Author: ax3l

include/openPMD/binding/python/Common.hpp

@@ -0,0 +1,55 @@
+/* Copyright 2023 The openPMD Community
+ *
+ * This header is used to centrally define classes that shall not violate the
+ * C++ one-definition-rule (ODR) for various Python translation units.
+ *
+ * Authors: Axel Huebl
+ * License: LGPL-3.0-or-later
+ */
+#pragma once
+
+#include "openPMD/Iteration.hpp"
+#include "openPMD/Mesh.hpp"
+#include "openPMD/ParticlePatches.hpp"
+#include "openPMD/ParticleSpecies.hpp"
+#include "openPMD/Record.hpp"
+#include "openPMD/Series.hpp"
+#include "openPMD/backend/BaseRecord.hpp"
+#include "openPMD/backend/BaseRecordComponent.hpp"
+#include "openPMD/backend/MeshRecordComponent.hpp"
+#include "openPMD/backend/PatchRecord.hpp"
+#include "openPMD/backend/PatchRecordComponent.hpp"
+
+#include <pybind11/gil.h>
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <pybind11/stl_bind.h>
+// not yet used:
+//   pybind11/functional.h  // for std::function
+
+// used exclusively in all our Python .cpp files
+namespace py = pybind11;
+using namespace openPMD;
+
+// opaque types
+using PyIterationContainer = Series::IterationsContainer_t;
+using PyMeshContainer = Container<Mesh>;
+using PyPartContainer = Container<ParticleSpecies>;
+using PyPatchContainer = Container<ParticlePatches>;
+using PyRecordContainer = Container<Record>;
+using PyPatchRecordContainer = Container<PatchRecord>;
+using PyRecordComponentContainer = Container<RecordComponent>;
+using PyMeshRecordComponentContainer = Container<MeshRecordComponent>;
+using PyPatchRecordComponentContainer = Container<PatchRecordComponent>;
+using PyBaseRecordComponentContainer = Container<BaseRecordComponent>;
+PYBIND11_MAKE_OPAQUE(PyIterationContainer)
+PYBIND11_MAKE_OPAQUE(PyMeshContainer)
+PYBIND11_MAKE_OPAQUE(PyPartContainer)
+PYBIND11_MAKE_OPAQUE(PyPatchContainer)
+PYBIND11_MAKE_OPAQUE(PyRecordContainer)
+PYBIND11_MAKE_OPAQUE(PyPatchRecordContainer)
+PYBIND11_MAKE_OPAQUE(PyRecordComponentContainer)
+PYBIND11_MAKE_OPAQUE(PyMeshRecordComponentContainer)
+PYBIND11_MAKE_OPAQUE(PyPatchRecordComponentContainer)
+PYBIND11_MAKE_OPAQUE(PyBaseRecordComponentContainer)

include/openPMD/binding/python/Numpy.hpp

@@ -22,9 +22,7 @@
 
 #include "openPMD/Datatype.hpp"
 
-#include <pybind11/numpy.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
+#include "Common.hpp"
 
 #include <exception>
 #include <string>

include/openPMD/binding/python/Pickle.hpp

@@ -24,8 +24,7 @@
 #include "openPMD/Series.hpp"
 #include "openPMD/backend/Attributable.hpp"
 
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
+#include "Common.hpp"
 
 #include <exception>
 #include <string>
@@ -46,8 +45,6 @@ template <typename... T_Args, typename T_SeriesAccessor>
 inline void
 add_pickle(pybind11::class_<T_Args...> &cl, T_SeriesAccessor &&seriesAccessor)
 {
-    namespace py = pybind11;
-
     // helper: get first class in py::class_ - that's the type we pickle
     using PickledClass =
         typename std::tuple_element<0, std::tuple<T_Args...> >::type;

src/binding/python/Access.cpp

@@ -18,13 +18,9 @@
  * and the GNU Lesser General Public License along with openPMD-api.
  * If not, see <http://www.gnu.org/licenses/>.
  */
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
-
 #include "openPMD/IO/Access.hpp"
 
-namespace py = pybind11;
-using namespace openPMD;
+#include "openPMD/binding/python/Common.hpp"
 
 void init_Access(py::module &m)
 {

src/binding/python/Attributable.cpp

@@ -22,11 +22,9 @@
 #include "openPMD/DatatypeHelpers.hpp"
 #include "openPMD/auxiliary/Variant.hpp"
 #include "openPMD/backend/Attribute.hpp"
-#include "openPMD/binding/python/Numpy.hpp"
 
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
-#include <pybind11/stl_bind.h>
+#include "openPMD/binding/python/Common.hpp"
+#include "openPMD/binding/python/Numpy.hpp"
 
 #include <algorithm>
 #include <array>
@@ -35,11 +33,7 @@
 #include <string>
 #include <vector>
 
-namespace py = pybind11;
-using namespace openPMD;
-
 using PyAttributeKeys = std::vector<std::string>;
-// PYBIND11_MAKE_OPAQUE(PyAttributeKeys)
 
 bool setAttributeFromBufferInfo(
     Attributable &attr, std::string const &key, py::buffer &a)

src/binding/python/BaseRecord.cpp

@@ -18,18 +18,14 @@
  * and the GNU Lesser General Public License along with openPMD-api.
  * If not, see <http://www.gnu.org/licenses/>.
  */
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
-
 #include "openPMD/backend/BaseRecord.hpp"
 #include "openPMD/backend/BaseRecordComponent.hpp"
 #include "openPMD/backend/Container.hpp"
 #include "openPMD/backend/MeshRecordComponent.hpp"
 #include "openPMD/backend/PatchRecordComponent.hpp"
-#include "openPMD/binding/python/UnitDimension.hpp"
 
-namespace py = pybind11;
-using namespace openPMD;
+#include "openPMD/binding/python/Common.hpp"
+#include "openPMD/binding/python/UnitDimension.hpp"
 
 void init_BaseRecord(py::module &m)
 {

src/binding/python/BaseRecordComponent.cpp

@@ -18,19 +18,14 @@
  * and the GNU Lesser General Public License along with openPMD-api.
  * If not, see <http://www.gnu.org/licenses/>.
  */
-#include <pybind11/numpy.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
-
-#include "openPMD/Datatype.hpp"
 #include "openPMD/backend/BaseRecordComponent.hpp"
+#include "openPMD/Datatype.hpp"
+
+#include "openPMD/binding/python/Common.hpp"
 #include "openPMD/binding/python/Numpy.hpp"
 
 #include <sstream>
 
-namespace py = pybind11;
-using namespace openPMD;
-
 void init_BaseRecordComponent(py::module &m)
 {
     py::class_<BaseRecordComponent, Attributable>(m, "Base_Record_Component")

src/binding/python/ChunkInfo.cpp

@@ -18,17 +18,13 @@
  * and the GNU Lesser General Public License along with openPMD-api.
  * If not, see <http://www.gnu.org/licenses/>.
  */
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
-
 #include "openPMD/ChunkInfo.hpp"
 
+#include "openPMD/binding/python/Common.hpp"
+
 #include <exception>
 #include <string>
 
-namespace py = pybind11;
-using namespace openPMD;
-
 void init_Chunk(py::module &m)
 {
     py::class_<ChunkInfo>(m, "ChunkInfo")

src/binding/python/Container.cpp

@@ -23,33 +23,16 @@
  *
  * BSD-style license, see pybind11 LICENSE file.
  */
-
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
-#include <pybind11/stl_bind.h>
-
-#include "openPMD/Iteration.hpp"
-#include "openPMD/Mesh.hpp"
-#include "openPMD/ParticlePatches.hpp"
-#include "openPMD/ParticleSpecies.hpp"
-#include "openPMD/Record.hpp"
-#include "openPMD/Series.hpp"
-#include "openPMD/backend/BaseRecord.hpp"
-#include "openPMD/backend/BaseRecordComponent.hpp"
 #include "openPMD/backend/Container.hpp"
-#include "openPMD/backend/MeshRecordComponent.hpp"
-#include "openPMD/backend/PatchRecord.hpp"
-#include "openPMD/backend/PatchRecordComponent.hpp"
+
+#include "openPMD/binding/python/Common.hpp"
 
 #include <cstddef>
 #include <memory>
 #include <sstream>
 #include <string>
 #include <utility>
 
-namespace py = pybind11;
-using namespace openPMD;
-
 namespace detail
 {
 /* based on std_bind.h in pybind11
@@ -156,27 +139,6 @@ bind_container(py::handle scope, std::string const &name, Args &&...args)
 }
 } // namespace detail
 
-using PyIterationContainer = Series::IterationsContainer_t;
-using PyMeshContainer = Container<Mesh>;
-using PyPartContainer = Container<ParticleSpecies>;
-using PyPatchContainer = Container<ParticlePatches>;
-using PyRecordContainer = Container<Record>;
-using PyPatchRecordContainer = Container<PatchRecord>;
-using PyRecordComponentContainer = Container<RecordComponent>;
-using PyMeshRecordComponentContainer = Container<MeshRecordComponent>;
-using PyPatchRecordComponentContainer = Container<PatchRecordComponent>;
-using PyBaseRecordComponentContainer = Container<BaseRecordComponent>;
-PYBIND11_MAKE_OPAQUE(PyIterationContainer)
-PYBIND11_MAKE_OPAQUE(PyMeshContainer)
-PYBIND11_MAKE_OPAQUE(PyPartContainer)
-PYBIND11_MAKE_OPAQUE(PyPatchContainer)
-PYBIND11_MAKE_OPAQUE(PyRecordContainer)
-PYBIND11_MAKE_OPAQUE(PyPatchRecordContainer)
-PYBIND11_MAKE_OPAQUE(PyRecordComponentContainer)
-PYBIND11_MAKE_OPAQUE(PyMeshRecordComponentContainer)
-PYBIND11_MAKE_OPAQUE(PyPatchRecordComponentContainer)
-PYBIND11_MAKE_OPAQUE(PyBaseRecordComponentContainer)
-
 void init_Container(py::module &m)
 {
     ::detail::bind_container<PyIterationContainer>(m, "Iteration_Container");

src/binding/python/Dataset.cpp

@@ -18,17 +18,13 @@
  * and the GNU Lesser General Public License along with openPMD-api.
  * If not, see <http://www.gnu.org/licenses/>.
  */
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
-
 #include "openPMD/Dataset.hpp"
+
+#include "openPMD/binding/python/Common.hpp"
 #include "openPMD/binding/python/Numpy.hpp"
 
 #include <string>
 
-namespace py = pybind11;
-using namespace openPMD;
-
 void init_Dataset(py::module &m)
 {
     py::class_<Dataset>(m, "Dataset")