diff --git a/src/Base/Array4.cpp b/src/Base/Array4.cpp index b014795d..1807372b 100644 --- a/src/Base/Array4.cpp +++ b/src/Base/Array4.cpp @@ -159,7 +159,7 @@ void make_Array4(py::module &m, std::string typestr) // __array_function__ feature requires NumPy 1.16 or later. - // Nvidia GPUs: __cuda_array_interface__ v2 + // Nvidia GPUs: __cuda_array_interface__ v3 // https://numba.readthedocs.io/en/latest/cuda/cuda_array_interface.html .def_property_readonly("__cuda_array_interface__", [](Array4 const & a4) { auto d = array_interface(a4); diff --git a/src/Base/PODVector.cpp b/src/Base/PODVector.cpp index a8b5a541..0a746561 100644 --- a/src/Base/PODVector.cpp +++ b/src/Base/PODVector.cpp @@ -16,11 +16,33 @@ namespace py = pybind11; using namespace amrex; +namespace +{ + /** CPU: __array_interface__ v3 + * + * https://numpy.org/doc/stable/reference/arrays.interface.html + */ + template > + py::dict + array_interface(PODVector const & podvector) + { + auto d = py::dict(); + bool const read_only = false; + d["data"] = py::make_tuple(std::intptr_t(podvector.dataPtr()), read_only); + d["shape"] = py::make_tuple(podvector.size()); + d["strides"] = py::none(); + d["typestr"] = py::format_descriptor::format(); + d["version"] = 3; + return d; + } +} + template > -void make_PODVector(py::module &m, std::string typestr) +void make_PODVector(py::module &m, std::string typestr, std::string allocstr) { - using PODVector_type=PODVector; - auto const podv_name = std::string("PODVector_").append(typestr); + using PODVector_type = PODVector; + auto const podv_name = std::string("PODVector_").append(typestr) + .append("_").append(allocstr); py::class_(m, podv_name.c_str()) .def("__repr__", @@ -60,12 +82,26 @@ void make_PODVector(py::module &m, std::string typestr) // swap .def_property_readonly("__array_interface__", [](PODVector_type const & podvector) { - auto d = py::dict(); - bool const read_only = false; - d["data"] = py::make_tuple(std::intptr_t(podvector.dataPtr()), read_only); - d["shape"] = py::make_tuple(podvector.size()); - d["strides"] = py::none(); - d["typestr"] = py::format_descriptor::format(); + return array_interface(podvector); + }) + .def_property_readonly("__cuda_array_interface__", [](PODVector_type const & podvector) { + // Nvidia GPUs: __cuda_array_interface__ v3 + // https://numba.readthedocs.io/en/latest/cuda/cuda_array_interface.html + auto d = array_interface(podvector); + + // data: + // Because the user of the interface may or may not be in the same context, the most common case is to use cuPointerGetAttribute with CU_POINTER_ATTRIBUTE_DEVICE_POINTER in the CUDA driver API (or the equivalent CUDA Runtime API) to retrieve a device pointer that is usable in the currently active context. + // TODO For zero-size arrays, use 0 here. + + // None or integer + // An optional stream upon which synchronization must take place at the point of consumption, either by synchronizing on the stream or enqueuing operations on the data on the given stream. Integer values in this entry are as follows: + // 0: This is disallowed as it would be ambiguous between None and the default stream, and also between the legacy and per-thread default streams. Any use case where 0 might be given should either use None, 1, or 2 instead for clarity. + // 1: The legacy default stream. + // 2: The per-thread default stream. + // Any other integer: a cudaStream_t represented as a Python integer. + // When None, no synchronization is required. + d["stream"] = py::none(); + d["version"] = 3; return d; }) @@ -75,6 +111,20 @@ void make_PODVector(py::module &m, std::string typestr) ; } +template +void make_PODVector(py::module &m, std::string typestr) +{ + // see Src/Base/AMReX_GpuContainers.H + make_PODVector> (m, typestr, "std"); + make_PODVector> (m, typestr, "arena"); + make_PODVector> (m, typestr, "pinned"); +#ifdef AMREX_USE_GPU + make_PODVector> (m, typestr, "device"); + make_PODVector> (m, typestr, "managed"); + make_PODVector> (m, typestr, "async"); +#endif +} + void init_PODVector(py::module& m) { make_PODVector (m, "real"); make_PODVector (m, "int"); diff --git a/src/Base/Vector.cpp b/src/Base/Vector.cpp index 900d649b..8d1000f9 100644 --- a/src/Base/Vector.cpp +++ b/src/Base/Vector.cpp @@ -20,11 +20,31 @@ namespace py = pybind11; using namespace amrex; +namespace +{ + /** CPU: __array_interface__ v3 + * + * https://numpy.org/doc/stable/reference/arrays.interface.html + */ + template > + py::dict + array_interface(Vector const & vector) + { + auto d = py::dict(); + bool const read_only = false; + d["data"] = py::make_tuple(std::intptr_t(vector.dataPtr()), read_only); + d["shape"] = py::make_tuple(vector.size()); + d["strides"] = py::none(); + d["typestr"] = py::format_descriptor::format(); + d["version"] = 3; + return d; + } +} template > void make_Vector(py::module &m, std::string typestr) { - using Vector_type=Vector; + using Vector_type = Vector; auto const v_name = std::string("Vector_").append(typestr); py::class_(m, v_name.c_str()) @@ -47,15 +67,30 @@ void make_Vector(py::module &m, std::string typestr) .def("size", &Vector_type::size) .def_property_readonly("__array_interface__", [](Vector_type const & vector) { - auto d = py::dict(); - bool const read_only = false; - d["data"] = py::make_tuple(std::intptr_t(vector.dataPtr()), read_only); - d["shape"] = py::make_tuple(vector.size()); - d["strides"] = py::none(); - d["typestr"] = py::format_descriptor::format(); + return array_interface(vector); + }) + .def_property_readonly("__cuda_array_interface__", [](Vector_type const & vector) { + // Nvidia GPUs: __cuda_array_interface__ v3 + // https://numba.readthedocs.io/en/latest/cuda/cuda_array_interface.html + auto d = array_interface(vector); + + // data: + // Because the user of the interface may or may not be in the same context, the most common case is to use cuPointerGetAttribute with CU_POINTER_ATTRIBUTE_DEVICE_POINTER in the CUDA driver API (or the equivalent CUDA Runtime API) to retrieve a device pointer that is usable in the currently active context. + // TODO For zero-size arrays, use 0 here. + + // None or integer + // An optional stream upon which synchronization must take place at the point of consumption, either by synchronizing on the stream or enqueuing operations on the data on the given stream. Integer values in this entry are as follows: + // 0: This is disallowed as it would be ambiguous between None and the default stream, and also between the legacy and per-thread default streams. Any use case where 0 might be given should either use None, 1, or 2 instead for clarity. + // 1: The legacy default stream. + // 2: The per-thread default stream. + // Any other integer: a cudaStream_t represented as a Python integer. + // When None, no synchronization is required. + d["stream"] = py::none(); + d["version"] = 3; return d; }) + // setter & getter .def("__setitem__", [](Vector_type & vector, int const idx, T const value){ vector[idx] = value; }) .def("__getitem__", [](Vector_type & v, int const idx){ return v[idx]; }) diff --git a/src/Particle/ArrayOfStructs.cpp b/src/Particle/ArrayOfStructs.cpp index 634c01e3..c1855126 100644 --- a/src/Particle/ArrayOfStructs.cpp +++ b/src/Particle/ArrayOfStructs.cpp @@ -14,16 +14,63 @@ namespace py = pybind11; using namespace amrex; +namespace +{ + /** CPU: __array_interface__ v3 + * + * https://numpy.org/doc/stable/reference/arrays.interface.html + */ + template class Allocator=DefaultAllocator> + py::dict + array_interface(ArrayOfStructs const & aos) + { + using ParticleType = Particle; + using RealType = typename ParticleType::RealType; + + auto d = py::dict(); + bool const read_only = false; + d["data"] = py::make_tuple(std::intptr_t(aos.dataPtr()), read_only); + d["shape"] = py::make_tuple(aos.size()); + d["strides"] = py::make_tuple(sizeof(ParticleType)); + d["typestr"] = "|V" + std::to_string(sizeof(ParticleType)); + py::list descr; + descr.append(py::make_tuple("x", py::format_descriptor::format())); +#if (AMREX_SPACEDIM >= 2) + descr.append(py::make_tuple("y", py::format_descriptor::format())); +#endif +#if (AMREX_SPACEDIM >= 3) + descr.append(py::make_tuple("z", py::format_descriptor::format())); +#endif + if (NReal > 0) { + for(int ii=0; ii < NReal; ii++) { + descr.append(py::make_tuple("rdata_"+std::to_string(ii),py::format_descriptor::format())); + } + } + descr.append(py::make_tuple("cpuid", py::format_descriptor::format()) ); + if (NInt > 0) { + for(int ii=0; ii < NInt; ++ii) { + descr.append(py::make_tuple("idata_"+std::to_string(ii),py::format_descriptor::format())); + } + } + + d["descr"] = descr; + d["version"] = 3; + return d; + } +} template class Allocator=DefaultAllocator> -void make_ArrayOfStructs(py::module &m) +void make_ArrayOfStructs(py::module &m, std::string allocstr) { - using AOSType = ArrayOfStructs; + using AOSType = ArrayOfStructs; using ParticleType = Particle; - using RealType = typename ParticleType::RealType; - auto const aos_name = std::string("ArrayOfStructs_").append(std::to_string(NReal) + "_" + std::to_string(NInt)); + auto const aos_name = std::string("ArrayOfStructs_") + .append(std::to_string(NReal)).append("_") + .append(std::to_string(NInt)).append("_") + .append(allocstr); py::class_(m, aos_name.c_str()) .def(py::init()) // TODO: @@ -41,35 +88,29 @@ void make_ArrayOfStructs(py::module &m) .def("push_back", &AOSType::push_back) .def("pop_back", &AOSType::pop_back) .def("back", py::overload_cast<>(&AOSType::back),"get back member. Problem!!!!! this is perfo") + // setter & getter .def_property_readonly("__array_interface__", [](AOSType const & aos) { - auto d = py::dict(); - bool const read_only = false; - d["data"] = py::make_tuple(std::intptr_t(aos.dataPtr()), read_only); - d["shape"] = py::make_tuple(aos.size()); - d["strides"] = py::make_tuple(sizeof(ParticleType)); - d["typestr"] = "|V" + std::to_string(sizeof(ParticleType)); - py::list descr; - descr.append(py::make_tuple("x", py::format_descriptor::format())); -#if (AMREX_SPACEDIM >= 2) - descr.append(py::make_tuple("y", py::format_descriptor::format())); -#endif -#if (AMREX_SPACEDIM >= 3) - descr.append(py::make_tuple("z", py::format_descriptor::format())); -#endif - if (NReal > 0) { - for(int ii=0; ii < NReal; ii++) { - descr.append(py::make_tuple("rdata_"+std::to_string(ii),py::format_descriptor::format())); - } - } - descr.append(py::make_tuple("cpuid", py::format_descriptor::format()) ); - if (NInt > 0) { - for(int ii=0; ii < NInt; ++ii) { - descr.append(py::make_tuple("idata_"+std::to_string(ii),py::format_descriptor::format())); - } - } + return array_interface(aos); + }) + .def_property_readonly("__cuda_array_interface__", [](AOSType const & aos) { + // Nvidia GPUs: __cuda_array_interface__ v3 + // https://numba.readthedocs.io/en/latest/cuda/cuda_array_interface.html + auto d = array_interface(aos); + + // data: + // Because the user of the interface may or may not be in the same context, the most common case is to use cuPointerGetAttribute with CU_POINTER_ATTRIBUTE_DEVICE_POINTER in the CUDA driver API (or the equivalent CUDA Runtime API) to retrieve a device pointer that is usable in the currently active context. + // TODO For zero-size arrays, use 0 here. + + // None or integer + // An optional stream upon which synchronization must take place at the point of consumption, either by synchronizing on the stream or enqueuing operations on the data on the given stream. Integer values in this entry are as follows: + // 0: This is disallowed as it would be ambiguous between None and the default stream, and also between the legacy and per-thread default streams. Any use case where 0 might be given should either use None, 1, or 2 instead for clarity. + // 1: The legacy default stream. + // 2: The per-thread default stream. + // Any other integer: a cudaStream_t represented as a Python integer. + // When None, no synchronization is required. + d["stream"] = py::none(); - d["descr"] = descr; d["version"] = 3; return d; }) @@ -79,9 +120,22 @@ void make_ArrayOfStructs(py::module &m) ; } +template +void make_ArrayOfStructs(py::module &m) +{ + // see Src/Base/AMReX_GpuContainers.H + make_ArrayOfStructs (m, "std"); + make_ArrayOfStructs (m, "arena"); + make_ArrayOfStructs (m, "pinned"); +#ifdef AMREX_USE_GPU + make_ArrayOfStructs (m, "device"); + make_ArrayOfStructs (m, "managed"); + make_ArrayOfStructs (m, "async"); +#endif +} + void init_ArrayOfStructs(py::module& m) { - make_ArrayOfStructs< 0, 0> (m); - make_ArrayOfStructs< 7, 0> (m); - make_ArrayOfStructs< 1, 1> (m); - make_ArrayOfStructs< 2, 1> (m); + make_ArrayOfStructs<0, 0> (m); // WarpX 22.07, ImpactX 22.07, HiPACE++ 22.07 + make_ArrayOfStructs<1, 1> (m); // test in ParticleContainer + make_ArrayOfStructs<2, 1> (m); // test } diff --git a/src/Particle/ParticleContainer.cpp b/src/Particle/ParticleContainer.cpp index 3c59032e..c333b7c3 100644 --- a/src/Particle/ParticleContainer.cpp +++ b/src/Particle/ParticleContainer.cpp @@ -24,7 +24,7 @@ using namespace amrex; template -void make_Base_Iterators (py::module &m) +void make_Base_Iterators (py::module &m, std::string allocstr) { using iterator_base = T_ParIterBase; using container = typename iterator_base::ContainerType; @@ -33,7 +33,10 @@ void make_Base_Iterators (py::module &m) constexpr int NArrayReal = container::NArrayReal; constexpr int NArrayInt = container::NArrayInt; - std::string particle_it_base_name = std::string("ParIterBase_").append(std::to_string(NStructReal) + "_" + std::to_string(NStructInt) + "_" + std::to_string(NArrayReal) + "_" + std::to_string(NArrayInt)); + std::string particle_it_base_name = std::string("ParIterBase_") + + std::to_string(NStructReal) + "_" + std::to_string(NStructInt) + "_" + + std::to_string(NArrayReal) + "_" + std::to_string(NArrayInt) + "_" + + allocstr; if (is_const) particle_it_base_name = "Const" + particle_it_base_name; py::class_(m, particle_it_base_name.c_str()) .def(py::init(), @@ -70,7 +73,7 @@ void make_Base_Iterators (py::module &m) } template class Allocator=DefaultAllocator> -void make_Iterators (py::module &m) +void make_Iterators (py::module &m, std::string allocstr) { using iterator = T_ParIter; using container = typename iterator::ContainerType; @@ -80,11 +83,13 @@ void make_Iterators (py::module &m) constexpr int NArrayInt = container::NArrayInt; using iterator_base = amrex::ParIterBase; - make_Base_Iterators< is_const, iterator_base >(m); + make_Base_Iterators< is_const, iterator_base >(m, allocstr); auto particle_it_name = std::string("Par"); if (is_const) particle_it_name += "Const"; - particle_it_name += std::string("Iter_").append(std::to_string(NStructReal) + "_" + std::to_string(NStructInt) + "_" + std::to_string(NArrayReal) + "_" + std::to_string(NArrayInt)); + particle_it_name += std::string("Iter_") + std::to_string(NStructReal) + "_" + + std::to_string(NStructInt) + "_" + std::to_string(NArrayReal) + "_" + + std::to_string(NArrayInt) + "_" + allocstr; py::class_(m, particle_it_name.c_str()) .def("__repr__", [particle_it_name](iterator const & pti) { @@ -101,20 +106,14 @@ void make_Iterators (py::module &m) ; } -template class Allocator=DefaultAllocator> -void make_ParticleContainer_and_Iterators (py::module &m) -{ - using ParticleContainerType = ParticleContainer< - T_NStructReal, T_NStructInt, T_NArrayReal, T_NArrayInt, - Allocator - >; - - using ParticleTileType = typename ParticleContainerType::ParticleTileType; - using AoS = typename ParticleTileType::AoS; - using ParticleInitData = typename ParticleContainerType::ParticleInitData; +template +void make_ParticleInitData (py::module &m) { + using ParticleInitData = ParticleInitType; - auto const particle_init_data_type = std::string("ParticleInitType_").append(std::to_string(T_NStructReal) + "_" + std::to_string(T_NStructInt) + "_" + std::to_string(T_NArrayReal) + "_" + std::to_string(T_NArrayInt)); + auto const particle_init_data_type = + std::string("ParticleInitType_") + std::to_string(T_NStructReal) + "_" + + std::to_string(T_NStructInt) + "_" + std::to_string(T_NArrayReal) + "_" + + std::to_string(T_NArrayInt); py::class_(m, particle_init_data_type.c_str()) .def(py::init<>()) .def_readwrite("real_struct_data", &ParticleInitData::real_struct_data) @@ -122,8 +121,23 @@ void make_ParticleContainer_and_Iterators (py::module &m) .def_readwrite("real_array_data", &ParticleInitData::real_array_data) .def_readwrite("int_array_data", &ParticleInitData::int_array_data) ; +} + +template class Allocator=DefaultAllocator> +void make_ParticleContainer_and_Iterators (py::module &m, std::string allocstr) +{ + using ParticleContainerType = ParticleContainer< + T_NStructReal, T_NStructInt, T_NArrayReal, T_NArrayInt, + Allocator + >; + using ParticleInitData = typename ParticleContainerType::ParticleInitData; + using ParticleTileType = typename ParticleContainerType::ParticleTileType; + //using AoS = typename ParticleContainerType::AoS; - auto const particle_container_type = std::string("ParticleContainer_").append(std::to_string(T_NStructReal) + "_" + std::to_string(T_NStructInt) + "_" + std::to_string(T_NArrayReal) + "_" + std::to_string(T_NArrayInt)); + auto const particle_container_type = std::string("ParticleContainer_") + std::to_string(T_NStructReal) + "_" + + std::to_string(T_NStructInt) + "_" + std::to_string(T_NArrayReal) + "_" + + std::to_string(T_NArrayInt) + "_" + allocstr; py::class_(m, particle_container_type.c_str()) .def(py::init()) .def(py::init()) @@ -213,18 +227,19 @@ void make_ParticleContainer_and_Iterators (py::module &m) .def("RemoveParticlesAtLevel", &ParticleContainerType::RemoveParticlesAtLevel) .def("RemoveParticlesNotAtFinestLevel", &ParticleContainerType::RemoveParticlesNotAtFinestLevel) // void CreateVirtualParticles (int level, AoS& virts) const; - // .def("CreateVirtualParticles", py::overload_cast(&ParticleContainerType::CreateVirtualParticles), py::const_) - .def("CreateVirtualParticles", [](ParticleContainerType& pc, int level, AoS& virts){ return pc.CreateVirtualParticles(level, virts);}) - // void CreateGhostParticles (int level, int ngrow, AoS& ghosts) const; - // .def("CreateGhostParticles", &ParticleContainerType::CreateGhostParticles) - .def("CreateGhostParticles", [](ParticleContainerType& pc, int level, int ngrow, AoS& ghosts) { - return pc.CreateGhostParticles(level, ngrow, ghosts); }) - .def("AddParticlesAtLevel", [](ParticleContainerType& pc, AoS& particles, int level, int nGrow=0) { - pc.AddParticlesAtLevel(particles, level, nGrow); - }) - - // void CreateGhostParticles (int level, int ngrow, ParticleTileType& ghosts) const; - // void AddParticlesAtLevel (ParticleTileType& particles, int level, int nGrow=0); + //.def("CreateVirtualParticles", py::overload_cast(&ParticleContainerType::CreateVirtualParticles, py::const_), + // py::arg("level"), py::arg("virts")) + .def("CreateVirtualParticles", py::overload_cast(&ParticleContainerType::CreateVirtualParticles, py::const_), + py::arg("level"), py::arg("virts")) + //.def("CreateGhostParticles", py::overload_cast(&ParticleContainerType::CreateGhostParticles, py::const_), + // py::arg("level"), py::arg("ngrow"), py::arg("ghosts")) + .def("CreateGhostParticles", py::overload_cast(&ParticleContainerType::CreateGhostParticles, py::const_), + py::arg("level"), py::arg("ngrow"), py::arg("ghosts")) + //.def("AddParticlesAtLevel", py::overload_cast(&ParticleContainerType::AddParticlesAtLevel), + // py::arg("particles"), py::arg("level"), py::arg("ngrow")=0) + .def("AddParticlesAtLevel", py::overload_cast(&ParticleContainerType::AddParticlesAtLevel), + py::arg("particles"), py::arg("level"), py::arg("ngrow")=0) + .def("clearParticles", &ParticleContainerType::clearParticles) // template ::value, int> foo = 0> @@ -322,11 +337,32 @@ void make_ParticleContainer_and_Iterators (py::module &m) ; using iterator = amrex::ParIter; - make_Iterators< false, iterator, Allocator >(m); + make_Iterators< false, iterator, Allocator >(m, allocstr); using const_iterator = amrex::ParConstIter; - make_Iterators< true, const_iterator, Allocator >(m); + make_Iterators< true, const_iterator, Allocator >(m, allocstr); } +template +void make_ParticleContainer_and_Iterators (py::module &m) +{ + make_ParticleInitData(m); + + // see Src/Base/AMReX_GpuContainers.H + make_ParticleContainer_and_Iterators(m, "std"); + make_ParticleContainer_and_Iterators(m, "arena"); + make_ParticleContainer_and_Iterators(m, "pinned"); +#ifdef AMREX_USE_GPU + make_ParticleContainer_and_Iterators(m, "device"); + make_ParticleContainer_and_Iterators(m, "managed"); + make_ParticleContainer_and_Iterators(m, "async"); +#endif +} void init_ParticleContainer(py::module& m) { // TODO: we might need to move all or most of the defines in here into a @@ -334,6 +370,5 @@ void init_ParticleContainer(py::module& m) { make_ParticleContainer_and_Iterators< 1, 1, 2, 1> (m); make_ParticleContainer_and_Iterators< 0, 0, 4, 0> (m); // HiPACE++ 22.07 make_ParticleContainer_and_Iterators< 0, 0, 5, 0> (m); // ImpactX 22.07 - make_ParticleContainer_and_Iterators< 0, 0, 7, 0> (m); make_ParticleContainer_and_Iterators< 0, 0, 37, 1> (m); // HiPACE++ 22.07 } diff --git a/src/Particle/ParticleTile.cpp b/src/Particle/ParticleTile.cpp index a08b9064..62374dba 100644 --- a/src/Particle/ParticleTile.cpp +++ b/src/Particle/ParticleTile.cpp @@ -20,30 +20,40 @@ using namespace amrex; template void make_Particle(py::module &m); +template +void make_ParticleTileData(py::module &m) { + using ParticleTileDataType = ParticleTileData; + using SuperParticleType = Particle; + + auto const particle_tile_data_type = + std::string("ParticleTileData_") + std::to_string(NStructReal) + "_" + + std::to_string(NStructInt) + "_" + std::to_string(NArrayReal) + "_" + + std::to_string(NArrayInt); + py::class_(m, particle_tile_data_type.c_str()) + .def(py::init()) + .def_readonly("m_size", &ParticleTileDataType::m_size) + .def_readonly("m_num_runtime_real", &ParticleTileDataType::m_num_runtime_real) + .def_readonly("m_num_runtime_int", &ParticleTileDataType::m_num_runtime_int) + .def("getSuperParticle", &ParticleTileDataType::getSuperParticle) + .def("setSuperParticle", &ParticleTileDataType::setSuperParticle) + // setter & getter + .def("__setitem__", [](ParticleTileDataType &pdt, int const v, + SuperParticleType const value) { pdt.setSuperParticle(value, v); }) + .def("__getitem__", + [](ParticleTileDataType &pdt, int const v) { return pdt.getSuperParticle(v); }); +} + template class Allocator=DefaultAllocator> -void make_ParticleTile(py::module &m) +void make_ParticleTile(py::module &m, std::string allocstr) { - using ParticleTileDataType = ParticleTileData; - using ParticleTileType=ParticleTile; + using ParticleTileType = ParticleTile; using ParticleType = Particle; - using SuperParticleType = Particle; - auto const particle_tile_data_type = std::string("ParticleTileData_").append(std::to_string(NStructReal) + "_" + std::to_string(NStructInt) + "_" + std::to_string(NArrayReal) + "_" + std::to_string(NArrayInt)); - py::class_(m, particle_tile_data_type.c_str()) - .def(py::init()) - .def_readonly("m_size", &ParticleTileDataType::m_size) - .def_readonly("m_num_runtime_real", &ParticleTileDataType::m_num_runtime_real) - .def_readonly("m_num_runtime_int", &ParticleTileDataType::m_num_runtime_int) - .def("getSuperParticle", &ParticleTileDataType::getSuperParticle) - .def("setSuperParticle", &ParticleTileDataType::setSuperParticle) - // setter & getter - .def("__setitem__", [](ParticleTileDataType & pdt, int const v, SuperParticleType const value){ pdt.setSuperParticle( value, v); }) - .def("__getitem__", [](ParticleTileDataType & pdt, int const v){ return pdt.getSuperParticle(v); }) - ; - - auto const particle_tile_type = std::string("ParticleTile_").append(std::to_string(NStructReal) + "_" + std::to_string(NStructInt) + "_" + std::to_string(NArrayReal) + "_" + std::to_string(NArrayInt)); + auto const particle_tile_type = std::string("ParticleTile_") + std::to_string(NStructReal) + "_" + + std::to_string(NStructInt) + "_" + std::to_string(NArrayReal) + "_" + + std::to_string(NArrayInt) + "_" + allocstr; py::class_(m, particle_tile_type.c_str()) .def(py::init()) .def_readonly_static("NAR", &ParticleTileType::NAR) @@ -96,6 +106,28 @@ void make_ParticleTile(py::module &m) ; } +template +void make_ParticleTile(py::module &m) +{ + make_ParticleTileData(m); + + // see Src/Base/AMReX_GpuContainers.H + make_ParticleTile(m, "std"); + make_ParticleTile(m, "arena"); + make_ParticleTile(m, "pinned"); +#ifdef AMREX_USE_GPU + make_ParticleTile(m, "device"); + make_ParticleTile(m, "managed"); + make_ParticleTile(m, "async"); +#endif +} + void init_ParticleTile(py::module& m) { // TODO: we might need to move all or most of the defines in here into a // test/example submodule, so they do not collide with downstream projects diff --git a/src/Particle/StructOfArrays.cpp b/src/Particle/StructOfArrays.cpp index fca90e46..c698e4bd 100644 --- a/src/Particle/StructOfArrays.cpp +++ b/src/Particle/StructOfArrays.cpp @@ -17,11 +17,12 @@ using namespace amrex; template class Allocator=DefaultAllocator> -void make_StructOfArrays(py::module &m) +void make_StructOfArrays(py::module &m, std::string allocstr) { - using SOAType = StructOfArrays; + using SOAType = StructOfArrays; - auto const soa_name = std::string("StructOfArrays_").append(std::to_string(NReal) + "_" + std::to_string(NInt)); + auto const soa_name = std::string("StructOfArrays_") + std::to_string(NReal) + "_" + + std::to_string(NInt) + "_" + allocstr; py::class_(m, soa_name.c_str()) .def(py::init()) .def("define", &SOAType::define) @@ -41,11 +42,23 @@ void make_StructOfArrays(py::module &m) ; } +template +void make_StructOfArrays(py::module &m) +{ + // see Src/Base/AMReX_GpuContainers.H + make_StructOfArrays(m, "std"); + make_StructOfArrays(m, "arena"); + make_StructOfArrays(m, "pinned"); +#ifdef AMREX_USE_GPU + make_StructOfArrays(m, "device"); + make_StructOfArrays(m, "managed"); + make_StructOfArrays(m, "async"); +#endif +} void init_StructOfArrays(py::module& m) { - make_StructOfArrays< 2, 1 > (m); - make_StructOfArrays< 4, 0 > (m); // HiPACE++ 22.07 - make_StructOfArrays< 5, 0 > (m); // ImpactX 22.07 - make_StructOfArrays< 7, 0 > (m); - make_StructOfArrays< 37, 1> (m); // HiPACE++ 22.07 + make_StructOfArrays< 2, 1>(m); + make_StructOfArrays< 4, 0>(m); // HiPACE++ 22.07 + make_StructOfArrays< 5, 0>(m); // ImpactX 22.07 + make_StructOfArrays<37, 1>(m); // HiPACE++ 22.07 } diff --git a/tests/test_aos.py b/tests/test_aos.py index 59ef4ef3..0085dc4f 100644 --- a/tests/test_aos.py +++ b/tests/test_aos.py @@ -7,7 +7,7 @@ def test_aos_init(): - aos = amrex.ArrayOfStructs_2_1() + aos = amrex.ArrayOfStructs_2_1_std() assert aos.numParticles() == 0 assert aos.numTotalParticles() == aos.numRealParticles() == 0 @@ -15,7 +15,7 @@ def test_aos_init(): def test_aos_push_pop(): - aos = amrex.ArrayOfStructs_2_1() + aos = amrex.ArrayOfStructs_2_1_std() p1 = amrex.Particle_2_1() p1.set_rdata([1.5, 2.2]) p1.set_idata([3]) @@ -50,7 +50,7 @@ def test_aos_push_pop(): def test_array_interface(): - aos = amrex.ArrayOfStructs_2_1() + aos = amrex.ArrayOfStructs_2_1_std() p1 = amrex.Particle_2_1() p1.setPos([1, 2, 3]) p1.set_rdata([4.5, 5.2]) diff --git a/tests/test_particleContainer.py b/tests/test_particleContainer.py index bdd7d6cd..db17e60e 100644 --- a/tests/test_particleContainer.py +++ b/tests/test_particleContainer.py @@ -13,7 +13,7 @@ def Npart(): @pytest.fixture(scope="function") def empty_particle_container(std_geometry, distmap, boxarr): - pc = amrex.ParticleContainer_1_1_2_1(std_geometry, distmap, boxarr) + pc = amrex.ParticleContainer_1_1_2_1_std(std_geometry, distmap, boxarr) return pc @@ -29,7 +29,7 @@ def std_particle(): @pytest.fixture(scope="function") def particle_container(Npart, std_geometry, distmap, boxarr, std_real_box): - pc = amrex.ParticleContainer_1_1_2_1(std_geometry, distmap, boxarr) + pc = amrex.ParticleContainer_1_1_2_1_std(std_geometry, distmap, boxarr) myt = amrex.ParticleInitType_1_1_2_1() myt.real_struct_data = [0.5] myt.int_struct_data = [5] @@ -62,17 +62,17 @@ def test_particleInitType(): def test_n_particles(particle_container, Npart): pc = particle_container assert pc.OK() - assert pc.NStructReal == amrex.ParticleContainer_1_1_2_1.NStructReal == 1 - assert pc.NStructInt == amrex.ParticleContainer_1_1_2_1.NStructInt == 1 - assert pc.NArrayReal == amrex.ParticleContainer_1_1_2_1.NArrayReal == 2 - assert pc.NArrayInt == amrex.ParticleContainer_1_1_2_1.NArrayInt == 1 + assert pc.NStructReal == amrex.ParticleContainer_1_1_2_1_std.NStructReal == 1 + assert pc.NStructInt == amrex.ParticleContainer_1_1_2_1_std.NStructInt == 1 + assert pc.NArrayReal == amrex.ParticleContainer_1_1_2_1_std.NArrayReal == 2 + assert pc.NArrayInt == amrex.ParticleContainer_1_1_2_1_std.NArrayInt == 1 assert ( pc.NumberOfParticlesAtLevel(0) == np.sum(pc.NumberOfParticlesInGrid(0)) == Npart ) def test_pc_init(): - pc = amrex.ParticleContainer_1_1_2_1() + pc = amrex.ParticleContainer_1_1_2_1_std() print("bytespread", pc.ByteSpread()) print("capacity", pc.PrintCapacity()) @@ -93,10 +93,10 @@ def test_pc_init(): print("define particle container") pc.Define(gm, dm, ba) assert pc.OK() - assert pc.NStructReal == amrex.ParticleContainer_1_1_2_1.NStructReal == 1 - assert pc.NStructInt == amrex.ParticleContainer_1_1_2_1.NStructInt == 1 - assert pc.NArrayReal == amrex.ParticleContainer_1_1_2_1.NArrayReal == 2 - assert pc.NArrayInt == amrex.ParticleContainer_1_1_2_1.NArrayInt == 1 + assert pc.NStructReal == amrex.ParticleContainer_1_1_2_1_std.NStructReal == 1 + assert pc.NStructInt == amrex.ParticleContainer_1_1_2_1_std.NStructInt == 1 + assert pc.NArrayReal == amrex.ParticleContainer_1_1_2_1_std.NArrayReal == 2 + assert pc.NArrayInt == amrex.ParticleContainer_1_1_2_1_std.NArrayInt == 1 print("bytespread", pc.ByteSpread()) print("capacity", pc.PrintCapacity()) @@ -123,7 +123,7 @@ def test_pc_init(): print("Iterate particle boxes & set values") lvl = 0 - for pti in amrex.ParIter_1_1_2_1(pc, level=lvl): + for pti in amrex.ParIter_1_1_2_1_std(pc, level=lvl): print("...") assert pti.num_particles == 1 assert pti.num_real_particles == 1 @@ -151,7 +151,7 @@ def test_pc_init(): assert np.allclose(int_arrays[0], np.array([2])) # read-only - for pti in amrex.ParConstIter_1_1_2_1(pc, level=lvl): + for pti in amrex.ParConstIter_1_1_2_1_std(pc, level=lvl): assert pti.num_particles == 1 assert pti.num_real_particles == 1 assert pti.num_neighbor_particles == 0 diff --git a/tests/test_particleTile.py b/tests/test_particleTile.py index a48c8c09..ad1e37bc 100644 --- a/tests/test_particleTile.py +++ b/tests/test_particleTile.py @@ -15,7 +15,7 @@ def test_ptile_data(): def test_ptile_funs(): - pt = amrex.ParticleTile_1_1_2_1() + pt = amrex.ParticleTile_1_1_2_1_std() assert pt.empty() and pt.size() == 0 assert pt.numParticles() == pt.numRealParticles() == pt.numNeighborParticles() == 0 @@ -35,7 +35,7 @@ def test_ptile_funs(): ################ def test_ptile_pushback_ptiledata(): - pt = amrex.ParticleTile_1_1_2_1() + pt = amrex.ParticleTile_1_1_2_1_std() p = amrex.Particle_1_1(1.0, 2.0, 3, 4.0, 5) sp = amrex.Particle_3_2(5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11, 12) pt.push_back(p) @@ -62,7 +62,7 @@ def test_ptile_pushback_ptiledata(): @pytest.mark.skipif(amrex.Config.spacedim != 3, reason="Requires AMREX_SPACEDIM = 3") def test_ptile_access(): - pt = amrex.ParticleTile_1_1_2_1() + pt = amrex.ParticleTile_1_1_2_1_std() sp1 = amrex.Particle_3_2() pt.push_back(sp1) pt.push_back(sp1) @@ -81,7 +81,7 @@ def test_ptile_access(): def test_ptile_soa(): - pt = amrex.ParticleTile_1_1_2_1() + pt = amrex.ParticleTile_1_1_2_1_std() pt.push_back_real(1, 2.1) pt.push_back_real([1.1, 1.3]) @@ -120,7 +120,7 @@ def test_ptile_soa(): @pytest.mark.skipif(amrex.Config.spacedim != 3, reason="Requires AMREX_SPACEDIM = 3") def test_ptile_aos(): - pt = amrex.ParticleTile_1_1_2_1() + pt = amrex.ParticleTile_1_1_2_1_std() p1 = amrex.Particle_1_1() p2 = amrex.Particle_1_1() p1.x = 3.0 diff --git a/tests/test_podvector.py b/tests/test_podvector.py index 37dafd67..6307b440 100644 --- a/tests/test_podvector.py +++ b/tests/test_podvector.py @@ -7,7 +7,7 @@ def test_podvector_init(): - podv = amrex.PODVector_real() + podv = amrex.PODVector_real_std() print(podv.__array_interface__) # podv[0] = 1 # podv[2] = 3 @@ -28,7 +28,7 @@ def test_podvector_init(): def test_array_interface(): - podv = amrex.PODVector_int() + podv = amrex.PODVector_int_std() podv.push_back(1) podv.push_back(2) podv.push_back(1) diff --git a/tests/test_soa.py b/tests/test_soa.py index cc7a0421..b5cea7a0 100644 --- a/tests/test_soa.py +++ b/tests/test_soa.py @@ -7,7 +7,7 @@ def test_soa_init(): - soa = amrex.StructOfArrays_2_1() + soa = amrex.StructOfArrays_2_1_std() print("--test init --") print("num real components", soa.NumRealComps()) print("num int components", soa.NumIntComps()) @@ -51,7 +51,7 @@ def test_soa_init(): def test_soa_from_tile(): - pt = amrex.ParticleTile_1_1_2_1() + pt = amrex.ParticleTile_1_1_2_1_std() p = amrex.Particle_1_1(1.0, 2.0, 3, rdata_0=4.0, idata_1=5) sp = amrex.Particle_3_2( 5.0, 6.0, 7.0, rdata_0=8.0, rdata_1=9.0, rdata_2=10.0, idata_0=11, idata_1=12