PureSoA: Disable AoS Access (#3290)

## Summary

When using a PureSoA particle container, trying to access the AoS
components will now result in a compile-time error.

Previously it still compiled, and I suspect it even allocated memory for
AoS.
New error message:
```
/hipace/src/Hipace.cpp:1499:39: error: ‘struct amrex::ThisParticleTileHasNoAoS’ has no member named ‘begin’
 1499 |         const auto& pos_structs = aos.begin() + nreal;
      |                                       ^~~~~
```
Using this I found more places that needed to be updated to account for
PureSoA.


PureSoA still has the following issues:
- [x] I believe it’s not possible to use the binning functions (in
`DenseBins.H`) as they require a particle pointer.
- [ ] The `ParticleTileData` struct that is used to access SoA data can
be very large, over 300 bytes in some cases. I recommend changing the
array-of-pointers to a full 2D array. This way ParticleTileData could be
as small as 24 bytes (real pointer, int pointer, stride).


## Additional background

Follow-up to #2878.

## Checklist

The proposed changes:
- [x] fix a bug or incorrect behavior in AMReX
- [ ] add new capabilities to AMReX
- [ ] changes answers in the test suite to more than roundoff level
- [ ] are likely to significantly affect the results of downstream AMReX
users
- [ ] include documentation in the code and/or rst files, if appropriate

---------

Co-authored-by: Axel Huebl <[email protected]>

Author: AlexanderSinn

Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H

@@ -581,14 +581,14 @@ int ParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::GetMeshMetadata(
       for (const auto& kv : pmap)
       {
         // loop over particles in ParticleTile
-        const auto& aos = kv.second.GetArrayOfStructs();
-        const auto &parts = aos();
+        auto& particle_tile = kv.second;
+        auto ptd = particle_tile.getConstParticleTileData();
 
         // visit only the "real" particles, skip the "neighbor" particles.
-        long long numReal = aos.numRealParticles();
+        long long numReal = particle_tile.numRealParticles();
         for (long long i = 0; i < numReal; ++i)
         {
-          const auto &part = parts[i];
+          const auto &part = ptd[i];
           if (part.id() > 0)
           {
 #if (AMREX_SPACEDIM == 1)
@@ -869,7 +869,7 @@ int ParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::AddParticlesSOARea
     for (MyParIter pti(*this->m_particles, level); pti.isValid(); ++pti)
     {
       auto& particle_attributes = pti.GetStructOfArrays();
-      auto& aos = pti.GetArrayOfStructs();
+      auto ptd = pti.GetParticleTile().getParticleTileData();
 
       auto numReal = pti.numParticles();
       // shuffle from the AMReX component order
@@ -881,7 +881,7 @@ int ParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::AddParticlesSOARea
 
         for (long long i = 0; i < numReal; ++i)
         {
-          const auto &part = aos[i];
+          const auto &part = ptd[i];
           if (part.id() > 0)
           {
             pData[i*nComps + j] = realData[i];
@@ -944,7 +944,7 @@ int ParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::AddParticlesSOAInt
     for (MyParIter pti(*this->m_particles, level); pti.isValid(); ++pti)
     {
       auto& particle_attributes = pti.GetStructOfArrays();
-      auto& aos = pti.GetArrayOfStructs();
+      auto ptd = pti.GetParticleTile().getParticleTileData();
 
       auto numReal = pti.numParticles();
       // shuffle from the AMReX component order
@@ -953,7 +953,7 @@ int ParticleDataAdaptor<ParticleType, NArrayReal, NArrayInt>::AddParticlesSOAInt
 
       for (long long i = 0; i < numReal; ++i)
       {
-        const auto &part = aos[i];
+        const auto &part = ptd[i];
         if (part.id() > 0)
         {
           pData[i] = intData[i];

Src/Particle/AMReX_ParticleContainerI.H

@@ -1653,8 +1653,8 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
                 if (npart != 0) {
                     Long last = npart - 1;
                     Long pindex = 0;
+                    auto ptd = particle_tile->getParticleTileData();
                     while (pindex <= last) {
-                        auto ptd = particle_tile->getParticleTileData();
                         ParticleType p(ptd,pindex);
 
                         if ((remove_negative == false) && (p.id() < 0)) {
@@ -1669,7 +1669,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
                             for (int comp = 0; comp < NumIntComps(); comp++) {
                                 soa.GetIntData(comp)[pindex] = soa.GetIntData(comp)[last];
                             }
-                            correctCellVectors(last, pindex, grid, aos[pindex]);
+                            correctCellVectors(last, pindex, grid, ptd[pindex]);
                             --last;
                             continue;
                         }
@@ -1685,7 +1685,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
                             for (int comp = 0; comp < NumIntComps(); comp++) {
                                 soa.GetIntData(comp)[pindex] = soa.GetIntData(comp)[last];
                             }
-                            correctCellVectors(last, pindex, grid, aos[pindex]);
+                            correctCellVectors(last, pindex, grid, ptd[pindex]);
                             --last;
                             continue;
                         }
@@ -1739,7 +1739,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
                             for (int comp = 0; comp < NumIntComps(); comp++) {
                                 soa.GetIntData(comp)[pindex] = soa.GetIntData(comp)[last];
                             }
-                            correctCellVectors(last, pindex, grid, aos[pindex]);
+                            correctCellVectors(last, pindex, grid, ptd[pindex]);
                             --last;
                             continue;
                         }
@@ -2206,13 +2206,15 @@ RedistributeMPI (std::map<int, Vector<char> >& not_ours,
               const auto& src_tile = kv.second;
 
               auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)];
-              auto old_size = dst_tile.GetArrayOfStructs().size();
+              auto old_size = dst_tile.size();
               auto new_size = old_size + src_tile.size();
               dst_tile.resize(new_size);
 
-              Gpu::copyAsync(Gpu::hostToDevice,
-                             src_tile.begin(), src_tile.end(),
-                             dst_tile.GetArrayOfStructs().begin() + old_size);
+              if constexpr(! ParticleType::is_soa_particle) {
+                  Gpu::copyAsync(Gpu::hostToDevice,
+                                 src_tile.begin(), src_tile.end(),
+                                 dst_tile.GetArrayOfStructs().begin() + old_size);
+              }
 
               for (int i = 0; i < NumRealComps(); ++i) {
                   Gpu::copyAsync(Gpu::hostToDevice,

Src/Particle/AMReX_ParticleInit.H

@@ -1143,7 +1143,7 @@ InitRandom (Long                    icount,
                 const auto& src_tile = kv.second;
 
                 auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)];
-                auto old_size = dst_tile.GetArrayOfStructs().size();
+                auto old_size = dst_tile.size();
                 auto new_size = old_size;
                 if constexpr(!ParticleType::is_soa_particle)
                 {
@@ -1286,7 +1286,7 @@ InitRandom (Long                    icount,
                 const auto& src_tile = kv.second;
 
                 auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)];
-                auto old_size = dst_tile.GetArrayOfStructs().size();
+                auto old_size = dst_tile.size();
                 auto new_size = old_size;
                 if constexpr(!ParticleType::is_soa_particle)
                 {

Src/Particle/AMReX_ParticleTile.H

@@ -634,6 +634,12 @@ struct ConstParticleTileData
     }
 };
 
+struct ThisParticleTileHasNoParticleVector {};
+
+struct ThisParticleTileHasNoAoS {
+    using ParticleVector = ThisParticleTileHasNoParticleVector;
+};
+
 template <typename T_ParticleType, int NArrayReal, int NArrayInt,
           template<class> class Allocator=DefaultAllocator>
 struct ParticleTile
@@ -651,7 +657,10 @@ struct ParticleTile
 
     using SuperParticleType = Particle<NStructReal + NArrayReal, NStructInt + NArrayInt>;
 
-    using AoS = ArrayOfStructs<ParticleType, Allocator>;
+    using AoS = typename std::conditional<
+        ParticleType::is_soa_particle,
+        ThisParticleTileHasNoAoS,
+        ArrayOfStructs<ParticleType, Allocator>>::type;
     //using ParticleVector = typename AoS::ParticleVector;
 
     using SoA = StructOfArrays<NArrayReal, NArrayInt, Allocator>;

Tests/Particles/RedistributeSOA/main.cpp

@@ -132,7 +132,7 @@ class TestParticleContainer
             }
 
             auto& particle_tile = DefineAndReturnParticleTile(lev, mfi.index(), mfi.LocalTileIndex());
-            auto old_size = particle_tile.GetArrayOfStructs().size();
+            auto old_size = particle_tile.size();
             auto new_size = old_size + host_real[0].size();
             particle_tile.resize(new_size);
 

Tests/Particles/SENSEI_Insitu_SOA/main.cpp

@@ -115,7 +115,7 @@ class TestParticleContainer
             }
 
             auto& particle_tile = DefineAndReturnParticleTile(lev, mfi.index(), mfi.LocalTileIndex());
-            auto old_size = particle_tile.GetArrayOfStructs().size();
+            auto old_size = particle_tile.size();
             auto new_size = old_size + host_real[0].size();
             particle_tile.resize(new_size);