BUG: Fixing multiple processor patch problems in parallel transfer of

particles.

src/OpenFOAM/meshes/ProcessorTopology/ProcessorTopology.C

@@ -27,6 +27,7 @@ License
 #include "ListOps.H"
 #include "Pstream.H"
 #include "commSchedule.H"
+#include "boolList.H"
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
@@ -42,6 +43,8 @@ Foam::labelList Foam::ProcessorTopology<Patch, ProcPatch>::procNeighbours
 
     label maxNb = 0;
 
+    boolList isNeighbourProc(Pstream::nProcs(), false);
+
     forAll(patches, patchi)
     {
         const Patch& patch = patches[patchi];
@@ -51,19 +54,34 @@ Foam::labelList Foam::ProcessorTopology<Patch, ProcPatch>::procNeighbours
             const ProcPatch& procPatch =
                 refCast<const ProcPatch>(patch);
 
-            nNeighbours++;
+            label pNeighbProcNo = procPatch.neighbProcNo();
 
-            maxNb = max(maxNb, procPatch.neighbProcNo());
+            if (!isNeighbourProc[pNeighbProcNo])
+            {
+                nNeighbours++;
+
+                maxNb = max(maxNb, procPatch.neighbProcNo());
+
+                isNeighbourProc[pNeighbProcNo] = true;
+            }
         }
     }
 
-    labelList neighbours(nNeighbours);
+    labelList neighbours(nNeighbours, -1);
+
+    nNeighbours = 0;
+
+    forAll(isNeighbourProc, procI)
+    {
+        if (isNeighbourProc[procI])
+        {
+            neighbours[nNeighbours++] = procI;
+        }
+    }
 
     procPatchMap_.setSize(maxNb + 1);
     procPatchMap_ = -1;
 
-    nNeighbours = 0;
-
     forAll(patches, patchi)
     {
         const Patch& patch = patches[patchi];
@@ -73,8 +91,6 @@ Foam::labelList Foam::ProcessorTopology<Patch, ProcPatch>::procNeighbours
             const ProcPatch& procPatch =
                 refCast<const ProcPatch>(patch);
 
-            neighbours[nNeighbours++] = procPatch.neighbProcNo();
-
             // Construct reverse map
             procPatchMap_[procPatch.neighbProcNo()] = patchi;
         }

src/OpenFOAM/meshes/ProcessorTopology/ProcessorTopology.H

@@ -30,6 +30,9 @@ Description
 
     *this[procI] gives the list of neighbouring processors.
 
+    TODO: This does not currently correctly support multiple processor
+    patches connecting two processors.
+
 SourceFiles
     ProcessorTopology.C
 

src/lagrangian/basic/Cloud/Cloud.C

@@ -102,8 +102,18 @@ template<class TrackingData>
 void Foam::Cloud<ParticleType>::move(TrackingData& td)
 {
     const globalMeshData& pData = polyMesh_.globalData();
-    const labelList& processorPatches = pData.processorPatches();
+    const labelList& neighbourProcs = pData[Pstream::myProcNo()];
-    const labelList& processorPatchIndices = pData.processorPatchIndices();
+    const labelList& procPatches = pData.processorPatches();
+    const labelList& procPatchIndices = pData.processorPatchIndices();
+    const labelList& procPatchNeighbours = pData.processorPatchNeighbours();
+    const polyBoundaryMesh& pbm = pMesh().boundaryMesh();
+
+    labelList neighbourProcIndices(Pstream::nProcs(), -1);
+
+    forAll(neighbourProcs, i)
+    {
+        neighbourProcIndices[neighbourProcs[i]] = i;
+    }
 
     // Initialise the stepFraction moved for the particles
     forAllIter(typename Cloud<ParticleType>, *this, pIter)
@@ -114,9 +124,19 @@ void Foam::Cloud<ParticleType>::move(TrackingData& td)
     // While there are particles to transfer
     while (true)
     {
-        // List of lists of particles to be transfered for all the processor
+        // List of lists of particles to be transfered for all of the
-        // patches
+        // neighbour processors
-        List<IDLList<ParticleType> > transferList(processorPatches.size());
+        List<IDLList<ParticleType> > particleTransferLists
+        (
+            neighbourProcs.size()
+        );
+
+        // List of destination processorPatches indices for all of the
+        // neighbour processors
+        List<DynamicList<label> > patchIndexTransferLists
+        (
+            neighbourProcs.size()
+        );
 
         // Loop over all particles
         forAllIter(typename Cloud<ParticleType>, *this, pIter)
@@ -134,15 +154,28 @@ void Foam::Cloud<ParticleType>::move(TrackingData& td)
                 // boundary face
                 if (Pstream::parRun() && p.facei_ >= pMesh().nInternalFaces())
                 {
-                    label patchi = pMesh().boundaryMesh().whichPatch(p.facei_);
+                    label patchi = pbm.whichPatch(p.facei_);
-                    label n = processorPatchIndices[patchi];
 
                     // ... and the face is on a processor patch
                     // prepare it for transfer
-                    if (n != -1)
+                    if (procPatchIndices[patchi] != -1)
                     {
+                        label n = neighbourProcIndices
+                        [
+                            refCast<const processorPolyPatch>
+                            (
+                                pbm[patchi]
+                            ).neighbProcNo()
+                        ];
+
                         p.prepareForParallelTransfer(patchi, td);
-                        transferList[n].append(this->remove(&p));
+
+                        particleTransferLists[n].append(this->remove(&p));
+
+                        patchIndexTransferLists[n].append
+                        (
+                            procPatchNeighbours[patchi]
+                        );
                     }
                 }
             }
@@ -157,31 +190,30 @@ void Foam::Cloud<ParticleType>::move(TrackingData& td)
             break;
         }
 
-
         // Allocate transfer buffers
         PstreamBuffers pBufs(Pstream::nonBlocking);
 
         // Stream into send buffers
-        forAll(transferList, i)
+        forAll(particleTransferLists, i)
         {
-            if (transferList[i].size())
+            if (particleTransferLists[i].size())
             {
                 UOPstream particleStream
                 (
-                    refCast<const processorPolyPatch>
+                    neighbourProcs[i],
-                    (
-                        pMesh().boundaryMesh()[processorPatches[i]]
-                    ).neighbProcNo(),
                     pBufs
                 );
 
-                particleStream << transferList[i];
+                particleStream
+                    << labelList(patchIndexTransferLists[i])
+                    << particleTransferLists[i];
             }
         }
 
         // Set up transfers when in non-blocking mode. Returns sizes (in bytes)
         // to be sent/received.
         labelListList allNTrans(Pstream::nProcs());
+
         pBufs.finishedSends(allNTrans);
 
         bool transfered = false;
@@ -203,34 +235,35 @@ void Foam::Cloud<ParticleType>::move(TrackingData& td)
             break;
         }
 
-
         // Retrieve from receive buffers
-        forAll(processorPatches, i)
+        forAll(neighbourProcs, i)
         {
-            label patchi = processorPatches[i];
+            label neighbProci = neighbourProcs[i];
 
-            const processorPolyPatch& procPatch =
+            label nRec = allNTrans[neighbProci][Pstream::myProcNo()];
-                refCast<const processorPolyPatch>
-                (pMesh().boundaryMesh()[patchi]);
 
-            label neighbProci = procPatch.neighbProcNo();
+            if (nRec)
-
-            label nRecPs = allNTrans[neighbProci][Pstream::myProcNo()];
-
-            if (nRecPs)
             {
                 UIPstream particleStream(neighbProci, pBufs);
 
+                labelList receivePatchIndex(particleStream);
+
                 IDLList<ParticleType> newParticles
                 (
                     particleStream,
                     typename ParticleType::iNew(*this)
                 );
 
+                label pI = 0;
+
                 forAllIter(typename Cloud<ParticleType>, newParticles, newpIter)
                 {
                     ParticleType& newp = newpIter();
+
+                    label patchi = procPatches[receivePatchIndex[pI++]];
+
                     newp.correctAfterParallelTransfer(patchi, td);
+
                     addParticle(newParticles.remove(&newp));
                 }
             }