From f4ebc90a02464c66b89d76f99f8d8912a4fe10c6 Mon Sep 17 00:00:00 2001
From: Mark Olesen <Mark.Olesen@esi-group.com>
Date: Wed, 23 Feb 2022 15:42:58 +0100
Subject: [PATCH] ENH: simplify gather code for decompositionMethods

- use globalIndex for gather/scatter. Reuse code patterns from globalIndex
  for handling of the xadjcy arrays.
---
 .../metisLikeDecomp/metisLikeDecomp.C         | 170 +++++++++++-------
 .../simpleGeomDecomp/simpleGeomDecomp.C       | 156 +++-------------
 2 files changed, 123 insertions(+), 203 deletions(-)

diff --git a/src/parallel/decompose/decompositionMethods/metisLikeDecomp/metisLikeDecomp.C b/src/parallel/decompose/decompositionMethods/metisLikeDecomp/metisLikeDecomp.C
index 39672ca1f5..6aaf26760a 100644
--- a/src/parallel/decompose/decompositionMethods/metisLikeDecomp/metisLikeDecomp.C
+++ b/src/parallel/decompose/decompositionMethods/metisLikeDecomp/metisLikeDecomp.C
@@ -55,51 +55,114 @@ Foam::label Foam::metisLikeDecomp::decomposeGeneral
         Info<< type() << "Decomp : running in parallel."
             << " Decomposing all of graph on master processor." << endl;
     }
-    const globalIndex globalCells(xadj.size()-1);
-    label nTotalConnections = returnReduce(adjncy.size(), sumOp<label>());
 
-    // Send all to master. Use scheduled to save some storage.
+    const globalIndex globalAdjncy(adjncy.size());
+    const globalIndex globalCells(xadj.size()-1);
+
+    List<label> allAdjncy(globalAdjncy.gather(adjncy));
+
+    // Gathering xadj to master is similar to globalIndex gather()
+    // except for the following:
+    //
+    //   - gathered list is size+1
+    //   - apply local to global renumbering
+
+    const UPstream::commsTypes commsType = UPstream::commsTypes::nonBlocking;
+    const label startOfRequests = UPstream::nRequests();
+
+
+    List<label> allXadj;
     if (Pstream::master())
     {
-        List<label> allAdjncy(nTotalConnections);
-        List<label> allXadj(globalCells.totalSize()+1);
-        List<scalar> allWeights(globalCells.totalSize());
+        allXadj.resize(globalCells.totalSize()+1);
+        allXadj.last() = globalAdjncy.totalSize();  // Final end offset
 
-        // Insert my own
-        label nTotalCells = 0;
-        forAll(cWeights, celli)
-        {
-            allXadj[nTotalCells] = xadj[celli];
-            allWeights[nTotalCells++] = cWeights[celli];
-        }
-        nTotalConnections = 0;
-        forAll(adjncy, i)
-        {
-            allAdjncy[nTotalConnections++] = adjncy[i];
-        }
+        // My values - no renumbering required
+        SubList<label>(allXadj, globalCells.localSize(0)) =
+            SubList<label>(xadj, globalCells.localSize(0));
 
-        for (const int slave : Pstream::subProcs())
+        for (const int proci : globalCells.subProcs())
         {
-            IPstream fromSlave(Pstream::commsTypes::scheduled, slave);
-            List<label> nbrAdjncy(fromSlave);
-            List<label> nbrXadj(fromSlave);
-            List<scalar> nbrWeights(fromSlave);
+            SubList<label> procSlot(allXadj, globalCells.range(proci));
 
-            // Append.
-            forAll(nbrXadj, celli)
+            if (procSlot.empty())
             {
-                allXadj[nTotalCells] = nTotalConnections+nbrXadj[celli];
-                allWeights[nTotalCells++] = nbrWeights[celli];
+                // Nothing to do
             }
-            // No need to renumber xadj since already global.
-            forAll(nbrAdjncy, i)
+            else
             {
-                allAdjncy[nTotalConnections++] = nbrAdjncy[i];
+                IPstream::read
+                (
+                    commsType,
+                    proci,
+                    procSlot.data_bytes(),
+                    procSlot.size_bytes(),
+                    UPstream::msgType(),
+                    UPstream::worldComm
+                );
             }
         }
-        allXadj[nTotalCells] = nTotalConnections;
+    }
+    else
+    {
+        // Send my part of the graph (local numbering)
 
-        labelList allDecomp;
+        if (xadj.size() <= 1)
+        {
+            // Nothing to do
+        }
+        else
+        {
+            SubList<label> procSlot(xadj, xadj.size()-1);
+
+            OPstream::write
+            (
+                commsType,
+                UPstream::masterNo(),
+                procSlot.cdata_bytes(),
+                procSlot.size_bytes(),
+                UPstream::msgType(),
+                UPstream::worldComm
+            );
+        }
+    }
+
+    if (commsType == UPstream::commsTypes::nonBlocking)
+    {
+        // Wait for all to finish
+        UPstream::waitRequests(startOfRequests);
+    }
+
+    // Local to global renumbering
+    if (Pstream::master())
+    {
+        for (const int proci : globalCells.subProcs())
+        {
+            SubList<label> procSlot(allXadj, globalCells.range(proci));
+
+            globalAdjncy.inplaceToGlobal(proci, procSlot);
+        }
+    }
+
+    // Ignore zero-sized weights ... and poorly sized ones too
+    List<scalar> allWeights;
+    if
+    (
+        returnReduce
+        (
+            (cWeights.size() == globalCells.localSize()), andOp<bool>()
+        )
+    )
+    {
+        allWeights = globalCells.gather(cWeights);
+    }
+
+
+    // Global decomposition
+    labelList allDecomp;
+
+    if (Pstream::master())
+    {
         decomposeSerial
         (
             allAdjncy,
@@ -108,45 +171,14 @@ Foam::label Foam::metisLikeDecomp::decomposeGeneral
             allDecomp
         );
 
-
-        // Send allFinalDecomp back
-        for (const int slave : Pstream::subProcs())
-        {
-            OPstream toSlave(Pstream::commsTypes::scheduled, slave);
-            toSlave << SubList<label>
-            (
-                allDecomp,
-                globalCells.localSize(slave),
-                globalCells.localStart(slave)
-            );
-        }
-
-        // Get my own part (always first)
-        decomp = SubList<label>(allDecomp, globalCells.localSize());
+        allAdjncy.clear();    // Not needed anymore
+        allXadj.clear();      // ...
+        allWeights.clear();   // ...
     }
-    else
-    {
-        // Send my part of the graph (already in global numbering)
-        {
-            OPstream toMaster
-            (
-                Pstream::commsTypes::scheduled,
-                Pstream::masterNo()
-            );
-            toMaster
-                << adjncy
-                << SubList<label>(xadj, xadj.size()-1)
-                << cWeights;
-        }
 
-        // Receive back decomposition
-        IPstream fromMaster
-        (
-            Pstream::commsTypes::scheduled,
-            Pstream::masterNo()
-        );
-        fromMaster >> decomp;
-    }
+    // The processor-local decomposition (output)
+    decomp.resize_nocopy(globalCells.localSize());
+    globalCells.scatter(allDecomp, decomp);
 
     return 0;
 }
diff --git a/src/parallel/decompose/decompositionMethods/simpleGeomDecomp/simpleGeomDecomp.C b/src/parallel/decompose/decompositionMethods/simpleGeomDecomp/simpleGeomDecomp.C
index 022dbbef62..1c663167cd 100644
--- a/src/parallel/decompose/decompositionMethods/simpleGeomDecomp/simpleGeomDecomp.C
+++ b/src/parallel/decompose/decompositionMethods/simpleGeomDecomp/simpleGeomDecomp.C
@@ -337,73 +337,18 @@ Foam::labelList Foam::simpleGeomDecomp::decompose
     }
     else
     {
-        globalIndex globalNumbers(points.size());
+        const globalIndex globalNumbers(points.size());
 
-        // Collect all points on master
+        pointField allPoints(globalNumbers.gather(points));
+
+        labelList allDecomp;
         if (Pstream::master())
         {
-            pointField allPoints(globalNumbers.totalSize());
-
-            label nTotalPoints = 0;
-            // Master first
-            SubField<point>(allPoints, points.size()) = points;
-            nTotalPoints += points.size();
-
-            // Add received
-            for (const int subproci : Pstream::subProcs())
-            {
-                IPstream fromProc(Pstream::commsTypes::scheduled, subproci);
-                pointField nbrPoints(fromProc);
-                SubField<point>
-                (
-                    allPoints,
-                    nbrPoints.size(),
-                    nTotalPoints
-                ) = nbrPoints;
-                nTotalPoints += nbrPoints.size();
-            }
-
-            // Decompose
-            labelList finalDecomp(decomposeOneProc(allPoints));
-
-            // Send back
-            for (const int subproci : Pstream::subProcs())
-            {
-                OPstream toProc(Pstream::commsTypes::scheduled, subproci);
-                toProc << SubField<label>
-                (
-                    finalDecomp,
-                    globalNumbers.localSize(subproci),
-                    globalNumbers.localStart(subproci)
-                );
-            }
-            // Get my own part
-            finalDecomp.setSize(points.size());
-
-            return finalDecomp;
+            allDecomp = decomposeOneProc(allPoints);
+            allPoints.clear();  // Not needed anymore
         }
-        else
-        {
-            // Send my points
-            {
-                OPstream toMaster
-                (
-                    Pstream::commsTypes::scheduled,
-                    Pstream::masterNo()
-                );
-                toMaster<< points;
-            }
 
-            // Receive back decomposition
-            IPstream fromMaster
-            (
-                Pstream::commsTypes::scheduled,
-                Pstream::masterNo()
-            );
-            labelList finalDecomp(fromMaster);
-
-            return finalDecomp;
-        }
+        return globalNumbers.scatter(allDecomp);
     }
 }
 
@@ -414,88 +359,31 @@ Foam::labelList Foam::simpleGeomDecomp::decompose
     const scalarField& weights
 ) const
 {
-    if (!Pstream::parRun())
+    if (returnReduce((points.size() != weights.size()), orOp<bool>()))
+    {
+        // Ignore zero-sized weights ... and poorly sized ones too
+        return decompose(points);
+    }
+    else if (!Pstream::parRun())
     {
         return decomposeOneProc(points, weights);
     }
     else
     {
-        globalIndex globalNumbers(points.size());
+        const globalIndex globalNumbers(points.size());
 
-        // Collect all points on master
+        pointField allPoints(globalNumbers.gather(points));
+        scalarField allWeights(globalNumbers.gather(weights));
+
+        labelList allDecomp;
         if (Pstream::master())
         {
-            pointField allPoints(globalNumbers.totalSize());
-            scalarField allWeights(globalNumbers.totalSize());
-
-            label nTotalPoints = 0;
-            // Master first
-            SubField<point>(allPoints, points.size()) = points;
-            SubField<scalar>(allWeights, points.size()) = weights;
-            nTotalPoints += points.size();
-
-            // Add received
-            for (const int subproci : Pstream::subProcs())
-            {
-                IPstream fromProc(Pstream::commsTypes::scheduled, subproci);
-                pointField nbrPoints(fromProc);
-                scalarField nbrWeights(fromProc);
-                SubField<point>
-                (
-                    allPoints,
-                    nbrPoints.size(),
-                    nTotalPoints
-                ) = nbrPoints;
-                SubField<scalar>
-                (
-                    allWeights,
-                    nbrWeights.size(),
-                    nTotalPoints
-                ) = nbrWeights;
-                nTotalPoints += nbrPoints.size();
-            }
-
-            // Decompose
-            labelList finalDecomp(decomposeOneProc(allPoints, allWeights));
-
-            // Send back
-            for (const int subproci : Pstream::subProcs())
-            {
-                OPstream toProc(Pstream::commsTypes::scheduled, subproci);
-                toProc << SubField<label>
-                (
-                    finalDecomp,
-                    globalNumbers.localSize(subproci),
-                    globalNumbers.localStart(subproci)
-                );
-            }
-            // Get my own part
-            finalDecomp.setSize(points.size());
-
-            return finalDecomp;
+            allDecomp = decomposeOneProc(allPoints, allWeights);
+            allPoints.clear();  // Not needed anymore
+            allWeights.clear(); // ...
         }
-        else
-        {
-            // Send my points
-            {
-                OPstream toMaster
-                (
-                    Pstream::commsTypes::scheduled,
-                    Pstream::masterNo()
-                );
-                toMaster<< points << weights;
-            }
 
-            // Receive back decomposition
-            IPstream fromMaster
-            (
-                Pstream::commsTypes::scheduled,
-                Pstream::masterNo()
-            );
-            labelList finalDecomp(fromMaster);
-
-            return finalDecomp;
-        }
+        return globalNumbers.scatter(allDecomp);
     }
 }