ENH: support gather of indirect list via globalIndex

applications/test/globalIndex/Test-globalIndex.C

@@ -28,14 +28,16 @@ Application
     globalIndexTest
 
 Description
-    Simple demonstration and test application for the globalIndex class.
+    Simple tests for the globalIndex class.
 
 \*---------------------------------------------------------------------------*/
 
 #include "globalIndex.H"
+#include "globalMeshData.H"
 #include "argList.H"
 #include "Time.H"
 #include "polyMesh.H"
+#include "IndirectList.H"
 #include "IOstreams.H"
 #include "Random.H"
 
@@ -218,6 +220,34 @@ int main(int argc, char *argv[])
         }
     }
 
+    Info<< "Gather indirect list of boundary points (patch = 0)\n";
+    {
+        const polyPatch& pp = mesh.boundaryMesh()[0];
+
+        // Map mesh point index to local (compact) point index
+        labelList pointToGlobal;
+        labelList uniqueMeshPointLabels;
+
+        autoPtr<globalIndex> globalPointsPtr =
+            mesh.globalData().mergePoints
+            (
+                pp.meshPoints(),
+                pp.meshPointMap(),
+                pointToGlobal,
+                uniqueMeshPointLabels
+            );
+
+        Info<< "local-sizes: " << globalPointsPtr().sizes() << nl;
+
+        UIndirectList<point> procPoints(mesh.points(), uniqueMeshPointLabels);
+        pointField patchPoints;
+
+        globalPointsPtr().gather(procPoints, patchPoints);
+
+        Info<< "gathered point field = " << patchPoints.size() << " points\n";
+    }
+
+    Info<< "\nEnd\n" << endl;
     return 0;
 }
 

src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.H

@@ -237,7 +237,7 @@ public:
         // Other
 
             //- Collect data in processor order on master (== procIDs[0]).
-            //  Needs offsets only on master.
+            //  Offsets needed on master only.
             template<class Container, class Type>
             static void gather
             (
@@ -247,12 +247,25 @@ public:
                 const UList<Type>& fld,
                 List<Type>& allFld,
                 const int tag = UPstream::msgType(),
-                const Pstream::commsTypes commsType =
+                const Pstream::commsTypes = Pstream::commsTypes::nonBlocking
-                    Pstream::commsTypes::nonBlocking
+            );
+
+            //- Collect indirect data in processor order on master
+            //  Offsets needed on master only.
+            template<class Type, class Addr>
+            static void gather
+            (
+                const labelUList& offsets,
+                const label comm,
+                const UList<int>& procIDs,
+                const IndirectListBase<Type, Addr>& fld,
+                List<Type>& allFld,
+                const int tag = UPstream::msgType(),
+                const Pstream::commsTypes = Pstream::commsTypes::scheduled
             );
 
             //- Collect data in processor order on master (== procIDs[0]).
-            //  Needs offsets only on master.
+            //  Offsets needed on master only.
             template<class Container, class Type>
             void gather
             (
@@ -276,8 +289,18 @@ public:
                 const UList<Type>& fld,
                 List<Type>& allFld,
                 const int tag = UPstream::msgType(),
-                const Pstream::commsTypes commsType =
+                const Pstream::commsTypes = Pstream::commsTypes::nonBlocking
-                    Pstream::commsTypes::nonBlocking
+            ) const;
+
+            //- Collect data indirectly in processor order on master.
+            //  Does communication with default communicator and message tag.
+            template<class Type, class Addr>
+            void gather
+            (
+                const IndirectListBase<Type, Addr>& fld,
+                List<Type>& allFld,
+                const int tag = UPstream::msgType(),
+                const Pstream::commsTypes = Pstream::commsTypes::scheduled
             ) const;
 
             //- Collect data in processor order on master.
@@ -288,8 +311,7 @@ public:
                 const UList<Type>& fld,
                 List<Type>& allFld,
                 const int tag = UPstream::msgType(),
-                const Pstream::commsTypes commsType =
+                const Pstream::commsTypes = Pstream::commsTypes::nonBlocking
-                    Pstream::commsTypes::nonBlocking
             );
 
 
@@ -303,8 +325,7 @@ public:
                 const Container& procIDs,
                 List<Type>& fld,
                 const int tag = UPstream::msgType(),
-                const Pstream::commsTypes commsType =
+                const Pstream::commsTypes = Pstream::commsTypes::nonBlocking
-                    Pstream::commsTypes::nonBlocking
             );
 
             //- Inplace collect in processor order on master (== procIDs[0]).
@@ -331,8 +352,7 @@ public:
             (
                 List<Type>& fld,
                 const int tag = UPstream::msgType(),
-                const Pstream::commsTypes commsType =
+                const Pstream::commsTypes = Pstream::commsTypes::nonBlocking
-                    Pstream::commsTypes::nonBlocking
             ) const;
 
             //- Inplace collect data in processor order on master
@@ -343,8 +363,7 @@ public:
             (
                 List<Type>& fld,
                 const int tag = UPstream::msgType(),
-                const Pstream::commsTypes commsType =
+                const Pstream::commsTypes = Pstream::commsTypes::nonBlocking
-                    Pstream::commsTypes::nonBlocking
             );
 
 
@@ -358,8 +377,7 @@ public:
                 const UList<Type>& allFld,
                 UList<Type>& fld,
                 const int tag = UPstream::msgType(),
-                const Pstream::commsTypes commsType =
+                const Pstream::commsTypes = Pstream::commsTypes::nonBlocking
-                    Pstream::commsTypes::nonBlocking
             );
 
             //- Distribute data in processor order. Requires fld to be sized!
@@ -386,8 +404,7 @@ public:
                 const UList<Type>& allFld,
                 UList<Type>& fld,
                 const int tag = UPstream::msgType(),
-                const Pstream::commsTypes commsType =
+                const Pstream::commsTypes = Pstream::commsTypes::nonBlocking
-                    Pstream::commsTypes::nonBlocking
             ) const;
 
             //- Get (potentially remote) data. Elements required given as

src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndexTemplates.C

@@ -42,9 +42,21 @@ void Foam::globalIndex::gather
     const Pstream::commsTypes commsType
 )
 {
+    if
+    (
+        !is_contiguous<Type>::value
+     && commsType == Pstream::commsTypes::nonBlocking
+    )
+    {
+        FatalErrorInFunction
+            << "Cannot use nonBlocking with non-contiguous data"
+            << exit(FatalError);
+        // Could also warn and change to scheduled etc...
+    }
+
     if (Pstream::myProcNo(comm) == procIDs[0])
     {
-        allFld.setSize(off.last());
+        allFld.resize(off.last());
 
         // Assign my local data
         SubList<Type>(allFld, fld.size(), 0) = fld;
@@ -73,7 +85,7 @@ void Foam::globalIndex::gather
                 }
                 else
                 {
-                    IPstream fromSlave
+                    IPstream fromProc
                     (
                         commsType,
                         procIDs[i],
@@ -81,22 +93,15 @@ void Foam::globalIndex::gather
                         tag,
                         comm
                     );
-                    fromSlave >> procSlot;
+                    fromProc >> procSlot;
                 }
             }
         }
         else
         {
-            // nonBlocking
+            // nonBlocking && is_contiguous == true (already checked)
 
-            if (!is_contiguous<Type>::value)
+            const label startOfRequests = Pstream::nRequests();
-            {
-                FatalErrorInFunction
-                    << "nonBlocking not supported for non-contiguous data"
-                    << exit(FatalError);
-            }
-
-            label startOfRequests = Pstream::nRequests();
 
             // Set up reads
             for (label i = 1; i < procIDs.size(); ++i)
@@ -108,7 +113,7 @@ void Foam::globalIndex::gather
                     commsType,
                     procIDs[i],
                     reinterpret_cast<char*>(procSlot.data()),
-                    procSlot.byteSize(),
+                    procSlot.size_bytes(),
                     tag,
                     comm
                 );
@@ -153,16 +158,9 @@ void Foam::globalIndex::gather
         }
         else
         {
-            // nonBlocking
+            // nonBlocking && is_contiguous == true (already checked)
 
-            if (!is_contiguous<Type>::value)
+            const label startOfRequests = Pstream::nRequests();
-            {
-                FatalErrorInFunction
-                    << "nonBlocking not supported for non-contiguous data"
-                    << exit(FatalError);
-            }
-
-            label startOfRequests = Pstream::nRequests();
 
             // Set up write
             OPstream::write
@@ -182,6 +180,64 @@ void Foam::globalIndex::gather
 }
 
 
+template<class Type, class Addr>
+void Foam::globalIndex::gather
+(
+    const labelUList& off,
+    const label comm,
+    const UList<int>& procIDs,
+    const IndirectListBase<Type, Addr>& fld,
+    List<Type>& allFld,
+    const int tag,
+    const Pstream::commsTypes commsType
+)
+{
+    if (commsType == Pstream::commsTypes::nonBlocking)
+    {
+        WarningInFunction
+            << "Cannot use nonBlocking with indirect list of data"
+            << exit(FatalError);
+        // Could also warn and change to scheduled etc...
+    }
+
+    if (Pstream::myProcNo(comm) == procIDs[0])
+    {
+        allFld.resize(off.last());
+
+        // Assign my local data
+        SubList<Type>(allFld, fld.size(), 0) = fld;
+
+        // Already verified commsType != nonBlocking
+        for (label i = 1; i < procIDs.size(); ++i)
+        {
+            SubList<Type> procSlot(allFld, off[i+1]-off[i], off[i]);
+
+            IPstream fromProc
+            (
+                commsType,
+                procIDs[i],
+                0,
+                tag,
+                comm
+            );
+            fromProc >> procSlot;
+        }
+    }
+    else
+    {
+        OPstream toMaster
+        (
+            commsType,
+            procIDs[0],
+            0,
+            tag,
+            comm
+        );
+        toMaster << fld;
+    }
+}
+
+
 template<class Type>
 void Foam::globalIndex::gather
 (
@@ -203,6 +259,28 @@ void Foam::globalIndex::gather
 }
 
 
+template<class Type, class Addr>
+void Foam::globalIndex::gather
+(
+    const IndirectListBase<Type, Addr>& fld,
+    List<Type>& allFld,
+    const int tag,
+    const Pstream::commsTypes commsType
+) const
+{
+    gather
+    (
+        offsets_,
+        UPstream::worldComm,
+        UPstream::procID(UPstream::worldComm),
+        fld,
+        allFld,
+        tag,
+        commsType
+    );
+}
+
+
 template<class Type>
 void Foam::globalIndex::gatherOp
 (
@@ -293,6 +371,18 @@ void Foam::globalIndex::scatter
     const Pstream::commsTypes commsType
 )
 {
+    if
+    (
+        !is_contiguous<Type>::value
+     && commsType == Pstream::commsTypes::nonBlocking
+    )
+    {
+        FatalErrorInFunction
+            << "Cannot use nonBlocking with non-contiguous data"
+            << exit(FatalError);
+        // Could also warn and change to scheduled etc...
+    }
+
     if (Pstream::myProcNo(comm) == procIDs[0])
     {
         fld.deepCopy(SubList<Type>(allFld, off[1]-off[0]));
@@ -305,12 +395,7 @@ void Foam::globalIndex::scatter
         {
             for (label i = 1; i < procIDs.size(); ++i)
             {
-                const SubList<Type> procSlot
+                const SubList<Type> procSlot(allFld, off[i+1]-off[i], off[i]);
-                (
-                    allFld,
-                    off[i+1]-off[i],
-                    off[i]
-                );
 
                 if (is_contiguous<Type>::value)
                 {
@@ -326,7 +411,7 @@ void Foam::globalIndex::scatter
                 }
                 else
                 {
-                    OPstream toSlave
+                    OPstream toProc
                     (
                         commsType,
                         procIDs[i],
@@ -334,32 +419,20 @@ void Foam::globalIndex::scatter
                         tag,
                         comm
                     );
-                    toSlave << procSlot;
+                    toProc << procSlot;
                 }
             }
         }
         else
         {
-            // nonBlocking
+            // nonBlocking && is_contiguous == true (already checked)
 
-            if (!is_contiguous<Type>::value)
+            const label startOfRequests = Pstream::nRequests();
-            {
-                FatalErrorInFunction
-                    << "nonBlocking not supported for non-contiguous data"
-                    << exit(FatalError);
-            }
-
-            label startOfRequests = Pstream::nRequests();
 
             // Set up writes
             for (label i = 1; i < procIDs.size(); ++i)
             {
-                const SubList<Type> procSlot
+                const SubList<Type> procSlot(allFld, off[i+1]-off[i], off[i]);
-                (
-                    allFld,
-                    off[i+1]-off[i],
-                    off[i]
-                );
 
                 OPstream::write
                 (
@@ -411,16 +484,9 @@ void Foam::globalIndex::scatter
         }
         else
         {
-            // nonBlocking
+            // nonBlocking && is_contiguous == true (already checked)
 
-            if (!is_contiguous<Type>::value)
+            const label startOfRequests = Pstream::nRequests();
-            {
-                FatalErrorInFunction
-                    << "nonBlocking not supported for non-contiguous data"
-                    << exit(FatalError);
-            }
-
-            label startOfRequests = Pstream::nRequests();
 
             // Set up read
             IPstream::read
@@ -472,7 +538,7 @@ void Foam::globalIndex::get
     const int tag
 ) const
 {
-    allFld.setSize(globalIds.size());
+    allFld.resize(globalIds.size());
     if (globalIds.size())
     {
         // Sort according to processor