ENH: syncTools: specialisation for contiguous data. Fixes #1986.

applications/test/syncTools/Test-syncTools.C

@@ -3,9 +3,10 @@
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
     \\  /    A nd           | www.openfoam.com
-     \\/     M anipulation  | Copyright (C) 2021 OpenCFD Ltd.
+     \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
+    Copyright (C) 2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -38,6 +39,7 @@ Description
 #include "Random.H"
 #include "PackedList.H"
 #include "flipOp.H"
+#include "pointList.H"
 
 using namespace Foam;
 
@@ -224,8 +226,6 @@ void testSparseData(const polyMesh& mesh, Random& rndGen)
             fullData[meshPointi] = pt;
         }
 
-        //Pout<< "sparseData:" << sparseData << endl;
-
         syncTools::syncPointMap
         (
             mesh,
@@ -310,8 +310,6 @@ void testSparseData(const polyMesh& mesh, Random& rndGen)
             fullData[meshEdgeI] = pt;
         }
 
-        //Pout<< "sparseData:" << sparseData << endl;
-
         syncTools::syncEdgeMap
         (
             mesh,
@@ -585,7 +583,7 @@ public:
 };
 void testEdgeFlip2(const polyMesh& mesh, Random& rndGen)
 {
-    Pout<< nl << "Testing edge-wise (oriented) data synchronisation." << endl;
+    Info<< nl << "Testing edge-wise (oriented) data synchronisation." << endl;
 
     const edgeList& edges = mesh.edges();
     const pointField& points = mesh.points();
@@ -766,6 +764,64 @@ void testEdgeFlip(const polyMesh& mesh, Random& rndGen)
 }
 
 
+class pointListOps
+{
+public:
+
+    void operator()(pointList& lhs, const pointList& rhs) const
+    {
+        forAll(lhs, i)
+        {
+            point& l = lhs[i];
+            const point& r = rhs[i];
+            maxMagSqrEqOp<vector>()(l, r);
+        }
+    }
+
+    void operator()
+    (
+        const vectorTensorTransform& vt,
+        const bool forward,
+        List<pointList>& fld
+    ) const
+    {
+        if (forward)
+        {
+            for (auto& elems : fld)
+            {
+                for (auto& elem : elems)
+                {
+                    elem = vt.transformPosition(elem);
+                }
+            }
+        }
+        else
+        {
+            for (auto& elems : fld)
+            {
+                for (auto& elem : elems)
+                {
+                    elem = vt.invTransformPosition(elem);
+                }
+            }
+        }
+    }
+
+    //- Transform patch-based field
+    void operator()(const coupledPolyPatch& cpp, List<pointList>& fld) const
+    {
+        forAll(fld, facei)
+        {
+            pointList& pts = fld[facei];
+            for (auto& pt : pts)
+            {
+                cpp.transformPosition(pt, facei);
+            }
+        }
+    }
+};
+
+
 void testFaceSync(const polyMesh& mesh, Random& rndGen)
 {
     Info<< nl << "Testing face-wise data synchronisation." << endl;
@@ -795,6 +851,52 @@ void testFaceSync(const polyMesh& mesh, Random& rndGen)
         }
     }
 
+
+    // Test non-contiguous data (uses streaming)
+
+    {
+        List<pointList> syncedFc(mesh.nFaces());
+
+        const pointField& fcs = mesh.faceCentres();
+        forAll(fcs, facei)
+        {
+            const point& fc = fcs[facei];
+            syncedFc[facei].setSize(2, fc);
+        }
+
+        SubList<pointList> bndValues
+        (
+            syncedFc,
+            mesh.nBoundaryFaces(),
+            mesh.nInternalFaces()
+        );
+        syncTools::syncBoundaryFaceList
+        (
+            mesh,
+            bndValues,
+            pointListOps(), //does maxMagSqrEqOp<pointList>()
+            pointListOps()  //transforms all
+        );
+
+        forAll(syncedFc, facei)
+        {
+            const point& fc = fcs[facei];
+            const pointList& fld = syncedFc[facei];
+            forAll(fld, i)
+            {
+                if (mag(fld[i] - fc) > SMALL)
+                {
+                    FatalErrorInFunction
+                        << "Face " << facei
+                        << " original centre " << fc
+                        << " synced centre " << fld[i]
+                        << exit(FatalError);
+                }
+            }
+        }
+    }
+
+
     // Test masterFaces
 
     {

src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C

@@ -1021,6 +1021,101 @@ void Foam::syncTools::syncBoundaryFaceList
     const polyBoundaryMesh& patches = mesh.boundaryMesh();
 
     if (parRun)
+    {
+        if
+        (
+            is_contiguous<T>::value
+         && Pstream::defaultCommsType == Pstream::commsTypes::nonBlocking
+        )
+        {
+            const label startRequest = UPstream::nRequests();
+
+            // Receive buffer
+            List<T> receivedValues(mesh.nBoundaryFaces());
+
+            // Set up reads
+            for (const polyPatch& pp : patches)
+            {
+                if (isA<processorPolyPatch>(pp) && pp.size() > 0)
+                {
+                    const processorPolyPatch& procPatch =
+                        refCast<const processorPolyPatch>(pp);
+
+                    SubList<T> fld
+                    (
+                        receivedValues,
+                        procPatch.size(),
+                        procPatch.offset()
+                    );
+
+                    IPstream::read
+                    (
+                        Pstream::commsTypes::nonBlocking,
+                        procPatch.neighbProcNo(),
+                        reinterpret_cast<char*>(fld.data()),
+                        fld.byteSize()
+                    );
+                }
+            }
+
+            // Set up writes
+            for (const polyPatch& pp : patches)
+            {
+                if (isA<processorPolyPatch>(pp) && pp.size() > 0)
+                {
+                    const processorPolyPatch& procPatch =
+                        refCast<const processorPolyPatch>(pp);
+
+                    const SubList<T> fld
+                    (
+                        faceValues,
+                        procPatch.size(),
+                        procPatch.offset()
+                    );
+
+                    OPstream::write
+                    (
+                        Pstream::commsTypes::nonBlocking,
+                        procPatch.neighbProcNo(),
+                        reinterpret_cast<const char*>(fld.cdata()),
+                        fld.byteSize()
+                    );
+                }
+            }
+
+            // Wait for all comms to finish
+            Pstream::waitRequests(startRequest);
+
+            // Combine with existing data
+            for (const polyPatch& pp : patches)
+            {
+                if (isA<processorPolyPatch>(pp) && pp.size() > 0)
+                {
+                    const processorPolyPatch& procPatch =
+                        refCast<const processorPolyPatch>(pp);
+                    SubList<T> recvFld
+                    (
+                        receivedValues,
+                        procPatch.size(),
+                        procPatch.offset()
+                    );
+                    const List<T>& fakeList = recvFld;
+                    top(procPatch, const_cast<List<T>&>(fakeList));
+
+                    SubList<T> patchValues
+                    (
+                        faceValues,
+                        procPatch.size(),
+                        procPatch.offset()
+                    );
+                    forAll(patchValues, i)
+                    {
+                        cop(patchValues[i], recvFld[i]);
+                    }
+                }
+            }
+        }
+        else
         {
             PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
 
@@ -1037,7 +1132,8 @@ void Foam::syncTools::syncBoundaryFaceList
 
                     // Send slice of values on the patch
                     UOPstream toNbr(procPatch.neighbProcNo(), pBufs);
-                toNbr<< SubList<T>(faceValues, procPatch.size(), patchStart);
+                    toNbr <<
+                        SubList<T>(faceValues, procPatch.size(), patchStart);
                 }
             }
 
@@ -1054,7 +1150,7 @@ void Foam::syncTools::syncBoundaryFaceList
                     const processorPolyPatch& procPatch =
                         refCast<const processorPolyPatch>(pp);
 
-                Field<T> nbrVals(procPatch.size());
+                    List<T> nbrVals(procPatch.size());
 
                     UIPstream fromNbr(procPatch.neighbProcNo(), pBufs);
                     fromNbr >> nbrVals;
@@ -1063,13 +1159,14 @@ void Foam::syncTools::syncBoundaryFaceList
 
                     label bFacei = procPatch.start()-boundaryOffset;
 
-                for (T& nbrVal : nbrVals)
+                    for (const T& nbrVal : nbrVals)
                     {
                         cop(faceValues[bFacei++], nbrVal);
                     }
                 }
             }
         }
+    }
 
     // Do the cyclics.
     for (const polyPatch& pp : patches)
@@ -1089,10 +1186,10 @@ void Foam::syncTools::syncBoundaryFaceList
                 const label nbrStart = nbrPatch.start()-boundaryOffset;
 
                 // Transform (copy of) data on both sides
-                Field<T> ownVals(SubList<T>(faceValues, patchSize, ownStart));
+                List<T> ownVals(SubList<T>(faceValues, patchSize, ownStart));
                 top(nbrPatch, ownVals);
 
-                Field<T> nbrVals(SubList<T>(faceValues, patchSize, nbrStart));
+                List<T> nbrVals(SubList<T>(faceValues, patchSize, nbrStart));
                 top(cycPatch, nbrVals);
 
                 label bFacei = ownStart;
@@ -1145,10 +1242,39 @@ void Foam::syncTools::syncFaceList
 
     if (parRun)
     {
-        PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
+        const label startRequest = UPstream::nRequests();
 
-        // Send
+        // Receive buffers
+        PtrList<PackedList<Width>> recvInfos(patches.size());
 
+        // Set up reads
+        for (const polyPatch& pp : patches)
+        {
+            if (isA<processorPolyPatch>(pp) && pp.size())
+            {
+                const processorPolyPatch& procPatch =
+                    refCast<const processorPolyPatch>(pp);
+
+                const label patchSize = procPatch.size();
+                const label patchi = procPatch.index();
+
+                recvInfos.set(patchi, new PackedList<Width>(patchSize));
+                PackedList<Width>& recvInfo = recvInfos[patchi];
+
+                IPstream::read
+                (
+                    Pstream::commsTypes::nonBlocking,
+                    procPatch.neighbProcNo(),
+                    reinterpret_cast<char*>(recvInfo.storage().data()),
+                    recvInfo.byteSize()
+                );
+            }
+        }
+
+        // Send buffers
+        PtrList<PackedList<Width>> sendInfos(patches.size());
+
+        // Set up writes
         for (const polyPatch& pp : patches)
         {
             if (isA<processorPolyPatch>(pp) && pp.size())
@@ -1161,18 +1287,29 @@ void Foam::syncTools::syncFaceList
                     procPatch.start()-boundaryOffset,
                     procPatch.size()
                 );
+                const label patchi = procPatch.index();
 
-                // Send slice of values on the patch
+                sendInfos.set
-                UOPstream toNbr(procPatch.neighbProcNo(), pBufs);
+                (
-                toNbr<< PackedList<Width>(faceValues, range);
+                    patchi,
+                    new PackedList<Width>(faceValues, range)
+                );
+                PackedList<Width>& sendInfo = sendInfos[patchi];
+
+                OPstream::write
+                (
+                    Pstream::commsTypes::nonBlocking,
+                    procPatch.neighbProcNo(),
+                    reinterpret_cast<const char*>(sendInfo.storage().cdata()),
+                    sendInfo.byteSize()
+                );
             }
         }
 
-        pBufs.finishedSends();
+        // Wait for all comms to finish
-
+        Pstream::waitRequests(startRequest);
-
-        // Receive and combine.
 
+        // Combine with existing data
         for (const polyPatch& pp : patches)
         {
             if (isA<processorPolyPatch>(pp) && pp.size())
@@ -1181,13 +1318,8 @@ void Foam::syncTools::syncFaceList
                     refCast<const processorPolyPatch>(pp);
 
                 const label patchSize = procPatch.size();
-
+                const label patchi = procPatch.index();
-                // Recv slice of values on the patch
+                const PackedList<Width>& recvInfo = recvInfos[patchi];
-                PackedList<Width> recvInfo(patchSize);
-                {
-                    UIPstream fromNbr(procPatch.neighbProcNo(), pBufs);
-                    fromNbr >> recvInfo;
-                }
 
                 // Combine (bitwise)
                 label bFacei = procPatch.start()-boundaryOffset;
@@ -1203,6 +1335,66 @@ void Foam::syncTools::syncFaceList
                 }
             }
         }
+
+
+        //PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
+        //
+        //// Send
+        //
+        //for (const polyPatch& pp : patches)
+        //{
+        //    if (isA<processorPolyPatch>(pp) && pp.size())
+        //    {
+        //        const processorPolyPatch& procPatch =
+        //            refCast<const processorPolyPatch>(pp);
+        //
+        //        const labelRange range
+        //        (
+        //            procPatch.start()-boundaryOffset,
+        //            procPatch.size()
+        //        );
+        //
+        //        // Send slice of values on the patch
+        //        UOPstream toNbr(procPatch.neighbProcNo(), pBufs);
+        //        toNbr<< PackedList<Width>(faceValues, range);
+        //    }
+        //}
+        //
+        //pBufs.finishedSends();
+        //
+        //
+        //// Receive and combine.
+        //
+        //for (const polyPatch& pp : patches)
+        //{
+        //    if (isA<processorPolyPatch>(pp) && pp.size())
+        //    {
+        //        const processorPolyPatch& procPatch =
+        //            refCast<const processorPolyPatch>(pp);
+        //
+        //        const label patchSize = procPatch.size();
+        //
+        //        // Recv slice of values on the patch
+        //        PackedList<Width> recvInfo(patchSize);
+        //        {
+        //            UIPstream fromNbr(procPatch.neighbProcNo(), pBufs);
+        //            fromNbr >> recvInfo;
+        //        }
+        //
+        //        // Combine (bitwise)
+        //        label bFacei = procPatch.start()-boundaryOffset;
+        //        for (label i = 0; i < patchSize; ++i)
+        //        {
+        //            unsigned int recvVal = recvInfo[i];
+        //            unsigned int faceVal = faceValues[bFacei];
+        //
+        //            cop(faceVal, recvVal);
+        //            faceValues.set(bFacei, faceVal);
+        //
+        //            ++bFacei;
+        //        }
+        //    }
+        //}
     }